Never read sysfs through Go's poller, and a GPU that stops answering is skipped

On grid the sampler sat 42 minutes in runtime_pollWait on the Arc A380's
hwmon fan1_input: kernfs accepts epoll, so os.ReadFile made the fd
non-blocking, i915 answered EAGAIN once, and the poller waited for a
readiness kernfs only signals on sysfs_notify. Every graph on the machine
froze while the API carried on.

readStr now reads with plain blocking syscalls. Each GPU's sysfs reads run
under a three-second deadline; a device that does not answer is marked
stalled, logged, and left alone until restart, with the box saying so.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

internal/metrics/gpu.go

@@ -21,6 +21,7 @@ import (
 	"bufio"
 	"encoding/json"
 	"io"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -54,7 +55,8 @@ type GPU struct {
 	MemUsed    uint64             `json:"mem_used,omitempty"` // VRAM used, or the clients' resident bytes where the driver has no VRAM counter
 	MemTotal   uint64             `json:"mem_total,omitempty"`
 	Clients    []GPUClient        `json:"clients"`
-	ClientsOK  bool               `json:"clients_ok"` // false when fdinfo of other users' processes could not be read
+	ClientsOK  bool               `json:"clients_ok"`        // false when fdinfo of other users' processes could not be read
+	Stalled    bool               `json:"stalled,omitempty"` // a sysfs read of this card never returned; its readings are paused until restart
 }
 
 // GPUClient is one process with the device open.
@@ -73,7 +75,7 @@ var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "rade
 
 type gpuDev struct {
 	card, dev, key, bus, driver, name, hwmon string
-	integrated                               bool
+	integrated, stalled                      bool
 	prevIdle, prevEnergy                     float64
 	prevAt                                   time.Time
 }
@@ -117,7 +119,7 @@ func (c *Collector) gpus(now time.Time, cpuTemp float64) []GPU {
 	live := map[string]bool{}
 	for _, d := range g.devs {
 		x := GPU{Name: d.name, Driver: d.driver, Bus: d.bus, Integrated: d.integrated, Clients: []GPUClient{}, ClientsOK: g.ok}
-		d.read(&x, now)
+		d.readWithin(&x, now)
 		if x.Temp == 0 && d.integrated && cpuTemp > 0 {
 			x.Temp, x.TempSrc = cpuTemp, "package"
 		}
@@ -305,6 +307,31 @@ func lspciName(desc string) string {
 	return strings.TrimSpace(vendor + " " + desc)
 }
 
+// readWithin is read with a deadline. A sysfs read that does not come back must not take
+// the whole sampler with it — every graph on every page of this machine stops — so the
+// device is marked stalled, its goroutine abandoned, and the rest of the sample goes on.
+const gpuReadWithin = 3 * time.Second
+
+func (d *gpuDev) readWithin(x *GPU, now time.Time) {
+	if d.stalled {
+		x.Stalled = true
+		return
+	}
+	done := make(chan GPU, 1)
+	go func() {
+		y := *x
+		d.read(&y, now)
+		done <- y
+	}()
+	select {
+	case y := <-done:
+		*x = y
+	case <-time.After(gpuReadWithin):
+		d.stalled, x.Stalled = true, true
+		log.Printf("gpu: %s (%s): a sysfs read did not return within %s; its readings are paused until restart", d.name, d.card, gpuReadWithin)
+	}
+}
+
 // read fills the device-level numbers: clock, idle/busy, hwmon.
 func (d *gpuDev) read(x *GPU, now time.Time) {
 	dt := now.Sub(d.prevAt).Seconds()

internal/metrics/metrics.go

@@ -1352,12 +1352,32 @@ func Signal(pid int, sig syscall.Signal) error {
 
 // ---- small helpers ----
 
+// readStr reads a /proc or /sys file whole, with plain blocking syscalls and never through
+// the runtime poller. os.ReadFile would hand a sysfs file to the poller — kernfs accepts
+// epoll — and a driver that answers a non-blocking read with EAGAIN (i915's fan attribute
+// did, on grid, 2026-10-02) then leaves the reader waiting for a readiness kernfs only
+// signals on sysfs_notify: never. The sampler sat in that wait for 42 minutes.
 func readStr(p string) string {
-	b, err := os.ReadFile(p)
+	fd, err := syscall.Open(p, syscall.O_RDONLY|syscall.O_CLOEXEC, 0)
 	if err != nil {
 		return ""
 	}
-	return string(b)
+	defer syscall.Close(fd)
+	var out []byte
+	buf := make([]byte, 4096)
+	for {
+		n, err := syscall.Read(fd, buf)
+		if n > 0 {
+			out = append(out, buf[:n]...)
+		}
+		if err == syscall.EINTR {
+			continue
+		}
+		if err != nil || n == 0 {
+			break
+		}
+	}
+	return string(out)
 }
 
 func readInt(p string) (int64, bool) {