Never read sysfs through Go's poller, and a GPU that stops answering is skipped

On grid the sampler sat 42 minutes in runtime_pollWait on the Arc A380's
hwmon fan1_input: kernfs accepts epoll, so os.ReadFile made the fd
non-blocking, i915 answered EAGAIN once, and the poller waited for a
readiness kernfs only signals on sysfs_notify. Every graph on the machine
froze while the API carried on.

readStr now reads with plain blocking syscalls. Each GPU's sysfs reads run
under a three-second deadline; a device that does not answer is marked
stalled, logged, and left alone until restart, with the box saying so.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

internal/metrics/gpu.go

@@ -21,6 +21,7 @@ import (
 	"bufio"
 	"encoding/json"
 	"io"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -54,7 +55,8 @@ type GPU struct {
 	MemUsed    uint64             `json:"mem_used,omitempty"` // VRAM used, or the clients' resident bytes where the driver has no VRAM counter
 	MemTotal   uint64             `json:"mem_total,omitempty"`
 	Clients    []GPUClient        `json:"clients"`
-	ClientsOK  bool               `json:"clients_ok"` // false when fdinfo of other users' processes could not be read
+	ClientsOK  bool               `json:"clients_ok"`        // false when fdinfo of other users' processes could not be read
+	Stalled    bool               `json:"stalled,omitempty"` // a sysfs read of this card never returned; its readings are paused until restart
 }
 
 // GPUClient is one process with the device open.
@@ -73,7 +75,7 @@ var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "rade
 
 type gpuDev struct {
 	card, dev, key, bus, driver, name, hwmon string
-	integrated                               bool
+	integrated, stalled                      bool
 	prevIdle, prevEnergy                     float64
 	prevAt                                   time.Time
 }
@@ -117,7 +119,7 @@ func (c *Collector) gpus(now time.Time, cpuTemp float64) []GPU {
 	live := map[string]bool{}
 	for _, d := range g.devs {
 		x := GPU{Name: d.name, Driver: d.driver, Bus: d.bus, Integrated: d.integrated, Clients: []GPUClient{}, ClientsOK: g.ok}
-		d.read(&x, now)
+		d.readWithin(&x, now)
 		if x.Temp == 0 && d.integrated && cpuTemp > 0 {
 			x.Temp, x.TempSrc = cpuTemp, "package"
 		}
@@ -305,6 +307,31 @@ func lspciName(desc string) string {
 	return strings.TrimSpace(vendor + " " + desc)
 }
 
+// readWithin is read with a deadline. A sysfs read that does not come back must not take
+// the whole sampler with it — every graph on every page of this machine stops — so the
+// device is marked stalled, its goroutine abandoned, and the rest of the sample goes on.
+const gpuReadWithin = 3 * time.Second
+
+func (d *gpuDev) readWithin(x *GPU, now time.Time) {
+	if d.stalled {
+		x.Stalled = true
+		return
+	}
+	done := make(chan GPU, 1)
+	go func() {
+		y := *x
+		d.read(&y, now)
+		done <- y
+	}()
+	select {
+	case y := <-done:
+		*x = y
+	case <-time.After(gpuReadWithin):
+		d.stalled, x.Stalled = true, true
+		log.Printf("gpu: %s (%s): a sysfs read did not return within %s; its readings are paused until restart", d.name, d.card, gpuReadWithin)
+	}
+}
+
 // read fills the device-level numbers: clock, idle/busy, hwmon.
 func (d *gpuDev) read(x *GPU, now time.Time) {
 	dt := now.Sub(d.prevAt).Seconds()

internal/metrics/metrics.go

@@ -1352,12 +1352,32 @@ func Signal(pid int, sig syscall.Signal) error {
 
 // ---- small helpers ----
 
+// readStr reads a /proc or /sys file whole, with plain blocking syscalls and never through
+// the runtime poller. os.ReadFile would hand a sysfs file to the poller — kernfs accepts
+// epoll — and a driver that answers a non-blocking read with EAGAIN (i915's fan attribute
+// did, on grid, 2026-10-02) then leaves the reader waiting for a readiness kernfs only
+// signals on sysfs_notify: never. The sampler sat in that wait for 42 minutes.
 func readStr(p string) string {
-	b, err := os.ReadFile(p)
+	fd, err := syscall.Open(p, syscall.O_RDONLY|syscall.O_CLOEXEC, 0)
 	if err != nil {
 		return ""
 	}
-	return string(b)
+	defer syscall.Close(fd)
+	var out []byte
+	buf := make([]byte, 4096)
+	for {
+		n, err := syscall.Read(fd, buf)
+		if n > 0 {
+			out = append(out, buf[:n]...)
+		}
+		if err == syscall.EINTR {
+			continue
+		}
+		if err != nil || n == 0 {
+			break
+		}
+	}
+	return string(out)
 }
 
 func readInt(p string) (int64, bool) {

web/src/Top.tsx

@@ -521,7 +521,8 @@ const ENGINES: [string, string][] = [['render', 'render'], ['compute', 'compute'
 function GPUBox({ g, values, tick, advance, cap }: { g: GPU; values: number[]; tick: number; advance: number; cap: number }) {
   const engines = ENGINES.filter(([k]) => g.engines && g.engines[k] !== undefined)
   const temp = g.temp !== undefined && g.temp > 0 ? g.temp : undefined
-  const how = g.usage_src === 'idle' ? 'busy, from idle residency' : g.usage_src === 'busy' ? 'busy, as the driver reports it' : g.usage_src === 'clients' ? 'the busiest engine, summed over its clients' : 'this driver exposes no activity counter'
+  const how = g.stalled ? 'a sysfs read of this card stopped answering; its readings are paused until the console restarts'
+    : g.usage_src === 'idle' ? 'busy, from idle residency' : g.usage_src === 'busy' ? 'busy, as the driver reports it' : g.usage_src === 'clients' ? 'the busiest engine, summed over its clients' : 'this driver exposes no activity counter'
   return (
     <div className="t-gpu">
       <div className="t-gpu-graph">

web/src/api.ts

@@ -83,7 +83,7 @@ export interface GPU {
   usage: number; usage_src?: 'idle' | 'busy' | 'clients'; engines?: Record<string, number>
   temp?: number; temp_src?: 'gpu' | 'package'; power_w?: number; power_cap_w?: number; fan_rpm?: number
   mhz?: number; req_mhz?: number; max_mhz?: number; mem_used?: number; mem_total?: number
-  clients: GPUClient[]; clients_ok: boolean
+  clients: GPUClient[]; clients_ok: boolean; stalled?: boolean
 }
 export interface ProcTable { t: number; cpu: number; cores: number[]; cpu_hist: number[]; cpu_model?: string; cpu_mhz?: number; cpu_max_mhz?: number; cpu_temp?: number; core_ids?: number[]; load: [number, number, number]; mem: MemInfo; swap: MemInfo; net: NetIf[]; gpus?: GPU[]; uptime: number; procs: Proc[]; total: number; running: number }
 export interface Sample {