diff --git a/internal/metrics/gpu.go b/internal/metrics/gpu.go index 1a9e9a0..0021241 100644 --- a/internal/metrics/gpu.go +++ b/internal/metrics/gpu.go @@ -21,6 +21,7 @@ import ( "bufio" "encoding/json" "io" + "log" "os" "os/exec" "path/filepath" @@ -54,7 +55,8 @@ type GPU struct { MemUsed uint64 `json:"mem_used,omitempty"` // VRAM used, or the clients' resident bytes where the driver has no VRAM counter MemTotal uint64 `json:"mem_total,omitempty"` Clients []GPUClient `json:"clients"` - ClientsOK bool `json:"clients_ok"` // false when fdinfo of other users' processes could not be read + ClientsOK bool `json:"clients_ok"` // false when fdinfo of other users' processes could not be read + Stalled bool `json:"stalled,omitempty"` // a sysfs read of this card never returned; its readings are paused until restart } // GPUClient is one process with the device open. @@ -73,7 +75,7 @@ var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "rade type gpuDev struct { card, dev, key, bus, driver, name, hwmon string - integrated bool + integrated, stalled bool prevIdle, prevEnergy float64 prevAt time.Time } @@ -117,7 +119,7 @@ func (c *Collector) gpus(now time.Time, cpuTemp float64) []GPU { live := map[string]bool{} for _, d := range g.devs { x := GPU{Name: d.name, Driver: d.driver, Bus: d.bus, Integrated: d.integrated, Clients: []GPUClient{}, ClientsOK: g.ok} - d.read(&x, now) + d.readWithin(&x, now) if x.Temp == 0 && d.integrated && cpuTemp > 0 { x.Temp, x.TempSrc = cpuTemp, "package" } @@ -305,6 +307,31 @@ func lspciName(desc string) string { return strings.TrimSpace(vendor + " " + desc) } +// readWithin is read with a deadline. A sysfs read that does not come back must not take +// the whole sampler with it — every graph on every page of this machine stops — so the +// device is marked stalled, its goroutine abandoned, and the rest of the sample goes on. +const gpuReadWithin = 3 * time.Second + +func (d *gpuDev) readWithin(x *GPU, now time.Time) { + if d.stalled { + x.Stalled = true + return + } + done := make(chan GPU, 1) + go func() { + y := *x + d.read(&y, now) + done <- y + }() + select { + case y := <-done: + *x = y + case <-time.After(gpuReadWithin): + d.stalled, x.Stalled = true, true + log.Printf("gpu: %s (%s): a sysfs read did not return within %s; its readings are paused until restart", d.name, d.card, gpuReadWithin) + } +} + // read fills the device-level numbers: clock, idle/busy, hwmon. func (d *gpuDev) read(x *GPU, now time.Time) { dt := now.Sub(d.prevAt).Seconds() diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 5292302..6b037a1 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -1352,12 +1352,32 @@ func Signal(pid int, sig syscall.Signal) error { // ---- small helpers ---- +// readStr reads a /proc or /sys file whole, with plain blocking syscalls and never through +// the runtime poller. os.ReadFile would hand a sysfs file to the poller — kernfs accepts +// epoll — and a driver that answers a non-blocking read with EAGAIN (i915's fan attribute +// did, on grid, 2026-10-02) then leaves the reader waiting for a readiness kernfs only +// signals on sysfs_notify: never. The sampler sat in that wait for 42 minutes. func readStr(p string) string { - b, err := os.ReadFile(p) + fd, err := syscall.Open(p, syscall.O_RDONLY|syscall.O_CLOEXEC, 0) if err != nil { return "" } - return string(b) + defer syscall.Close(fd) + var out []byte + buf := make([]byte, 4096) + for { + n, err := syscall.Read(fd, buf) + if n > 0 { + out = append(out, buf[:n]...) + } + if err == syscall.EINTR { + continue + } + if err != nil || n == 0 { + break + } + } + return string(out) } func readInt(p string) (int64, bool) { diff --git a/web/src/Top.tsx b/web/src/Top.tsx index c8549c3..9e115ec 100644 --- a/web/src/Top.tsx +++ b/web/src/Top.tsx @@ -521,7 +521,8 @@ const ENGINES: [string, string][] = [['render', 'render'], ['compute', 'compute' function GPUBox({ g, values, tick, advance, cap }: { g: GPU; values: number[]; tick: number; advance: number; cap: number }) { const engines = ENGINES.filter(([k]) => g.engines && g.engines[k] !== undefined) const temp = g.temp !== undefined && g.temp > 0 ? g.temp : undefined - const how = g.usage_src === 'idle' ? 'busy, from idle residency' : g.usage_src === 'busy' ? 'busy, as the driver reports it' : g.usage_src === 'clients' ? 'the busiest engine, summed over its clients' : 'this driver exposes no activity counter' + const how = g.stalled ? 'a sysfs read of this card stopped answering; its readings are paused until the console restarts' + : g.usage_src === 'idle' ? 'busy, from idle residency' : g.usage_src === 'busy' ? 'busy, as the driver reports it' : g.usage_src === 'clients' ? 'the busiest engine, summed over its clients' : 'this driver exposes no activity counter' return (