hamlab/internal/metrics/gpu.go

package metrics

// GPU telemetry, the way btop and intel_gpu_top get it without a vendor library: the
// device from sysfs, and who is using it from DRM fdinfo.
//
//   - The device: /sys/class/drm/card<n> with a driver we know to be a GPU. Clock from the
//     driver's frequency files, temperature/power/fan from the hwmon under the device, and
//     how busy it is from the one counter every Intel GT exports without root — RC6 idle
//     residency, so busy = 100 − idle. amdgpu says it outright (gpu_busy_percent).
//   - The clients: every process holding /dev/dri/* open has a drm-engine-<name> running
//     total (nanoseconds) in /proc/<pid>/fdinfo/<fd>; the delta between two reads is that
//     process's share of each engine, and the sum across clients is the engine's load. Other
//     users' fdinfo is unreadable, so when the daemon isn't root it keeps one root child
//     (`hamlab drm-clients`) that does the walk, asked over a pipe — one sudo per daemon
//     start, not one per second.
//
// A GPU with no counter of either kind (Asahi, today) is still listed: a name is better
// than silence about hardware the machine has.

import (
	"bufio"
	"encoding/json"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"hamlab/internal/priv"
)

// GPU is one graphics device as a Sample reports it.
type GPU struct {
	Name       string             `json:"name"`
	Driver     string             `json:"driver"`
	Bus        string             `json:"bus,omitempty"` // PCI address; empty for a platform device
	Integrated bool               `json:"integrated,omitempty"`
	Usage      float64            `json:"usage"`               // % busy
	UsageSrc   string             `json:"usage_src,omitempty"` // "idle" (100 − idle residency), "busy" (the driver's own %), "clients" (the busiest engine); "" when nothing measures it
	Engines    map[string]float64 `json:"engines,omitempty"`   // render/video/copy/enhance… % from the clients, normalised by engine count
	Temp       float64            `json:"temp,omitempty"`
	TempSrc    string             `json:"temp_src,omitempty"` // "gpu" from the device's own sensor, "package" the CPU die an integrated GPU shares
	PowerW     float64            `json:"power_w,omitempty"`
	PowerCapW  float64            `json:"power_cap_w,omitempty"`
	FanRPM     int                `json:"fan_rpm,omitempty"`
	FanPct     *float64           `json:"fan_pct,omitempty"` // fan speed as a % of its range: nvidia-smi's fan.speed, or the hwmon's pwm1; nil when the card has no such reading (0 is a stopped fan)
	MHz        float64            `json:"mhz,omitempty"`     // actual clock (0 when power-gated)
	ReqMHz     float64            `json:"req_mhz,omitempty"` // what the driver asked for
	MaxMHz     float64            `json:"max_mhz,omitempty"`
	MemUsed    uint64             `json:"mem_used,omitempty"` // VRAM used, or the clients' resident bytes where the driver has no VRAM counter
	MemTotal   uint64             `json:"mem_total,omitempty"`
	Clients    []GPUClient        `json:"clients"`
	ClientsOK  bool               `json:"clients_ok"`        // false when fdinfo of other users' processes could not be read
	Stalled    bool               `json:"stalled,omitempty"` // a read of this card has been out for gpuStallAfter; its readings are paused until it returns
}

// GPUClient is one process with the device open.
type GPUClient struct {
	PID     int                `json:"pid"`
	Name    string             `json:"name"`
	Usage   float64            `json:"usage"` // its busiest engine, %
	Engines map[string]float64 `json:"engines,omitempty"`
	Mem     uint64             `json:"mem,omitempty"` // resident bytes
}

// gpuDrivers are DRM drivers that drive a GPU, not just a scanout (simpledrm, ast, the
// Apple display coprocessor) — those have no engines and nothing to say.
var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "radeon": true, "nouveau": true, "nvidia-drm": true, "nvidia": true,
	"asahi": true, "panfrost": true, "panthor": true, "v3d": true, "msm": true, "etnaviv": true, "lima": true, "powervr": true, "virtio_gpu": true}

type gpuDev struct {
	card, dev, key, bus, driver, name, hwmon string
	integrated, stalled                      bool
	prevIdle, prevEnergy                     float64
	prevAt                                   time.Time
	nv                                       []GPUClient  // nvidia: the process table from the last read (pmon), fdinfo says nothing there
	nvRead                                   []GPUClient  // the same, as the read in flight left it: only that read touches it
	pending                                  chan gpuRead // the read in flight, nil when none is
	pendingAt                                time.Time
	last                                     *GPU                  // the last read that came back
	readFn                                   func(*GPU, time.Time) // nil is read (tests stand a slow one in)
}

type gpuRead struct {
	x  GPU
	nv []GPUClient
}

type gpuState struct {
	devs     []*gpuDev
	listedAt time.Time
	clients  map[string]clientPrev  // device key + client id → engine totals at the previous scan
	last     []drmFD                // the newest client walk
	scan     func() ([]drmFD, bool) // the walk; nil is scanClients (tests stand a fake list in)
	helper   *drmHelper
	ok       bool // the last walk saw every process
}

type clientPrev struct {
	ns map[string]uint64
	at time.Time
}

const gpuRelist = time.Minute // how often the card list is re-read: hotplug is rare, the readdir is cheap

// gpus is the GPU section of a sample. cpuTemp stands in for an integrated GPU's
// temperature: it is the same die.
func (c *Collector) gpus(now time.Time, cpuTemp float64) []GPU {
	g := &c.gpu
	if g.devs == nil || now.Sub(g.listedAt) >= gpuRelist {
		g.devs, g.listedAt = listGPUs(g.devs), now
	}
	if len(g.devs) == 0 {
		return []GPU{}
	}
	if g.clients == nil {
		g.clients = map[string]clientPrev{}
	}
	// The clients are read on every sample, like everything else in it: their engine shares
	// move on the same tick as the device's own numbers, at whatever rate is set.
	scan := c.scanClients
	if g.scan != nil {
		scan = g.scan
	}
	g.last, g.ok = scan()
	out := make([]GPU, 0, len(g.devs))
	live := map[string]bool{}
	for _, d := range g.devs {
		x := GPU{Name: d.name, Driver: d.driver, Bus: d.bus, Integrated: d.integrated, Clients: []GPUClient{}, ClientsOK: g.ok}
		d.readWithin(&x, now)
		if x.Temp == 0 && d.integrated && cpuTemp > 0 {
			x.Temp, x.TempSrc = cpuTemp, "package"
		}
		// the clients: one entry per client id, under the first pid that holds it (a child
		// that inherited the fd shares the client)
		byID := map[string]*GPUClient{}
		var order []string
		engines := map[string]float64{}
		var resident uint64
		if d.driver == "nvidia" { // no engine counters in its fdinfo; nvidia-smi pmon is the client list
			x.Clients, x.ClientsOK = append(x.Clients, d.nv...), true
		}
		for _, f := range g.last {
			if d.driver == "nvidia" {
				break
			}
			fi := parseFdinfo(f.Info)
			if fi.pdev != "" && fi.pdev != d.bus || fi.pdev == "" && fi.driver != d.driver {
				continue
			}
			key := d.key + "/" + fi.client
			live[key] = true
			if byID[fi.client] != nil {
				continue
			}
			cl := &GPUClient{PID: f.PID, Name: f.Comm, Mem: fi.mem}
			byID[fi.client] = cl
			order = append(order, fi.client)
			resident += fi.mem
			if prev, ok := g.clients[key]; ok {
				if dt := now.Sub(prev.at).Nanoseconds(); dt > 0 {
					for e, ns := range fi.engines {
						p, ok := prev.ns[e]
						if !ok || ns < p {
							continue
						}
						pct := float64(ns-p) / float64(dt) * 100
						if n := fi.capacity[e]; n > 1 {
							pct /= float64(n)
						}
						pct = round1(pct)
						if cl.Engines == nil {
							cl.Engines = map[string]float64{}
						}
						cl.Engines[e] = pct
						engines[e] += pct
						if pct > cl.Usage {
							cl.Usage = pct
						}
					}
				}
			}
			g.clients[key] = clientPrev{ns: fi.engines, at: now}
		}
		for _, id := range order {
			x.Clients = append(x.Clients, *byID[id])
		}
		sort.SliceStable(x.Clients, func(i, j int) bool { return x.Clients[i].Usage > x.Clients[j].Usage })
		if len(engines) > 0 {
			x.Engines = map[string]float64{}
			for e, v := range engines {
				if v > 100 {
					v = 100
				}
				x.Engines[e] = round1(v)
			}
		}
		if x.UsageSrc == "" && len(x.Engines) > 0 {
			for _, v := range x.Engines {
				if v > x.Usage {
					x.Usage = v
				}
			}
			x.UsageSrc = "clients"
		}
		if x.MemTotal == 0 && x.MemUsed == 0 {
			x.MemUsed = resident
		}
		out = append(out, x)
	}
	for k := range g.clients { // a client that closed the device
		if !live[k] {
			delete(g.clients, k)
		}
	}
	return out
}

// listGPUs enumerates the cards, keeping the counters of those already known.
func listGPUs(known []*gpuDev) []*gpuDev {
	ents, _ := os.ReadDir("/sys/class/drm")
	old := map[string]*gpuDev{}
	for _, d := range known {
		old[d.card] = d
	}
	out := []*gpuDev{}
	for _, e := range ents {
		n := e.Name()
		if !strings.HasPrefix(n, "card") || strings.Contains(n, "-") {
			continue
		}
		card := "/sys/class/drm/" + n
		dev, err := filepath.EvalSymlinks(card + "/device")
		if err != nil {
			continue
		}
		driver := ueventValue(dev+"/uevent", "DRIVER")
		if driver == "" { // some drivers hang the DRIVER key off the parent
			driver = filepath.Base(readLink(dev + "/driver"))
		}
		if !gpuDrivers[driver] {
			continue
		}
		if d, ok := old[card]; ok && d.dev == dev {
			out = append(out, d)
			continue
		}
		d := &gpuDev{card: card, dev: dev, driver: driver}
		if pci := ueventValue(dev+"/uevent", "PCI_SLOT_NAME"); pci != "" {
			d.bus, d.key = pci, pci
			// an on-die GPU hangs off root bus 0 (Intel's 00:02.0, an AMD APU's 00:08.0 bridge)
			d.integrated = strings.HasPrefix(pci, "0000:00:")
		} else {
			d.key = "driver:" + driver
			d.integrated = true
		}
		if hw, _ := filepath.Glob(dev + "/hwmon/hwmon*"); len(hw) > 0 {
			d.hwmon = hw[0]
		}
		d.name = gpuName(dev, d.bus, driver)
		out = append(out, d)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].card < out[j].card })
	return out
}

// RenderNodeName is the GPU behind a render node ("/dev/dri/renderD128" → "Intel Arc A380"),
// named the same way the Processes tab names it; "" when sysfs has nothing for it.
func RenderNodeName(node string) string {
	dev, err := filepath.EvalSymlinks("/sys/class/drm/" + filepath.Base(node) + "/device")
	if err != nil {
		return ""
	}
	driver := ueventValue(dev+"/uevent", "DRIVER")
	if driver == "" {
		driver = filepath.Base(readLink(dev + "/driver"))
	}
	return gpuName(dev, ueventValue(dev+"/uevent", "PCI_SLOT_NAME"), driver)
}

// RenderNodeIntegrated reports a render node whose GPU sits on the CPU die (PCI root bus 0,
// or no PCI bus at all), as opposed to a card on the bus.
func RenderNodeIntegrated(node string) bool {
	dev, err := filepath.EvalSymlinks("/sys/class/drm/" + filepath.Base(node) + "/device")
	if err != nil {
		return true
	}
	pci := ueventValue(dev+"/uevent", "PCI_SLOT_NAME")
	return pci == "" || strings.HasPrefix(pci, "0000:00:")
}

var gpuVendors = map[string]string{"0x8086": "Intel", "0x1002": "AMD", "0x10de": "NVIDIA", "0x1af4": "Virtio", "0x106b": "Apple"}

var (
	revRe     = regexp.MustCompile(`\s*\(rev [^)]*\)`)
	bracketRe = regexp.MustCompile(`\[([^\]]+)\]`)
)

// gpuName is "Intel Arc A380": from the embedded PCI ID table first, so the name is the
// same on every machine; from lspci for a card newer than the table; and failing both,
// the vendor and the raw device ID.
func gpuName(dev, bus, driver string) string {
	if bus != "" {
		vid := strings.TrimPrefix(strings.TrimSpace(readStr(dev+"/vendor")), "0x")
		did := strings.TrimPrefix(strings.TrimSpace(readStr(dev+"/device")), "0x")
		if vn, dn := pciName(vid, did); dn != "" {
			return lspciName(vn + " " + dn)
		}
		if out, ok := runOut("lspci", "-s", bus); ok {
			if _, desc, found := strings.Cut(out, ": "); found {
				return lspciName(desc)
			}
		}
	}
	vendor := gpuVendors[strings.TrimSpace(readStr(dev+"/vendor"))]
	if compat := readStr(dev + "/of_node/compatible"); compat != "" { // "apple,agx-t8103\0apple,agx-g13g"
		first := strings.SplitN(compat, "\x00", 2)[0]
		if v, model, ok := strings.Cut(first, ","); ok {
			if vendor == "" && v != "" {
				vendor = strings.ToUpper(v[:1]) + v[1:]
			}
			return vendor + " GPU (" + model + ")"
		}
	}
	if vendor == "" && driver != "" {
		vendor = strings.ToUpper(driver[:1]) + driver[1:]
	}
	if id := strings.TrimPrefix(strings.TrimSpace(readStr(dev+"/device")), "0x"); id != "" {
		return strings.TrimSpace(vendor + " GPU " + id)
	}
	return strings.TrimSpace(vendor + " GPU")
}

// lspciName turns "Intel Corporation DG2 [Arc A380] (rev 05)" into "Intel Arc A380".
func lspciName(desc string) string {
	desc = revRe.ReplaceAllString(strings.TrimSpace(desc), "")
	vendor := ""
	for _, v := range []struct{ long, short string }{{"Intel Corporation", "Intel"}, {"Advanced Micro Devices, Inc. [AMD/ATI]", "AMD"}, {"Advanced Micro Devices, Inc. [AMD]", "AMD"}, {"NVIDIA Corporation", "NVIDIA"}, {"Red Hat, Inc.", "Virtio"}} {
		if strings.HasPrefix(desc, v.long) {
			vendor = v.short
			desc = strings.TrimSpace(desc[len(v.long):])
			break
		}
	}
	if vendor == "" { // vendor not recognised: its first word
		if i := strings.IndexByte(desc, ' '); i > 0 {
			vendor, desc = desc[:i], strings.TrimSpace(desc[i+1:])
		}
	}
	if m := bracketRe.FindStringSubmatch(desc); m != nil {
		desc = m[1]
	}
	return strings.TrimSpace(vendor + " " + desc)
}

// readWithin is read with a deadline. A read that does not come back must not take the
// whole sampler with it — every graph on every page of this machine stops — so one still out
// after gpuReadWithin is left running, the sample goes on with the card's last readings, and
// no second read starts until it returns. Late is normal for nvidia-smi on a busy card; only a
// read out for gpuStallAfter marks the card stalled, and that clears the moment it returns. A
// sysfs read that never returns keeps it stalled, at the cost of one goroutine.
var (
	gpuReadWithin = 3 * time.Second
	gpuStallAfter = 15 * time.Second // past both of readNvidia's nvidia-smi calls timing out (runOut, 5 s each)
)

func (d *gpuDev) readWithin(x *GPU, now time.Time) {
	var r gpuRead
	back := false
	if d.pending == nil {
		done := make(chan gpuRead, 1)
		y := *x
		read := d.read
		if d.readFn != nil {
			read = d.readFn
		}
		go func() {
			read(&y, now)
			done <- gpuRead{y, d.nvRead}
		}()
		d.pending, d.pendingAt = done, now
		select {
		case r, back = <-done:
		case <-time.After(gpuReadWithin):
		}
	} else { // started on an earlier sample: taken if it is back, never waited on again
		select {
		case r, back = <-d.pending:
		default:
		}
	}
	ok := x.ClientsOK
	defer func() { x.ClientsOK = ok }()
	if back {
		if d.stalled {
			log.Printf("gpu: %s (%s): its read came back after %s; readings resume", d.name, d.card, now.Sub(d.pendingAt).Round(time.Second))
		}
		d.pending, d.nv, d.stalled, d.last = nil, r.nv, false, &r.x
		*x = r.x
		return
	}
	if d.last != nil {
		*x = *d.last
	}
	if now.Sub(d.pendingAt) >= gpuStallAfter {
		if !d.stalled {
			log.Printf("gpu: %s (%s): a read has not returned in %s; its readings are paused until it does", d.name, d.card, gpuStallAfter)
		}
		d.stalled, x.Stalled = true, true
	}
}

// read fills the device-level numbers: clock, idle/busy, hwmon.
func (d *gpuDev) read(x *GPU, now time.Time) {
	dt := now.Sub(d.prevAt).Seconds()
	switch d.driver {
	case "nvidia":
		d.readNvidia(x)
	case "i915":
		x.MHz = readF(d.card + "/gt_act_freq_mhz")
		x.ReqMHz = readF(d.card + "/gt_cur_freq_mhz")
		x.MaxMHz = readF(d.card + "/gt_max_freq_mhz")
		idle, ok := readInt(d.card + "/gt/gt0/rc6_residency_ms")
		if !ok {
			idle, ok = readInt(d.card + "/power/rc6_residency_ms")
		}
		if ok {
			d.busyFromIdle(x, float64(idle), dt)
		}
	case "xe":
		gt := d.dev + "/tile0/gt0"
		x.MHz = readF(gt + "/freq0/act_freq")
		x.ReqMHz = readF(gt + "/freq0/cur_freq")
		x.MaxMHz = readF(gt + "/freq0/max_freq")
		if idle, ok := readInt(gt + "/gtidle/idle_residency_ms"); ok {
			d.busyFromIdle(x, float64(idle), dt)
		}
	case "amdgpu":
		if v, ok := readInt(d.dev + "/gpu_busy_percent"); ok {
			x.Usage, x.UsageSrc = float64(v), "busy"
		}
		x.MemUsed = uint64(readF(d.dev + "/mem_info_vram_used"))
		x.MemTotal = uint64(readF(d.dev + "/mem_info_vram_total"))
		// "1: 1800Mhz *" — the starred level is the current one, the last the highest
		for _, line := range strings.Split(readStr(d.dev+"/pp_dpm_sclk"), "\n") {
			f := strings.Fields(line)
			if len(f) < 2 {
				continue
			}
			mhz := leadingNum(f[1])
			x.MaxMHz = mhz
			if strings.HasSuffix(line, "*") {
				x.MHz = mhz
			}
		}
	}
	if d.hwmon != "" {
		if t := readF(d.hwmon + "/temp1_input"); t > 0 {
			x.Temp, x.TempSrc = t/1000, "gpu"
		}
		if p := readF(d.hwmon + "/power1_input"); p > 0 {
			x.PowerW = round1(p / 1e6)
		} else if p := readF(d.hwmon + "/power1_average"); p > 0 {
			x.PowerW = round1(p / 1e6)
		} else if e := readF(d.hwmon + "/energy1_input"); e > 0 { // µJ, cumulative
			if d.prevEnergy > 0 && e >= d.prevEnergy && dt > 0 && !d.prevAt.IsZero() {
				x.PowerW = round1((e - d.prevEnergy) / 1e6 / dt)
			}
			d.prevEnergy = e
		}
		if p := readF(d.hwmon + "/power1_cap"); p > 0 {
			x.PowerCapW = p / 1e6
		} else if p := readF(d.hwmon + "/power1_max"); p > 0 {
			x.PowerCapW = p / 1e6
		}
		x.FanRPM = int(readF(d.hwmon + "/fan1_input"))
		if pwm, err := strconv.ParseFloat(strings.TrimSpace(readStr(d.hwmon+"/pwm1")), 64); err == nil {
			top := readF(d.hwmon + "/pwm1_max")
			if top <= 0 {
				top = 255
			}
			v := round1(pwm / top * 100)
			x.FanPct = &v
		}
	}
	d.prevAt = now
}

// busyFromIdle: the GT's idle residency is a running ms total; the share of the interval
// it did not cover is how busy the device was.
func (d *gpuDev) busyFromIdle(x *GPU, idle, dt float64) {
	x.UsageSrc = "idle"
	if d.prevAt.IsZero() || dt <= 0 || idle < d.prevIdle {
		d.prevIdle = idle
		return
	}
	busy := 100 - (idle-d.prevIdle)/(dt*1000)*100
	if busy < 0 {
		busy = 0
	}
	if busy > 100 {
		busy = 100
	}
	x.Usage = round1(busy)
	d.prevIdle = idle
}

// ---- DRM fdinfo: who has the device open ----

// drmFD is one client of a DRM device: the fdinfo text of one open fd, with its owner.
type drmFD struct {
	PID  int    `json:"pid"`
	Comm string `json:"comm"`
	Info string `json:"info"`
}

type fdinfo struct {
	driver, pdev, client string
	engines              map[string]uint64 // ns, cumulative
	capacity             map[string]int
	mem                  uint64 // bytes
}

func parseFdinfo(s string) fdinfo {
	fi := fdinfo{engines: map[string]uint64{}, capacity: map[string]int{}}
	var resident, memory uint64
	for _, line := range strings.Split(s, "\n") {
		k, v, ok := strings.Cut(line, ":")
		if !ok || !strings.HasPrefix(k, "drm-") {
			continue
		}
		v = strings.TrimSpace(v)
		switch {
		case k == "drm-driver":
			fi.driver = v
		case k == "drm-pdev":
			fi.pdev = v
		case k == "drm-client-id":
			fi.client = v
		case strings.HasPrefix(k, "drm-engine-capacity-"):
			fi.capacity[k[len("drm-engine-capacity-"):]], _ = strconv.Atoi(v)
		case strings.HasPrefix(k, "drm-engine-"):
			n, _ := strconv.ParseUint(strings.TrimSuffix(v, " ns"), 10, 64)
			fi.engines[k[len("drm-engine-"):]] = n
		case strings.HasPrefix(k, "drm-resident-"):
			resident += kib(v)
		case strings.HasPrefix(k, "drm-memory-"):
			memory += kib(v)
		}
	}
	fi.mem = resident
	if fi.mem == 0 {
		fi.mem = memory
	}
	return fi
}

// kib parses "52624 KiB" (or MiB, or a bare count of bytes) to bytes.
func kib(v string) uint64 {
	f := strings.Fields(v)
	if len(f) == 0 {
		return 0
	}
	n, _ := strconv.ParseUint(f[0], 10, 64)
	if len(f) > 1 {
		switch f[1] {
		case "KiB":
			n <<= 10
		case "MiB":
			n <<= 20
		case "GiB":
			n <<= 30
		}
	}
	return n
}

// drmScan walks /proc for every fd open on /dev/dri/*, one entry per client per process.
// Run as root it sees everyone; as a user, only that user's processes.
func drmScan() (out []drmFD, complete bool) {
	ents, err := os.ReadDir("/proc")
	if err != nil {
		return []drmFD{}, false
	}
	complete = true
	out = []drmFD{}
	for _, e := range ents {
		pid, err := strconv.Atoi(e.Name())
		if err != nil {
			continue
		}
		fds, err := os.ReadDir("/proc/" + e.Name() + "/fd")
		if err != nil {
			if !os.IsNotExist(err) {
				complete = false
			}
			continue
		}
		seen := map[string]bool{}
		comm := ""
		for _, fd := range fds {
			target, err := os.Readlink("/proc/" + e.Name() + "/fd/" + fd.Name())
			if err != nil || !strings.HasPrefix(target, "/dev/dri/") {
				continue
			}
			info := readStr("/proc/" + e.Name() + "/fdinfo/" + fd.Name())
			_, rest, ok := strings.Cut(info, "drm-client-id:")
			if !ok {
				continue
			}
			id := strings.TrimSpace(strings.SplitN(rest, "\n", 2)[0])
			if seen[id] {
				continue
			}
			seen[id] = true
			if comm == "" {
				comm = strings.TrimSpace(readStr("/proc/" + e.Name() + "/comm"))
			}
			out = append(out, drmFD{PID: pid, Comm: comm, Info: info})
		}
	}
	sort.SliceStable(out, func(i, j int) bool { return out[i].PID < out[j].PID })
	return out, complete
}

// scanClients walks in-process when that sees everything (root), else through the helper.
func (c *Collector) scanClients() ([]drmFD, bool) {
	if priv.Mode() == "root" || !priv.Available() {
		return drmScan()
	}
	if c.gpu.helper == nil {
		c.gpu.helper = &drmHelper{}
	}
	if fds, ok := c.gpu.helper.scan(); ok {
		return fds, true
	}
	return drmScan() // the helper is down: our own processes are better than nothing
}

// drmHelper is the root child that answers "scan" requests. One line in, one JSON line out.
type drmHelper struct {
	mu       sync.Mutex
	cmd      *exec.Cmd
	in       io.WriteCloser
	out      *bufio.Reader
	failedAt time.Time
}

const helperRetry = 30 * time.Second

func (h *drmHelper) scan() ([]drmFD, bool) {
	h.mu.Lock()
	defer h.mu.Unlock()
	if h.cmd == nil {
		if !h.failedAt.IsZero() && time.Since(h.failedAt) < helperRetry {
			return nil, false
		}
		if !h.start() {
			h.failedAt = time.Now()
			return nil, false
		}
	}
	// a stuck child must not stall the sampler: it is killed, and tried again later
	cmd := h.cmd
	stop := time.AfterFunc(5*time.Second, func() { cmd.Process.Kill() })
	defer stop.Stop()
	if _, err := io.WriteString(h.in, "\n"); err != nil {
		h.fail()
		return nil, false
	}
	line, err := h.out.ReadBytes('\n')
	if err != nil {
		h.fail()
		return nil, false
	}
	var fds []drmFD
	if json.Unmarshal(line, &fds) != nil {
		h.fail()
		return nil, false
	}
	if fds == nil {
		fds = []drmFD{}
	}
	return fds, true
}

func (h *drmHelper) start() bool {
	self, err := os.Executable()
	if err != nil {
		return false
	}
	full := priv.Wrap(self, "drm-clients")
	cmd := exec.Command(full[0], full[1:]...)
	in, err := cmd.StdinPipe()
	if err != nil {
		return false
	}
	out, err := cmd.StdoutPipe()
	if err != nil {
		return false
	}
	if err := cmd.Start(); err != nil {
		return false
	}
	h.cmd, h.in, h.out = cmd, in, bufio.NewReaderSize(out, 1<<20)
	return true
}

func (h *drmHelper) fail() {
	if h.cmd != nil {
		h.in.Close()
		h.cmd.Process.Kill()
		go h.cmd.Wait()
	}
	h.cmd, h.in, h.out = nil, nil, nil
	h.failedAt = time.Now()
}

// DRMClientsHelper is `hamlab drm-clients`: for every line read on stdin, one JSON line of
// drmFD on stdout. Ends when stdin does — when the daemon that started it has gone.
func DRMClientsHelper(stdin io.Reader, stdout io.Writer) int {
	sc := bufio.NewScanner(stdin)
	w := bufio.NewWriter(stdout)
	for sc.Scan() {
		fds, _ := drmScan()
		b, _ := json.Marshal(fds)
		w.Write(b)
		w.WriteByte('\n')
		if w.Flush() != nil {
			return 1
		}
	}
	return 0
}

// ---- small helpers ----

func readF(p string) float64 {
	v, err := strconv.ParseFloat(strings.TrimSpace(readStr(p)), 64)
	if err != nil {
		return 0
	}
	return v
}

func readLink(p string) string {
	t, err := os.Readlink(p)
	if err != nil {
		return ""
	}
	return t
}

func ueventValue(p, key string) string {
	for _, line := range strings.Split(readStr(p), "\n") {
		if k, v, ok := strings.Cut(line, "="); ok && k == key {
			return strings.TrimSpace(v)
		}
	}
	return ""
}

// The proprietary NVIDIA driver exports no busy counter in sysfs and no engine totals in
// fdinfo, so the device and its processes come from nvidia-smi — two short invocations per
// sample (~20 ms and ~75 ms on a GTX 965M), no root needed. The PCI address selects the
// card, so two NVIDIA GPUs stay apart.
func (d *gpuDev) readNvidia(x *GPU) {
	if out, ok := runOut("nvidia-smi", "--id="+d.bus, "--query-gpu=utilization.gpu,utilization.encoder,utilization.decoder,memory.used,memory.total,clocks.sm,clocks.max.sm,temperature.gpu,power.draw,power.limit,fan.speed", "--format=csv,noheader,nounits"); ok {
		parseNvidiaQuery(x, out)
	}
	if out, ok := runOut("nvidia-smi", "pmon", "-c", "1", "-s", "um", "--id="+d.bus); ok {
		d.nvRead = parseNvidiaPmon(out)
	}
}

// nvNum is one nvidia-smi CSV field; "[N/A]" (no sensor, no limit) is "nothing".
func nvNum(s string) (float64, bool) {
	s = strings.TrimSpace(s)
	if s == "" || strings.Contains(s, "N/A") {
		return 0, false
	}
	v, err := strconv.ParseFloat(s, 64)
	return v, err == nil
}

// parseNvidiaQuery fills a GPU from one --query-gpu line in the field order readNvidia asks
// for: gpu%, encoder%, decoder%, mem used MiB, mem total MiB, SM MHz, max SM MHz, °C, W, W cap,
// fan % ("[N/A]" on a card with no fan of its own: a laptop's, a passive datacentre card).
func parseNvidiaQuery(x *GPU, line string) {
	f := strings.Split(strings.TrimSpace(line), ",")
	if len(f) < 10 {
		return
	}
	if v, ok := nvNum(f[0]); ok {
		x.Usage, x.UsageSrc = v, "busy"
		x.Engines = map[string]float64{"render": v}
		enc, okE := nvNum(f[1])
		dec, okD := nvNum(f[2])
		if okE || okD {
			x.Engines["video"] = max(enc, dec)
		}
	}
	if v, ok := nvNum(f[3]); ok {
		x.MemUsed = uint64(v) << 20
	}
	if v, ok := nvNum(f[4]); ok {
		x.MemTotal = uint64(v) << 20
	}
	if v, ok := nvNum(f[5]); ok {
		x.MHz = v
	}
	if v, ok := nvNum(f[6]); ok {
		x.MaxMHz = v
	}
	if v, ok := nvNum(f[7]); ok && v > 0 {
		x.Temp, x.TempSrc = v, "gpu"
	}
	if v, ok := nvNum(f[8]); ok {
		x.PowerW = round1(v)
	}
	if v, ok := nvNum(f[9]); ok {
		x.PowerCapW = v
	}
	if len(f) > 10 {
		if v, ok := nvNum(f[10]); ok {
			x.FanPct = &v
		}
	}
}

// parseNvidiaPmon reads `nvidia-smi pmon -c 1 -s um`: a header naming the columns, then one
// row per process — "-" where a process has no share. Columns vary between driver versions,
// so they are taken by name. Busiest first, like the fdinfo clients.
func parseNvidiaPmon(out string) []GPUClient {
	col := map[string]int{}
	var clients []GPUClient
	for _, line := range strings.Split(out, "\n") {
		f := strings.Fields(line)
		if len(f) == 0 {
			continue
		}
		if f[0] == "#" {
			if len(f) > 1 && f[1] == "gpu" { // "# gpu pid type sm mem enc dec jpg ofa fb ccpm command"
				col = map[string]int{}
				for i, name := range f[1:] {
					col[name] = i
				}
			}
			continue
		}
		get := func(name string) (float64, bool) {
			i, ok := col[name]
			if !ok || i >= len(f) || f[i] == "-" {
				return 0, false
			}
			v, err := strconv.ParseFloat(f[i], 64)
			return v, err == nil
		}
		pid, ok := get("pid")
		if !ok || pid <= 0 {
			continue
		}
		cl := GPUClient{PID: int(pid), Engines: map[string]float64{}}
		if i, ok := col["command"]; ok && i < len(f) {
			cl.Name = strings.Join(f[i:], " ")
		}
		if v, ok := get("sm"); ok {
			cl.Engines["render"] = v
		}
		enc, okE := get("enc")
		dec, okD := get("dec")
		if okE || okD {
			cl.Engines["video"] = max(enc, dec)
		}
		for _, v := range cl.Engines {
			if v > cl.Usage {
				cl.Usage = v
			}
		}
		if len(cl.Engines) == 0 {
			cl.Engines = nil
		}
		if v, ok := get("fb"); ok {
			cl.Mem = uint64(v) << 20
		}
		clients = append(clients, cl)
	}
	sort.SliceStable(clients, func(i, j int) bool { return clients[i].Usage > clients[j].Usage })
	return clients
}