metrics: NVIDIA GPUs via nvidia-smi — device busy, VRAM, clocks, temp, power and the pmon process table

The proprietary driver's DRM card says DRIVER=nvidia, which the driver list did not
know, so the card was never enumerated; and it exposes no sysfs busy counter or
fdinfo engine totals. Read it with nvidia-smi instead (--query-gpu per sample,
pmon -c 1 -s um for clients), by PCI address. Parsers unit-tested, including an
older pmon column layout and [N/A] fields.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

PLAN.md

@@ -314,7 +314,13 @@ something long in one, switch away and back, refresh — it's all still there.
   exposes without root; `intel_gpu_top` shows the same number) or amdgpu's own percent;
   per-process and per-engine shares are DRM fdinfo deltas, the way nvtop reads them. Other
   users' fdinfo needs root, so a daemon running as `core` keeps **one** root child
-  (`hamlab drm-clients`, over a pipe) rather than a sudo a second. The Arc A380 on i915 6.12
+  (`hamlab drm-clients`, over a pipe) rather than a sudo a second. **NVIDIA's proprietary
+  driver (2026-10-23)** has neither a sysfs busy counter nor fdinfo engine totals, and its DRM
+  card reports `DRIVER=nvidia` (not `nvidia-drm`), so until then such a card was not even
+  listed; now it is read through `nvidia-smi` — `--query-gpu` for busy/encoder/decoder, VRAM,
+  SM clock, temperature and power, and `pmon -c 1 -s um` for the per-process table (sm/enc/dec
+  shares, framebuffer MiB) — two ~20–75 ms calls per sample, selected by PCI address, no root.
+  First machine: `link`, the Surface Studio's GTX 965M. The Arc A380 on i915 6.12
   exposes power, energy and fan but **no temperature**, so its box shows none; an integrated
   GPU shows the CPU package temperature marked `*`. Asahi (the Mac mini) has neither a busy
   counter nor fdinfo in 6.19, so it lists the device and says so. No NVIDIA path: nothing here

internal/metrics/gpu.go

@@ -70,7 +70,7 @@ type GPUClient struct {
 
 // gpuDrivers are DRM drivers that drive a GPU, not just a scanout (simpledrm, ast, the
 // Apple display coprocessor) — those have no engines and nothing to say.
-var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "radeon": true, "nouveau": true, "nvidia-drm": true,
+var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "radeon": true, "nouveau": true, "nvidia-drm": true, "nvidia": true,
 	"asahi": true, "panfrost": true, "panthor": true, "v3d": true, "msm": true, "etnaviv": true, "lima": true, "powervr": true, "virtio_gpu": true}
 
 type gpuDev struct {
@@ -78,6 +78,7 @@ type gpuDev struct {
 	integrated, stalled                      bool
 	prevIdle, prevEnergy                     float64
 	prevAt                                   time.Time
+	nv                                       []GPUClient // nvidia: the process table from the last read (pmon), fdinfo says nothing there
 }
 
 type gpuState struct {
@@ -129,7 +130,13 @@ func (c *Collector) gpus(now time.Time, cpuTemp float64) []GPU {
 		var order []string
 		engines := map[string]float64{}
 		var resident uint64
+		if d.driver == "nvidia" { // no engine counters in its fdinfo; nvidia-smi pmon is the client list
+			x.Clients, x.ClientsOK = append(x.Clients, d.nv...), true
+		}
 		for _, f := range g.last {
+			if d.driver == "nvidia" {
+				break
+			}
 			fi := parseFdinfo(f.Info)
 			if fi.pdev != "" && fi.pdev != d.bus || fi.pdev == "" && fi.driver != d.driver {
 				continue
@@ -368,6 +375,8 @@ func (d *gpuDev) readWithin(x *GPU, now time.Time) {
 func (d *gpuDev) read(x *GPU, now time.Time) {
 	dt := now.Sub(d.prevAt).Seconds()
 	switch d.driver {
+	case "nvidia":
+		d.readNvidia(x)
 	case "i915":
 		x.MHz = readF(d.card + "/gt_act_freq_mhz")
 		x.ReqMHz = readF(d.card + "/gt_cur_freq_mhz")
@@ -703,3 +712,126 @@ func ueventValue(p, key string) string {
 	}
 	return ""
 }
+
+// The proprietary NVIDIA driver exports no busy counter in sysfs and no engine totals in
+// fdinfo, so the device and its processes come from nvidia-smi — two short invocations per
+// sample (~20 ms and ~75 ms on a GTX 965M), no root needed. The PCI address selects the
+// card, so two NVIDIA GPUs stay apart.
+func (d *gpuDev) readNvidia(x *GPU) {
+	if out, ok := runOut("nvidia-smi", "--id="+d.bus, "--query-gpu=utilization.gpu,utilization.encoder,utilization.decoder,memory.used,memory.total,clocks.sm,clocks.max.sm,temperature.gpu,power.draw,power.limit", "--format=csv,noheader,nounits"); ok {
+		parseNvidiaQuery(x, out)
+	}
+	if out, ok := runOut("nvidia-smi", "pmon", "-c", "1", "-s", "um", "--id="+d.bus); ok {
+		d.nv = parseNvidiaPmon(out)
+	}
+}
+
+// nvNum is one nvidia-smi CSV field; "[N/A]" (no sensor, no limit) is "nothing".
+func nvNum(s string) (float64, bool) {
+	s = strings.TrimSpace(s)
+	if s == "" || strings.Contains(s, "N/A") {
+		return 0, false
+	}
+	v, err := strconv.ParseFloat(s, 64)
+	return v, err == nil
+}
+
+// parseNvidiaQuery fills a GPU from one --query-gpu line in the field order readNvidia asks
+// for: gpu%, encoder%, decoder%, mem used MiB, mem total MiB, SM MHz, max SM MHz, °C, W, W cap.
+func parseNvidiaQuery(x *GPU, line string) {
+	f := strings.Split(strings.TrimSpace(line), ",")
+	if len(f) < 10 {
+		return
+	}
+	if v, ok := nvNum(f[0]); ok {
+		x.Usage, x.UsageSrc = v, "busy"
+		x.Engines = map[string]float64{"render": v}
+		enc, okE := nvNum(f[1])
+		dec, okD := nvNum(f[2])
+		if okE || okD {
+			x.Engines["video"] = max(enc, dec)
+		}
+	}
+	if v, ok := nvNum(f[3]); ok {
+		x.MemUsed = uint64(v) << 20
+	}
+	if v, ok := nvNum(f[4]); ok {
+		x.MemTotal = uint64(v) << 20
+	}
+	if v, ok := nvNum(f[5]); ok {
+		x.MHz = v
+	}
+	if v, ok := nvNum(f[6]); ok {
+		x.MaxMHz = v
+	}
+	if v, ok := nvNum(f[7]); ok && v > 0 {
+		x.Temp, x.TempSrc = v, "gpu"
+	}
+	if v, ok := nvNum(f[8]); ok {
+		x.PowerW = round1(v)
+	}
+	if v, ok := nvNum(f[9]); ok {
+		x.PowerCapW = v
+	}
+}
+
+// parseNvidiaPmon reads `nvidia-smi pmon -c 1 -s um`: a header naming the columns, then one
+// row per process — "-" where a process has no share. Columns vary between driver versions,
+// so they are taken by name. Busiest first, like the fdinfo clients.
+func parseNvidiaPmon(out string) []GPUClient {
+	col := map[string]int{}
+	var clients []GPUClient
+	for _, line := range strings.Split(out, "\n") {
+		f := strings.Fields(line)
+		if len(f) == 0 {
+			continue
+		}
+		if f[0] == "#" {
+			if len(f) > 1 && f[1] == "gpu" { // "# gpu pid type sm mem enc dec jpg ofa fb ccpm command"
+				col = map[string]int{}
+				for i, name := range f[1:] {
+					col[name] = i
+				}
+			}
+			continue
+		}
+		get := func(name string) (float64, bool) {
+			i, ok := col[name]
+			if !ok || i >= len(f) || f[i] == "-" {
+				return 0, false
+			}
+			v, err := strconv.ParseFloat(f[i], 64)
+			return v, err == nil
+		}
+		pid, ok := get("pid")
+		if !ok || pid <= 0 {
+			continue
+		}
+		cl := GPUClient{PID: int(pid), Engines: map[string]float64{}}
+		if i, ok := col["command"]; ok && i < len(f) {
+			cl.Name = strings.Join(f[i:], " ")
+		}
+		if v, ok := get("sm"); ok {
+			cl.Engines["render"] = v
+		}
+		enc, okE := get("enc")
+		dec, okD := get("dec")
+		if okE || okD {
+			cl.Engines["video"] = max(enc, dec)
+		}
+		for _, v := range cl.Engines {
+			if v > cl.Usage {
+				cl.Usage = v
+			}
+		}
+		if len(cl.Engines) == 0 {
+			cl.Engines = nil
+		}
+		if v, ok := get("fb"); ok {
+			cl.Mem = uint64(v) << 20
+		}
+		clients = append(clients, cl)
+	}
+	sort.SliceStable(clients, func(i, j int) bool { return clients[i].Usage > clients[j].Usage })
+	return clients
+}

internal/metrics/gpu_test.go

@@ -170,3 +170,51 @@ func TestPCIName(t *testing.T) {
 		t.Errorf("an unknown device should be empty, got %q", d)
 	}
 }
+
+func TestParseNvidiaQuery(t *testing.T) {
+	var g GPU
+	parseNvidiaQuery(&g, "31, 0, 12, 1488, 2048, 949, 1097, 57, 23.41, [N/A]\n")
+	if g.Usage != 31 || g.UsageSrc != "busy" || g.Engines["render"] != 31 || g.Engines["video"] != 12 {
+		t.Errorf("usage: %+v", g)
+	}
+	if g.MemUsed != 1488<<20 || g.MemTotal != 2048<<20 || g.MHz != 949 || g.MaxMHz != 1097 {
+		t.Errorf("mem/clock: %+v", g)
+	}
+	if g.Temp != 57 || g.TempSrc != "gpu" || g.PowerW != 23.4 || g.PowerCapW != 0 {
+		t.Errorf("temp/power (cap must stay 0 for [N/A]): %+v", g)
+	}
+	var short GPU
+	parseNvidiaQuery(&short, "garbage")
+	if short.UsageSrc != "" {
+		t.Errorf("a short line must leave the GPU untouched: %+v", short)
+	}
+}
+
+const pmonA = `# gpu         pid   type     sm    mem    enc    dec    jpg    ofa     fb   ccpm    command 
+# Idx           #    C/G      %      %      %      %      %      %     MB     MB    name 
+    0       1063     G      -      -      -      -      -      -    701      0    Xorg           
+    0       1480     G     27     22      -      -      -      -    270      0    cinnamon       
+    0      13318     G      9      7     40      -      -      -    454      0    firefox-esr    
+`
+
+func TestParseNvidiaPmon(t *testing.T) {
+	cl := parseNvidiaPmon(pmonA)
+	if len(cl) != 3 {
+		t.Fatalf("want 3 clients, got %+v", cl)
+	}
+	// busiest first: firefox's encoder share (40) beats cinnamon's sm (27)
+	if cl[0].PID != 13318 || cl[0].Name != "firefox-esr" || cl[0].Usage != 40 || cl[0].Engines["render"] != 9 || cl[0].Engines["video"] != 40 || cl[0].Mem != 454<<20 {
+		t.Errorf("firefox: %+v", cl[0])
+	}
+	if cl[1].PID != 1480 || cl[1].Usage != 27 || cl[1].Engines["render"] != 27 {
+		t.Errorf("cinnamon: %+v", cl[1])
+	}
+	if cl[2].PID != 1063 || cl[2].Usage != 0 || cl[2].Engines != nil || cl[2].Mem != 701<<20 {
+		t.Errorf("xorg (no shares, only memory): %+v", cl[2])
+	}
+	// an older driver without the jpg/ofa/ccpm columns
+	old := "# gpu        pid  type    sm   mem   enc   dec   fb   command\n# Idx          #   C/G     %     %     %     %   MB   name\n    0      42    C    55    10     -     -  100   python3\n"
+	if cl := parseNvidiaPmon(old); len(cl) != 1 || cl[0].PID != 42 || cl[0].Usage != 55 || cl[0].Mem != 100<<20 || cl[0].Name != "python3" {
+		t.Errorf("old layout: %+v", cl)
+	}
+}