metrics: NVIDIA GPUs via nvidia-smi — device busy, VRAM, clocks, temp, power and the pmon process table

The proprietary driver's DRM card says DRIVER=nvidia, which the driver list did not know, so the card was never enumerated; and it exposes no sysfs busy counter or fdinfo engine totals. Read it with nvidia-smi instead (--query-gpu per sample, pmon -c 1 -s um for clients), by PCI address. Parsers unit-tested, including an older pmon column layout and [N/A] fields. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-23 00:29:46 -07:00
parent 450a9d3327
commit 8fc71496f0
3 changed files with 188 additions and 2 deletions
@@ -170,3 +170,51 @@ func TestPCIName(t *testing.T) {
 		t.Errorf("an unknown device should be empty, got %q", d)
 	}
 }
+
+func TestParseNvidiaQuery(t *testing.T) {
+	var g GPU
+	parseNvidiaQuery(&g, "31, 0, 12, 1488, 2048, 949, 1097, 57, 23.41, [N/A]\n")
+	if g.Usage != 31 || g.UsageSrc != "busy" || g.Engines["render"] != 31 || g.Engines["video"] != 12 {
+		t.Errorf("usage: %+v", g)
+	}
+	if g.MemUsed != 1488<<20 || g.MemTotal != 2048<<20 || g.MHz != 949 || g.MaxMHz != 1097 {
+		t.Errorf("mem/clock: %+v", g)
+	}
+	if g.Temp != 57 || g.TempSrc != "gpu" || g.PowerW != 23.4 || g.PowerCapW != 0 {
+		t.Errorf("temp/power (cap must stay 0 for [N/A]): %+v", g)
+	}
+	var short GPU
+	parseNvidiaQuery(&short, "garbage")
+	if short.UsageSrc != "" {
+		t.Errorf("a short line must leave the GPU untouched: %+v", short)
+	}
+}
+
+const pmonA = `# gpu         pid   type     sm    mem    enc    dec    jpg    ofa     fb   ccpm    command 
+# Idx           #    C/G      %      %      %      %      %      %     MB     MB    name 
+    0       1063     G      -      -      -      -      -      -    701      0    Xorg           
+    0       1480     G     27     22      -      -      -      -    270      0    cinnamon       
+    0      13318     G      9      7     40      -      -      -    454      0    firefox-esr    
+`
+
+func TestParseNvidiaPmon(t *testing.T) {
+	cl := parseNvidiaPmon(pmonA)
+	if len(cl) != 3 {
+		t.Fatalf("want 3 clients, got %+v", cl)
+	}
+	// busiest first: firefox's encoder share (40) beats cinnamon's sm (27)
+	if cl[0].PID != 13318 || cl[0].Name != "firefox-esr" || cl[0].Usage != 40 || cl[0].Engines["render"] != 9 || cl[0].Engines["video"] != 40 || cl[0].Mem != 454<<20 {
+		t.Errorf("firefox: %+v", cl[0])
+	}
+	if cl[1].PID != 1480 || cl[1].Usage != 27 || cl[1].Engines["render"] != 27 {
+		t.Errorf("cinnamon: %+v", cl[1])
+	}
+	if cl[2].PID != 1063 || cl[2].Usage != 0 || cl[2].Engines != nil || cl[2].Mem != 701<<20 {
+		t.Errorf("xorg (no shares, only memory): %+v", cl[2])
+	}
+	// an older driver without the jpg/ofa/ccpm columns
+	old := "# gpu        pid  type    sm   mem   enc   dec   fb   command\n# Idx          #   C/G     %     %     %     %   MB   name\n    0      42    C    55    10     -     -  100   python3\n"
+	if cl := parseNvidiaPmon(old); len(cl) != 1 || cl[0].PID != 42 || cl[0].Usage != 55 || cl[0].Mem != 100<<20 || cl[0].Name != "python3" {
+		t.Errorf("old layout: %+v", cl)
+	}
+}