GPU boxes on the Processes tab: busy, engines, temps, and who has the card open

Every DRM device with a real GPU driver gets a box under the cpu box, laid out like it: the busy curve, a meter for the device and one per engine, temperature, clock, power and fan where the card has them, and the processes holding it open with their share. The process table gains a Gpu% column and sort. Device busy is Intel's RC6 idle residency inverted (what intel_gpu_top shows, and readable without root) or amdgpu's gpu_busy_percent. Per-process shares are DRM fdinfo deltas, as nvtop reads them; other users' fdinfo needs root, so a daemon running as a user keeps one root child (hamlab drm-clients) over a pipe rather than paying a sudo a second. Everything is also a gpu.* sensor. What cannot be measured is left out, not shown as zero: an Arc A380 on i915 6.12 has no temperature sensor, an integrated GPU wears the package temperature marked *, and Asahi has neither a busy counter nor fdinfo yet. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-02 18:18:02 -07:00
parent 0e82719070
commit db40220365
11 changed files with 981 additions and 19 deletions
@@ -0,0 +1,136 @@
+package metrics
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+)
+
+const fdinfoA = `pos:	0
+flags:	02100002
+drm-driver:	i915
+drm-client-id:	544
+drm-pdev:	0000:00:02.0
+drm-total-system0:	52624 KiB
+drm-resident-system0:	50140 KiB
+drm-resident-stolen-system0:	32 KiB
+drm-engine-render:	1000000000 ns
+drm-engine-copy:	0 ns
+drm-engine-video:	2000000000 ns
+drm-engine-capacity-video:	2
+drm-engine-video-enhance:	0 ns
+`
+
+func TestParseFdinfo(t *testing.T) {
+	fi := parseFdinfo(fdinfoA)
+	if fi.driver != "i915" || fi.pdev != "0000:00:02.0" || fi.client != "544" {
+		t.Fatalf("identity: %+v", fi)
+	}
+	if fi.engines["video"] != 2000000000 || fi.capacity["video"] != 2 {
+		t.Errorf("video engine: %+v", fi)
+	}
+	if fi.mem != (50140+32)<<10 {
+		t.Errorf("resident = %d", fi.mem)
+	}
+}
+
+func TestLspciName(t *testing.T) {
+	for in, want := range map[string]string{
+		"Intel Corporation DG2 [Arc A380] (rev 05)":                                              "Intel Arc A380",
+		"Intel Corporation RocketLake-S GT1 [UHD Graphics 750] (rev 04)":                         "Intel UHD Graphics 750",
+		"Intel Corporation Alder Lake-N [UHD Graphics]":                                          "Intel UHD Graphics",
+		"Advanced Micro Devices, Inc. [AMD/ATI] Navi 23 [Radeon RX 6600/6600 XT/6600M] (rev c7)": "AMD Radeon RX 6600/6600 XT/6600M",
+		"NVIDIA Corporation GA106 [GeForce RTX 3060 Lite Hash Rate] (rev a1)":                    "NVIDIA GeForce RTX 3060 Lite Hash Rate",
+		"Moore Threads Technology Co.,Ltd MTT S80":                                               "Moore Threads Technology Co.,Ltd MTT S80",
+	} {
+		if got := lspciName(in); got != want {
+			t.Errorf("%q → %q, want %q", in, got, want)
+		}
+	}
+}
+
+// Two scans a second apart: a client that ran its render engine for half of it and both
+// video engines flat out reads 50% render, 100% video, and the device's engines follow.
+func TestGPUClientsShare(t *testing.T) {
+	c := NewCollector()
+	d := &gpuDev{card: "/nonexistent/card9", dev: "/nonexistent", key: "0000:00:02.0", bus: "0000:00:02.0", driver: "i915", name: "Test GPU", integrated: true}
+	c.gpu.devs = []*gpuDev{d}
+	c.gpu.listedAt = time.Now().Add(time.Hour) // keep the fake list
+	c.gpu.clients = map[string]clientPrev{}
+	t0 := time.Now()
+	c.gpu.last = []drmFD{{PID: 7, Comm: "ffmpeg", Info: fdinfoA}}
+	c.gpu.scanAt = t0 // no walk: the fake list stands
+	first := c.gpus(t0, 40)
+	if len(first) != 1 || len(first[0].Clients) != 1 || first[0].Clients[0].Usage != 0 {
+		t.Fatalf("first pass: %+v", first)
+	}
+	if first[0].Temp != 40 || first[0].TempSrc != "package" {
+		t.Errorf("an integrated GPU wears the package temperature: %+v", first[0])
+	}
+	later := fdinfoA
+	later = replace(later, "drm-engine-render:	1000000000 ns", "drm-engine-render:	1500000000 ns")
+	later = replace(later, "drm-engine-video:	2000000000 ns", "drm-engine-video:	4000000000 ns")
+	c.gpu.last = []drmFD{{PID: 7, Comm: "ffmpeg", Info: later}}
+	c.gpu.scanAt = t0.Add(time.Second)
+	got := c.gpus(t0.Add(time.Second), 40)
+	cl := got[0].Clients[0]
+	if cl.Engines["render"] != 50 || cl.Engines["video"] != 100 || cl.Usage != 100 {
+		t.Errorf("client share: %+v", cl)
+	}
+	if got[0].Engines["video"] != 100 || got[0].Usage != 100 || got[0].UsageSrc != "clients" {
+		t.Errorf("device from clients: %+v", got[0])
+	}
+	if got[0].MemUsed != cl.Mem {
+		t.Errorf("resident memory should be the clients' sum: %d vs %d", got[0].MemUsed, cl.Mem)
+	}
+	// the Processes table joins the client to its row
+	c.ring = []Sample{{T: 1, GPUs: got}}
+	c.prevProc = nil
+	table := c.Processes()
+	if len(table.GPUs) != 1 {
+		t.Errorf("table carries the GPUs: %+v", table.GPUs)
+	}
+}
+
+func TestFlattenGPU(t *testing.T) {
+	s := Sample{T: 5, GPUs: []GPU{{Name: "Intel Arc A380", Driver: "i915", Bus: "0000:03:00.0", Usage: 12, UsageSrc: "idle", Engines: map[string]float64{"video": 30}, PowerW: 9.5, PowerCapW: 55, MHz: 2450, MaxMHz: 2450, Clients: []GPUClient{{PID: 1}}}}}
+	got := map[string]any{}
+	for _, x := range flatten(s, nil) {
+		got[x.ID] = x.Value
+	}
+	for id, want := range map[string]any{"gpu.intel_arc_a380.usage": 12.0, "gpu.intel_arc_a380.video": 30.0, "gpu.intel_arc_a380.power": 9.5, "gpu.intel_arc_a380.freq": 2450.0, "gpu.intel_arc_a380.clients": 1.0} {
+		if got[id] != want {
+			t.Errorf("%s = %v, want %v", id, got[id], want)
+		}
+	}
+	if _, ok := got["gpu.intel_arc_a380.temp"]; ok {
+		t.Error("no temperature sensor, no temperature")
+	}
+}
+
+func replace(s, old, new string) string {
+	for i := 0; i+len(old) <= len(s); i++ {
+		if s[i:i+len(old)] == old {
+			return s[:i] + new + s[i+len(old):]
+		}
+	}
+	return s
+}
+
+// TestLiveGPU reads this machine's GPUs for real: HAMLAB_LIVE=1 go test -run TestLiveGPU -v ./internal/metrics
+func TestLiveGPU(t *testing.T) {
+	if os.Getenv("HAMLAB_LIVE") == "" {
+		t.Skip("set HAMLAB_LIVE=1 to read the real hardware")
+	}
+	c := NewCollector()
+	c.sample()
+	time.Sleep(1100 * time.Millisecond)
+	c.sample()
+	s, _ := c.Latest()
+	b, _ := json.MarshalIndent(s.GPUs, "", "  ")
+	t.Logf("%s", b)
+	for _, x := range flatten(s, func(id string) bool { return len(id) > 4 && id[:4] == "gpu." }) {
+		t.Logf("%s = %v %s", x.ID, x.Value, x.Unit)
+	}
+}