metrics: NVIDIA GPUs via nvidia-smi — device busy, VRAM, clocks, temp, power and the pmon process table

The proprietary driver's DRM card says DRIVER=nvidia, which the driver list did not
know, so the card was never enumerated; and it exposes no sysfs busy counter or
fdinfo engine totals. Read it with nvidia-smi instead (--query-gpu per sample,
pmon -c 1 -s um for clients), by PCI address. Parsers unit-tested, including an
older pmon column layout and [N/A] fields.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

internal/metrics/gpu.go

@@ -70,7 +70,7 @@ type GPUClient struct {
 
 // gpuDrivers are DRM drivers that drive a GPU, not just a scanout (simpledrm, ast, the
 // Apple display coprocessor) — those have no engines and nothing to say.
-var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "radeon": true, "nouveau": true, "nvidia-drm": true,
+var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "radeon": true, "nouveau": true, "nvidia-drm": true, "nvidia": true,
 	"asahi": true, "panfrost": true, "panthor": true, "v3d": true, "msm": true, "etnaviv": true, "lima": true, "powervr": true, "virtio_gpu": true}
 
 type gpuDev struct {
@@ -78,6 +78,7 @@ type gpuDev struct {
 	integrated, stalled                      bool
 	prevIdle, prevEnergy                     float64
 	prevAt                                   time.Time
+	nv                                       []GPUClient // nvidia: the process table from the last read (pmon), fdinfo says nothing there
 }
 
 type gpuState struct {
@@ -129,7 +130,13 @@ func (c *Collector) gpus(now time.Time, cpuTemp float64) []GPU {
 		var order []string
 		engines := map[string]float64{}
 		var resident uint64
+		if d.driver == "nvidia" { // no engine counters in its fdinfo; nvidia-smi pmon is the client list
+			x.Clients, x.ClientsOK = append(x.Clients, d.nv...), true
+		}
 		for _, f := range g.last {
+			if d.driver == "nvidia" {
+				break
+			}
 			fi := parseFdinfo(f.Info)
 			if fi.pdev != "" && fi.pdev != d.bus || fi.pdev == "" && fi.driver != d.driver {
 				continue
@@ -368,6 +375,8 @@ func (d *gpuDev) readWithin(x *GPU, now time.Time) {
 func (d *gpuDev) read(x *GPU, now time.Time) {
 	dt := now.Sub(d.prevAt).Seconds()
 	switch d.driver {
+	case "nvidia":
+		d.readNvidia(x)
 	case "i915":
 		x.MHz = readF(d.card + "/gt_act_freq_mhz")
 		x.ReqMHz = readF(d.card + "/gt_cur_freq_mhz")
@@ -703,3 +712,126 @@ func ueventValue(p, key string) string {
 	}
 	return ""
 }
+
+// The proprietary NVIDIA driver exports no busy counter in sysfs and no engine totals in
+// fdinfo, so the device and its processes come from nvidia-smi — two short invocations per
+// sample (~20 ms and ~75 ms on a GTX 965M), no root needed. The PCI address selects the
+// card, so two NVIDIA GPUs stay apart.
+func (d *gpuDev) readNvidia(x *GPU) {
+	if out, ok := runOut("nvidia-smi", "--id="+d.bus, "--query-gpu=utilization.gpu,utilization.encoder,utilization.decoder,memory.used,memory.total,clocks.sm,clocks.max.sm,temperature.gpu,power.draw,power.limit", "--format=csv,noheader,nounits"); ok {
+		parseNvidiaQuery(x, out)
+	}
+	if out, ok := runOut("nvidia-smi", "pmon", "-c", "1", "-s", "um", "--id="+d.bus); ok {
+		d.nv = parseNvidiaPmon(out)
+	}
+}
+
+// nvNum is one nvidia-smi CSV field; "[N/A]" (no sensor, no limit) is "nothing".
+func nvNum(s string) (float64, bool) {
+	s = strings.TrimSpace(s)
+	if s == "" || strings.Contains(s, "N/A") {
+		return 0, false
+	}
+	v, err := strconv.ParseFloat(s, 64)
+	return v, err == nil
+}
+
+// parseNvidiaQuery fills a GPU from one --query-gpu line in the field order readNvidia asks
+// for: gpu%, encoder%, decoder%, mem used MiB, mem total MiB, SM MHz, max SM MHz, °C, W, W cap.
+func parseNvidiaQuery(x *GPU, line string) {
+	f := strings.Split(strings.TrimSpace(line), ",")
+	if len(f) < 10 {
+		return
+	}
+	if v, ok := nvNum(f[0]); ok {
+		x.Usage, x.UsageSrc = v, "busy"
+		x.Engines = map[string]float64{"render": v}
+		enc, okE := nvNum(f[1])
+		dec, okD := nvNum(f[2])
+		if okE || okD {
+			x.Engines["video"] = max(enc, dec)
+		}
+	}
+	if v, ok := nvNum(f[3]); ok {
+		x.MemUsed = uint64(v) << 20
+	}
+	if v, ok := nvNum(f[4]); ok {
+		x.MemTotal = uint64(v) << 20
+	}
+	if v, ok := nvNum(f[5]); ok {
+		x.MHz = v
+	}
+	if v, ok := nvNum(f[6]); ok {
+		x.MaxMHz = v
+	}
+	if v, ok := nvNum(f[7]); ok && v > 0 {
+		x.Temp, x.TempSrc = v, "gpu"
+	}
+	if v, ok := nvNum(f[8]); ok {
+		x.PowerW = round1(v)
+	}
+	if v, ok := nvNum(f[9]); ok {
+		x.PowerCapW = v
+	}
+}
+
+// parseNvidiaPmon reads `nvidia-smi pmon -c 1 -s um`: a header naming the columns, then one
+// row per process — "-" where a process has no share. Columns vary between driver versions,
+// so they are taken by name. Busiest first, like the fdinfo clients.
+func parseNvidiaPmon(out string) []GPUClient {
+	col := map[string]int{}
+	var clients []GPUClient
+	for _, line := range strings.Split(out, "\n") {
+		f := strings.Fields(line)
+		if len(f) == 0 {
+			continue
+		}
+		if f[0] == "#" {
+			if len(f) > 1 && f[1] == "gpu" { // "# gpu pid type sm mem enc dec jpg ofa fb ccpm command"
+				col = map[string]int{}
+				for i, name := range f[1:] {
+					col[name] = i
+				}
+			}
+			continue
+		}
+		get := func(name string) (float64, bool) {
+			i, ok := col[name]
+			if !ok || i >= len(f) || f[i] == "-" {
+				return 0, false
+			}
+			v, err := strconv.ParseFloat(f[i], 64)
+			return v, err == nil
+		}
+		pid, ok := get("pid")
+		if !ok || pid <= 0 {
+			continue
+		}
+		cl := GPUClient{PID: int(pid), Engines: map[string]float64{}}
+		if i, ok := col["command"]; ok && i < len(f) {
+			cl.Name = strings.Join(f[i:], " ")
+		}
+		if v, ok := get("sm"); ok {
+			cl.Engines["render"] = v
+		}
+		enc, okE := get("enc")
+		dec, okD := get("dec")
+		if okE || okD {
+			cl.Engines["video"] = max(enc, dec)
+		}
+		for _, v := range cl.Engines {
+			if v > cl.Usage {
+				cl.Usage = v
+			}
+		}
+		if len(cl.Engines) == 0 {
+			cl.Engines = nil
+		}
+		if v, ok := get("fb"); ok {
+			cl.Mem = uint64(v) << 20
+		}
+		clients = append(clients, cl)
+	}
+	sort.SliceStable(clients, func(i, j int) bool { return clients[i].Usage > clients[j].Usage })
+	return clients
+}