diff --git a/PLAN.md b/PLAN.md index a3acba4..2fb3216 100644 --- a/PLAN.md +++ b/PLAN.md @@ -314,7 +314,13 @@ something long in one, switch away and back, refresh — it's all still there. exposes without root; `intel_gpu_top` shows the same number) or amdgpu's own percent; per-process and per-engine shares are DRM fdinfo deltas, the way nvtop reads them. Other users' fdinfo needs root, so a daemon running as `core` keeps **one** root child - (`hamlab drm-clients`, over a pipe) rather than a sudo a second. The Arc A380 on i915 6.12 + (`hamlab drm-clients`, over a pipe) rather than a sudo a second. **NVIDIA's proprietary + driver (2026-10-23)** has neither a sysfs busy counter nor fdinfo engine totals, and its DRM + card reports `DRIVER=nvidia` (not `nvidia-drm`), so until then such a card was not even + listed; now it is read through `nvidia-smi` — `--query-gpu` for busy/encoder/decoder, VRAM, + SM clock, temperature and power, and `pmon -c 1 -s um` for the per-process table (sm/enc/dec + shares, framebuffer MiB) — two ~20–75 ms calls per sample, selected by PCI address, no root. + First machine: `link`, the Surface Studio's GTX 965M. The Arc A380 on i915 6.12 exposes power, energy and fan but **no temperature**, so its box shows none; an integrated GPU shows the CPU package temperature marked `*`. Asahi (the Mac mini) has neither a busy counter nor fdinfo in 6.19, so it lists the device and says so. No NVIDIA path: nothing here diff --git a/internal/metrics/gpu.go b/internal/metrics/gpu.go index 05fed0d..23acb7a 100644 --- a/internal/metrics/gpu.go +++ b/internal/metrics/gpu.go @@ -70,7 +70,7 @@ type GPUClient struct { // gpuDrivers are DRM drivers that drive a GPU, not just a scanout (simpledrm, ast, the // Apple display coprocessor) — those have no engines and nothing to say. -var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "radeon": true, "nouveau": true, "nvidia-drm": true, +var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "radeon": true, "nouveau": true, "nvidia-drm": true, "nvidia": true, "asahi": true, "panfrost": true, "panthor": true, "v3d": true, "msm": true, "etnaviv": true, "lima": true, "powervr": true, "virtio_gpu": true} type gpuDev struct { @@ -78,6 +78,7 @@ type gpuDev struct { integrated, stalled bool prevIdle, prevEnergy float64 prevAt time.Time + nv []GPUClient // nvidia: the process table from the last read (pmon), fdinfo says nothing there } type gpuState struct { @@ -129,7 +130,13 @@ func (c *Collector) gpus(now time.Time, cpuTemp float64) []GPU { var order []string engines := map[string]float64{} var resident uint64 + if d.driver == "nvidia" { // no engine counters in its fdinfo; nvidia-smi pmon is the client list + x.Clients, x.ClientsOK = append(x.Clients, d.nv...), true + } for _, f := range g.last { + if d.driver == "nvidia" { + break + } fi := parseFdinfo(f.Info) if fi.pdev != "" && fi.pdev != d.bus || fi.pdev == "" && fi.driver != d.driver { continue @@ -368,6 +375,8 @@ func (d *gpuDev) readWithin(x *GPU, now time.Time) { func (d *gpuDev) read(x *GPU, now time.Time) { dt := now.Sub(d.prevAt).Seconds() switch d.driver { + case "nvidia": + d.readNvidia(x) case "i915": x.MHz = readF(d.card + "/gt_act_freq_mhz") x.ReqMHz = readF(d.card + "/gt_cur_freq_mhz") @@ -703,3 +712,126 @@ func ueventValue(p, key string) string { } return "" } + +// The proprietary NVIDIA driver exports no busy counter in sysfs and no engine totals in +// fdinfo, so the device and its processes come from nvidia-smi — two short invocations per +// sample (~20 ms and ~75 ms on a GTX 965M), no root needed. The PCI address selects the +// card, so two NVIDIA GPUs stay apart. +func (d *gpuDev) readNvidia(x *GPU) { + if out, ok := runOut("nvidia-smi", "--id="+d.bus, "--query-gpu=utilization.gpu,utilization.encoder,utilization.decoder,memory.used,memory.total,clocks.sm,clocks.max.sm,temperature.gpu,power.draw,power.limit", "--format=csv,noheader,nounits"); ok { + parseNvidiaQuery(x, out) + } + if out, ok := runOut("nvidia-smi", "pmon", "-c", "1", "-s", "um", "--id="+d.bus); ok { + d.nv = parseNvidiaPmon(out) + } +} + +// nvNum is one nvidia-smi CSV field; "[N/A]" (no sensor, no limit) is "nothing". +func nvNum(s string) (float64, bool) { + s = strings.TrimSpace(s) + if s == "" || strings.Contains(s, "N/A") { + return 0, false + } + v, err := strconv.ParseFloat(s, 64) + return v, err == nil +} + +// parseNvidiaQuery fills a GPU from one --query-gpu line in the field order readNvidia asks +// for: gpu%, encoder%, decoder%, mem used MiB, mem total MiB, SM MHz, max SM MHz, °C, W, W cap. +func parseNvidiaQuery(x *GPU, line string) { + f := strings.Split(strings.TrimSpace(line), ",") + if len(f) < 10 { + return + } + if v, ok := nvNum(f[0]); ok { + x.Usage, x.UsageSrc = v, "busy" + x.Engines = map[string]float64{"render": v} + enc, okE := nvNum(f[1]) + dec, okD := nvNum(f[2]) + if okE || okD { + x.Engines["video"] = max(enc, dec) + } + } + if v, ok := nvNum(f[3]); ok { + x.MemUsed = uint64(v) << 20 + } + if v, ok := nvNum(f[4]); ok { + x.MemTotal = uint64(v) << 20 + } + if v, ok := nvNum(f[5]); ok { + x.MHz = v + } + if v, ok := nvNum(f[6]); ok { + x.MaxMHz = v + } + if v, ok := nvNum(f[7]); ok && v > 0 { + x.Temp, x.TempSrc = v, "gpu" + } + if v, ok := nvNum(f[8]); ok { + x.PowerW = round1(v) + } + if v, ok := nvNum(f[9]); ok { + x.PowerCapW = v + } +} + +// parseNvidiaPmon reads `nvidia-smi pmon -c 1 -s um`: a header naming the columns, then one +// row per process — "-" where a process has no share. Columns vary between driver versions, +// so they are taken by name. Busiest first, like the fdinfo clients. +func parseNvidiaPmon(out string) []GPUClient { + col := map[string]int{} + var clients []GPUClient + for _, line := range strings.Split(out, "\n") { + f := strings.Fields(line) + if len(f) == 0 { + continue + } + if f[0] == "#" { + if len(f) > 1 && f[1] == "gpu" { // "# gpu pid type sm mem enc dec jpg ofa fb ccpm command" + col = map[string]int{} + for i, name := range f[1:] { + col[name] = i + } + } + continue + } + get := func(name string) (float64, bool) { + i, ok := col[name] + if !ok || i >= len(f) || f[i] == "-" { + return 0, false + } + v, err := strconv.ParseFloat(f[i], 64) + return v, err == nil + } + pid, ok := get("pid") + if !ok || pid <= 0 { + continue + } + cl := GPUClient{PID: int(pid), Engines: map[string]float64{}} + if i, ok := col["command"]; ok && i < len(f) { + cl.Name = strings.Join(f[i:], " ") + } + if v, ok := get("sm"); ok { + cl.Engines["render"] = v + } + enc, okE := get("enc") + dec, okD := get("dec") + if okE || okD { + cl.Engines["video"] = max(enc, dec) + } + for _, v := range cl.Engines { + if v > cl.Usage { + cl.Usage = v + } + } + if len(cl.Engines) == 0 { + cl.Engines = nil + } + if v, ok := get("fb"); ok { + cl.Mem = uint64(v) << 20 + } + clients = append(clients, cl) + } + sort.SliceStable(clients, func(i, j int) bool { return clients[i].Usage > clients[j].Usage }) + return clients +} diff --git a/internal/metrics/gpu_test.go b/internal/metrics/gpu_test.go index 75bfbc8..a08b140 100644 --- a/internal/metrics/gpu_test.go +++ b/internal/metrics/gpu_test.go @@ -170,3 +170,51 @@ func TestPCIName(t *testing.T) { t.Errorf("an unknown device should be empty, got %q", d) } } + +func TestParseNvidiaQuery(t *testing.T) { + var g GPU + parseNvidiaQuery(&g, "31, 0, 12, 1488, 2048, 949, 1097, 57, 23.41, [N/A]\n") + if g.Usage != 31 || g.UsageSrc != "busy" || g.Engines["render"] != 31 || g.Engines["video"] != 12 { + t.Errorf("usage: %+v", g) + } + if g.MemUsed != 1488<<20 || g.MemTotal != 2048<<20 || g.MHz != 949 || g.MaxMHz != 1097 { + t.Errorf("mem/clock: %+v", g) + } + if g.Temp != 57 || g.TempSrc != "gpu" || g.PowerW != 23.4 || g.PowerCapW != 0 { + t.Errorf("temp/power (cap must stay 0 for [N/A]): %+v", g) + } + var short GPU + parseNvidiaQuery(&short, "garbage") + if short.UsageSrc != "" { + t.Errorf("a short line must leave the GPU untouched: %+v", short) + } +} + +const pmonA = `# gpu pid type sm mem enc dec jpg ofa fb ccpm command +# Idx # C/G % % % % % % MB MB name + 0 1063 G - - - - - - 701 0 Xorg + 0 1480 G 27 22 - - - - 270 0 cinnamon + 0 13318 G 9 7 40 - - - 454 0 firefox-esr +` + +func TestParseNvidiaPmon(t *testing.T) { + cl := parseNvidiaPmon(pmonA) + if len(cl) != 3 { + t.Fatalf("want 3 clients, got %+v", cl) + } + // busiest first: firefox's encoder share (40) beats cinnamon's sm (27) + if cl[0].PID != 13318 || cl[0].Name != "firefox-esr" || cl[0].Usage != 40 || cl[0].Engines["render"] != 9 || cl[0].Engines["video"] != 40 || cl[0].Mem != 454<<20 { + t.Errorf("firefox: %+v", cl[0]) + } + if cl[1].PID != 1480 || cl[1].Usage != 27 || cl[1].Engines["render"] != 27 { + t.Errorf("cinnamon: %+v", cl[1]) + } + if cl[2].PID != 1063 || cl[2].Usage != 0 || cl[2].Engines != nil || cl[2].Mem != 701<<20 { + t.Errorf("xorg (no shares, only memory): %+v", cl[2]) + } + // an older driver without the jpg/ofa/ccpm columns + old := "# gpu pid type sm mem enc dec fb command\n# Idx # C/G % % % % MB name\n 0 42 C 55 10 - - 100 python3\n" + if cl := parseNvidiaPmon(old); len(cl) != 1 || cl[0].PID != 42 || cl[0].Usage != 55 || cl[0].Mem != 100<<20 || cl[0].Name != "python3" { + t.Errorf("old layout: %+v", cl) + } +}