mitchell/hamlab
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 14 Wiki Activity

metrics: NVIDIA GPUs via nvidia-smi — device busy, VRAM, clocks, temp, power and the pmon process table

Browse Source 
The proprietary driver's DRM card says DRIVER=nvidia, which the driver list did not
know, so the card was never enumerated; and it exposes no sysfs busy counter or
fdinfo engine totals. Read it with nvidia-smi instead (--query-gpu per sample,
pmon -c 1 -s um for clients), by PCI address. Parsers unit-tested, including an
older pmon column layout and [N/A] fields.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
2026-10-23 00:29:46 -07:00
co-authored by Claude Fable 5.1
parent 450a9d3327
commit 8fc71496f0
3 changed files with 188 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/metrics/gpu.go
+133 -1
View File
@@ -70,7 +70,7 @@ type GPUClient struct {
// gpuDrivers are DRM drivers that drive a GPU, not just a scanout (simpledrm, ast, the
// Apple display coprocessor) — those have no engines and nothing to say.
var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "radeon": true, "nouveau": true, "nvidia-drm": true,
var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "radeon": true, "nouveau": true, "nvidia-drm": true, "nvidia": true,
"asahi": true, "panfrost": true, "panthor": true, "v3d": true, "msm": true, "etnaviv": true, "lima": true, "powervr": true, "virtio_gpu": true}
type gpuDev struct {
@@ -78,6 +78,7 @@ type gpuDev struct {
integrated, stalled bool
prevIdle, prevEnergy float64
prevAt time.Time
nv []GPUClient // nvidia: the process table from the last read (pmon), fdinfo says nothing there
}
type gpuState struct {
@@ -129,7 +130,13 @@ func (c *Collector) gpus(now time.Time, cpuTemp float64) []GPU {
var order []string
engines := map[string]float64{}
var resident uint64
if d.driver == "nvidia" { // no engine counters in its fdinfo; nvidia-smi pmon is the client list
x.Clients, x.ClientsOK = append(x.Clients, d.nv...), true
}
for _, f := range g.last {
if d.driver == "nvidia" {
break
}
fi := parseFdinfo(f.Info)
if fi.pdev != "" && fi.pdev != d.bus || fi.pdev == "" && fi.driver != d.driver {
continue
@@ -368,6 +375,8 @@ func (d *gpuDev) readWithin(x *GPU, now time.Time) {
func (d *gpuDev) read(x *GPU, now time.Time) {
dt := now.Sub(d.prevAt).Seconds()
switch d.driver {
case "nvidia":
d.readNvidia(x)
case "i915":
x.MHz = readF(d.card + "/gt_act_freq_mhz")
x.ReqMHz = readF(d.card + "/gt_cur_freq_mhz")
@@ -703,3 +712,126 @@ func ueventValue(p, key string) string {
}
return ""
}
// The proprietary NVIDIA driver exports no busy counter in sysfs and no engine totals in
// fdinfo, so the device and its processes come from nvidia-smi — two short invocations per
// sample (~20 ms and ~75 ms on a GTX 965M), no root needed. The PCI address selects the
// card, so two NVIDIA GPUs stay apart.
func (d *gpuDev) readNvidia(x *GPU) {
if out, ok := runOut("nvidia-smi", "--id="+d.bus, "--query-gpu=utilization.gpu,utilization.encoder,utilization.decoder,memory.used,memory.total,clocks.sm,clocks.max.sm,temperature.gpu,power.draw,power.limit", "--format=csv,noheader,nounits"); ok {
parseNvidiaQuery(x, out)
}
if out, ok := runOut("nvidia-smi", "pmon", "-c", "1", "-s", "um", "--id="+d.bus); ok {
d.nv = parseNvidiaPmon(out)
}
}
// nvNum is one nvidia-smi CSV field; "[N/A]" (no sensor, no limit) is "nothing".
func nvNum(s string) (float64, bool) {
s = strings.TrimSpace(s)
if s == "" || strings.Contains(s, "N/A") {
return 0, false
}
v, err := strconv.ParseFloat(s, 64)
return v, err == nil
}
// parseNvidiaQuery fills a GPU from one --query-gpu line in the field order readNvidia asks
// for: gpu%, encoder%, decoder%, mem used MiB, mem total MiB, SM MHz, max SM MHz, °C, W, W cap.
func parseNvidiaQuery(x *GPU, line string) {
f := strings.Split(strings.TrimSpace(line), ",")
if len(f) < 10 {
return
}
if v, ok := nvNum(f[0]); ok {
x.Usage, x.UsageSrc = v, "busy"
x.Engines = map[string]float64{"render": v}
enc, okE := nvNum(f[1])
dec, okD := nvNum(f[2])
if okE || okD {
x.Engines["video"] = max(enc, dec)
}
}
if v, ok := nvNum(f[3]); ok {
x.MemUsed = uint64(v) << 20
}
if v, ok := nvNum(f[4]); ok {
x.MemTotal = uint64(v) << 20
}
if v, ok := nvNum(f[5]); ok {
x.MHz = v
}
if v, ok := nvNum(f[6]); ok {
x.MaxMHz = v
}
if v, ok := nvNum(f[7]); ok && v > 0 {
x.Temp, x.TempSrc = v, "gpu"
}
if v, ok := nvNum(f[8]); ok {
x.PowerW = round1(v)
}
if v, ok := nvNum(f[9]); ok {
x.PowerCapW = v
}
}
// parseNvidiaPmon reads `nvidia-smi pmon -c 1 -s um`: a header naming the columns, then one
// row per process — "-" where a process has no share. Columns vary between driver versions,
// so they are taken by name. Busiest first, like the fdinfo clients.
func parseNvidiaPmon(out string) []GPUClient {
col := map[string]int{}
var clients []GPUClient
for _, line := range strings.Split(out, "\n") {
f := strings.Fields(line)
if len(f) == 0 {
continue
}
if f[0] == "#" {
if len(f) > 1 && f[1] == "gpu" { // "# gpu pid type sm mem enc dec jpg ofa fb ccpm command"
col = map[string]int{}
for i, name := range f[1:] {
col[name] = i
}
}
continue
}
get := func(name string) (float64, bool) {
i, ok := col[name]
if !ok || i >= len(f) || f[i] == "-" {
return 0, false
}
v, err := strconv.ParseFloat(f[i], 64)
return v, err == nil
}
pid, ok := get("pid")
if !ok || pid <= 0 {
continue
}
cl := GPUClient{PID: int(pid), Engines: map[string]float64{}}
if i, ok := col["command"]; ok && i < len(f) {
cl.Name = strings.Join(f[i:], " ")
}
if v, ok := get("sm"); ok {
cl.Engines["render"] = v
}
enc, okE := get("enc")
dec, okD := get("dec")
if okE || okD {
cl.Engines["video"] = max(enc, dec)
}
for _, v := range cl.Engines {
if v > cl.Usage {
cl.Usage = v
}
}
if len(cl.Engines) == 0 {
cl.Engines = nil
}
if v, ok := get("fb"); ok {
cl.Mem = uint64(v) << 20
}
clients = append(clients, cl)
}
sort.SliceStable(clients, func(i, j int) bool { return clients[i].Usage > clients[j].Usage })
return clients
}
internal/metrics/gpu_test.go
+48
View File
@@ -170,3 +170,51 @@ func TestPCIName(t *testing.T) {
t.Errorf("an unknown device should be empty, got %q", d)
}
}
func TestParseNvidiaQuery(t *testing.T) {
var g GPU
parseNvidiaQuery(&g, "31, 0, 12, 1488, 2048, 949, 1097, 57, 23.41, [N/A]\n")
if g.Usage != 31 || g.UsageSrc != "busy" || g.Engines["render"] != 31 || g.Engines["video"] != 12 {
t.Errorf("usage: %+v", g)
}
if g.MemUsed != 1488<<20 || g.MemTotal != 2048<<20 || g.MHz != 949 || g.MaxMHz != 1097 {
t.Errorf("mem/clock: %+v", g)
}
if g.Temp != 57 || g.TempSrc != "gpu" || g.PowerW != 23.4 || g.PowerCapW != 0 {
t.Errorf("temp/power (cap must stay 0 for [N/A]): %+v", g)
}
var short GPU
parseNvidiaQuery(&short, "garbage")
if short.UsageSrc != "" {
t.Errorf("a short line must leave the GPU untouched: %+v", short)
}
}
const pmonA = `# gpu pid type sm mem enc dec jpg ofa fb ccpm command
# Idx # C/G % % % % % % MB MB name
0 1063 G - - - - - - 701 0 Xorg
0 1480 G 27 22 - - - - 270 0 cinnamon
0 13318 G 9 7 40 - - - 454 0 firefox-esr
`
func TestParseNvidiaPmon(t *testing.T) {
cl := parseNvidiaPmon(pmonA)
if len(cl) != 3 {
t.Fatalf("want 3 clients, got %+v", cl)
}
// busiest first: firefox's encoder share (40) beats cinnamon's sm (27)
if cl[0].PID != 13318 || cl[0].Name != "firefox-esr" || cl[0].Usage != 40 || cl[0].Engines["render"] != 9 || cl[0].Engines["video"] != 40 || cl[0].Mem != 454<<20 {
t.Errorf("firefox: %+v", cl[0])
}
if cl[1].PID != 1480 || cl[1].Usage != 27 || cl[1].Engines["render"] != 27 {
t.Errorf("cinnamon: %+v", cl[1])
}
if cl[2].PID != 1063 || cl[2].Usage != 0 || cl[2].Engines != nil || cl[2].Mem != 701<<20 {
t.Errorf("xorg (no shares, only memory): %+v", cl[2])
}
// an older driver without the jpg/ofa/ccpm columns
old := "# gpu pid type sm mem enc dec fb command\n# Idx # C/G % % % % MB name\n 0 42 C 55 10 - - 100 python3\n"
if cl := parseNvidiaPmon(old); len(cl) != 1 || cl[0].PID != 42 || cl[0].Usage != 55 || cl[0].Mem != 100<<20 || cl[0].Name != "python3" {
t.Errorf("old layout: %+v", cl)
}
}