mitchell/hamlab
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 14 Wiki Activity

GPU boxes on the Processes tab: busy, engines, temps, and who has the card open

Browse Source 
Every DRM device with a real GPU driver gets a box under the cpu box, laid out
like it: the busy curve, a meter for the device and one per engine, temperature,
clock, power and fan where the card has them, and the processes holding it open
with their share. The process table gains a Gpu% column and sort.

Device busy is Intel's RC6 idle residency inverted (what intel_gpu_top shows,
and readable without root) or amdgpu's gpu_busy_percent. Per-process shares are
DRM fdinfo deltas, as nvtop reads them; other users' fdinfo needs root, so a
daemon running as a user keeps one root child (hamlab drm-clients) over a pipe
rather than paying a sudo a second. Everything is also a gpu.* sensor.

What cannot be measured is left out, not shown as zero: an Arc A380 on i915 6.12
has no temperature sensor, an integrated GPU wears the package temperature
marked *, and Asahi has neither a busy counter nor fdinfo yet.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
2026-10-02 18:18:02 -07:00
co-authored by Claude Fable 5.1
parent 0e82719070
commit db40220365
11 changed files with 981 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/metrics/metrics.go
+23 -4
View File
@@ -122,9 +122,10 @@ type Proc struct {
Name string `json:"name"`
Cmd string `json:"cmd,omitempty"`
User string `json:"user"`
CPU float64 `json:"cpu"` // percent of one core
CPUTime float64 `json:"cpu_time"` // seconds, lifetime
RSS uint64 `json:"rss"` // bytes
CPU float64 `json:"cpu"` // percent of one core
GPU float64 `json:"gpu,omitempty"` // its busiest GPU engine, percent; summed over the GPUs it has open
CPUTime float64 `json:"cpu_time"` // seconds, lifetime
RSS uint64 `json:"rss"` // bytes
Threads int `json:"threads"`
State string `json:"state"`
Nice int `json:"nice"`
@@ -145,6 +146,7 @@ type Sample struct {
Disks []Disk `json:"disks"`
FS []FS `json:"fs"`
UPS *UPS `json:"ups"`
GPUs []GPU `json:"gpus"`
Uptime float64 `json:"uptime"`
fsT, upsT int64 // when the slow sections were really read; sensors report these, not T
@@ -173,6 +175,7 @@ type Collector struct {
procAt time.Time
lastTable *ProcTable
upsProbe func() *UPS // nil when no UPS daemon is installed
gpu gpuState
subMu sync.Mutex
subs map[chan struct{}]struct{} // told after every sample; see Subscribe
@@ -356,6 +359,7 @@ func (c *Collector) sample() {
c.lastUPS, c.upsAt = c.upsProbe(), now
}
s.UPS, s.upsT = c.lastUPS, c.upsAt.UnixMilli()
s.GPUs = c.gpus(now, cpuTemp(s.Temps))
if first {
return // deltas are meaningless on the priming pass
@@ -1112,6 +1116,7 @@ type ProcTable struct {
Mem Mem `json:"mem"`
Swap Mem `json:"swap"`
Net []Net `json:"net"`
GPUs []GPU `json:"gpus"`
Uptime float64 `json:"uptime"`
Procs []Proc `json:"procs"`
Total int `json:"total"`
@@ -1274,6 +1279,20 @@ func (c *Collector) Processes() ProcTable {
}
}
c.prevProc, c.procAt = cur, now
// what each process is doing on the GPUs, from the newest sample's client lists
gpuBy := map[int]float64{}
gpus := []GPU{}
if n := len(c.ring); n > 0 && c.ring[n-1].GPUs != nil {
gpus = c.ring[n-1].GPUs
for _, g := range gpus {
for _, cl := range g.Clients {
gpuBy[cl.PID] += cl.Usage
}
}
}
for i := range procs {
procs[i].GPU = round1(gpuBy[procs[i].PID])
}
sort.Slice(procs, func(i, j int) bool {
if procs[i].CPU != procs[j].CPU {
return procs[i].CPU > procs[j].CPU
@@ -1281,7 +1300,7 @@ func (c *Collector) Processes() ProcTable {
return procs[i].RSS > procs[j].RSS
})
t := ProcTable{T: now.UnixMilli(), Procs: procs, Total: len(procs), Uptime: uptime(), Load: loadavg(),
CPUModel: cpuModel, CPUMHz: round1(cpuMHz()), CPUMaxMHz: cpuMaxMHz, CPUHist: []float64{}, Net: []Net{}}
CPUModel: cpuModel, CPUMHz: round1(cpuMHz()), CPUMaxMHz: cpuMaxMHz, CPUHist: []float64{}, Net: []Net{}, GPUs: gpus}
for _, p := range procs {
if p.State == "R" {
t.Running++