GPU boxes on the Processes tab: busy, engines, temps, and who has the card open
Every DRM device with a real GPU driver gets a box under the cpu box, laid out like it: the busy curve, a meter for the device and one per engine, temperature, clock, power and fan where the card has them, and the processes holding it open with their share. The process table gains a Gpu% column and sort. Device busy is Intel's RC6 idle residency inverted (what intel_gpu_top shows, and readable without root) or amdgpu's gpu_busy_percent. Per-process shares are DRM fdinfo deltas, as nvtop reads them; other users' fdinfo needs root, so a daemon running as a user keeps one root child (hamlab drm-clients) over a pipe rather than paying a sudo a second. Everything is also a gpu.* sensor. What cannot be measured is left out, not shown as zero: an Arc A380 on i915 6.12 has no temperature sensor, an integrated GPU wears the package temperature marked *, and Asahi has neither a busy counter nor fdinfo yet. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
0e82719070
commit
db40220365
@@ -0,0 +1,646 @@
|
||||
package metrics
|
||||
|
||||
// GPU telemetry, the way btop and intel_gpu_top get it without a vendor library: the
|
||||
// device from sysfs, and who is using it from DRM fdinfo.
|
||||
//
|
||||
// - The device: /sys/class/drm/card<n> with a driver we know to be a GPU. Clock from the
|
||||
// driver's frequency files, temperature/power/fan from the hwmon under the device, and
|
||||
// how busy it is from the one counter every Intel GT exports without root — RC6 idle
|
||||
// residency, so busy = 100 − idle. amdgpu says it outright (gpu_busy_percent).
|
||||
// - The clients: every process holding /dev/dri/* open has a drm-engine-<name> running
|
||||
// total (nanoseconds) in /proc/<pid>/fdinfo/<fd>; the delta between two reads is that
|
||||
// process's share of each engine, and the sum across clients is the engine's load. Other
|
||||
// users' fdinfo is unreadable, so when the daemon isn't root it keeps one root child
|
||||
// (`hamlab drm-clients`) that does the walk, asked over a pipe — one sudo per daemon
|
||||
// start, not one per second.
|
||||
//
|
||||
// A GPU with no counter of either kind (Asahi, today) is still listed: a name is better
|
||||
// than silence about hardware the machine has.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"hamlab/internal/priv"
|
||||
)
|
||||
|
||||
// GPU is one graphics device as a Sample reports it.
|
||||
type GPU struct {
|
||||
Name string `json:"name"`
|
||||
Driver string `json:"driver"`
|
||||
Bus string `json:"bus,omitempty"` // PCI address; empty for a platform device
|
||||
Integrated bool `json:"integrated,omitempty"`
|
||||
Usage float64 `json:"usage"` // % busy
|
||||
UsageSrc string `json:"usage_src,omitempty"` // "idle" (100 − idle residency), "busy" (the driver's own %), "clients" (the busiest engine); "" when nothing measures it
|
||||
Engines map[string]float64 `json:"engines,omitempty"` // render/video/copy/enhance… % from the clients, normalised by engine count
|
||||
Temp float64 `json:"temp,omitempty"`
|
||||
TempSrc string `json:"temp_src,omitempty"` // "gpu" from the device's own sensor, "package" the CPU die an integrated GPU shares
|
||||
PowerW float64 `json:"power_w,omitempty"`
|
||||
PowerCapW float64 `json:"power_cap_w,omitempty"`
|
||||
FanRPM int `json:"fan_rpm,omitempty"`
|
||||
MHz float64 `json:"mhz,omitempty"` // actual clock (0 when power-gated)
|
||||
ReqMHz float64 `json:"req_mhz,omitempty"` // what the driver asked for
|
||||
MaxMHz float64 `json:"max_mhz,omitempty"`
|
||||
MemUsed uint64 `json:"mem_used,omitempty"` // VRAM used, or the clients' resident bytes where the driver has no VRAM counter
|
||||
MemTotal uint64 `json:"mem_total,omitempty"`
|
||||
Clients []GPUClient `json:"clients"`
|
||||
ClientsOK bool `json:"clients_ok"` // false when fdinfo of other users' processes could not be read
|
||||
}
|
||||
|
||||
// GPUClient is one process with the device open.
|
||||
type GPUClient struct {
|
||||
PID int `json:"pid"`
|
||||
Name string `json:"name"`
|
||||
Usage float64 `json:"usage"` // its busiest engine, %
|
||||
Engines map[string]float64 `json:"engines,omitempty"`
|
||||
Mem uint64 `json:"mem,omitempty"` // resident bytes
|
||||
}
|
||||
|
||||
// gpuDrivers are DRM drivers that drive a GPU, not just a scanout (simpledrm, ast, the
|
||||
// Apple display coprocessor) — those have no engines and nothing to say.
|
||||
var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "radeon": true, "nouveau": true, "nvidia-drm": true,
|
||||
"asahi": true, "panfrost": true, "panthor": true, "v3d": true, "msm": true, "etnaviv": true, "lima": true, "powervr": true, "virtio_gpu": true}
|
||||
|
||||
type gpuDev struct {
|
||||
card, dev, key, bus, driver, name, hwmon string
|
||||
integrated bool
|
||||
prevIdle, prevEnergy float64
|
||||
prevAt time.Time
|
||||
}
|
||||
|
||||
type gpuState struct {
|
||||
devs []*gpuDev
|
||||
listedAt time.Time
|
||||
clients map[string]clientPrev // device key + client id → engine totals at the previous scan
|
||||
last []drmFD // the newest client walk, reused between walks
|
||||
scanAt time.Time
|
||||
helper *drmHelper
|
||||
ok bool // the last walk saw every process
|
||||
}
|
||||
|
||||
type clientPrev struct {
|
||||
ns map[string]uint64
|
||||
at time.Time
|
||||
}
|
||||
|
||||
const gpuRelist = time.Minute // how often the card list is re-read: hotplug is rare, the readdir is cheap
|
||||
const gpuScanEvery = time.Second // engine shares over less than this are noise
|
||||
|
||||
// gpus is the GPU section of a sample. cpuTemp stands in for an integrated GPU's
|
||||
// temperature: it is the same die.
|
||||
func (c *Collector) gpus(now time.Time, cpuTemp float64) []GPU {
|
||||
g := &c.gpu
|
||||
if g.devs == nil || now.Sub(g.listedAt) >= gpuRelist {
|
||||
g.devs, g.listedAt = listGPUs(g.devs), now
|
||||
}
|
||||
if len(g.devs) == 0 {
|
||||
return []GPU{}
|
||||
}
|
||||
if g.clients == nil {
|
||||
g.clients = map[string]clientPrev{}
|
||||
}
|
||||
if g.scanAt.IsZero() || now.Sub(g.scanAt) >= gpuScanEvery {
|
||||
g.last, g.ok = c.scanClients()
|
||||
g.scanAt = now
|
||||
}
|
||||
out := make([]GPU, 0, len(g.devs))
|
||||
live := map[string]bool{}
|
||||
for _, d := range g.devs {
|
||||
x := GPU{Name: d.name, Driver: d.driver, Bus: d.bus, Integrated: d.integrated, Clients: []GPUClient{}, ClientsOK: g.ok}
|
||||
d.read(&x, now)
|
||||
if x.Temp == 0 && d.integrated && cpuTemp > 0 {
|
||||
x.Temp, x.TempSrc = cpuTemp, "package"
|
||||
}
|
||||
// the clients: one entry per client id, under the first pid that holds it (a child
|
||||
// that inherited the fd shares the client)
|
||||
byID := map[string]*GPUClient{}
|
||||
var order []string
|
||||
engines := map[string]float64{}
|
||||
var resident uint64
|
||||
for _, f := range g.last {
|
||||
fi := parseFdinfo(f.Info)
|
||||
if fi.pdev != "" && fi.pdev != d.bus || fi.pdev == "" && fi.driver != d.driver {
|
||||
continue
|
||||
}
|
||||
key := d.key + "/" + fi.client
|
||||
live[key] = true
|
||||
if byID[fi.client] != nil {
|
||||
continue
|
||||
}
|
||||
cl := &GPUClient{PID: f.PID, Name: f.Comm, Mem: fi.mem}
|
||||
byID[fi.client] = cl
|
||||
order = append(order, fi.client)
|
||||
resident += fi.mem
|
||||
if prev, ok := g.clients[key]; ok {
|
||||
if dt := now.Sub(prev.at).Nanoseconds(); dt > 0 {
|
||||
for e, ns := range fi.engines {
|
||||
p, ok := prev.ns[e]
|
||||
if !ok || ns < p {
|
||||
continue
|
||||
}
|
||||
pct := float64(ns-p) / float64(dt) * 100
|
||||
if n := fi.capacity[e]; n > 1 {
|
||||
pct /= float64(n)
|
||||
}
|
||||
pct = round1(pct)
|
||||
if cl.Engines == nil {
|
||||
cl.Engines = map[string]float64{}
|
||||
}
|
||||
cl.Engines[e] = pct
|
||||
engines[e] += pct
|
||||
if pct > cl.Usage {
|
||||
cl.Usage = pct
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
g.clients[key] = clientPrev{ns: fi.engines, at: now}
|
||||
}
|
||||
for _, id := range order {
|
||||
x.Clients = append(x.Clients, *byID[id])
|
||||
}
|
||||
sort.SliceStable(x.Clients, func(i, j int) bool { return x.Clients[i].Usage > x.Clients[j].Usage })
|
||||
if len(engines) > 0 {
|
||||
x.Engines = map[string]float64{}
|
||||
for e, v := range engines {
|
||||
if v > 100 {
|
||||
v = 100
|
||||
}
|
||||
x.Engines[e] = round1(v)
|
||||
}
|
||||
}
|
||||
if x.UsageSrc == "" && len(x.Engines) > 0 {
|
||||
for _, v := range x.Engines {
|
||||
if v > x.Usage {
|
||||
x.Usage = v
|
||||
}
|
||||
}
|
||||
x.UsageSrc = "clients"
|
||||
}
|
||||
if x.MemTotal == 0 && x.MemUsed == 0 {
|
||||
x.MemUsed = resident
|
||||
}
|
||||
out = append(out, x)
|
||||
}
|
||||
for k := range g.clients { // a client that closed the device
|
||||
if !live[k] {
|
||||
delete(g.clients, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// listGPUs enumerates the cards, keeping the counters of those already known.
|
||||
func listGPUs(known []*gpuDev) []*gpuDev {
|
||||
ents, _ := os.ReadDir("/sys/class/drm")
|
||||
old := map[string]*gpuDev{}
|
||||
for _, d := range known {
|
||||
old[d.card] = d
|
||||
}
|
||||
out := []*gpuDev{}
|
||||
for _, e := range ents {
|
||||
n := e.Name()
|
||||
if !strings.HasPrefix(n, "card") || strings.Contains(n, "-") {
|
||||
continue
|
||||
}
|
||||
card := "/sys/class/drm/" + n
|
||||
dev, err := filepath.EvalSymlinks(card + "/device")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
driver := ueventValue(dev+"/uevent", "DRIVER")
|
||||
if driver == "" { // some drivers hang the DRIVER key off the parent
|
||||
driver = filepath.Base(readLink(dev + "/driver"))
|
||||
}
|
||||
if !gpuDrivers[driver] {
|
||||
continue
|
||||
}
|
||||
if d, ok := old[card]; ok && d.dev == dev {
|
||||
out = append(out, d)
|
||||
continue
|
||||
}
|
||||
d := &gpuDev{card: card, dev: dev, driver: driver}
|
||||
if pci := ueventValue(dev+"/uevent", "PCI_SLOT_NAME"); pci != "" {
|
||||
d.bus, d.key = pci, pci
|
||||
// an on-die GPU hangs off root bus 0 (Intel's 00:02.0, an AMD APU's 00:08.0 bridge)
|
||||
d.integrated = strings.HasPrefix(pci, "0000:00:")
|
||||
} else {
|
||||
d.key = "driver:" + driver
|
||||
d.integrated = true
|
||||
}
|
||||
if hw, _ := filepath.Glob(dev + "/hwmon/hwmon*"); len(hw) > 0 {
|
||||
d.hwmon = hw[0]
|
||||
}
|
||||
d.name = gpuName(dev, d.bus, driver)
|
||||
out = append(out, d)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].card < out[j].card })
|
||||
return out
|
||||
}
|
||||
|
||||
var gpuVendors = map[string]string{"0x8086": "Intel", "0x1002": "AMD", "0x10de": "NVIDIA", "0x1af4": "Virtio", "0x106b": "Apple"}
|
||||
|
||||
var (
|
||||
revRe = regexp.MustCompile(`\s*\(rev [^)]*\)`)
|
||||
bracketRe = regexp.MustCompile(`\[([^\]]+)\]`)
|
||||
)
|
||||
|
||||
// gpuName is "Intel Arc A380", from lspci's bracketed marketing name when it has one.
|
||||
func gpuName(dev, bus, driver string) string {
|
||||
if bus != "" {
|
||||
if out, ok := runOut("lspci", "-s", bus); ok {
|
||||
if _, desc, found := strings.Cut(out, ": "); found {
|
||||
return lspciName(desc)
|
||||
}
|
||||
}
|
||||
}
|
||||
vendor := gpuVendors[strings.TrimSpace(readStr(dev+"/vendor"))]
|
||||
if compat := readStr(dev + "/of_node/compatible"); compat != "" { // "apple,agx-t8103\0apple,agx-g13g"
|
||||
first := strings.SplitN(compat, "\x00", 2)[0]
|
||||
if v, model, ok := strings.Cut(first, ","); ok {
|
||||
if vendor == "" && v != "" {
|
||||
vendor = strings.ToUpper(v[:1]) + v[1:]
|
||||
}
|
||||
return vendor + " GPU (" + model + ")"
|
||||
}
|
||||
}
|
||||
if vendor == "" && driver != "" {
|
||||
vendor = strings.ToUpper(driver[:1]) + driver[1:]
|
||||
}
|
||||
if id := strings.TrimPrefix(strings.TrimSpace(readStr(dev+"/device")), "0x"); id != "" {
|
||||
return strings.TrimSpace(vendor + " GPU " + id)
|
||||
}
|
||||
return strings.TrimSpace(vendor + " GPU")
|
||||
}
|
||||
|
||||
// lspciName turns "Intel Corporation DG2 [Arc A380] (rev 05)" into "Intel Arc A380".
|
||||
func lspciName(desc string) string {
|
||||
desc = revRe.ReplaceAllString(strings.TrimSpace(desc), "")
|
||||
vendor := ""
|
||||
for _, v := range []struct{ long, short string }{{"Intel Corporation", "Intel"}, {"Advanced Micro Devices, Inc. [AMD/ATI]", "AMD"}, {"Advanced Micro Devices, Inc. [AMD]", "AMD"}, {"NVIDIA Corporation", "NVIDIA"}, {"Red Hat, Inc.", "Virtio"}} {
|
||||
if strings.HasPrefix(desc, v.long) {
|
||||
vendor = v.short
|
||||
desc = strings.TrimSpace(desc[len(v.long):])
|
||||
break
|
||||
}
|
||||
}
|
||||
if vendor == "" { // vendor not recognised: its first word
|
||||
if i := strings.IndexByte(desc, ' '); i > 0 {
|
||||
vendor, desc = desc[:i], strings.TrimSpace(desc[i+1:])
|
||||
}
|
||||
}
|
||||
if m := bracketRe.FindStringSubmatch(desc); m != nil {
|
||||
desc = m[1]
|
||||
}
|
||||
return strings.TrimSpace(vendor + " " + desc)
|
||||
}
|
||||
|
||||
// read fills the device-level numbers: clock, idle/busy, hwmon.
|
||||
func (d *gpuDev) read(x *GPU, now time.Time) {
|
||||
dt := now.Sub(d.prevAt).Seconds()
|
||||
switch d.driver {
|
||||
case "i915":
|
||||
x.MHz = readF(d.card + "/gt_act_freq_mhz")
|
||||
x.ReqMHz = readF(d.card + "/gt_cur_freq_mhz")
|
||||
x.MaxMHz = readF(d.card + "/gt_max_freq_mhz")
|
||||
idle, ok := readInt(d.card + "/gt/gt0/rc6_residency_ms")
|
||||
if !ok {
|
||||
idle, ok = readInt(d.card + "/power/rc6_residency_ms")
|
||||
}
|
||||
if ok {
|
||||
d.busyFromIdle(x, float64(idle), dt)
|
||||
}
|
||||
case "xe":
|
||||
gt := d.dev + "/tile0/gt0"
|
||||
x.MHz = readF(gt + "/freq0/act_freq")
|
||||
x.ReqMHz = readF(gt + "/freq0/cur_freq")
|
||||
x.MaxMHz = readF(gt + "/freq0/max_freq")
|
||||
if idle, ok := readInt(gt + "/gtidle/idle_residency_ms"); ok {
|
||||
d.busyFromIdle(x, float64(idle), dt)
|
||||
}
|
||||
case "amdgpu":
|
||||
if v, ok := readInt(d.dev + "/gpu_busy_percent"); ok {
|
||||
x.Usage, x.UsageSrc = float64(v), "busy"
|
||||
}
|
||||
x.MemUsed = uint64(readF(d.dev + "/mem_info_vram_used"))
|
||||
x.MemTotal = uint64(readF(d.dev + "/mem_info_vram_total"))
|
||||
// "1: 1800Mhz *" — the starred level is the current one, the last the highest
|
||||
for _, line := range strings.Split(readStr(d.dev+"/pp_dpm_sclk"), "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 2 {
|
||||
continue
|
||||
}
|
||||
mhz := leadingNum(f[1])
|
||||
x.MaxMHz = mhz
|
||||
if strings.HasSuffix(line, "*") {
|
||||
x.MHz = mhz
|
||||
}
|
||||
}
|
||||
}
|
||||
if d.hwmon != "" {
|
||||
if t := readF(d.hwmon + "/temp1_input"); t > 0 {
|
||||
x.Temp, x.TempSrc = t/1000, "gpu"
|
||||
}
|
||||
if p := readF(d.hwmon + "/power1_input"); p > 0 {
|
||||
x.PowerW = round1(p / 1e6)
|
||||
} else if p := readF(d.hwmon + "/power1_average"); p > 0 {
|
||||
x.PowerW = round1(p / 1e6)
|
||||
} else if e := readF(d.hwmon + "/energy1_input"); e > 0 { // µJ, cumulative
|
||||
if d.prevEnergy > 0 && e >= d.prevEnergy && dt > 0 && !d.prevAt.IsZero() {
|
||||
x.PowerW = round1((e - d.prevEnergy) / 1e6 / dt)
|
||||
}
|
||||
d.prevEnergy = e
|
||||
}
|
||||
if p := readF(d.hwmon + "/power1_cap"); p > 0 {
|
||||
x.PowerCapW = p / 1e6
|
||||
} else if p := readF(d.hwmon + "/power1_max"); p > 0 {
|
||||
x.PowerCapW = p / 1e6
|
||||
}
|
||||
x.FanRPM = int(readF(d.hwmon + "/fan1_input"))
|
||||
}
|
||||
d.prevAt = now
|
||||
}
|
||||
|
||||
// busyFromIdle: the GT's idle residency is a running ms total; the share of the interval
|
||||
// it did not cover is how busy the device was.
|
||||
func (d *gpuDev) busyFromIdle(x *GPU, idle, dt float64) {
|
||||
x.UsageSrc = "idle"
|
||||
if d.prevAt.IsZero() || dt <= 0 || idle < d.prevIdle {
|
||||
d.prevIdle = idle
|
||||
return
|
||||
}
|
||||
busy := 100 - (idle-d.prevIdle)/(dt*1000)*100
|
||||
if busy < 0 {
|
||||
busy = 0
|
||||
}
|
||||
if busy > 100 {
|
||||
busy = 100
|
||||
}
|
||||
x.Usage = round1(busy)
|
||||
d.prevIdle = idle
|
||||
}
|
||||
|
||||
// ---- DRM fdinfo: who has the device open ----
|
||||
|
||||
// drmFD is one client of a DRM device: the fdinfo text of one open fd, with its owner.
|
||||
type drmFD struct {
|
||||
PID int `json:"pid"`
|
||||
Comm string `json:"comm"`
|
||||
Info string `json:"info"`
|
||||
}
|
||||
|
||||
type fdinfo struct {
|
||||
driver, pdev, client string
|
||||
engines map[string]uint64 // ns, cumulative
|
||||
capacity map[string]int
|
||||
mem uint64 // bytes
|
||||
}
|
||||
|
||||
func parseFdinfo(s string) fdinfo {
|
||||
fi := fdinfo{engines: map[string]uint64{}, capacity: map[string]int{}}
|
||||
var resident, memory uint64
|
||||
for _, line := range strings.Split(s, "\n") {
|
||||
k, v, ok := strings.Cut(line, ":")
|
||||
if !ok || !strings.HasPrefix(k, "drm-") {
|
||||
continue
|
||||
}
|
||||
v = strings.TrimSpace(v)
|
||||
switch {
|
||||
case k == "drm-driver":
|
||||
fi.driver = v
|
||||
case k == "drm-pdev":
|
||||
fi.pdev = v
|
||||
case k == "drm-client-id":
|
||||
fi.client = v
|
||||
case strings.HasPrefix(k, "drm-engine-capacity-"):
|
||||
fi.capacity[k[len("drm-engine-capacity-"):]], _ = strconv.Atoi(v)
|
||||
case strings.HasPrefix(k, "drm-engine-"):
|
||||
n, _ := strconv.ParseUint(strings.TrimSuffix(v, " ns"), 10, 64)
|
||||
fi.engines[k[len("drm-engine-"):]] = n
|
||||
case strings.HasPrefix(k, "drm-resident-"):
|
||||
resident += kib(v)
|
||||
case strings.HasPrefix(k, "drm-memory-"):
|
||||
memory += kib(v)
|
||||
}
|
||||
}
|
||||
fi.mem = resident
|
||||
if fi.mem == 0 {
|
||||
fi.mem = memory
|
||||
}
|
||||
return fi
|
||||
}
|
||||
|
||||
// kib parses "52624 KiB" (or MiB, or a bare count of bytes) to bytes.
|
||||
func kib(v string) uint64 {
|
||||
f := strings.Fields(v)
|
||||
if len(f) == 0 {
|
||||
return 0
|
||||
}
|
||||
n, _ := strconv.ParseUint(f[0], 10, 64)
|
||||
if len(f) > 1 {
|
||||
switch f[1] {
|
||||
case "KiB":
|
||||
n <<= 10
|
||||
case "MiB":
|
||||
n <<= 20
|
||||
case "GiB":
|
||||
n <<= 30
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// drmScan walks /proc for every fd open on /dev/dri/*, one entry per client per process.
|
||||
// Run as root it sees everyone; as a user, only that user's processes.
|
||||
func drmScan() (out []drmFD, complete bool) {
|
||||
ents, err := os.ReadDir("/proc")
|
||||
if err != nil {
|
||||
return []drmFD{}, false
|
||||
}
|
||||
complete = true
|
||||
out = []drmFD{}
|
||||
for _, e := range ents {
|
||||
pid, err := strconv.Atoi(e.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
fds, err := os.ReadDir("/proc/" + e.Name() + "/fd")
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
complete = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
comm := ""
|
||||
for _, fd := range fds {
|
||||
target, err := os.Readlink("/proc/" + e.Name() + "/fd/" + fd.Name())
|
||||
if err != nil || !strings.HasPrefix(target, "/dev/dri/") {
|
||||
continue
|
||||
}
|
||||
info := readStr("/proc/" + e.Name() + "/fdinfo/" + fd.Name())
|
||||
_, rest, ok := strings.Cut(info, "drm-client-id:")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
id := strings.TrimSpace(strings.SplitN(rest, "\n", 2)[0])
|
||||
if seen[id] {
|
||||
continue
|
||||
}
|
||||
seen[id] = true
|
||||
if comm == "" {
|
||||
comm = strings.TrimSpace(readStr("/proc/" + e.Name() + "/comm"))
|
||||
}
|
||||
out = append(out, drmFD{PID: pid, Comm: comm, Info: info})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].PID < out[j].PID })
|
||||
return out, complete
|
||||
}
|
||||
|
||||
// scanClients walks in-process when that sees everything (root), else through the helper.
|
||||
func (c *Collector) scanClients() ([]drmFD, bool) {
|
||||
if priv.Mode() == "root" || !priv.Available() {
|
||||
return drmScan()
|
||||
}
|
||||
if c.gpu.helper == nil {
|
||||
c.gpu.helper = &drmHelper{}
|
||||
}
|
||||
if fds, ok := c.gpu.helper.scan(); ok {
|
||||
return fds, true
|
||||
}
|
||||
return drmScan() // the helper is down: our own processes are better than nothing
|
||||
}
|
||||
|
||||
// drmHelper is the root child that answers "scan" requests. One line in, one JSON line out.
|
||||
type drmHelper struct {
|
||||
mu sync.Mutex
|
||||
cmd *exec.Cmd
|
||||
in io.WriteCloser
|
||||
out *bufio.Reader
|
||||
failedAt time.Time
|
||||
}
|
||||
|
||||
const helperRetry = 30 * time.Second
|
||||
|
||||
func (h *drmHelper) scan() ([]drmFD, bool) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.cmd == nil {
|
||||
if !h.failedAt.IsZero() && time.Since(h.failedAt) < helperRetry {
|
||||
return nil, false
|
||||
}
|
||||
if !h.start() {
|
||||
h.failedAt = time.Now()
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
// a stuck child must not stall the sampler: it is killed, and tried again later
|
||||
cmd := h.cmd
|
||||
stop := time.AfterFunc(5*time.Second, func() { cmd.Process.Kill() })
|
||||
defer stop.Stop()
|
||||
if _, err := io.WriteString(h.in, "\n"); err != nil {
|
||||
h.fail()
|
||||
return nil, false
|
||||
}
|
||||
line, err := h.out.ReadBytes('\n')
|
||||
if err != nil {
|
||||
h.fail()
|
||||
return nil, false
|
||||
}
|
||||
var fds []drmFD
|
||||
if json.Unmarshal(line, &fds) != nil {
|
||||
h.fail()
|
||||
return nil, false
|
||||
}
|
||||
if fds == nil {
|
||||
fds = []drmFD{}
|
||||
}
|
||||
return fds, true
|
||||
}
|
||||
|
||||
func (h *drmHelper) start() bool {
|
||||
self, err := os.Executable()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
full := priv.Wrap(self, "drm-clients")
|
||||
cmd := exec.Command(full[0], full[1:]...)
|
||||
in, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
out, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return false
|
||||
}
|
||||
h.cmd, h.in, h.out = cmd, in, bufio.NewReaderSize(out, 1<<20)
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *drmHelper) fail() {
|
||||
if h.cmd != nil {
|
||||
h.in.Close()
|
||||
h.cmd.Process.Kill()
|
||||
go h.cmd.Wait()
|
||||
}
|
||||
h.cmd, h.in, h.out = nil, nil, nil
|
||||
h.failedAt = time.Now()
|
||||
}
|
||||
|
||||
// DRMClientsHelper is `hamlab drm-clients`: for every line read on stdin, one JSON line of
|
||||
// drmFD on stdout. Ends when stdin does — when the daemon that started it has gone.
|
||||
func DRMClientsHelper(stdin io.Reader, stdout io.Writer) int {
|
||||
sc := bufio.NewScanner(stdin)
|
||||
w := bufio.NewWriter(stdout)
|
||||
for sc.Scan() {
|
||||
fds, _ := drmScan()
|
||||
b, _ := json.Marshal(fds)
|
||||
w.Write(b)
|
||||
w.WriteByte('\n')
|
||||
if w.Flush() != nil {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ---- small helpers ----
|
||||
|
||||
func readF(p string) float64 {
|
||||
v, err := strconv.ParseFloat(strings.TrimSpace(readStr(p)), 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func readLink(p string) string {
|
||||
t, err := os.Readlink(p)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func ueventValue(p, key string) string {
|
||||
for _, line := range strings.Split(readStr(p), "\n") {
|
||||
if k, v, ok := strings.Cut(line, "="); ok && k == key {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user