Mitchell-JohnsonandClaude Fable 5.1
b5cecce435
On grid the sampler sat 42 minutes in runtime_pollWait on the Arc A380's hwmon fan1_input: kernfs accepts epoll, so os.ReadFile made the fd non-blocking, i915 answered EAGAIN once, and the poller waited for a readiness kernfs only signals on sysfs_notify. Every graph on the machine froze while the API carried on. readStr now reads with plain blocking syscalls. Each GPU's sysfs reads run under a three-second deadline; a device that does not answer is marked stalled, logged, and left alone until restart, with the box saying so. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
674 lines
20 KiB
Go
674 lines
20 KiB
Go
package metrics
|
||
|
||
// GPU telemetry, the way btop and intel_gpu_top get it without a vendor library: the
|
||
// device from sysfs, and who is using it from DRM fdinfo.
|
||
//
|
||
// - The device: /sys/class/drm/card<n> with a driver we know to be a GPU. Clock from the
|
||
// driver's frequency files, temperature/power/fan from the hwmon under the device, and
|
||
// how busy it is from the one counter every Intel GT exports without root — RC6 idle
|
||
// residency, so busy = 100 − idle. amdgpu says it outright (gpu_busy_percent).
|
||
// - The clients: every process holding /dev/dri/* open has a drm-engine-<name> running
|
||
// total (nanoseconds) in /proc/<pid>/fdinfo/<fd>; the delta between two reads is that
|
||
// process's share of each engine, and the sum across clients is the engine's load. Other
|
||
// users' fdinfo is unreadable, so when the daemon isn't root it keeps one root child
|
||
// (`hamlab drm-clients`) that does the walk, asked over a pipe — one sudo per daemon
|
||
// start, not one per second.
|
||
//
|
||
// A GPU with no counter of either kind (Asahi, today) is still listed: a name is better
|
||
// than silence about hardware the machine has.
|
||
|
||
import (
|
||
"bufio"
|
||
"encoding/json"
|
||
"io"
|
||
"log"
|
||
"os"
|
||
"os/exec"
|
||
"path/filepath"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"hamlab/internal/priv"
|
||
)
|
||
|
||
// GPU is one graphics device as a Sample reports it.
|
||
type GPU struct {
|
||
Name string `json:"name"`
|
||
Driver string `json:"driver"`
|
||
Bus string `json:"bus,omitempty"` // PCI address; empty for a platform device
|
||
Integrated bool `json:"integrated,omitempty"`
|
||
Usage float64 `json:"usage"` // % busy
|
||
UsageSrc string `json:"usage_src,omitempty"` // "idle" (100 − idle residency), "busy" (the driver's own %), "clients" (the busiest engine); "" when nothing measures it
|
||
Engines map[string]float64 `json:"engines,omitempty"` // render/video/copy/enhance… % from the clients, normalised by engine count
|
||
Temp float64 `json:"temp,omitempty"`
|
||
TempSrc string `json:"temp_src,omitempty"` // "gpu" from the device's own sensor, "package" the CPU die an integrated GPU shares
|
||
PowerW float64 `json:"power_w,omitempty"`
|
||
PowerCapW float64 `json:"power_cap_w,omitempty"`
|
||
FanRPM int `json:"fan_rpm,omitempty"`
|
||
MHz float64 `json:"mhz,omitempty"` // actual clock (0 when power-gated)
|
||
ReqMHz float64 `json:"req_mhz,omitempty"` // what the driver asked for
|
||
MaxMHz float64 `json:"max_mhz,omitempty"`
|
||
MemUsed uint64 `json:"mem_used,omitempty"` // VRAM used, or the clients' resident bytes where the driver has no VRAM counter
|
||
MemTotal uint64 `json:"mem_total,omitempty"`
|
||
Clients []GPUClient `json:"clients"`
|
||
ClientsOK bool `json:"clients_ok"` // false when fdinfo of other users' processes could not be read
|
||
Stalled bool `json:"stalled,omitempty"` // a sysfs read of this card never returned; its readings are paused until restart
|
||
}
|
||
|
||
// GPUClient is one process with the device open.
|
||
type GPUClient struct {
|
||
PID int `json:"pid"`
|
||
Name string `json:"name"`
|
||
Usage float64 `json:"usage"` // its busiest engine, %
|
||
Engines map[string]float64 `json:"engines,omitempty"`
|
||
Mem uint64 `json:"mem,omitempty"` // resident bytes
|
||
}
|
||
|
||
// gpuDrivers are DRM drivers that drive a GPU, not just a scanout (simpledrm, ast, the
|
||
// Apple display coprocessor) — those have no engines and nothing to say.
|
||
var gpuDrivers = map[string]bool{"i915": true, "xe": true, "amdgpu": true, "radeon": true, "nouveau": true, "nvidia-drm": true,
|
||
"asahi": true, "panfrost": true, "panthor": true, "v3d": true, "msm": true, "etnaviv": true, "lima": true, "powervr": true, "virtio_gpu": true}
|
||
|
||
type gpuDev struct {
|
||
card, dev, key, bus, driver, name, hwmon string
|
||
integrated, stalled bool
|
||
prevIdle, prevEnergy float64
|
||
prevAt time.Time
|
||
}
|
||
|
||
type gpuState struct {
|
||
devs []*gpuDev
|
||
listedAt time.Time
|
||
clients map[string]clientPrev // device key + client id → engine totals at the previous scan
|
||
last []drmFD // the newest client walk, reused between walks
|
||
scanAt time.Time
|
||
helper *drmHelper
|
||
ok bool // the last walk saw every process
|
||
}
|
||
|
||
type clientPrev struct {
|
||
ns map[string]uint64
|
||
at time.Time
|
||
}
|
||
|
||
const gpuRelist = time.Minute // how often the card list is re-read: hotplug is rare, the readdir is cheap
|
||
const gpuScanEvery = time.Second // engine shares over less than this are noise
|
||
|
||
// gpus is the GPU section of a sample. cpuTemp stands in for an integrated GPU's
|
||
// temperature: it is the same die.
|
||
func (c *Collector) gpus(now time.Time, cpuTemp float64) []GPU {
|
||
g := &c.gpu
|
||
if g.devs == nil || now.Sub(g.listedAt) >= gpuRelist {
|
||
g.devs, g.listedAt = listGPUs(g.devs), now
|
||
}
|
||
if len(g.devs) == 0 {
|
||
return []GPU{}
|
||
}
|
||
if g.clients == nil {
|
||
g.clients = map[string]clientPrev{}
|
||
}
|
||
if g.scanAt.IsZero() || now.Sub(g.scanAt) >= gpuScanEvery {
|
||
g.last, g.ok = c.scanClients()
|
||
g.scanAt = now
|
||
}
|
||
out := make([]GPU, 0, len(g.devs))
|
||
live := map[string]bool{}
|
||
for _, d := range g.devs {
|
||
x := GPU{Name: d.name, Driver: d.driver, Bus: d.bus, Integrated: d.integrated, Clients: []GPUClient{}, ClientsOK: g.ok}
|
||
d.readWithin(&x, now)
|
||
if x.Temp == 0 && d.integrated && cpuTemp > 0 {
|
||
x.Temp, x.TempSrc = cpuTemp, "package"
|
||
}
|
||
// the clients: one entry per client id, under the first pid that holds it (a child
|
||
// that inherited the fd shares the client)
|
||
byID := map[string]*GPUClient{}
|
||
var order []string
|
||
engines := map[string]float64{}
|
||
var resident uint64
|
||
for _, f := range g.last {
|
||
fi := parseFdinfo(f.Info)
|
||
if fi.pdev != "" && fi.pdev != d.bus || fi.pdev == "" && fi.driver != d.driver {
|
||
continue
|
||
}
|
||
key := d.key + "/" + fi.client
|
||
live[key] = true
|
||
if byID[fi.client] != nil {
|
||
continue
|
||
}
|
||
cl := &GPUClient{PID: f.PID, Name: f.Comm, Mem: fi.mem}
|
||
byID[fi.client] = cl
|
||
order = append(order, fi.client)
|
||
resident += fi.mem
|
||
if prev, ok := g.clients[key]; ok {
|
||
if dt := now.Sub(prev.at).Nanoseconds(); dt > 0 {
|
||
for e, ns := range fi.engines {
|
||
p, ok := prev.ns[e]
|
||
if !ok || ns < p {
|
||
continue
|
||
}
|
||
pct := float64(ns-p) / float64(dt) * 100
|
||
if n := fi.capacity[e]; n > 1 {
|
||
pct /= float64(n)
|
||
}
|
||
pct = round1(pct)
|
||
if cl.Engines == nil {
|
||
cl.Engines = map[string]float64{}
|
||
}
|
||
cl.Engines[e] = pct
|
||
engines[e] += pct
|
||
if pct > cl.Usage {
|
||
cl.Usage = pct
|
||
}
|
||
}
|
||
}
|
||
}
|
||
g.clients[key] = clientPrev{ns: fi.engines, at: now}
|
||
}
|
||
for _, id := range order {
|
||
x.Clients = append(x.Clients, *byID[id])
|
||
}
|
||
sort.SliceStable(x.Clients, func(i, j int) bool { return x.Clients[i].Usage > x.Clients[j].Usage })
|
||
if len(engines) > 0 {
|
||
x.Engines = map[string]float64{}
|
||
for e, v := range engines {
|
||
if v > 100 {
|
||
v = 100
|
||
}
|
||
x.Engines[e] = round1(v)
|
||
}
|
||
}
|
||
if x.UsageSrc == "" && len(x.Engines) > 0 {
|
||
for _, v := range x.Engines {
|
||
if v > x.Usage {
|
||
x.Usage = v
|
||
}
|
||
}
|
||
x.UsageSrc = "clients"
|
||
}
|
||
if x.MemTotal == 0 && x.MemUsed == 0 {
|
||
x.MemUsed = resident
|
||
}
|
||
out = append(out, x)
|
||
}
|
||
for k := range g.clients { // a client that closed the device
|
||
if !live[k] {
|
||
delete(g.clients, k)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// listGPUs enumerates the cards, keeping the counters of those already known.
|
||
func listGPUs(known []*gpuDev) []*gpuDev {
|
||
ents, _ := os.ReadDir("/sys/class/drm")
|
||
old := map[string]*gpuDev{}
|
||
for _, d := range known {
|
||
old[d.card] = d
|
||
}
|
||
out := []*gpuDev{}
|
||
for _, e := range ents {
|
||
n := e.Name()
|
||
if !strings.HasPrefix(n, "card") || strings.Contains(n, "-") {
|
||
continue
|
||
}
|
||
card := "/sys/class/drm/" + n
|
||
dev, err := filepath.EvalSymlinks(card + "/device")
|
||
if err != nil {
|
||
continue
|
||
}
|
||
driver := ueventValue(dev+"/uevent", "DRIVER")
|
||
if driver == "" { // some drivers hang the DRIVER key off the parent
|
||
driver = filepath.Base(readLink(dev + "/driver"))
|
||
}
|
||
if !gpuDrivers[driver] {
|
||
continue
|
||
}
|
||
if d, ok := old[card]; ok && d.dev == dev {
|
||
out = append(out, d)
|
||
continue
|
||
}
|
||
d := &gpuDev{card: card, dev: dev, driver: driver}
|
||
if pci := ueventValue(dev+"/uevent", "PCI_SLOT_NAME"); pci != "" {
|
||
d.bus, d.key = pci, pci
|
||
// an on-die GPU hangs off root bus 0 (Intel's 00:02.0, an AMD APU's 00:08.0 bridge)
|
||
d.integrated = strings.HasPrefix(pci, "0000:00:")
|
||
} else {
|
||
d.key = "driver:" + driver
|
||
d.integrated = true
|
||
}
|
||
if hw, _ := filepath.Glob(dev + "/hwmon/hwmon*"); len(hw) > 0 {
|
||
d.hwmon = hw[0]
|
||
}
|
||
d.name = gpuName(dev, d.bus, driver)
|
||
out = append(out, d)
|
||
}
|
||
sort.Slice(out, func(i, j int) bool { return out[i].card < out[j].card })
|
||
return out
|
||
}
|
||
|
||
var gpuVendors = map[string]string{"0x8086": "Intel", "0x1002": "AMD", "0x10de": "NVIDIA", "0x1af4": "Virtio", "0x106b": "Apple"}
|
||
|
||
var (
|
||
revRe = regexp.MustCompile(`\s*\(rev [^)]*\)`)
|
||
bracketRe = regexp.MustCompile(`\[([^\]]+)\]`)
|
||
)
|
||
|
||
// gpuName is "Intel Arc A380", from lspci's bracketed marketing name when it has one.
|
||
func gpuName(dev, bus, driver string) string {
|
||
if bus != "" {
|
||
if out, ok := runOut("lspci", "-s", bus); ok {
|
||
if _, desc, found := strings.Cut(out, ": "); found {
|
||
return lspciName(desc)
|
||
}
|
||
}
|
||
}
|
||
vendor := gpuVendors[strings.TrimSpace(readStr(dev+"/vendor"))]
|
||
if compat := readStr(dev + "/of_node/compatible"); compat != "" { // "apple,agx-t8103\0apple,agx-g13g"
|
||
first := strings.SplitN(compat, "\x00", 2)[0]
|
||
if v, model, ok := strings.Cut(first, ","); ok {
|
||
if vendor == "" && v != "" {
|
||
vendor = strings.ToUpper(v[:1]) + v[1:]
|
||
}
|
||
return vendor + " GPU (" + model + ")"
|
||
}
|
||
}
|
||
if vendor == "" && driver != "" {
|
||
vendor = strings.ToUpper(driver[:1]) + driver[1:]
|
||
}
|
||
if id := strings.TrimPrefix(strings.TrimSpace(readStr(dev+"/device")), "0x"); id != "" {
|
||
return strings.TrimSpace(vendor + " GPU " + id)
|
||
}
|
||
return strings.TrimSpace(vendor + " GPU")
|
||
}
|
||
|
||
// lspciName turns "Intel Corporation DG2 [Arc A380] (rev 05)" into "Intel Arc A380".
|
||
func lspciName(desc string) string {
|
||
desc = revRe.ReplaceAllString(strings.TrimSpace(desc), "")
|
||
vendor := ""
|
||
for _, v := range []struct{ long, short string }{{"Intel Corporation", "Intel"}, {"Advanced Micro Devices, Inc. [AMD/ATI]", "AMD"}, {"Advanced Micro Devices, Inc. [AMD]", "AMD"}, {"NVIDIA Corporation", "NVIDIA"}, {"Red Hat, Inc.", "Virtio"}} {
|
||
if strings.HasPrefix(desc, v.long) {
|
||
vendor = v.short
|
||
desc = strings.TrimSpace(desc[len(v.long):])
|
||
break
|
||
}
|
||
}
|
||
if vendor == "" { // vendor not recognised: its first word
|
||
if i := strings.IndexByte(desc, ' '); i > 0 {
|
||
vendor, desc = desc[:i], strings.TrimSpace(desc[i+1:])
|
||
}
|
||
}
|
||
if m := bracketRe.FindStringSubmatch(desc); m != nil {
|
||
desc = m[1]
|
||
}
|
||
return strings.TrimSpace(vendor + " " + desc)
|
||
}
|
||
|
||
// readWithin is read with a deadline. A sysfs read that does not come back must not take
|
||
// the whole sampler with it — every graph on every page of this machine stops — so the
|
||
// device is marked stalled, its goroutine abandoned, and the rest of the sample goes on.
|
||
const gpuReadWithin = 3 * time.Second
|
||
|
||
func (d *gpuDev) readWithin(x *GPU, now time.Time) {
|
||
if d.stalled {
|
||
x.Stalled = true
|
||
return
|
||
}
|
||
done := make(chan GPU, 1)
|
||
go func() {
|
||
y := *x
|
||
d.read(&y, now)
|
||
done <- y
|
||
}()
|
||
select {
|
||
case y := <-done:
|
||
*x = y
|
||
case <-time.After(gpuReadWithin):
|
||
d.stalled, x.Stalled = true, true
|
||
log.Printf("gpu: %s (%s): a sysfs read did not return within %s; its readings are paused until restart", d.name, d.card, gpuReadWithin)
|
||
}
|
||
}
|
||
|
||
// read fills the device-level numbers: clock, idle/busy, hwmon.
|
||
func (d *gpuDev) read(x *GPU, now time.Time) {
|
||
dt := now.Sub(d.prevAt).Seconds()
|
||
switch d.driver {
|
||
case "i915":
|
||
x.MHz = readF(d.card + "/gt_act_freq_mhz")
|
||
x.ReqMHz = readF(d.card + "/gt_cur_freq_mhz")
|
||
x.MaxMHz = readF(d.card + "/gt_max_freq_mhz")
|
||
idle, ok := readInt(d.card + "/gt/gt0/rc6_residency_ms")
|
||
if !ok {
|
||
idle, ok = readInt(d.card + "/power/rc6_residency_ms")
|
||
}
|
||
if ok {
|
||
d.busyFromIdle(x, float64(idle), dt)
|
||
}
|
||
case "xe":
|
||
gt := d.dev + "/tile0/gt0"
|
||
x.MHz = readF(gt + "/freq0/act_freq")
|
||
x.ReqMHz = readF(gt + "/freq0/cur_freq")
|
||
x.MaxMHz = readF(gt + "/freq0/max_freq")
|
||
if idle, ok := readInt(gt + "/gtidle/idle_residency_ms"); ok {
|
||
d.busyFromIdle(x, float64(idle), dt)
|
||
}
|
||
case "amdgpu":
|
||
if v, ok := readInt(d.dev + "/gpu_busy_percent"); ok {
|
||
x.Usage, x.UsageSrc = float64(v), "busy"
|
||
}
|
||
x.MemUsed = uint64(readF(d.dev + "/mem_info_vram_used"))
|
||
x.MemTotal = uint64(readF(d.dev + "/mem_info_vram_total"))
|
||
// "1: 1800Mhz *" — the starred level is the current one, the last the highest
|
||
for _, line := range strings.Split(readStr(d.dev+"/pp_dpm_sclk"), "\n") {
|
||
f := strings.Fields(line)
|
||
if len(f) < 2 {
|
||
continue
|
||
}
|
||
mhz := leadingNum(f[1])
|
||
x.MaxMHz = mhz
|
||
if strings.HasSuffix(line, "*") {
|
||
x.MHz = mhz
|
||
}
|
||
}
|
||
}
|
||
if d.hwmon != "" {
|
||
if t := readF(d.hwmon + "/temp1_input"); t > 0 {
|
||
x.Temp, x.TempSrc = t/1000, "gpu"
|
||
}
|
||
if p := readF(d.hwmon + "/power1_input"); p > 0 {
|
||
x.PowerW = round1(p / 1e6)
|
||
} else if p := readF(d.hwmon + "/power1_average"); p > 0 {
|
||
x.PowerW = round1(p / 1e6)
|
||
} else if e := readF(d.hwmon + "/energy1_input"); e > 0 { // µJ, cumulative
|
||
if d.prevEnergy > 0 && e >= d.prevEnergy && dt > 0 && !d.prevAt.IsZero() {
|
||
x.PowerW = round1((e - d.prevEnergy) / 1e6 / dt)
|
||
}
|
||
d.prevEnergy = e
|
||
}
|
||
if p := readF(d.hwmon + "/power1_cap"); p > 0 {
|
||
x.PowerCapW = p / 1e6
|
||
} else if p := readF(d.hwmon + "/power1_max"); p > 0 {
|
||
x.PowerCapW = p / 1e6
|
||
}
|
||
x.FanRPM = int(readF(d.hwmon + "/fan1_input"))
|
||
}
|
||
d.prevAt = now
|
||
}
|
||
|
||
// busyFromIdle: the GT's idle residency is a running ms total; the share of the interval
|
||
// it did not cover is how busy the device was.
|
||
func (d *gpuDev) busyFromIdle(x *GPU, idle, dt float64) {
|
||
x.UsageSrc = "idle"
|
||
if d.prevAt.IsZero() || dt <= 0 || idle < d.prevIdle {
|
||
d.prevIdle = idle
|
||
return
|
||
}
|
||
busy := 100 - (idle-d.prevIdle)/(dt*1000)*100
|
||
if busy < 0 {
|
||
busy = 0
|
||
}
|
||
if busy > 100 {
|
||
busy = 100
|
||
}
|
||
x.Usage = round1(busy)
|
||
d.prevIdle = idle
|
||
}
|
||
|
||
// ---- DRM fdinfo: who has the device open ----
|
||
|
||
// drmFD is one client of a DRM device: the fdinfo text of one open fd, with its owner.
|
||
type drmFD struct {
|
||
PID int `json:"pid"`
|
||
Comm string `json:"comm"`
|
||
Info string `json:"info"`
|
||
}
|
||
|
||
type fdinfo struct {
|
||
driver, pdev, client string
|
||
engines map[string]uint64 // ns, cumulative
|
||
capacity map[string]int
|
||
mem uint64 // bytes
|
||
}
|
||
|
||
func parseFdinfo(s string) fdinfo {
|
||
fi := fdinfo{engines: map[string]uint64{}, capacity: map[string]int{}}
|
||
var resident, memory uint64
|
||
for _, line := range strings.Split(s, "\n") {
|
||
k, v, ok := strings.Cut(line, ":")
|
||
if !ok || !strings.HasPrefix(k, "drm-") {
|
||
continue
|
||
}
|
||
v = strings.TrimSpace(v)
|
||
switch {
|
||
case k == "drm-driver":
|
||
fi.driver = v
|
||
case k == "drm-pdev":
|
||
fi.pdev = v
|
||
case k == "drm-client-id":
|
||
fi.client = v
|
||
case strings.HasPrefix(k, "drm-engine-capacity-"):
|
||
fi.capacity[k[len("drm-engine-capacity-"):]], _ = strconv.Atoi(v)
|
||
case strings.HasPrefix(k, "drm-engine-"):
|
||
n, _ := strconv.ParseUint(strings.TrimSuffix(v, " ns"), 10, 64)
|
||
fi.engines[k[len("drm-engine-"):]] = n
|
||
case strings.HasPrefix(k, "drm-resident-"):
|
||
resident += kib(v)
|
||
case strings.HasPrefix(k, "drm-memory-"):
|
||
memory += kib(v)
|
||
}
|
||
}
|
||
fi.mem = resident
|
||
if fi.mem == 0 {
|
||
fi.mem = memory
|
||
}
|
||
return fi
|
||
}
|
||
|
||
// kib parses "52624 KiB" (or MiB, or a bare count of bytes) to bytes.
|
||
func kib(v string) uint64 {
|
||
f := strings.Fields(v)
|
||
if len(f) == 0 {
|
||
return 0
|
||
}
|
||
n, _ := strconv.ParseUint(f[0], 10, 64)
|
||
if len(f) > 1 {
|
||
switch f[1] {
|
||
case "KiB":
|
||
n <<= 10
|
||
case "MiB":
|
||
n <<= 20
|
||
case "GiB":
|
||
n <<= 30
|
||
}
|
||
}
|
||
return n
|
||
}
|
||
|
||
// drmScan walks /proc for every fd open on /dev/dri/*, one entry per client per process.
|
||
// Run as root it sees everyone; as a user, only that user's processes.
|
||
func drmScan() (out []drmFD, complete bool) {
|
||
ents, err := os.ReadDir("/proc")
|
||
if err != nil {
|
||
return []drmFD{}, false
|
||
}
|
||
complete = true
|
||
out = []drmFD{}
|
||
for _, e := range ents {
|
||
pid, err := strconv.Atoi(e.Name())
|
||
if err != nil {
|
||
continue
|
||
}
|
||
fds, err := os.ReadDir("/proc/" + e.Name() + "/fd")
|
||
if err != nil {
|
||
if !os.IsNotExist(err) {
|
||
complete = false
|
||
}
|
||
continue
|
||
}
|
||
seen := map[string]bool{}
|
||
comm := ""
|
||
for _, fd := range fds {
|
||
target, err := os.Readlink("/proc/" + e.Name() + "/fd/" + fd.Name())
|
||
if err != nil || !strings.HasPrefix(target, "/dev/dri/") {
|
||
continue
|
||
}
|
||
info := readStr("/proc/" + e.Name() + "/fdinfo/" + fd.Name())
|
||
_, rest, ok := strings.Cut(info, "drm-client-id:")
|
||
if !ok {
|
||
continue
|
||
}
|
||
id := strings.TrimSpace(strings.SplitN(rest, "\n", 2)[0])
|
||
if seen[id] {
|
||
continue
|
||
}
|
||
seen[id] = true
|
||
if comm == "" {
|
||
comm = strings.TrimSpace(readStr("/proc/" + e.Name() + "/comm"))
|
||
}
|
||
out = append(out, drmFD{PID: pid, Comm: comm, Info: info})
|
||
}
|
||
}
|
||
sort.SliceStable(out, func(i, j int) bool { return out[i].PID < out[j].PID })
|
||
return out, complete
|
||
}
|
||
|
||
// scanClients walks in-process when that sees everything (root), else through the helper.
|
||
func (c *Collector) scanClients() ([]drmFD, bool) {
|
||
if priv.Mode() == "root" || !priv.Available() {
|
||
return drmScan()
|
||
}
|
||
if c.gpu.helper == nil {
|
||
c.gpu.helper = &drmHelper{}
|
||
}
|
||
if fds, ok := c.gpu.helper.scan(); ok {
|
||
return fds, true
|
||
}
|
||
return drmScan() // the helper is down: our own processes are better than nothing
|
||
}
|
||
|
||
// drmHelper is the root child that answers "scan" requests. One line in, one JSON line out.
|
||
type drmHelper struct {
|
||
mu sync.Mutex
|
||
cmd *exec.Cmd
|
||
in io.WriteCloser
|
||
out *bufio.Reader
|
||
failedAt time.Time
|
||
}
|
||
|
||
const helperRetry = 30 * time.Second
|
||
|
||
func (h *drmHelper) scan() ([]drmFD, bool) {
|
||
h.mu.Lock()
|
||
defer h.mu.Unlock()
|
||
if h.cmd == nil {
|
||
if !h.failedAt.IsZero() && time.Since(h.failedAt) < helperRetry {
|
||
return nil, false
|
||
}
|
||
if !h.start() {
|
||
h.failedAt = time.Now()
|
||
return nil, false
|
||
}
|
||
}
|
||
// a stuck child must not stall the sampler: it is killed, and tried again later
|
||
cmd := h.cmd
|
||
stop := time.AfterFunc(5*time.Second, func() { cmd.Process.Kill() })
|
||
defer stop.Stop()
|
||
if _, err := io.WriteString(h.in, "\n"); err != nil {
|
||
h.fail()
|
||
return nil, false
|
||
}
|
||
line, err := h.out.ReadBytes('\n')
|
||
if err != nil {
|
||
h.fail()
|
||
return nil, false
|
||
}
|
||
var fds []drmFD
|
||
if json.Unmarshal(line, &fds) != nil {
|
||
h.fail()
|
||
return nil, false
|
||
}
|
||
if fds == nil {
|
||
fds = []drmFD{}
|
||
}
|
||
return fds, true
|
||
}
|
||
|
||
func (h *drmHelper) start() bool {
|
||
self, err := os.Executable()
|
||
if err != nil {
|
||
return false
|
||
}
|
||
full := priv.Wrap(self, "drm-clients")
|
||
cmd := exec.Command(full[0], full[1:]...)
|
||
in, err := cmd.StdinPipe()
|
||
if err != nil {
|
||
return false
|
||
}
|
||
out, err := cmd.StdoutPipe()
|
||
if err != nil {
|
||
return false
|
||
}
|
||
if err := cmd.Start(); err != nil {
|
||
return false
|
||
}
|
||
h.cmd, h.in, h.out = cmd, in, bufio.NewReaderSize(out, 1<<20)
|
||
return true
|
||
}
|
||
|
||
func (h *drmHelper) fail() {
|
||
if h.cmd != nil {
|
||
h.in.Close()
|
||
h.cmd.Process.Kill()
|
||
go h.cmd.Wait()
|
||
}
|
||
h.cmd, h.in, h.out = nil, nil, nil
|
||
h.failedAt = time.Now()
|
||
}
|
||
|
||
// DRMClientsHelper is `hamlab drm-clients`: for every line read on stdin, one JSON line of
|
||
// drmFD on stdout. Ends when stdin does — when the daemon that started it has gone.
|
||
func DRMClientsHelper(stdin io.Reader, stdout io.Writer) int {
|
||
sc := bufio.NewScanner(stdin)
|
||
w := bufio.NewWriter(stdout)
|
||
for sc.Scan() {
|
||
fds, _ := drmScan()
|
||
b, _ := json.Marshal(fds)
|
||
w.Write(b)
|
||
w.WriteByte('\n')
|
||
if w.Flush() != nil {
|
||
return 1
|
||
}
|
||
}
|
||
return 0
|
||
}
|
||
|
||
// ---- small helpers ----
|
||
|
||
func readF(p string) float64 {
|
||
v, err := strconv.ParseFloat(strings.TrimSpace(readStr(p)), 64)
|
||
if err != nil {
|
||
return 0
|
||
}
|
||
return v
|
||
}
|
||
|
||
func readLink(p string) string {
|
||
t, err := os.Readlink(p)
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
return t
|
||
}
|
||
|
||
func ueventValue(p, key string) string {
|
||
for _, line := range strings.Split(readStr(p), "\n") {
|
||
if k, v, ok := strings.Cut(line, "="); ok && k == key {
|
||
return strings.TrimSpace(v)
|
||
}
|
||
}
|
||
return ""
|
||
}
|