Every DRM device with a real GPU driver gets a box under the cpu box, laid out like it: the busy curve, a meter for the device and one per engine, temperature, clock, power and fan where the card has them, and the processes holding it open with their share. The process table gains a Gpu% column and sort. Device busy is Intel's RC6 idle residency inverted (what intel_gpu_top shows, and readable without root) or amdgpu's gpu_busy_percent. Per-process shares are DRM fdinfo deltas, as nvtop reads them; other users' fdinfo needs root, so a daemon running as a user keeps one root child (hamlab drm-clients) over a pipe rather than paying a sudo a second. Everything is also a gpu.* sensor. What cannot be measured is left out, not shown as zero: an Arc A380 on i915 6.12 has no temperature sensor, an integrated GPU wears the package temperature marked *, and Asahi has neither a busy counter nor fdinfo yet. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
230 lines
8.8 KiB
Go
230 lines
8.8 KiB
Go
package metrics
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
|
|
"hamlab/internal/sensors"
|
|
)
|
|
|
|
// The collector as a sensors.Provider: every number in a Sample, one sensor each. The
|
|
// IDs are built from names the hardware reports (chip, label, interface, device, mount),
|
|
// never from enumeration order like hwmon3, which moves between boots.
|
|
|
|
// flattener turns one Sample into sensors. With want set it builds only those, so walking
|
|
// the whole ring for a history costs string compares, not allocations.
|
|
type flattener struct {
|
|
want func(id string) bool
|
|
seen map[string]int
|
|
out []sensors.Sensor
|
|
}
|
|
|
|
type bounds struct{ min, max, warn, crit *float64 }
|
|
|
|
var pct = bounds{min: sensors.F(0), max: sensors.F(100)}
|
|
|
|
func upTo(max float64) bounds { return bounds{min: sensors.F(0), max: sensors.F(max)} }
|
|
|
|
func (f *flattener) add(id, name, kind, unit string, v any, t int64, b bounds, labels ...string) {
|
|
f.seen[id]++
|
|
if n := f.seen[id]; n > 1 { // two identical chips: the second is coretemp_2
|
|
id = fmt.Sprintf("%s_%d", id, n)
|
|
}
|
|
if f.want != nil && !f.want(id) {
|
|
return
|
|
}
|
|
s := sensors.Sensor{ID: id, Name: name, Kind: kind, Unit: unit, Value: v, T: t, Min: b.min, Max: b.max, Warn: b.warn, Crit: b.crit}
|
|
if len(labels) > 0 {
|
|
s.Labels = make(map[string]string, len(labels)/2)
|
|
for i := 0; i+1 < len(labels); i += 2 {
|
|
s.Labels[labels[i]] = labels[i+1]
|
|
}
|
|
}
|
|
f.out = append(f.out, s)
|
|
}
|
|
|
|
func ratio(part, whole uint64) float64 {
|
|
if whole == 0 {
|
|
return 0
|
|
}
|
|
return round1(float64(part) / float64(whole) * 100)
|
|
}
|
|
|
|
func flatten(s Sample, want func(id string) bool) []sensors.Sensor {
|
|
f := &flattener{want: want, seen: map[string]int{}}
|
|
t, none := s.T, bounds{}
|
|
|
|
f.add("cpu.usage", "CPU", "usage", "%", s.CPU, t, pct)
|
|
for i, c := range s.Cores {
|
|
n := strconv.Itoa(i)
|
|
f.add("cpu.core."+n+".usage", "Core "+n, "usage", "%", c, t, pct, "core", n)
|
|
}
|
|
if c := cpuTemp(s.Temps); c > 0 {
|
|
f.add("cpu.temp", "CPU temperature", "temperature", "°C", c, t, none)
|
|
}
|
|
for i, w := range []string{"1", "5", "15"} {
|
|
f.add("load."+w, "Load "+w+" min", "load", "", s.Load[i], t, none)
|
|
}
|
|
f.add("uptime", "Uptime", "duration", "s", s.Uptime, t, none)
|
|
f.add("hamlab.sample_interval", "Sampling interval", "duration", "ms", float64(s.Interval), t, none)
|
|
|
|
for _, m := range []struct {
|
|
id, name string
|
|
v Mem
|
|
}{{"mem", "Memory", s.Mem}, {"swap", "Swap", s.Swap}} {
|
|
f.add(m.id+".total", m.name+" total", "bytes", "B", float64(m.v.Total), t, none)
|
|
f.add(m.id+".used", m.name+" used", "bytes", "B", float64(m.v.Used), t, upTo(float64(m.v.Total)))
|
|
f.add(m.id+".avail", m.name+" available", "bytes", "B", float64(m.v.Avail), t, upTo(float64(m.v.Total)))
|
|
f.add(m.id+".used_pct", m.name+" used", "usage", "%", ratio(m.v.Used, m.v.Total), t, pct)
|
|
}
|
|
f.add("mem.free", "Memory free", "bytes", "B", float64(s.Mem.Free), t, upTo(float64(s.Mem.Total)))
|
|
f.add("mem.cached", "Memory cached", "bytes", "B", float64(s.Mem.Cached), t, upTo(float64(s.Mem.Total)))
|
|
f.add("mem.buffers", "Memory buffers", "bytes", "B", float64(s.Mem.Buffers), t, upTo(float64(s.Mem.Total)))
|
|
|
|
for _, x := range s.Temps {
|
|
b := bounds{}
|
|
if x.Max > 0 {
|
|
b.warn = sensors.F(x.Max)
|
|
}
|
|
if x.Crit > 0 {
|
|
b.crit = sensors.F(x.Crit)
|
|
}
|
|
f.add(sensors.ID("temp", sensors.Seg(x.Chip), sensors.Seg(x.Label)), x.Chip+" "+x.Label, "temperature", "°C", x.C, t, b, "chip", x.Chip, "label", x.Label)
|
|
}
|
|
for _, x := range s.Fans {
|
|
f.add(sensors.ID("fan", sensors.Seg(x.Chip), sensors.Seg(x.Label)), x.Chip+" "+x.Label, "fan", "rpm", float64(x.RPM), t, none, "chip", x.Chip, "label", x.Label)
|
|
}
|
|
|
|
for _, x := range s.Net {
|
|
p := "net." + sensors.Seg(x.Iface)
|
|
f.add(p+".rx", x.Iface+" receive", "rate", "B/s", x.RxBps, t, none, "iface", x.Iface)
|
|
f.add(p+".tx", x.Iface+" transmit", "rate", "B/s", x.TxBps, t, none, "iface", x.Iface)
|
|
f.add(p+".rx_total", x.Iface+" received", "bytes", "B", float64(x.RxTotal), t, none, "iface", x.Iface)
|
|
f.add(p+".tx_total", x.Iface+" transmitted", "bytes", "B", float64(x.TxTotal), t, none, "iface", x.Iface)
|
|
f.add(p+".up", x.Iface+" link", "state", "", x.Up, t, none, "iface", x.Iface)
|
|
}
|
|
|
|
for _, x := range s.Disks {
|
|
p := "disk." + sensors.Seg(x.Name)
|
|
f.add(p+".read", x.Name+" read", "rate", "B/s", x.ReadBps, t, none, "device", x.Name, "model", x.Model)
|
|
f.add(p+".write", x.Name+" write", "rate", "B/s", x.WriteBps, t, none, "device", x.Name, "model", x.Model)
|
|
f.add(p+".util", x.Name+" busy", "usage", "%", x.UtilPct, t, pct, "device", x.Name, "model", x.Model)
|
|
f.add(p+".size", x.Name+" size", "bytes", "B", float64(x.Size), t, none, "device", x.Name, "model", x.Model)
|
|
if x.Temp > 0 {
|
|
f.add(p+".temp", x.Name+" temperature", "temperature", "°C", x.Temp, t, none, "device", x.Name, "model", x.Model)
|
|
}
|
|
}
|
|
|
|
for _, x := range s.FS {
|
|
p := "fs." + sensors.Seg(x.Mount)
|
|
// remote says the bytes are on another machine (a CIFS or NFS share), so a view of
|
|
// this machine's hardware can leave it out without losing the sensor
|
|
lbl := []string{"mount", x.Mount, "fstype", x.Type}
|
|
if x.Remote {
|
|
lbl = append(lbl, "remote", "1")
|
|
}
|
|
if x.Disk != "" {
|
|
lbl = append(lbl, "disk", x.Disk)
|
|
}
|
|
f.add(p+".size", x.Mount+" size", "bytes", "B", float64(x.Size), s.fsT, none, lbl...)
|
|
f.add(p+".used", x.Mount+" used", "bytes", "B", float64(x.Used), s.fsT, upTo(float64(x.Size)), lbl...)
|
|
f.add(p+".used_pct", x.Mount+" used", "usage", "%", ratio(x.Used, x.Size), s.fsT, pct, lbl...)
|
|
}
|
|
|
|
for _, g := range s.GPUs {
|
|
p := "gpu." + sensors.Seg(g.Name)
|
|
lbl := []string{"name", g.Name, "driver", g.Driver}
|
|
if g.Bus != "" {
|
|
lbl = append(lbl, "bus", g.Bus)
|
|
}
|
|
if g.Integrated {
|
|
lbl = append(lbl, "integrated", "1")
|
|
}
|
|
if g.UsageSrc != "" {
|
|
f.add(p+".usage", g.Name, "usage", "%", g.Usage, t, pct, lbl...)
|
|
}
|
|
for _, e := range []string{"render", "video", "copy", "video-enhance", "compute"} {
|
|
if v, ok := g.Engines[e]; ok {
|
|
f.add(p+"."+sensors.Seg(e), g.Name+" "+e, "usage", "%", v, t, pct, lbl...)
|
|
}
|
|
}
|
|
if g.Temp > 0 {
|
|
f.add(p+".temp", g.Name+" temperature", "temperature", "°C", g.Temp, t, none, append(lbl, "source", g.TempSrc)...)
|
|
}
|
|
if g.PowerW > 0 || g.PowerCapW > 0 {
|
|
b := none
|
|
if g.PowerCapW > 0 {
|
|
b = upTo(g.PowerCapW)
|
|
}
|
|
f.add(p+".power", g.Name+" power", "power", "W", g.PowerW, t, b, lbl...)
|
|
}
|
|
if g.FanRPM > 0 {
|
|
f.add(p+".fan", g.Name+" fan", "fan", "rpm", float64(g.FanRPM), t, none, lbl...)
|
|
}
|
|
if g.MaxMHz > 0 {
|
|
f.add(p+".freq", g.Name+" clock", "frequency", "MHz", g.MHz, t, upTo(g.MaxMHz), lbl...)
|
|
}
|
|
if g.MemTotal > 0 {
|
|
f.add(p+".mem_used", g.Name+" memory used", "bytes", "B", float64(g.MemUsed), t, upTo(float64(g.MemTotal)), lbl...)
|
|
f.add(p+".mem_total", g.Name+" memory", "bytes", "B", float64(g.MemTotal), t, none, lbl...)
|
|
} else if g.MemUsed > 0 {
|
|
f.add(p+".mem_used", g.Name+" memory used", "bytes", "B", float64(g.MemUsed), t, none, lbl...)
|
|
}
|
|
f.add(p+".clients", g.Name+" clients", "count", "", float64(len(g.Clients)), t, none, lbl...)
|
|
}
|
|
|
|
if u := s.UPS; u != nil {
|
|
ut := s.upsT
|
|
f.add("ups.battery", "UPS battery", "usage", "%", u.BatteryPct, ut, pct, "model", u.Model)
|
|
f.add("ups.runtime", "UPS runtime", "duration", "min", u.RuntimeMin, ut, none, "model", u.Model)
|
|
f.add("ups.load", "UPS load", "power", "W", u.LoadW, ut, none, "model", u.Model)
|
|
f.add("ups.load_pct", "UPS load", "usage", "%", u.LoadPct, ut, pct, "model", u.Model)
|
|
f.add("ups.in_v", "UPS input", "voltage", "V", u.InV, ut, none, "model", u.Model)
|
|
f.add("ups.out_v", "UPS output", "voltage", "V", u.OutV, ut, none, "model", u.Model)
|
|
f.add("ups.on_battery", "UPS on battery", "state", "", u.OnBattery, ut, none, "model", u.Model)
|
|
f.add("ups.state", "UPS state", "state", "", u.State, ut, none, "model", u.Model)
|
|
f.add("ups.source", "UPS power source", "state", "", u.Source, ut, none, "model", u.Model)
|
|
f.add("ups.last_event", "UPS last event", "state", "", u.LastEvent, ut, none, "model", u.Model, "driver", u.Driver)
|
|
}
|
|
return f.out
|
|
}
|
|
|
|
// Sensors implements sensors.Provider.
|
|
func (c *Collector) Sensors() []sensors.Sensor {
|
|
s, ok := c.Latest()
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return flatten(s, nil)
|
|
}
|
|
|
|
// SensorHistories implements sensors.Historian: the ring, seen through the sensors that
|
|
// match. A sensor that didn't exist in a sample (a disk plugged in since) simply has a
|
|
// shorter series; nothing is zero-filled. States have no curve and are left out.
|
|
func (c *Collector) SensorHistories(match func(id string) bool) []sensors.Series {
|
|
c.mu.RLock()
|
|
defer c.mu.RUnlock()
|
|
by := map[string]*sensors.Series{}
|
|
var order []string
|
|
for _, smp := range c.ring {
|
|
for _, x := range flatten(smp, match) {
|
|
v, ok := x.Value.(float64)
|
|
if !ok {
|
|
continue
|
|
}
|
|
s := by[x.ID]
|
|
if s == nil {
|
|
s = &sensors.Series{ID: x.ID, Unit: x.Unit, T: []int64{}, V: []float64{}}
|
|
by[x.ID] = s
|
|
order = append(order, x.ID)
|
|
}
|
|
s.T, s.V = append(s.T, smp.T), append(s.V, v)
|
|
}
|
|
}
|
|
out := make([]sensors.Series, 0, len(order))
|
|
for _, id := range order {
|
|
out = append(out, *by[id])
|
|
}
|
|
return out
|
|
}
|