New sensor cpu.power: watts from the RAPL energy counters (Intel, and AMD Zen through the same driver), summed over every package and averaged into history. Parts of a package, psys and the MMIO duplicate are left out. The counters are root-only, so a daemon that isn't root reads them through `hamlab cpu-energy`, a root helper started once with sudo or doas and asked over a pipe each sample, the same shape as the GPU client walk's helper. It reads only the counters it finds itself. No counters or no root means no reading, and the CPU box shows nothing extra. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
260 lines
10 KiB
Go
260 lines
10 KiB
Go
package metrics
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
|
|
"hamlab/internal/sensors"
|
|
)
|
|
|
|
// The collector as a sensors.Provider: every number in a Sample, one sensor each. The
|
|
// IDs are built from names the hardware reports (chip, label, interface, device, mount),
|
|
// never from enumeration order like hwmon3, which moves between boots.
|
|
|
|
// flattener turns one Sample into sensors. With want set it builds only those, so walking
|
|
// the whole ring for a history costs string compares, not allocations.
|
|
type flattener struct {
|
|
want func(id string) bool
|
|
seen map[string]int
|
|
out []sensors.Sensor
|
|
}
|
|
|
|
type bounds struct{ min, max, warn, crit *float64 }
|
|
|
|
var pct = bounds{min: sensors.F(0), max: sensors.F(100)}
|
|
|
|
func upTo(max float64) bounds { return bounds{min: sensors.F(0), max: sensors.F(max)} }
|
|
|
|
func (f *flattener) add(id, name, kind, unit string, v any, t int64, b bounds, labels ...string) {
|
|
f.seen[id]++
|
|
if n := f.seen[id]; n > 1 { // two identical chips: the second is coretemp_2
|
|
id = fmt.Sprintf("%s_%d", id, n)
|
|
}
|
|
if f.want != nil && !f.want(id) {
|
|
return
|
|
}
|
|
s := sensors.Sensor{ID: id, Name: name, Kind: kind, Unit: unit, Value: v, T: t, Min: b.min, Max: b.max, Warn: b.warn, Crit: b.crit}
|
|
if len(labels) > 0 {
|
|
s.Labels = make(map[string]string, len(labels)/2)
|
|
for i := 0; i+1 < len(labels); i += 2 {
|
|
s.Labels[labels[i]] = labels[i+1]
|
|
}
|
|
}
|
|
f.out = append(f.out, s)
|
|
}
|
|
|
|
func ratio(part, whole uint64) float64 {
|
|
if whole == 0 {
|
|
return 0
|
|
}
|
|
return round1(float64(part) / float64(whole) * 100)
|
|
}
|
|
|
|
func flatten(s Sample, want func(id string) bool) []sensors.Sensor {
|
|
f := &flattener{want: want, seen: map[string]int{}}
|
|
t, none := s.T, bounds{}
|
|
|
|
f.add("cpu.usage", "CPU", "usage", "%", s.CPU, t, pct)
|
|
for i, c := range s.Cores {
|
|
n := strconv.Itoa(i)
|
|
f.add("cpu.core."+n+".usage", "Core "+n, "usage", "%", c, t, pct, "core", n)
|
|
}
|
|
if c := cpuTemp(s.Temps); c > 0 {
|
|
f.add("cpu.temp", "CPU temperature", "temperature", "°C", c, t, none)
|
|
}
|
|
if s.CPUPower > 0 {
|
|
f.add("cpu.power", "CPU power", "power", "W", s.CPUPower, t, none)
|
|
}
|
|
for i, w := range []string{"1", "5", "15"} {
|
|
f.add("load."+w, "Load "+w+" min", "load", "", s.Load[i], t, none)
|
|
}
|
|
f.add("uptime", "Uptime", "duration", "s", s.Uptime, t, none)
|
|
f.add("hamlab.sample_interval", "Sampling interval", "duration", "ms", float64(s.Interval), t, none)
|
|
|
|
for _, m := range []struct {
|
|
id, name string
|
|
v Mem
|
|
}{{"mem", "Memory", s.Mem}, {"swap", "Swap", s.Swap}} {
|
|
f.add(m.id+".total", m.name+" total", "bytes", "B", float64(m.v.Total), t, none)
|
|
f.add(m.id+".used", m.name+" used", "bytes", "B", float64(m.v.Used), t, upTo(float64(m.v.Total)))
|
|
f.add(m.id+".avail", m.name+" available", "bytes", "B", float64(m.v.Avail), t, upTo(float64(m.v.Total)))
|
|
f.add(m.id+".used_pct", m.name+" used", "usage", "%", ratio(m.v.Used, m.v.Total), t, pct)
|
|
}
|
|
f.add("mem.free", "Memory free", "bytes", "B", float64(s.Mem.Free), t, upTo(float64(s.Mem.Total)))
|
|
f.add("mem.cached", "Memory cached", "bytes", "B", float64(s.Mem.Cached), t, upTo(float64(s.Mem.Total)))
|
|
f.add("mem.buffers", "Memory buffers", "bytes", "B", float64(s.Mem.Buffers), t, upTo(float64(s.Mem.Total)))
|
|
|
|
for _, x := range s.Temps {
|
|
b := bounds{}
|
|
if x.Max > 0 {
|
|
b.warn = sensors.F(x.Max)
|
|
}
|
|
if x.Crit > 0 {
|
|
b.crit = sensors.F(x.Crit)
|
|
}
|
|
f.add(sensors.ID("temp", sensors.Seg(x.Chip), sensors.Seg(x.Label)), x.Chip+" "+x.Label, "temperature", "°C", x.C, t, b, "chip", x.Chip, "label", x.Label)
|
|
}
|
|
for _, x := range s.Fans {
|
|
f.add(sensors.ID("fan", sensors.Seg(x.Chip), sensors.Seg(x.Label)), x.Chip+" "+x.Label, "fan", "rpm", float64(x.RPM), t, none, "chip", x.Chip, "label", x.Label)
|
|
}
|
|
|
|
for _, x := range s.Net {
|
|
p := "net." + sensors.Seg(x.Iface)
|
|
f.add(p+".rx", x.Iface+" receive", "rate", "B/s", x.RxBps, t, none, "iface", x.Iface)
|
|
f.add(p+".tx", x.Iface+" transmit", "rate", "B/s", x.TxBps, t, none, "iface", x.Iface)
|
|
f.add(p+".rx_total", x.Iface+" received", "bytes", "B", float64(x.RxTotal), t, none, "iface", x.Iface)
|
|
f.add(p+".tx_total", x.Iface+" transmitted", "bytes", "B", float64(x.TxTotal), t, none, "iface", x.Iface)
|
|
f.add(p+".up", x.Iface+" link", "state", "", x.Up, t, none, "iface", x.Iface)
|
|
}
|
|
|
|
for _, x := range s.Disks {
|
|
p := "disk." + sensors.Seg(x.Name)
|
|
f.add(p+".read", x.Name+" read", "rate", "B/s", x.ReadBps, t, none, "device", x.Name, "model", x.Model)
|
|
f.add(p+".write", x.Name+" write", "rate", "B/s", x.WriteBps, t, none, "device", x.Name, "model", x.Model)
|
|
f.add(p+".util", x.Name+" busy", "usage", "%", x.UtilPct, t, pct, "device", x.Name, "model", x.Model)
|
|
f.add(p+".size", x.Name+" size", "bytes", "B", float64(x.Size), t, none, "device", x.Name, "model", x.Model)
|
|
if x.Temp > 0 {
|
|
f.add(p+".temp", x.Name+" temperature", "temperature", "°C", x.Temp, t, none, "device", x.Name, "model", x.Model)
|
|
}
|
|
}
|
|
|
|
for _, x := range s.FS {
|
|
p := "fs." + sensors.Seg(x.Mount)
|
|
// remote says the bytes are on another machine (a CIFS or NFS share), so a view of
|
|
// this machine's hardware can leave it out without losing the sensor
|
|
lbl := []string{"mount", x.Mount, "fstype", x.Type}
|
|
if x.Remote {
|
|
lbl = append(lbl, "remote", "1")
|
|
}
|
|
if x.Disk != "" {
|
|
lbl = append(lbl, "disk", x.Disk)
|
|
}
|
|
f.add(p+".size", x.Mount+" size", "bytes", "B", float64(x.Size), s.fsT, none, lbl...)
|
|
f.add(p+".used", x.Mount+" used", "bytes", "B", float64(x.Used), s.fsT, upTo(float64(x.Size)), lbl...)
|
|
f.add(p+".used_pct", x.Mount+" used", "usage", "%", ratio(x.Used, x.Size), s.fsT, pct, lbl...)
|
|
}
|
|
|
|
for _, g := range s.GPUs {
|
|
p := "gpu." + sensors.Seg(g.Name)
|
|
lbl := []string{"name", g.Name, "driver", g.Driver}
|
|
if g.Bus != "" {
|
|
lbl = append(lbl, "bus", g.Bus)
|
|
}
|
|
if g.Integrated {
|
|
lbl = append(lbl, "integrated", "1")
|
|
}
|
|
if g.UsageSrc != "" {
|
|
f.add(p+".usage", g.Name, "usage", "%", g.Usage, t, pct, lbl...)
|
|
}
|
|
for _, e := range []string{"render", "video", "copy", "video-enhance", "compute"} {
|
|
if v, ok := g.Engines[e]; ok {
|
|
f.add(p+"."+sensors.Seg(e), g.Name+" "+e, "usage", "%", v, t, pct, lbl...)
|
|
}
|
|
}
|
|
if g.Temp > 0 {
|
|
f.add(p+".temp", g.Name+" temperature", "temperature", "°C", g.Temp, t, none, append(lbl, "source", g.TempSrc)...)
|
|
}
|
|
if g.PowerW > 0 || g.PowerCapW > 0 {
|
|
b := none
|
|
if g.PowerCapW > 0 {
|
|
b = upTo(g.PowerCapW)
|
|
}
|
|
f.add(p+".power", g.Name+" power", "power", "W", g.PowerW, t, b, lbl...)
|
|
}
|
|
if g.FanRPM > 0 {
|
|
f.add(p+".fan", g.Name+" fan", "fan", "rpm", float64(g.FanRPM), t, none, lbl...)
|
|
}
|
|
if g.MaxMHz > 0 {
|
|
f.add(p+".freq", g.Name+" clock", "frequency", "MHz", g.MHz, t, upTo(g.MaxMHz), lbl...)
|
|
}
|
|
if g.MemTotal > 0 {
|
|
f.add(p+".mem_used", g.Name+" memory used", "bytes", "B", float64(g.MemUsed), t, upTo(float64(g.MemTotal)), lbl...)
|
|
f.add(p+".mem_total", g.Name+" memory", "bytes", "B", float64(g.MemTotal), t, none, lbl...)
|
|
} else if g.MemUsed > 0 {
|
|
f.add(p+".mem_used", g.Name+" memory used", "bytes", "B", float64(g.MemUsed), t, none, lbl...)
|
|
}
|
|
f.add(p+".clients", g.Name+" clients", "count", "", float64(len(g.Clients)), t, none, lbl...)
|
|
}
|
|
|
|
if a := s.ARC; a != nil {
|
|
p := "zfs.arc."
|
|
cap := upTo(float64(a.Max))
|
|
f.add(p+"size", "ARC size", "bytes", "B", float64(a.Size), t, cap)
|
|
f.add(p+"target", "ARC target", "bytes", "B", float64(a.Target), t, cap)
|
|
f.add(p+"min", "ARC floor", "bytes", "B", float64(a.Min), t, none)
|
|
f.add(p+"max", "ARC ceiling", "bytes", "B", float64(a.Max), t, none)
|
|
f.add(p+"mru", "ARC recently used", "bytes", "B", float64(a.MRU), t, cap)
|
|
f.add(p+"mfu", "ARC frequently used", "bytes", "B", float64(a.MFU), t, cap)
|
|
f.add(p+"metadata", "ARC metadata", "bytes", "B", float64(a.Metadata), t, cap)
|
|
f.add(p+"data", "ARC data", "bytes", "B", float64(a.Data), t, cap)
|
|
if a.HitPctKnown {
|
|
f.add(p+"hit_pct", "ARC hit rate", "usage", "%", a.HitPct, t, pct)
|
|
}
|
|
f.add(p+"reads", "ARC reads", "rate", "/s", a.ReadsPerSec, t, none)
|
|
f.add(p+"misses", "ARC misses", "rate", "/s", a.MissesPerSec, t, none)
|
|
f.add(p+"lifetime_hit_pct", "ARC hit rate since boot", "usage", "%", a.LifetimeHitPct, t, pct)
|
|
f.add(p+"no_grow", "ARC held by memory pressure", "state", "", a.NoGrow, t, none)
|
|
if a.Compressed > 0 {
|
|
f.add(p+"compress_ratio", "ARC compression", "ratio", "x", round1(float64(a.Uncompressed)/float64(a.Compressed)*10)/10, t, none)
|
|
}
|
|
if a.L2Size > 0 {
|
|
f.add(p+"l2_size", "L2ARC size", "bytes", "B", float64(a.L2Size), t, none)
|
|
f.add(p+"l2_hit_pct", "L2ARC hit rate", "usage", "%", a.L2HitPct, t, pct)
|
|
}
|
|
}
|
|
|
|
if u := s.UPS; u != nil {
|
|
ut := s.upsT
|
|
f.add("ups.battery", "UPS battery", "usage", "%", u.BatteryPct, ut, pct, "model", u.Model)
|
|
f.add("ups.runtime", "UPS runtime", "duration", "min", u.RuntimeMin, ut, none, "model", u.Model)
|
|
f.add("ups.load", "UPS load", "power", "W", u.LoadW, ut, none, "model", u.Model)
|
|
f.add("ups.load_pct", "UPS load", "usage", "%", u.LoadPct, ut, pct, "model", u.Model)
|
|
f.add("ups.in_v", "UPS input", "voltage", "V", u.InV, ut, none, "model", u.Model)
|
|
f.add("ups.out_v", "UPS output", "voltage", "V", u.OutV, ut, none, "model", u.Model)
|
|
f.add("ups.on_battery", "UPS on battery", "state", "", u.OnBattery, ut, none, "model", u.Model)
|
|
f.add("ups.state", "UPS state", "state", "", u.State, ut, none, "model", u.Model)
|
|
f.add("ups.source", "UPS power source", "state", "", u.Source, ut, none, "model", u.Model)
|
|
f.add("ups.last_event", "UPS last event", "state", "", u.LastEvent, ut, none, "model", u.Model, "driver", u.Driver)
|
|
}
|
|
return f.out
|
|
}
|
|
|
|
// Sensors implements sensors.Provider.
|
|
func (c *Collector) Sensors() []sensors.Sensor {
|
|
s, ok := c.Latest()
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return flatten(s, nil)
|
|
}
|
|
|
|
// SensorHistories implements sensors.Historian: the ring, seen through the sensors that
|
|
// match. A sensor that didn't exist in a sample (a disk plugged in since) simply has a
|
|
// shorter series; nothing is zero-filled. States have no curve and are left out.
|
|
func (c *Collector) SensorHistories(match func(id string) bool) []sensors.Series {
|
|
c.mu.RLock()
|
|
defer c.mu.RUnlock()
|
|
by := map[string]*sensors.Series{}
|
|
var order []string
|
|
for _, smp := range c.ring {
|
|
for _, x := range flatten(smp, match) {
|
|
v, ok := x.Value.(float64)
|
|
if !ok {
|
|
continue
|
|
}
|
|
s := by[x.ID]
|
|
if s == nil {
|
|
s = &sensors.Series{ID: x.ID, Unit: x.Unit, T: []int64{}, V: []float64{}}
|
|
by[x.ID] = s
|
|
order = append(order, x.ID)
|
|
}
|
|
s.T, s.V = append(s.T, smp.T), append(s.V, v)
|
|
}
|
|
}
|
|
out := make([]sensors.Series, 0, len(order))
|
|
for _, id := range order {
|
|
out = append(out, *by[id])
|
|
}
|
|
return out
|
|
}
|