A Cache section between the pools and the schedules. One card for the ARC: size against its ceiling with the target and floor, sparklines of size and hit rate from the sampler's history, what it holds (MRU/MFU/dirty, data vs metadata, compression, the ghost lists), the prefetcher and dbuf cache, the memory-pressure counters, and the since-boot breakdown by demand/prefetch and data/metadata. Then a card per pool for L2ARC, SLOG, special vdev, dedup table and spares — the devices and their fill where they exist, and where they don't, plainly none and what that means (the ZIL writing to the pool's own vdevs, with how much it has written). Live numbers come from the sampler, which reads arcstats every tick and publishes zfs.arc.* sensors; the fuller picture is read on request, with zpool list -Hpv and zfs get run without sudo and cached for thirty seconds. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
257 lines
10 KiB
Go
257 lines
10 KiB
Go
package metrics
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
|
|
"hamlab/internal/sensors"
|
|
)
|
|
|
|
// The collector as a sensors.Provider: every number in a Sample, one sensor each. The
|
|
// IDs are built from names the hardware reports (chip, label, interface, device, mount),
|
|
// never from enumeration order like hwmon3, which moves between boots.
|
|
|
|
// flattener turns one Sample into sensors. With want set it builds only those, so walking
|
|
// the whole ring for a history costs string compares, not allocations.
|
|
type flattener struct {
|
|
want func(id string) bool
|
|
seen map[string]int
|
|
out []sensors.Sensor
|
|
}
|
|
|
|
type bounds struct{ min, max, warn, crit *float64 }
|
|
|
|
var pct = bounds{min: sensors.F(0), max: sensors.F(100)}
|
|
|
|
func upTo(max float64) bounds { return bounds{min: sensors.F(0), max: sensors.F(max)} }
|
|
|
|
func (f *flattener) add(id, name, kind, unit string, v any, t int64, b bounds, labels ...string) {
|
|
f.seen[id]++
|
|
if n := f.seen[id]; n > 1 { // two identical chips: the second is coretemp_2
|
|
id = fmt.Sprintf("%s_%d", id, n)
|
|
}
|
|
if f.want != nil && !f.want(id) {
|
|
return
|
|
}
|
|
s := sensors.Sensor{ID: id, Name: name, Kind: kind, Unit: unit, Value: v, T: t, Min: b.min, Max: b.max, Warn: b.warn, Crit: b.crit}
|
|
if len(labels) > 0 {
|
|
s.Labels = make(map[string]string, len(labels)/2)
|
|
for i := 0; i+1 < len(labels); i += 2 {
|
|
s.Labels[labels[i]] = labels[i+1]
|
|
}
|
|
}
|
|
f.out = append(f.out, s)
|
|
}
|
|
|
|
func ratio(part, whole uint64) float64 {
|
|
if whole == 0 {
|
|
return 0
|
|
}
|
|
return round1(float64(part) / float64(whole) * 100)
|
|
}
|
|
|
|
func flatten(s Sample, want func(id string) bool) []sensors.Sensor {
|
|
f := &flattener{want: want, seen: map[string]int{}}
|
|
t, none := s.T, bounds{}
|
|
|
|
f.add("cpu.usage", "CPU", "usage", "%", s.CPU, t, pct)
|
|
for i, c := range s.Cores {
|
|
n := strconv.Itoa(i)
|
|
f.add("cpu.core."+n+".usage", "Core "+n, "usage", "%", c, t, pct, "core", n)
|
|
}
|
|
if c := cpuTemp(s.Temps); c > 0 {
|
|
f.add("cpu.temp", "CPU temperature", "temperature", "°C", c, t, none)
|
|
}
|
|
for i, w := range []string{"1", "5", "15"} {
|
|
f.add("load."+w, "Load "+w+" min", "load", "", s.Load[i], t, none)
|
|
}
|
|
f.add("uptime", "Uptime", "duration", "s", s.Uptime, t, none)
|
|
f.add("hamlab.sample_interval", "Sampling interval", "duration", "ms", float64(s.Interval), t, none)
|
|
|
|
for _, m := range []struct {
|
|
id, name string
|
|
v Mem
|
|
}{{"mem", "Memory", s.Mem}, {"swap", "Swap", s.Swap}} {
|
|
f.add(m.id+".total", m.name+" total", "bytes", "B", float64(m.v.Total), t, none)
|
|
f.add(m.id+".used", m.name+" used", "bytes", "B", float64(m.v.Used), t, upTo(float64(m.v.Total)))
|
|
f.add(m.id+".avail", m.name+" available", "bytes", "B", float64(m.v.Avail), t, upTo(float64(m.v.Total)))
|
|
f.add(m.id+".used_pct", m.name+" used", "usage", "%", ratio(m.v.Used, m.v.Total), t, pct)
|
|
}
|
|
f.add("mem.free", "Memory free", "bytes", "B", float64(s.Mem.Free), t, upTo(float64(s.Mem.Total)))
|
|
f.add("mem.cached", "Memory cached", "bytes", "B", float64(s.Mem.Cached), t, upTo(float64(s.Mem.Total)))
|
|
f.add("mem.buffers", "Memory buffers", "bytes", "B", float64(s.Mem.Buffers), t, upTo(float64(s.Mem.Total)))
|
|
|
|
for _, x := range s.Temps {
|
|
b := bounds{}
|
|
if x.Max > 0 {
|
|
b.warn = sensors.F(x.Max)
|
|
}
|
|
if x.Crit > 0 {
|
|
b.crit = sensors.F(x.Crit)
|
|
}
|
|
f.add(sensors.ID("temp", sensors.Seg(x.Chip), sensors.Seg(x.Label)), x.Chip+" "+x.Label, "temperature", "°C", x.C, t, b, "chip", x.Chip, "label", x.Label)
|
|
}
|
|
for _, x := range s.Fans {
|
|
f.add(sensors.ID("fan", sensors.Seg(x.Chip), sensors.Seg(x.Label)), x.Chip+" "+x.Label, "fan", "rpm", float64(x.RPM), t, none, "chip", x.Chip, "label", x.Label)
|
|
}
|
|
|
|
for _, x := range s.Net {
|
|
p := "net." + sensors.Seg(x.Iface)
|
|
f.add(p+".rx", x.Iface+" receive", "rate", "B/s", x.RxBps, t, none, "iface", x.Iface)
|
|
f.add(p+".tx", x.Iface+" transmit", "rate", "B/s", x.TxBps, t, none, "iface", x.Iface)
|
|
f.add(p+".rx_total", x.Iface+" received", "bytes", "B", float64(x.RxTotal), t, none, "iface", x.Iface)
|
|
f.add(p+".tx_total", x.Iface+" transmitted", "bytes", "B", float64(x.TxTotal), t, none, "iface", x.Iface)
|
|
f.add(p+".up", x.Iface+" link", "state", "", x.Up, t, none, "iface", x.Iface)
|
|
}
|
|
|
|
for _, x := range s.Disks {
|
|
p := "disk." + sensors.Seg(x.Name)
|
|
f.add(p+".read", x.Name+" read", "rate", "B/s", x.ReadBps, t, none, "device", x.Name, "model", x.Model)
|
|
f.add(p+".write", x.Name+" write", "rate", "B/s", x.WriteBps, t, none, "device", x.Name, "model", x.Model)
|
|
f.add(p+".util", x.Name+" busy", "usage", "%", x.UtilPct, t, pct, "device", x.Name, "model", x.Model)
|
|
f.add(p+".size", x.Name+" size", "bytes", "B", float64(x.Size), t, none, "device", x.Name, "model", x.Model)
|
|
if x.Temp > 0 {
|
|
f.add(p+".temp", x.Name+" temperature", "temperature", "°C", x.Temp, t, none, "device", x.Name, "model", x.Model)
|
|
}
|
|
}
|
|
|
|
for _, x := range s.FS {
|
|
p := "fs." + sensors.Seg(x.Mount)
|
|
// remote says the bytes are on another machine (a CIFS or NFS share), so a view of
|
|
// this machine's hardware can leave it out without losing the sensor
|
|
lbl := []string{"mount", x.Mount, "fstype", x.Type}
|
|
if x.Remote {
|
|
lbl = append(lbl, "remote", "1")
|
|
}
|
|
if x.Disk != "" {
|
|
lbl = append(lbl, "disk", x.Disk)
|
|
}
|
|
f.add(p+".size", x.Mount+" size", "bytes", "B", float64(x.Size), s.fsT, none, lbl...)
|
|
f.add(p+".used", x.Mount+" used", "bytes", "B", float64(x.Used), s.fsT, upTo(float64(x.Size)), lbl...)
|
|
f.add(p+".used_pct", x.Mount+" used", "usage", "%", ratio(x.Used, x.Size), s.fsT, pct, lbl...)
|
|
}
|
|
|
|
for _, g := range s.GPUs {
|
|
p := "gpu." + sensors.Seg(g.Name)
|
|
lbl := []string{"name", g.Name, "driver", g.Driver}
|
|
if g.Bus != "" {
|
|
lbl = append(lbl, "bus", g.Bus)
|
|
}
|
|
if g.Integrated {
|
|
lbl = append(lbl, "integrated", "1")
|
|
}
|
|
if g.UsageSrc != "" {
|
|
f.add(p+".usage", g.Name, "usage", "%", g.Usage, t, pct, lbl...)
|
|
}
|
|
for _, e := range []string{"render", "video", "copy", "video-enhance", "compute"} {
|
|
if v, ok := g.Engines[e]; ok {
|
|
f.add(p+"."+sensors.Seg(e), g.Name+" "+e, "usage", "%", v, t, pct, lbl...)
|
|
}
|
|
}
|
|
if g.Temp > 0 {
|
|
f.add(p+".temp", g.Name+" temperature", "temperature", "°C", g.Temp, t, none, append(lbl, "source", g.TempSrc)...)
|
|
}
|
|
if g.PowerW > 0 || g.PowerCapW > 0 {
|
|
b := none
|
|
if g.PowerCapW > 0 {
|
|
b = upTo(g.PowerCapW)
|
|
}
|
|
f.add(p+".power", g.Name+" power", "power", "W", g.PowerW, t, b, lbl...)
|
|
}
|
|
if g.FanRPM > 0 {
|
|
f.add(p+".fan", g.Name+" fan", "fan", "rpm", float64(g.FanRPM), t, none, lbl...)
|
|
}
|
|
if g.MaxMHz > 0 {
|
|
f.add(p+".freq", g.Name+" clock", "frequency", "MHz", g.MHz, t, upTo(g.MaxMHz), lbl...)
|
|
}
|
|
if g.MemTotal > 0 {
|
|
f.add(p+".mem_used", g.Name+" memory used", "bytes", "B", float64(g.MemUsed), t, upTo(float64(g.MemTotal)), lbl...)
|
|
f.add(p+".mem_total", g.Name+" memory", "bytes", "B", float64(g.MemTotal), t, none, lbl...)
|
|
} else if g.MemUsed > 0 {
|
|
f.add(p+".mem_used", g.Name+" memory used", "bytes", "B", float64(g.MemUsed), t, none, lbl...)
|
|
}
|
|
f.add(p+".clients", g.Name+" clients", "count", "", float64(len(g.Clients)), t, none, lbl...)
|
|
}
|
|
|
|
if a := s.ARC; a != nil {
|
|
p := "zfs.arc."
|
|
cap := upTo(float64(a.Max))
|
|
f.add(p+"size", "ARC size", "bytes", "B", float64(a.Size), t, cap)
|
|
f.add(p+"target", "ARC target", "bytes", "B", float64(a.Target), t, cap)
|
|
f.add(p+"min", "ARC floor", "bytes", "B", float64(a.Min), t, none)
|
|
f.add(p+"max", "ARC ceiling", "bytes", "B", float64(a.Max), t, none)
|
|
f.add(p+"mru", "ARC recently used", "bytes", "B", float64(a.MRU), t, cap)
|
|
f.add(p+"mfu", "ARC frequently used", "bytes", "B", float64(a.MFU), t, cap)
|
|
f.add(p+"metadata", "ARC metadata", "bytes", "B", float64(a.Metadata), t, cap)
|
|
f.add(p+"data", "ARC data", "bytes", "B", float64(a.Data), t, cap)
|
|
if a.HitPctKnown {
|
|
f.add(p+"hit_pct", "ARC hit rate", "usage", "%", a.HitPct, t, pct)
|
|
}
|
|
f.add(p+"reads", "ARC reads", "rate", "/s", a.ReadsPerSec, t, none)
|
|
f.add(p+"misses", "ARC misses", "rate", "/s", a.MissesPerSec, t, none)
|
|
f.add(p+"lifetime_hit_pct", "ARC hit rate since boot", "usage", "%", a.LifetimeHitPct, t, pct)
|
|
f.add(p+"no_grow", "ARC held by memory pressure", "state", "", a.NoGrow, t, none)
|
|
if a.Compressed > 0 {
|
|
f.add(p+"compress_ratio", "ARC compression", "ratio", "x", round1(float64(a.Uncompressed)/float64(a.Compressed)*10)/10, t, none)
|
|
}
|
|
if a.L2Size > 0 {
|
|
f.add(p+"l2_size", "L2ARC size", "bytes", "B", float64(a.L2Size), t, none)
|
|
f.add(p+"l2_hit_pct", "L2ARC hit rate", "usage", "%", a.L2HitPct, t, pct)
|
|
}
|
|
}
|
|
|
|
if u := s.UPS; u != nil {
|
|
ut := s.upsT
|
|
f.add("ups.battery", "UPS battery", "usage", "%", u.BatteryPct, ut, pct, "model", u.Model)
|
|
f.add("ups.runtime", "UPS runtime", "duration", "min", u.RuntimeMin, ut, none, "model", u.Model)
|
|
f.add("ups.load", "UPS load", "power", "W", u.LoadW, ut, none, "model", u.Model)
|
|
f.add("ups.load_pct", "UPS load", "usage", "%", u.LoadPct, ut, pct, "model", u.Model)
|
|
f.add("ups.in_v", "UPS input", "voltage", "V", u.InV, ut, none, "model", u.Model)
|
|
f.add("ups.out_v", "UPS output", "voltage", "V", u.OutV, ut, none, "model", u.Model)
|
|
f.add("ups.on_battery", "UPS on battery", "state", "", u.OnBattery, ut, none, "model", u.Model)
|
|
f.add("ups.state", "UPS state", "state", "", u.State, ut, none, "model", u.Model)
|
|
f.add("ups.source", "UPS power source", "state", "", u.Source, ut, none, "model", u.Model)
|
|
f.add("ups.last_event", "UPS last event", "state", "", u.LastEvent, ut, none, "model", u.Model, "driver", u.Driver)
|
|
}
|
|
return f.out
|
|
}
|
|
|
|
// Sensors implements sensors.Provider.
|
|
func (c *Collector) Sensors() []sensors.Sensor {
|
|
s, ok := c.Latest()
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return flatten(s, nil)
|
|
}
|
|
|
|
// SensorHistories implements sensors.Historian: the ring, seen through the sensors that
|
|
// match. A sensor that didn't exist in a sample (a disk plugged in since) simply has a
|
|
// shorter series; nothing is zero-filled. States have no curve and are left out.
|
|
func (c *Collector) SensorHistories(match func(id string) bool) []sensors.Series {
|
|
c.mu.RLock()
|
|
defer c.mu.RUnlock()
|
|
by := map[string]*sensors.Series{}
|
|
var order []string
|
|
for _, smp := range c.ring {
|
|
for _, x := range flatten(smp, match) {
|
|
v, ok := x.Value.(float64)
|
|
if !ok {
|
|
continue
|
|
}
|
|
s := by[x.ID]
|
|
if s == nil {
|
|
s = &sensors.Series{ID: x.ID, Unit: x.Unit, T: []int64{}, V: []float64{}}
|
|
by[x.ID] = s
|
|
order = append(order, x.ID)
|
|
}
|
|
s.T, s.V = append(s.T, smp.T), append(s.V, v)
|
|
}
|
|
}
|
|
out := make([]sensors.Series, 0, len(order))
|
|
for _, id := range order {
|
|
out = append(out, *by[id])
|
|
}
|
|
return out
|
|
}
|