Mitchell-JohnsonandClaude Fable 5.1
3c6f4d4639
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
580 lines
14 KiB
Go
580 lines
14 KiB
Go
// Package metrics samples host telemetry (CPU, memory, temps, fans, network, disks,
|
|
// filesystems, UPS) on a timer into an in-memory ring, for the Dashboard tab.
|
|
// Everything is read from /proc and /sys directly; the only external command is
|
|
// pwrstat (CyberPower PowerPanel), which is optional.
|
|
package metrics
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
Interval = 2 * time.Second
|
|
History = 300 // samples kept (10 min at 2 s)
|
|
fsEvery = 15 // sample fs usage every N ticks
|
|
upsEvery = 5 // sample UPS every N ticks
|
|
sectorSize = 512
|
|
)
|
|
|
|
type Mem struct {
|
|
Total uint64 `json:"total"`
|
|
Used uint64 `json:"used"`
|
|
Avail uint64 `json:"avail"`
|
|
}
|
|
|
|
type Temp struct {
|
|
Chip string `json:"chip"`
|
|
Label string `json:"label"`
|
|
C float64 `json:"c"`
|
|
Max float64 `json:"max,omitempty"`
|
|
Crit float64 `json:"crit,omitempty"`
|
|
}
|
|
|
|
type Fan struct {
|
|
Chip string `json:"chip"`
|
|
Label string `json:"label"`
|
|
RPM int `json:"rpm"`
|
|
}
|
|
|
|
type Net struct {
|
|
Iface string `json:"iface"`
|
|
Up bool `json:"up"`
|
|
RxBps float64 `json:"rx_bps"`
|
|
TxBps float64 `json:"tx_bps"`
|
|
RxTotal uint64 `json:"rx_total"`
|
|
TxTotal uint64 `json:"tx_total"`
|
|
}
|
|
|
|
type Disk struct {
|
|
Name string `json:"name"`
|
|
Model string `json:"model,omitempty"`
|
|
ReadBps float64 `json:"read_bps"`
|
|
WriteBps float64 `json:"write_bps"`
|
|
UtilPct float64 `json:"util_pct"`
|
|
Temp float64 `json:"temp,omitempty"`
|
|
}
|
|
|
|
type FS struct {
|
|
Mount string `json:"mount"`
|
|
Type string `json:"type"`
|
|
Size uint64 `json:"size"`
|
|
Used uint64 `json:"used"`
|
|
}
|
|
|
|
type UPS struct {
|
|
Model string `json:"model"`
|
|
State string `json:"state"`
|
|
Source string `json:"source"`
|
|
InV float64 `json:"in_v"`
|
|
OutV float64 `json:"out_v"`
|
|
BatteryPct float64 `json:"battery_pct"`
|
|
RuntimeMin float64 `json:"runtime_min"`
|
|
LoadW float64 `json:"load_w"`
|
|
LoadPct float64 `json:"load_pct"`
|
|
LastEvent string `json:"last_event,omitempty"`
|
|
}
|
|
|
|
type Sample struct {
|
|
T int64 `json:"t"` // unix ms
|
|
CPU float64 `json:"cpu"`
|
|
Cores []float64 `json:"cores"`
|
|
Load [3]float64 `json:"load"`
|
|
Mem Mem `json:"mem"`
|
|
Swap Mem `json:"swap"`
|
|
Temps []Temp `json:"temps"`
|
|
Fans []Fan `json:"fans"`
|
|
Net []Net `json:"net"`
|
|
Disks []Disk `json:"disks"`
|
|
FS []FS `json:"fs"`
|
|
UPS *UPS `json:"ups"`
|
|
Uptime float64 `json:"uptime"`
|
|
}
|
|
|
|
// Series is the compact history the UI seeds its sparklines from.
|
|
type Series struct {
|
|
T []int64 `json:"t"`
|
|
CPU []float64 `json:"cpu"`
|
|
Mem []uint64 `json:"mem"`
|
|
Net map[string]map[string][]float64 `json:"net"` // iface -> {rx,tx}
|
|
Disk map[string]map[string][]float64 `json:"disk"` // name -> {r,w,u}
|
|
Temp map[string][]float64 `json:"temp"` // "chip/label" -> °C
|
|
}
|
|
|
|
type Collector struct {
|
|
mu sync.RWMutex
|
|
ring []Sample
|
|
tick int
|
|
|
|
prevCPU []cpuTimes
|
|
prevNet map[string][2]uint64
|
|
prevDisk map[string][3]uint64 // sectors read, sectors written, io ticks ms
|
|
prevT time.Time
|
|
lastFS []FS
|
|
lastUPS *UPS
|
|
upsOK bool
|
|
}
|
|
|
|
func NewCollector() *Collector {
|
|
c := &Collector{prevNet: map[string][2]uint64{}, prevDisk: map[string][3]uint64{}}
|
|
c.upsOK = lookPath("pwrstat") && lookPath("sudo")
|
|
return c
|
|
}
|
|
|
|
// Run samples until ctx is cancelled.
|
|
func (c *Collector) Run(ctx context.Context) {
|
|
c.sample() // prime deltas
|
|
t := time.NewTicker(Interval)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
c.sample()
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *Collector) Latest() (Sample, bool) {
|
|
c.mu.RLock()
|
|
defer c.mu.RUnlock()
|
|
if len(c.ring) == 0 {
|
|
return Sample{}, false
|
|
}
|
|
return c.ring[len(c.ring)-1], true
|
|
}
|
|
|
|
func (c *Collector) History() Series {
|
|
c.mu.RLock()
|
|
defer c.mu.RUnlock()
|
|
n := len(c.ring)
|
|
s := Series{T: make([]int64, n), CPU: make([]float64, n), Mem: make([]uint64, n),
|
|
Net: map[string]map[string][]float64{}, Disk: map[string]map[string][]float64{}, Temp: map[string][]float64{}}
|
|
series := func(m map[string]map[string][]float64, key string, keys ...string) map[string][]float64 {
|
|
v, ok := m[key]
|
|
if !ok {
|
|
v = map[string][]float64{}
|
|
for _, k := range keys {
|
|
v[k] = make([]float64, n)
|
|
}
|
|
m[key] = v
|
|
}
|
|
return v
|
|
}
|
|
for i, smp := range c.ring {
|
|
s.T[i], s.CPU[i], s.Mem[i] = smp.T, smp.CPU, smp.Mem.Used
|
|
for _, x := range smp.Net {
|
|
v := series(s.Net, x.Iface, "rx", "tx")
|
|
v["rx"][i], v["tx"][i] = x.RxBps, x.TxBps
|
|
}
|
|
for _, d := range smp.Disks {
|
|
v := series(s.Disk, d.Name, "r", "w", "u")
|
|
v["r"][i], v["w"][i], v["u"][i] = d.ReadBps, d.WriteBps, d.UtilPct
|
|
}
|
|
for _, t := range smp.Temps {
|
|
k := t.Chip + "/" + t.Label
|
|
if s.Temp[k] == nil {
|
|
s.Temp[k] = make([]float64, n)
|
|
}
|
|
s.Temp[k][i] = t.C
|
|
}
|
|
}
|
|
return s
|
|
}
|
|
|
|
// ---- sampling ----
|
|
|
|
func (c *Collector) sample() {
|
|
now := time.Now()
|
|
dt := now.Sub(c.prevT).Seconds()
|
|
first := c.prevT.IsZero()
|
|
c.prevT = now
|
|
|
|
s := Sample{T: now.UnixMilli(), Temps: []Temp{}, Fans: []Fan{}, Net: []Net{}, Disks: []Disk{}}
|
|
s.CPU, s.Cores = c.cpu()
|
|
s.Load = loadavg()
|
|
s.Mem, s.Swap = meminfo()
|
|
s.Uptime = uptime()
|
|
s.Temps, s.Fans = hwmon()
|
|
s.Net = c.net(dt)
|
|
s.Disks = c.disks(dt, s.Temps)
|
|
|
|
if c.tick%fsEvery == 0 || c.lastFS == nil {
|
|
c.lastFS = filesystems()
|
|
}
|
|
s.FS = c.lastFS
|
|
if c.upsOK && (c.tick%upsEvery == 0 || c.lastUPS == nil) {
|
|
c.lastUPS = ups()
|
|
}
|
|
s.UPS = c.lastUPS
|
|
c.tick++
|
|
|
|
if first {
|
|
return // deltas are meaningless on the priming pass
|
|
}
|
|
c.mu.Lock()
|
|
c.ring = append(c.ring, s)
|
|
if len(c.ring) > History {
|
|
c.ring = c.ring[len(c.ring)-History:]
|
|
}
|
|
c.mu.Unlock()
|
|
}
|
|
|
|
type cpuTimes struct{ busy, total uint64 }
|
|
|
|
func (c *Collector) cpu() (float64, []float64) {
|
|
f, err := os.Open("/proc/stat")
|
|
if err != nil {
|
|
return 0, nil
|
|
}
|
|
defer f.Close()
|
|
var cur []cpuTimes
|
|
sc := bufio.NewScanner(f)
|
|
for sc.Scan() {
|
|
fs := strings.Fields(sc.Text())
|
|
if len(fs) < 8 || !strings.HasPrefix(fs[0], "cpu") {
|
|
continue
|
|
}
|
|
var vals [10]uint64
|
|
for i := 1; i < len(fs) && i <= 10; i++ {
|
|
vals[i-1], _ = strconv.ParseUint(fs[i], 10, 64)
|
|
}
|
|
idle := vals[3] + vals[4]
|
|
var total uint64
|
|
for _, v := range vals {
|
|
total += v
|
|
}
|
|
cur = append(cur, cpuTimes{busy: total - idle, total: total})
|
|
}
|
|
prev := c.prevCPU
|
|
c.prevCPU = cur
|
|
if len(prev) != len(cur) || len(cur) == 0 {
|
|
return 0, nil
|
|
}
|
|
pct := func(i int) float64 {
|
|
dt := float64(cur[i].total - prev[i].total)
|
|
if dt <= 0 {
|
|
return 0
|
|
}
|
|
return 100 * float64(cur[i].busy-prev[i].busy) / dt
|
|
}
|
|
cores := make([]float64, 0, len(cur)-1)
|
|
for i := 1; i < len(cur); i++ {
|
|
cores = append(cores, round1(pct(i)))
|
|
}
|
|
return round1(pct(0)), cores
|
|
}
|
|
|
|
func loadavg() (l [3]float64) {
|
|
b, err := os.ReadFile("/proc/loadavg")
|
|
if err != nil {
|
|
return
|
|
}
|
|
f := strings.Fields(string(b))
|
|
for i := 0; i < 3 && i < len(f); i++ {
|
|
l[i], _ = strconv.ParseFloat(f[i], 64)
|
|
}
|
|
return
|
|
}
|
|
|
|
func meminfo() (Mem, Mem) {
|
|
m := map[string]uint64{}
|
|
b, err := os.ReadFile("/proc/meminfo")
|
|
if err != nil {
|
|
return Mem{}, Mem{}
|
|
}
|
|
for _, line := range strings.Split(string(b), "\n") {
|
|
k, v, ok := strings.Cut(line, ":")
|
|
if !ok {
|
|
continue
|
|
}
|
|
f := strings.Fields(v)
|
|
if len(f) == 0 {
|
|
continue
|
|
}
|
|
n, _ := strconv.ParseUint(f[0], 10, 64)
|
|
m[k] = n * 1024
|
|
}
|
|
mem := Mem{Total: m["MemTotal"], Avail: m["MemAvailable"]}
|
|
mem.Used = mem.Total - mem.Avail
|
|
swap := Mem{Total: m["SwapTotal"], Avail: m["SwapFree"]}
|
|
swap.Used = swap.Total - swap.Avail
|
|
return mem, swap
|
|
}
|
|
|
|
func uptime() float64 {
|
|
b, err := os.ReadFile("/proc/uptime")
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
if f := strings.Fields(string(b)); len(f) > 0 {
|
|
v, _ := strconv.ParseFloat(f[0], 64)
|
|
return v
|
|
}
|
|
return 0
|
|
}
|
|
|
|
var sensorRe = regexp.MustCompile(`^(temp|fan)(\d+)_input$`)
|
|
|
|
func hwmon() ([]Temp, []Fan) {
|
|
temps, fans := []Temp{}, []Fan{}
|
|
dirs, _ := filepath.Glob("/sys/class/hwmon/hwmon*")
|
|
for _, d := range dirs {
|
|
chip := strings.TrimSpace(readStr(filepath.Join(d, "name")))
|
|
ents, err := os.ReadDir(d)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, e := range ents {
|
|
m := sensorRe.FindStringSubmatch(e.Name())
|
|
if m == nil {
|
|
continue
|
|
}
|
|
base := m[1] + m[2]
|
|
raw, ok := readInt(filepath.Join(d, e.Name()))
|
|
if !ok {
|
|
continue
|
|
}
|
|
label := strings.TrimSpace(readStr(filepath.Join(d, base+"_label")))
|
|
if label == "" {
|
|
label = base
|
|
}
|
|
if m[1] == "temp" {
|
|
t := Temp{Chip: chip, Label: label, C: float64(raw) / 1000}
|
|
// Limits outside 0-200 °C are unset sentinels (NVMe reports ~65000 for absent ones).
|
|
if v, ok := readInt(filepath.Join(d, base+"_max")); ok && v > 0 && v < 200000 {
|
|
t.Max = float64(v) / 1000
|
|
}
|
|
if v, ok := readInt(filepath.Join(d, base+"_crit")); ok && v > 0 && v < 200000 {
|
|
t.Crit = float64(v) / 1000
|
|
}
|
|
temps = append(temps, t)
|
|
} else {
|
|
fans = append(fans, Fan{Chip: chip, Label: label, RPM: int(raw)})
|
|
}
|
|
}
|
|
}
|
|
sort.SliceStable(temps, func(i, j int) bool {
|
|
if temps[i].Chip != temps[j].Chip {
|
|
return temps[i].Chip < temps[j].Chip
|
|
}
|
|
return temps[i].Label < temps[j].Label
|
|
})
|
|
return temps, fans
|
|
}
|
|
|
|
func skipIface(n string) bool {
|
|
return n == "lo" || strings.HasPrefix(n, "veth") || strings.HasPrefix(n, "br-") ||
|
|
strings.HasPrefix(n, "docker") || strings.HasPrefix(n, "virbr")
|
|
}
|
|
|
|
func (c *Collector) net(dt float64) []Net {
|
|
out := []Net{}
|
|
f, err := os.Open("/proc/net/dev")
|
|
if err != nil {
|
|
return out
|
|
}
|
|
defer f.Close()
|
|
sc := bufio.NewScanner(f)
|
|
for sc.Scan() {
|
|
name, rest, ok := strings.Cut(sc.Text(), ":")
|
|
if !ok {
|
|
continue
|
|
}
|
|
name = strings.TrimSpace(name)
|
|
if skipIface(name) {
|
|
continue
|
|
}
|
|
fs := strings.Fields(rest)
|
|
if len(fs) < 16 {
|
|
continue
|
|
}
|
|
rx, _ := strconv.ParseUint(fs[0], 10, 64)
|
|
tx, _ := strconv.ParseUint(fs[8], 10, 64)
|
|
n := Net{Iface: name, RxTotal: rx, TxTotal: tx}
|
|
st := strings.TrimSpace(readStr("/sys/class/net/" + name + "/operstate"))
|
|
n.Up = st == "up" || st == "unknown" // wireguard/tun report "unknown" while carrying traffic
|
|
if p, ok := c.prevNet[name]; ok && dt > 0 {
|
|
n.RxBps = round1(float64(rx-p[0]) / dt)
|
|
n.TxBps = round1(float64(tx-p[1]) / dt)
|
|
}
|
|
c.prevNet[name] = [2]uint64{rx, tx}
|
|
out = append(out, n)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Iface < out[j].Iface })
|
|
return out
|
|
}
|
|
|
|
var wholeDisk = regexp.MustCompile(`^(sd[a-z]+|vd[a-z]+|nvme\d+n\d+|mmcblk\d+)$`)
|
|
|
|
func (c *Collector) disks(dt float64, temps []Temp) []Disk {
|
|
out := []Disk{}
|
|
f, err := os.Open("/proc/diskstats")
|
|
if err != nil {
|
|
return out
|
|
}
|
|
defer f.Close()
|
|
sc := bufio.NewScanner(f)
|
|
for sc.Scan() {
|
|
fs := strings.Fields(sc.Text())
|
|
if len(fs) < 14 || !wholeDisk.MatchString(fs[2]) {
|
|
continue
|
|
}
|
|
name := fs[2]
|
|
rd, _ := strconv.ParseUint(fs[5], 10, 64)
|
|
wr, _ := strconv.ParseUint(fs[9], 10, 64)
|
|
ticks, _ := strconv.ParseUint(fs[12], 10, 64)
|
|
d := Disk{Name: name, Model: strings.TrimSpace(readStr("/sys/block/" + name + "/device/model"))}
|
|
if p, ok := c.prevDisk[name]; ok && dt > 0 {
|
|
d.ReadBps = round1(float64(rd-p[0]) * sectorSize / dt)
|
|
d.WriteBps = round1(float64(wr-p[1]) * sectorSize / dt)
|
|
d.UtilPct = round1(float64(ticks-p[2]) / (dt * 1000) * 100)
|
|
if d.UtilPct > 100 {
|
|
d.UtilPct = 100
|
|
}
|
|
}
|
|
c.prevDisk[name] = [3]uint64{rd, wr, ticks}
|
|
// NVMe exposes a "Composite" temp on an hwmon chip named "nvme"; attach it when there is exactly one such disk.
|
|
if strings.HasPrefix(name, "nvme") {
|
|
for _, t := range temps {
|
|
if t.Chip == "nvme" && t.Label == "Composite" {
|
|
d.Temp = t.C
|
|
}
|
|
}
|
|
}
|
|
out = append(out, d)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
|
return out
|
|
}
|
|
|
|
var realFS = map[string]bool{"ext4": true, "ext3": true, "ext2": true, "xfs": true, "btrfs": true, "zfs": true,
|
|
"f2fs": true, "vfat": true, "exfat": true, "ntfs": true, "ntfs3": true, "cifs": true, "nfs": true, "nfs4": true, "fuse.sshfs": true}
|
|
|
|
func filesystems() []FS {
|
|
out := []FS{}
|
|
b, err := os.ReadFile("/proc/mounts")
|
|
if err != nil {
|
|
return out
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, line := range strings.Split(string(b), "\n") {
|
|
fs := strings.Fields(line)
|
|
if len(fs) < 3 || !realFS[fs[2]] || seen[fs[1]] {
|
|
continue
|
|
}
|
|
seen[fs[1]] = true
|
|
mount := strings.ReplaceAll(fs[1], `\040`, " ")
|
|
// statfs on a dead network mount can block; bound it.
|
|
done := make(chan FS, 1)
|
|
go func() {
|
|
var st syscall.Statfs_t
|
|
if err := syscall.Statfs(mount, &st); err != nil {
|
|
done <- FS{}
|
|
return
|
|
}
|
|
bs := uint64(st.Bsize)
|
|
done <- FS{Mount: mount, Type: fs[2], Size: st.Blocks * bs, Used: (st.Blocks - st.Bfree) * bs}
|
|
}()
|
|
select {
|
|
case r := <-done:
|
|
if r.Size > 0 {
|
|
out = append(out, r)
|
|
}
|
|
case <-time.After(3 * time.Second):
|
|
out = append(out, FS{Mount: mount, Type: fs[2]}) // unreachable: size 0 signals it
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func ups() *UPS {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
b, err := exec.CommandContext(ctx, "sudo", "-n", "pwrstat", "-status").Output()
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
u := &UPS{}
|
|
got := false
|
|
num := func(s string) float64 {
|
|
s = strings.TrimSpace(s)
|
|
end := 0
|
|
for end < len(s) && (s[end] >= '0' && s[end] <= '9' || s[end] == '.') {
|
|
end++
|
|
}
|
|
v, _ := strconv.ParseFloat(s[:end], 64)
|
|
return v
|
|
}
|
|
for _, line := range strings.Split(string(b), "\n") {
|
|
k, v, ok := strings.Cut(line, "....")
|
|
if !ok {
|
|
continue
|
|
}
|
|
k = strings.TrimSpace(k)
|
|
v = strings.TrimSpace(strings.TrimLeft(v, "."))
|
|
got = true
|
|
switch k {
|
|
case "Model Name":
|
|
u.Model = v
|
|
case "State":
|
|
u.State = v
|
|
case "Power Supply by":
|
|
u.Source = v
|
|
case "Utility Voltage":
|
|
u.InV = num(v)
|
|
case "Output Voltage":
|
|
u.OutV = num(v)
|
|
case "Battery Capacity":
|
|
u.BatteryPct = num(v)
|
|
case "Remaining Runtime":
|
|
u.RuntimeMin = num(v)
|
|
case "Load":
|
|
u.LoadW = num(v)
|
|
if i := strings.Index(v, "("); i >= 0 {
|
|
u.LoadPct = num(v[i+1:])
|
|
}
|
|
case "Last Power Event":
|
|
u.LastEvent = v
|
|
}
|
|
}
|
|
if !got {
|
|
return nil
|
|
}
|
|
return u
|
|
}
|
|
|
|
// ---- small helpers ----
|
|
|
|
func readStr(p string) string {
|
|
b, err := os.ReadFile(p)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func readInt(p string) (int64, bool) {
|
|
s := strings.TrimSpace(readStr(p))
|
|
if s == "" {
|
|
return 0, false
|
|
}
|
|
v, err := strconv.ParseInt(s, 10, 64)
|
|
return v, err == nil
|
|
}
|
|
|
|
func round1(v float64) float64 { return float64(int64(v*10+0.5)) / 10 }
|
|
|
|
func lookPath(bin string) bool { _, err := exec.LookPath(bin); return err == nil }
|