hamlab/internal/metrics/metrics.go

// Package metrics samples host telemetry (CPU, memory, temps, fans, network, disks,
// filesystems, UPS) on a timer into an in-memory ring, for the Dashboard tab.
//
// Portability rules: everything comes from /proc and /sys, which every Linux kernel
// exposes the same way regardless of distro or architecture; nothing here needs
// lm-sensors, smartmontools or root. Sections whose source is absent (no fans, no
// UPS daemon, no hwmon at all on a VM) simply come back empty rather than failing.
// The only external commands are optional UPS daemons: pwrstat (CyberPower), upsc
// (NUT) and apcaccess (apcupsd), tried in that order.
package metrics

import (
	"bufio"
	"context"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

const (
	Interval   = 2 * time.Second
	History    = 300 // samples kept (10 min at 2 s)
	fsEvery    = 15  // sample fs usage every N ticks
	upsEvery   = 5   // sample UPS every N ticks
	sectorSize = 512 // /proc/diskstats always counts 512-byte sectors
)

type Mem struct {
	Total uint64 `json:"total"`
	Used  uint64 `json:"used"`
	Avail uint64 `json:"avail"`
}

type Temp struct {
	Chip  string  `json:"chip"`
	Label string  `json:"label"`
	C     float64 `json:"c"`
	Max   float64 `json:"max,omitempty"`
	Crit  float64 `json:"crit,omitempty"`
}

type Fan struct {
	Chip  string `json:"chip"`
	Label string `json:"label"`
	RPM   int    `json:"rpm"`
}

type Net struct {
	Iface   string  `json:"iface"`
	Up      bool    `json:"up"`
	RxBps   float64 `json:"rx_bps"`
	TxBps   float64 `json:"tx_bps"`
	RxTotal uint64  `json:"rx_total"`
	TxTotal uint64  `json:"tx_total"`
}

type Disk struct {
	Name     string  `json:"name"`
	Model    string  `json:"model,omitempty"`
	Size     uint64  `json:"size"`
	ReadBps  float64 `json:"read_bps"`
	WriteBps float64 `json:"write_bps"`
	UtilPct  float64 `json:"util_pct"`
	Temp     float64 `json:"temp,omitempty"`
}

type FS struct {
	Mount string `json:"mount"`
	Type  string `json:"type"`
	Size  uint64 `json:"size"`
	Used  uint64 `json:"used"`
}

type UPS struct {
	Driver     string  `json:"driver"` // pwrstat | nut | apcupsd
	Model      string  `json:"model"`
	State      string  `json:"state"`
	Source     string  `json:"source"`
	OnBattery  bool    `json:"on_battery"`
	InV        float64 `json:"in_v"`
	OutV       float64 `json:"out_v"`
	BatteryPct float64 `json:"battery_pct"`
	RuntimeMin float64 `json:"runtime_min"`
	LoadW      float64 `json:"load_w"`
	LoadPct    float64 `json:"load_pct"`
	LastEvent  string  `json:"last_event,omitempty"`
}

type Sample struct {
	T      int64      `json:"t"` // unix ms
	CPU    float64    `json:"cpu"`
	Cores  []float64  `json:"cores"`
	Load   [3]float64 `json:"load"`
	Mem    Mem        `json:"mem"`
	Swap   Mem        `json:"swap"`
	Temps  []Temp     `json:"temps"`
	Fans   []Fan      `json:"fans"`
	Net    []Net      `json:"net"`
	Disks  []Disk     `json:"disks"`
	FS     []FS       `json:"fs"`
	UPS    *UPS       `json:"ups"`
	Uptime float64    `json:"uptime"`
}

// Series is the compact history the UI seeds its sparklines from.
type Series struct {
	T    []int64                         `json:"t"`
	CPU  []float64                       `json:"cpu"`
	Mem  []uint64                        `json:"mem"`
	Net  map[string]map[string][]float64 `json:"net"`  // iface -> {rx,tx}
	Disk map[string]map[string][]float64 `json:"disk"` // name -> {r,w,u}
	Temp map[string][]float64            `json:"temp"` // "chip/label" -> °C
}

type Collector struct {
	mu   sync.RWMutex
	ring []Sample
	tick int

	prevCPU  []cpuTimes
	prevNet  map[string][2]uint64
	prevDisk map[string][3]uint64 // sectors read, sectors written, io ticks ms
	prevT    time.Time
	lastFS   []FS
	lastUPS  *UPS
	upsProbe func() *UPS // nil when no UPS daemon is installed
}

func NewCollector() *Collector {
	c := &Collector{prevNet: map[string][2]uint64{}, prevDisk: map[string][3]uint64{}}
	switch {
	case lookPath("pwrstat"):
		c.upsProbe = upsPwrstat
	case lookPath("upsc"):
		c.upsProbe = upsNUT
	case lookPath("apcaccess"):
		c.upsProbe = upsAPC
	}
	return c
}

// Run samples until ctx is cancelled.
func (c *Collector) Run(ctx context.Context) {
	c.sample() // prime deltas
	t := time.NewTicker(Interval)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
			c.sample()
		}
	}
}

func (c *Collector) Latest() (Sample, bool) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	if len(c.ring) == 0 {
		return Sample{}, false
	}
	return c.ring[len(c.ring)-1], true
}

func (c *Collector) History() Series {
	c.mu.RLock()
	defer c.mu.RUnlock()
	n := len(c.ring)
	s := Series{T: make([]int64, n), CPU: make([]float64, n), Mem: make([]uint64, n),
		Net: map[string]map[string][]float64{}, Disk: map[string]map[string][]float64{}, Temp: map[string][]float64{}}
	series := func(m map[string]map[string][]float64, key string, keys ...string) map[string][]float64 {
		v, ok := m[key]
		if !ok {
			v = map[string][]float64{}
			for _, k := range keys {
				v[k] = make([]float64, n)
			}
			m[key] = v
		}
		return v
	}
	for i, smp := range c.ring {
		s.T[i], s.CPU[i], s.Mem[i] = smp.T, smp.CPU, smp.Mem.Used
		for _, x := range smp.Net {
			v := series(s.Net, x.Iface, "rx", "tx")
			v["rx"][i], v["tx"][i] = x.RxBps, x.TxBps
		}
		for _, d := range smp.Disks {
			v := series(s.Disk, d.Name, "r", "w", "u")
			v["r"][i], v["w"][i], v["u"][i] = d.ReadBps, d.WriteBps, d.UtilPct
		}
		for _, t := range smp.Temps {
			k := t.Chip + "/" + t.Label
			if s.Temp[k] == nil {
				s.Temp[k] = make([]float64, n)
			}
			s.Temp[k][i] = t.C
		}
	}
	return s
}

// ---- sampling ----

func (c *Collector) sample() {
	now := time.Now()
	dt := now.Sub(c.prevT).Seconds()
	first := c.prevT.IsZero()
	c.prevT = now

	s := Sample{T: now.UnixMilli(), Temps: []Temp{}, Fans: []Fan{}, Net: []Net{}, Disks: []Disk{}}
	s.CPU, s.Cores = c.cpu()
	s.Load = loadavg()
	s.Mem, s.Swap = meminfo()
	s.Uptime = uptime()
	s.Temps, s.Fans = hwmon()
	if len(s.Temps) == 0 {
		s.Temps = thermalZones() // ARM SoCs and older kernels often expose only these
	}
	s.Net = c.net(dt)
	s.Disks = c.disks(dt)

	if c.tick%fsEvery == 0 || c.lastFS == nil {
		c.lastFS = filesystems()
	}
	s.FS = c.lastFS
	if c.upsProbe != nil && (c.tick%upsEvery == 0 || c.lastUPS == nil) {
		c.lastUPS = c.upsProbe()
	}
	s.UPS = c.lastUPS
	c.tick++

	if first {
		return // deltas are meaningless on the priming pass
	}
	c.mu.Lock()
	c.ring = append(c.ring, s)
	if len(c.ring) > History {
		c.ring = c.ring[len(c.ring)-History:]
	}
	c.mu.Unlock()
}

type cpuTimes struct{ busy, total uint64 }

func (c *Collector) cpu() (float64, []float64) {
	f, err := os.Open("/proc/stat")
	if err != nil {
		return 0, nil
	}
	defer f.Close()
	var cur []cpuTimes
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		fs := strings.Fields(sc.Text())
		if len(fs) < 8 || !strings.HasPrefix(fs[0], "cpu") {
			continue
		}
		var vals [10]uint64
		for i := 1; i < len(fs) && i <= 10; i++ {
			vals[i-1], _ = strconv.ParseUint(fs[i], 10, 64)
		}
		idle := vals[3] + vals[4]
		var total uint64
		for _, v := range vals {
			total += v
		}
		cur = append(cur, cpuTimes{busy: total - idle, total: total})
	}
	prev := c.prevCPU
	c.prevCPU = cur
	if len(prev) != len(cur) || len(cur) == 0 {
		return 0, nil
	}
	pct := func(i int) float64 {
		dt := float64(cur[i].total - prev[i].total)
		if dt <= 0 {
			return 0
		}
		return 100 * float64(cur[i].busy-prev[i].busy) / dt
	}
	cores := make([]float64, 0, len(cur)-1)
	for i := 1; i < len(cur); i++ {
		cores = append(cores, round1(pct(i)))
	}
	return round1(pct(0)), cores
}

func loadavg() (l [3]float64) {
	b, err := os.ReadFile("/proc/loadavg")
	if err != nil {
		return
	}
	f := strings.Fields(string(b))
	for i := 0; i < 3 && i < len(f); i++ {
		l[i], _ = strconv.ParseFloat(f[i], 64)
	}
	return
}

func meminfo() (Mem, Mem) {
	m := map[string]uint64{}
	b, err := os.ReadFile("/proc/meminfo")
	if err != nil {
		return Mem{}, Mem{}
	}
	for _, line := range strings.Split(string(b), "\n") {
		k, v, ok := strings.Cut(line, ":")
		if !ok {
			continue
		}
		f := strings.Fields(v)
		if len(f) == 0 {
			continue
		}
		n, _ := strconv.ParseUint(f[0], 10, 64)
		m[k] = n * 1024
	}
	mem := Mem{Total: m["MemTotal"], Avail: m["MemAvailable"]}
	if mem.Avail == 0 { // pre-3.14 kernels: approximate
		mem.Avail = m["MemFree"] + m["Buffers"] + m["Cached"]
	}
	mem.Used = mem.Total - mem.Avail
	swap := Mem{Total: m["SwapTotal"], Avail: m["SwapFree"]}
	swap.Used = swap.Total - swap.Avail
	return mem, swap
}

func uptime() float64 {
	b, err := os.ReadFile("/proc/uptime")
	if err != nil {
		return 0
	}
	if f := strings.Fields(string(b)); len(f) > 0 {
		v, _ := strconv.ParseFloat(f[0], 64)
		return v
	}
	return 0
}

// ---- temperatures and fans ----

var sensorRe = regexp.MustCompile(`^(temp|fan)(\d+)_input$`)

// tempLimit reads a *_max / *_crit file; values outside 0-200 °C are unset sentinels
// (NVMe reports ~65000 for absent thresholds).
func tempLimit(p string) float64 {
	if v, ok := readInt(p); ok && v > 0 && v < 200000 {
		return float64(v) / 1000
	}
	return 0
}

// hwmonDir reads every temp*/fan* sensor under one hwmon directory.
func hwmonDir(d string) (chip string, temps []Temp, fans []Fan) {
	chip = strings.TrimSpace(readStr(filepath.Join(d, "name")))
	ents, err := os.ReadDir(d)
	if err != nil {
		return
	}
	for _, e := range ents {
		m := sensorRe.FindStringSubmatch(e.Name())
		if m == nil {
			continue
		}
		base := m[1] + m[2]
		raw, ok := readInt(filepath.Join(d, e.Name()))
		if !ok {
			continue
		}
		label := strings.TrimSpace(readStr(filepath.Join(d, base+"_label")))
		if label == "" {
			label = base
		}
		if m[1] == "temp" {
			temps = append(temps, Temp{Chip: chip, Label: label, C: float64(raw) / 1000,
				Max: tempLimit(filepath.Join(d, base+"_max")), Crit: tempLimit(filepath.Join(d, base+"_crit"))})
		} else if raw > 0 || fileExists(filepath.Join(d, base+"_enable")) {
			fans = append(fans, Fan{Chip: chip, Label: label, RPM: int(raw)})
		}
	}
	return
}

func hwmon() ([]Temp, []Fan) {
	temps, fans := []Temp{}, []Fan{}
	dirs, _ := filepath.Glob("/sys/class/hwmon/hwmon*")
	for _, d := range dirs {
		_, t, f := hwmonDir(d)
		temps = append(temps, t...)
		fans = append(fans, f...)
	}
	sort.SliceStable(temps, func(i, j int) bool {
		if temps[i].Chip != temps[j].Chip {
			return temps[i].Chip < temps[j].Chip
		}
		return temps[i].Label < temps[j].Label
	})
	return temps, fans
}

func thermalZones() []Temp {
	out := []Temp{}
	dirs, _ := filepath.Glob("/sys/class/thermal/thermal_zone*")
	for _, d := range dirs {
		raw, ok := readInt(filepath.Join(d, "temp"))
		if !ok || raw <= 0 {
			continue
		}
		typ := strings.TrimSpace(readStr(filepath.Join(d, "type")))
		if typ == "" {
			typ = filepath.Base(d)
		}
		t := Temp{Chip: "thermal", Label: typ, C: float64(raw) / 1000}
		// trip points: trip_point_N_type in {critical, hot, passive, active}
		trips, _ := filepath.Glob(filepath.Join(d, "trip_point_*_type"))
		for _, tp := range trips {
			kind := strings.TrimSpace(readStr(tp))
			v := tempLimit(strings.TrimSuffix(tp, "_type") + "_temp")
			switch kind {
			case "critical":
				t.Crit = v
			case "hot", "passive":
				if t.Max == 0 || v < t.Max {
					t.Max = v
				}
			}
		}
		out = append(out, t)
	}
	return out
}

// blockTemp finds the drive's own temperature via the hwmon node the kernel attaches
// to the device: the nvme driver and drivetemp (SATA/SAS) both do this.
func blockTemp(name string) float64 {
	for _, pat := range []string{"/sys/block/" + name + "/device/hwmon*", "/sys/block/" + name + "/device/hwmon/hwmon*"} {
		dirs, _ := filepath.Glob(pat)
		for _, d := range dirs {
			if v, ok := readInt(filepath.Join(d, "temp1_input")); ok {
				return float64(v) / 1000
			}
		}
	}
	return 0
}

// ---- network ----

// Virtual interfaces that only ever carry traffic already counted on a real one.
var skipIfacePrefix = []string{"veth", "br-", "docker", "virbr", "vnet", "tap", "lxc", "lxd", "cni", "flannel", "cali", "kube", "podman", "dummy"}

func skipIface(n string) bool {
	if n == "lo" {
		return true
	}
	for _, p := range skipIfacePrefix {
		if strings.HasPrefix(n, p) {
			return true
		}
	}
	return false
}

func (c *Collector) net(dt float64) []Net {
	out := []Net{}
	f, err := os.Open("/proc/net/dev")
	if err != nil {
		return out
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		name, rest, ok := strings.Cut(sc.Text(), ":")
		if !ok {
			continue
		}
		name = strings.TrimSpace(name)
		if skipIface(name) {
			continue
		}
		fs := strings.Fields(rest)
		if len(fs) < 16 {
			continue
		}
		rx, _ := strconv.ParseUint(fs[0], 10, 64)
		tx, _ := strconv.ParseUint(fs[8], 10, 64)
		n := Net{Iface: name, RxTotal: rx, TxTotal: tx}
		st := strings.TrimSpace(readStr("/sys/class/net/" + name + "/operstate"))
		n.Up = st == "up" || st == "unknown" // wireguard/tun report "unknown" while carrying traffic
		if p, ok := c.prevNet[name]; ok && dt > 0 {
			n.RxBps = round1(float64(rx-p[0]) / dt)
			n.TxBps = round1(float64(tx-p[1]) / dt)
		}
		c.prevNet[name] = [2]uint64{rx, tx}
		out = append(out, n)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Iface < out[j].Iface })
	return out
}

// ---- disks ----

// Pseudo block devices with nothing physical behind them.
var skipDiskPrefix = []string{"loop", "ram", "zram", "nbd", "sr", "fd", "dm-", "md", "drbd", "zd", "pmem"}

func skipDisk(n string) bool {
	for _, p := range skipDiskPrefix {
		if strings.HasPrefix(n, p) {
			return true
		}
	}
	return false
}

func diskModel(name string) string {
	dev := "/sys/block/" + name + "/device/"
	for _, f := range []string{"model", "name", "vendor"} { // SCSI/NVMe: model; MMC/SD: name
		if s := strings.TrimSpace(readStr(dev + f)); s != "" {
			return s
		}
	}
	return ""
}

func (c *Collector) disks(dt float64) []Disk {
	out := []Disk{}
	// /sys/block lists whole devices only (partitions live underneath), so this is
	// the authoritative "is a disk" test across sd/nvme/mmcblk/vd/xvd/hd/…
	ents, err := os.ReadDir("/sys/block")
	if err != nil {
		return out
	}
	whole := map[string]bool{}
	for _, e := range ents {
		if !skipDisk(e.Name()) {
			whole[e.Name()] = true
		}
	}
	f, err := os.Open("/proc/diskstats")
	if err != nil {
		return out
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		fs := strings.Fields(sc.Text())
		if len(fs) < 14 || !whole[fs[2]] {
			continue
		}
		name := fs[2]
		rd, _ := strconv.ParseUint(fs[5], 10, 64)
		wr, _ := strconv.ParseUint(fs[9], 10, 64)
		ticks, _ := strconv.ParseUint(fs[12], 10, 64)
		d := Disk{Name: name, Model: diskModel(name), Temp: blockTemp(name)}
		if sz, ok := readInt("/sys/block/" + name + "/size"); ok {
			d.Size = uint64(sz) * sectorSize
		}
		if d.Size == 0 {
			continue // empty card reader / detached media
		}
		if p, ok := c.prevDisk[name]; ok && dt > 0 {
			d.ReadBps = round1(float64(rd-p[0]) * sectorSize / dt)
			d.WriteBps = round1(float64(wr-p[1]) * sectorSize / dt)
			d.UtilPct = round1(float64(ticks-p[2]) / (dt * 1000) * 100)
			if d.UtilPct > 100 {
				d.UtilPct = 100
			}
		}
		c.prevDisk[name] = [3]uint64{rd, wr, ticks}
		out = append(out, d)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
	return out
}

// ---- filesystems ----

// Kernel/virtual filesystems that aren't storage. Anything else that's mounted is
// shown, so new real filesystems (bcachefs, exotic FUSE) work without a code change.
var pseudoFS = map[string]bool{
	"proc": true, "sysfs": true, "devtmpfs": true, "devpts": true, "tmpfs": true, "ramfs": true,
	"cgroup": true, "cgroup2": true, "pstore": true, "bpf": true, "securityfs": true, "debugfs": true,
	"tracefs": true, "configfs": true, "fusectl": true, "hugetlbfs": true, "mqueue": true, "efivarfs": true,
	"binfmt_misc": true, "autofs": true, "rpc_pipefs": true, "nsfs": true, "overlay": true, "squashfs": true,
	"fuse.portal": true, "fuse.gvfsd-fuse": true, "fuse.lxcfs": true, "selinuxfs": true, "none": true, "zfs_snap": true,
}

// Mount prefixes that are container/snap/runtime internals, not the host's storage.
var skipMountPrefix = []string{"/var/lib/docker/", "/var/lib/containers/", "/var/lib/kubelet/", "/run/", "/snap/", "/proc/", "/sys/", "/dev/"}

func skipMount(m string) bool {
	for _, p := range skipMountPrefix {
		if strings.HasPrefix(m, p) {
			return true
		}
	}
	return false
}

func filesystems() []FS {
	out := []FS{}
	b, err := os.ReadFile("/proc/mounts")
	if err != nil {
		return out
	}
	seen := map[string]bool{}
	for _, line := range strings.Split(string(b), "\n") {
		fs := strings.Fields(line)
		if len(fs) < 3 || pseudoFS[fs[2]] {
			continue
		}
		mount := strings.ReplaceAll(fs[1], `\040`, " ")
		if seen[mount] || skipMount(mount) {
			continue
		}
		seen[mount] = true
		// statfs on a dead network mount can block; bound it.
		done := make(chan FS, 1)
		go func() {
			var st syscall.Statfs_t
			if err := syscall.Statfs(mount, &st); err != nil {
				done <- FS{}
				return
			}
			bs := uint64(st.Bsize)
			done <- FS{Mount: mount, Type: fs[2], Size: st.Blocks * bs, Used: (st.Blocks - st.Bfree) * bs}
		}()
		select {
		case r := <-done:
			if r.Size > 0 {
				out = append(out, r)
			}
		case <-time.After(3 * time.Second):
			out = append(out, FS{Mount: mount, Type: fs[2]}) // unreachable: size 0 signals it
		}
	}
	return out
}

// ---- UPS ----

func runOut(args ...string) (string, bool) {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	b, err := exec.CommandContext(ctx, args[0], args[1:]...).Output()
	if err != nil {
		return "", false
	}
	return string(b), true
}

// leadingNum parses the number at the start of "119 V", "47 min.", "170 Watt(17 %)".
func leadingNum(s string) float64 {
	s = strings.TrimSpace(s)
	end := 0
	for end < len(s) && (s[end] >= '0' && s[end] <= '9' || s[end] == '.') {
		end++
	}
	v, _ := strconv.ParseFloat(s[:end], 64)
	return v
}

// upsPwrstat: CyberPower PowerPanel. pwrstat only talks to its daemon as root, so fall
// back to passwordless sudo when plain execution is refused.
func upsPwrstat() *UPS {
	out, ok := runOut("pwrstat", "-status")
	if !ok {
		if out, ok = runOut("sudo", "-n", "pwrstat", "-status"); !ok {
			return nil
		}
	}
	u := &UPS{Driver: "pwrstat"}
	got := false
	for _, line := range strings.Split(out, "\n") {
		k, v, ok := strings.Cut(line, "....")
		if !ok {
			continue
		}
		k = strings.TrimSpace(k)
		v = strings.TrimSpace(strings.TrimLeft(v, "."))
		got = true
		switch k {
		case "Model Name":
			u.Model = v
		case "State":
			u.State = v
		case "Power Supply by":
			u.Source = v
			u.OnBattery = !strings.Contains(v, "Utility")
		case "Utility Voltage":
			u.InV = leadingNum(v)
		case "Output Voltage":
			u.OutV = leadingNum(v)
		case "Battery Capacity":
			u.BatteryPct = leadingNum(v)
		case "Remaining Runtime":
			u.RuntimeMin = leadingNum(v)
		case "Load":
			u.LoadW = leadingNum(v)
			if i := strings.Index(v, "("); i >= 0 {
				u.LoadPct = leadingNum(v[i+1:])
			}
		case "Last Power Event":
			u.LastEvent = v
		}
	}
	if !got {
		return nil
	}
	return u
}

// upsNUT: Network UPS Tools. `upsc -l` names the UPSes; `upsc <name>` dumps key: value.
func upsNUT() *UPS {
	list, ok := runOut("upsc", "-l")
	if !ok {
		return nil
	}
	name := ""
	for _, l := range strings.Split(list, "\n") {
		if l = strings.TrimSpace(l); l != "" {
			name = l
			break
		}
	}
	if name == "" {
		return nil
	}
	out, ok := runOut("upsc", name)
	if !ok {
		return nil
	}
	kv := map[string]string{}
	for _, l := range strings.Split(out, "\n") {
		if k, v, ok := strings.Cut(l, ":"); ok {
			kv[strings.TrimSpace(k)] = strings.TrimSpace(v)
		}
	}
	if len(kv) == 0 {
		return nil
	}
	u := &UPS{Driver: "nut", Model: strings.TrimSpace(kv["ups.mfr"] + " " + kv["ups.model"])}
	status := kv["ups.status"]
	u.OnBattery = strings.Contains(status, "OB")
	u.State = status
	u.Source = "Utility Power"
	if u.OnBattery {
		u.Source = "Battery"
	}
	u.InV = leadingNum(kv["input.voltage"])
	u.OutV = leadingNum(kv["output.voltage"])
	u.BatteryPct = leadingNum(kv["battery.charge"])
	u.RuntimeMin = leadingNum(kv["battery.runtime"]) / 60
	u.LoadPct = leadingNum(kv["ups.load"])
	u.LoadW = leadingNum(kv["ups.realpower"])
	if u.LoadW == 0 {
		u.LoadW = round1(leadingNum(kv["ups.realpower.nominal"]) * u.LoadPct / 100)
	}
	return u
}

// upsAPC: apcupsd. `apcaccess` prints "KEY : value" lines.
func upsAPC() *UPS {
	out, ok := runOut("apcaccess", "status")
	if !ok {
		return nil
	}
	kv := map[string]string{}
	for _, l := range strings.Split(out, "\n") {
		if k, v, ok := strings.Cut(l, ":"); ok {
			kv[strings.TrimSpace(k)] = strings.TrimSpace(v)
		}
	}
	if kv["STATUS"] == "" {
		return nil
	}
	u := &UPS{Driver: "apcupsd", Model: kv["MODEL"], State: kv["STATUS"]}
	u.OnBattery = strings.Contains(kv["STATUS"], "ONBATT")
	u.Source = "Utility Power"
	if u.OnBattery {
		u.Source = "Battery"
	}
	u.InV = leadingNum(kv["LINEV"])
	u.OutV = leadingNum(kv["OUTPUTV"])
	u.BatteryPct = leadingNum(kv["BCHARGE"])
	u.RuntimeMin = leadingNum(kv["TIMELEFT"])
	u.LoadPct = leadingNum(kv["LOADPCT"])
	u.LoadW = round1(leadingNum(kv["NOMPOWER"]) * u.LoadPct / 100)
	if kv["LASTXFER"] != "" {
		u.LastEvent = kv["LASTXFER"]
	}
	return u
}

// ---- small helpers ----

func readStr(p string) string {
	b, err := os.ReadFile(p)
	if err != nil {
		return ""
	}
	return string(b)
}

func readInt(p string) (int64, bool) {
	s := strings.TrimSpace(readStr(p))
	if s == "" {
		return 0, false
	}
	v, err := strconv.ParseInt(s, 10, 64)
	return v, err == nil
}

func fileExists(p string) bool { _, err := os.Stat(p); return err == nil }

func round1(v float64) float64 { return float64(int64(v*10+0.5)) / 10 }

func lookPath(bin string) bool { _, err := exec.LookPath(bin); return err == nil }