hamlab/internal/metrics/metrics.go

// Package metrics samples host telemetry (CPU, memory, temps, fans, network, disks,
// filesystems, UPS) on a timer into an in-memory ring, for the Dashboard tab.
//
// Portability rules: everything comes from /proc and /sys, which every Linux kernel
// exposes the same way regardless of distro or architecture; nothing here needs
// lm-sensors, smartmontools or root. Sections whose source is absent (no fans, no
// UPS daemon, no hwmon at all on a VM) simply come back empty rather than failing.
// The only external commands are optional UPS daemons: pwrstat (CyberPower), upsc
// (NUT) and apcaccess (apcupsd), tried in that order.
package metrics

import (
	"bufio"
	"context"
	"errors"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"hamlab/internal/priv"
)

const (
	DefaultInterval = 2 * time.Second
	MinInterval     = 100 * time.Millisecond
	MaxInterval     = 10 * time.Second
	History         = 300 // points kept per curve; the window below sets how far apart they are
	// How far back the kept history reaches. The point count is fixed, so this sets the spacing:
	// at a minute every sample is its own point, at a day each point is the average of ~5 minutes.
	DefaultWindow = 10 * time.Minute
	MinWindow     = 1 * time.Minute
	MaxWindow     = 24 * time.Hour
	fsEvery       = 30 * time.Second // filesystem usage changes slowly and statfs on network mounts isn't free
	upsEvery      = 10 * time.Second // UPS daemons are an exec each; don't hammer them at fast intervals
	sectorSize    = 512              // /proc/diskstats always counts 512-byte sectors
)

type Mem struct {
	Total   uint64 `json:"total"`
	Used    uint64 `json:"used"`
	Avail   uint64 `json:"avail"`
	Free    uint64 `json:"free,omitempty"`
	Cached  uint64 `json:"cached,omitempty"`
	Buffers uint64 `json:"buffers,omitempty"`
}

type Temp struct {
	Chip  string  `json:"chip"`
	Label string  `json:"label"`
	C     float64 `json:"c"`
	Max   float64 `json:"max,omitempty"`
	Crit  float64 `json:"crit,omitempty"`
}

type Fan struct {
	Chip  string `json:"chip"`
	Label string `json:"label"`
	RPM   int    `json:"rpm"`
}

type Net struct {
	Iface   string  `json:"iface"`
	Up      bool    `json:"up"`
	RxBps   float64 `json:"rx_bps"`
	TxBps   float64 `json:"tx_bps"`
	RxTotal uint64  `json:"rx_total"`
	TxTotal uint64  `json:"tx_total"`
}

type Disk struct {
	Name     string  `json:"name"`
	Model    string  `json:"model,omitempty"`
	Size     uint64  `json:"size"`
	ReadBps  float64 `json:"read_bps"`
	WriteBps float64 `json:"write_bps"`
	UtilPct  float64 `json:"util_pct"`
	Temp     float64 `json:"temp,omitempty"`
}

type FS struct {
	Mount string `json:"mount"`
	Type  string `json:"type"`
	Size  uint64 `json:"size"`
	Used  uint64 `json:"used"`
	// Remote is set when the filesystem lives on another machine — a CIFS or NFS share,
	// sshfs, and so on. Its bytes are somebody else's disk, so a panel about this machine's
	// hardware leaves it out, while the sensor stays for anyone who does want it.
	Remote bool `json:"remote,omitempty"`
	// Disk is the physical drive these bytes are on ("nvme0", "sda"), named as the sampler
	// names it; empty when that cannot be
	// said honestly: a ZFS pool or an md array spreads a filesystem over several drives, and
	// a remote mount is on none of them.
	Disk string `json:"disk,omitempty"`
}

type UPS struct {
	Driver     string  `json:"driver"` // pwrstat | nut | apcupsd
	Model      string  `json:"model"`
	State      string  `json:"state"`
	Source     string  `json:"source"`
	OnBattery  bool    `json:"on_battery"`
	InV        float64 `json:"in_v"`
	OutV       float64 `json:"out_v"`
	BatteryPct float64 `json:"battery_pct"`
	RuntimeMin float64 `json:"runtime_min"`
	LoadW      float64 `json:"load_w"`
	LoadPct    float64 `json:"load_pct"`
	LastEvent  string  `json:"last_event,omitempty"`
}

type Proc struct {
	PID     int     `json:"pid"`
	PPID    int     `json:"ppid"`
	Name    string  `json:"name"`
	Cmd     string  `json:"cmd,omitempty"`
	User    string  `json:"user"`
	CPU     float64 `json:"cpu"`           // percent of one core
	GPU     float64 `json:"gpu,omitempty"` // its busiest GPU engine, percent; summed over the GPUs it has open
	CPUTime float64 `json:"cpu_time"`      // seconds, lifetime
	RSS     uint64  `json:"rss"`           // bytes
	Threads int     `json:"threads"`
	State   string  `json:"state"`
	Nice    int     `json:"nice"`
	Started int64   `json:"started"` // unix ms
}

type Sample struct {
	T        int64      `json:"t"`           // unix ms
	Interval int64      `json:"interval_ms"` // sampling interval in effect
	CPU      float64    `json:"cpu"`
	Cores    []float64  `json:"cores"`
	Load     [3]float64 `json:"load"`
	Mem      Mem        `json:"mem"`
	Swap     Mem        `json:"swap"`
	Temps    []Temp     `json:"temps"`
	Fans     []Fan      `json:"fans"`
	Net      []Net      `json:"net"`
	Disks    []Disk     `json:"disks"`
	FS       []FS       `json:"fs"`
	UPS      *UPS       `json:"ups"`
	GPUs     []GPU      `json:"gpus"`
	ARC      *ARC       `json:"arc"` // the ZFS read cache; nil without the module
	Uptime   float64    `json:"uptime"`

	fsT, upsT int64 // when the slow sections were really read; sensors report these, not T
}

type Collector struct {
	mu       sync.RWMutex
	ring     []Sample // the kept history: one point per stride, each the average of its bucket
	last     Sample   // the newest raw sample, whatever the stride
	bucket   []Sample // samples since the last point was kept
	lastKept int64    // when that was, unix ms
	interval time.Duration
	window   time.Duration
	wake     chan struct{} // nudges Run when the interval changes

	prevCPU   []cpuTimes
	prevNet   map[string][2]uint64
	prevDisk  map[string][3]uint64 // sectors read, sectors written, io ticks ms
	prevT     time.Time
	lastFS    []FS
	fsAt      time.Time
	lastUPS   *UPS
	upsAt     time.Time
	prevProc  map[int]procTimes
	procInfo  map[int]procStatic
	procAt    time.Time
	lastTable *ProcTable
	upsProbe  func() *UPS // nil when no UPS daemon is installed
	gpu       gpuState
	prevARC   arcPrev

	subMu sync.Mutex
	subs  map[chan struct{}]struct{} // told after every sample; see Subscribe
}

func NewCollector() *Collector {
	c := &Collector{prevNet: map[string][2]uint64{}, prevDisk: map[string][3]uint64{},
		interval: DefaultInterval, window: DefaultWindow, wake: make(chan struct{}, 1)}
	switch {
	case lookPath("pwrstat"):
		c.upsProbe = upsPwrstat
	case lookPath("upsc"):
		c.upsProbe = upsNUT
	case lookPath("apcaccess"):
		c.upsProbe = upsAPC
	}
	return c
}

// Run samples until ctx is cancelled, at whatever interval is current.
func (c *Collector) Run(ctx context.Context) {
	c.sample() // prime deltas
	for {
		// The wait is what's left of the interval after the work, so a machine whose
		// sample takes 30 ms still delivers 4 a second at 250 ms, not 3.6.
		wait := c.Interval() - time.Since(c.prevT)
		if wait < 0 {
			wait = 0
		}
		t := time.NewTimer(wait)
		select {
		case <-ctx.Done():
			t.Stop()
			return
		case <-c.wake: // interval changed: restart the wait
			t.Stop()
		case <-t.C:
			c.sample()
		}
	}
}

// SetInterval changes the sampling period, clamped to [MinInterval, MaxInterval].
func (c *Collector) SetInterval(d time.Duration) {
	if d < MinInterval {
		d = MinInterval
	}
	if d > MaxInterval {
		d = MaxInterval
	}
	c.mu.Lock()
	changed := d != c.interval
	c.interval = d
	c.mu.Unlock()
	if changed {
		select {
		case c.wake <- struct{}{}:
		default:
		}
	}
}

func (c *Collector) Interval() time.Duration {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.interval
}

// SetWindow sets how far back the kept history reaches, clamped to [MinWindow, MaxWindow].
func (c *Collector) SetWindow(d time.Duration) {
	if d < MinWindow {
		d = MinWindow
	}
	if d > MaxWindow {
		d = MaxWindow
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	c.window = d
	c.trim()
}

func (c *Collector) Window() time.Duration {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.window
}

// Stride is how far apart the kept points are: the sampling interval until the window is
// wider than History samples of it, then the window split History ways.
func (c *Collector) Stride() time.Duration {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.stride()
}

func (c *Collector) stride() time.Duration {
	if s := c.window / History; s > c.interval {
		return s
	}
	return c.interval
}

// keep folds samples into the history, one point per stride. Averaged, not sampled: a CPU
// graph of the last six hours should show the load through each five minutes, not whichever
// two-second instant happened to land on the mark.
func (c *Collector) keep(s Sample) {
	c.bucket = append(c.bucket, s)
	// A point is due a stride after the last one (for the first, a stride after the interval its
	// first sample stands for). Half an interval of slack, because a timer never fires exactly on
	// time: without it a tick that lands 5 ms early is "not yet", and every point runs one sample
	// long. Measured from the last point kept, not by counting samples, so a machine that stalled
	// or slept simply keeps the next sample it gets.
	iv := c.interval.Milliseconds()
	since := c.lastKept
	if since == 0 {
		since = c.bucket[0].T - iv
	}
	if s.T-since < c.stride().Milliseconds()-iv/2 {
		return
	}
	c.lastKept = s.T
	c.ring = append(c.ring, fold(c.bucket))
	c.bucket = c.bucket[:0]
	c.trim()
}

// trim drops points past the count budget and past the window. Caller holds c.mu.
func (c *Collector) trim() {
	if len(c.ring) > History {
		c.ring = c.ring[len(c.ring)-History:]
	}
	n := len(c.ring)
	if n == 0 {
		return
	}
	cut := c.ring[n-1].T - c.window.Milliseconds()
	i := 0
	for i < n && c.ring[i].T < cut {
		i++
	}
	if i > 0 {
		c.ring = c.ring[i:]
	}
}

func (c *Collector) Latest() (Sample, bool) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	if c.last.T == 0 {
		return Sample{}, false
	}
	return c.last, true
}

// ---- sampling ----

func (c *Collector) sample() {
	now := time.Now()
	dt := now.Sub(c.prevT).Seconds()
	first := c.prevT.IsZero()
	c.prevT = now

	s := Sample{T: now.UnixMilli(), Interval: c.Interval().Milliseconds(), Temps: []Temp{}, Fans: []Fan{}, Net: []Net{}, Disks: []Disk{}}
	s.CPU, s.Cores = c.cpu()
	s.Load = loadavg()
	s.Mem, s.Swap = meminfo()
	s.Uptime = uptime()
	s.Temps, s.Fans = hwmon()
	if len(s.Temps) == 0 {
		s.Temps = thermalZones() // ARM SoCs and older kernels often expose only these
	}
	s.Net = c.net(dt)
	s.Disks = c.disks(dt)

	if c.lastFS == nil || now.Sub(c.fsAt) >= fsEvery {
		c.lastFS, c.fsAt = filesystems(), now
	}
	s.FS, s.fsT = c.lastFS, c.fsAt.UnixMilli()
	if c.upsProbe != nil && (c.upsAt.IsZero() || now.Sub(c.upsAt) >= upsEvery) {
		c.lastUPS, c.upsAt = c.upsProbe(), now
	}
	s.UPS, s.upsT = c.lastUPS, c.upsAt.UnixMilli()
	s.GPUs = c.gpus(now, cpuTemp(s.Temps))
	s.ARC = c.arc(now)

	if first {
		return // deltas are meaningless on the priming pass
	}
	c.mu.Lock()
	c.last = s
	c.keep(s)
	c.mu.Unlock()

	c.subMu.Lock()
	for ch := range c.subs {
		select {
		case ch <- struct{}{}:
		default: // a slow listener misses a tick; it reads the latest on the next one
		}
	}
	c.subMu.Unlock()
}

// Subscribe returns a channel that gets a nudge after every new sample, and the func
// that ends the subscription.
func (c *Collector) Subscribe() (<-chan struct{}, func()) {
	ch := make(chan struct{}, 1)
	c.subMu.Lock()
	if c.subs == nil {
		c.subs = map[chan struct{}]struct{}{}
	}
	c.subs[ch] = struct{}{}
	c.subMu.Unlock()
	return ch, func() { c.subMu.Lock(); delete(c.subs, ch); c.subMu.Unlock() }
}

type cpuTimes struct{ busy, total uint64 }

func (c *Collector) cpu() (float64, []float64) {
	f, err := os.Open("/proc/stat")
	if err != nil {
		return 0, nil
	}
	defer f.Close()
	var cur []cpuTimes
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		fs := strings.Fields(sc.Text())
		if len(fs) < 8 || !strings.HasPrefix(fs[0], "cpu") {
			continue
		}
		var vals [10]uint64
		for i := 1; i < len(fs) && i <= 10; i++ {
			vals[i-1], _ = strconv.ParseUint(fs[i], 10, 64)
		}
		idle := vals[3] + vals[4]
		var total uint64
		for _, v := range vals {
			total += v
		}
		cur = append(cur, cpuTimes{busy: total - idle, total: total})
	}
	prev := c.prevCPU
	c.prevCPU = cur
	if len(prev) != len(cur) || len(cur) == 0 {
		return 0, nil
	}
	pct := func(i int) float64 {
		dt := float64(cur[i].total - prev[i].total)
		if dt <= 0 {
			return 0
		}
		return 100 * float64(cur[i].busy-prev[i].busy) / dt
	}
	cores := make([]float64, 0, len(cur)-1)
	for i := 1; i < len(cur); i++ {
		cores = append(cores, round1(pct(i)))
	}
	return round1(pct(0)), cores
}

func loadavg() (l [3]float64) {
	b, err := os.ReadFile("/proc/loadavg")
	if err != nil {
		return
	}
	f := strings.Fields(string(b))
	for i := 0; i < 3 && i < len(f); i++ {
		l[i], _ = strconv.ParseFloat(f[i], 64)
	}
	return
}

func meminfo() (Mem, Mem) {
	m := map[string]uint64{}
	b, err := os.ReadFile("/proc/meminfo")
	if err != nil {
		return Mem{}, Mem{}
	}
	for _, line := range strings.Split(string(b), "\n") {
		k, v, ok := strings.Cut(line, ":")
		if !ok {
			continue
		}
		f := strings.Fields(v)
		if len(f) == 0 {
			continue
		}
		n, _ := strconv.ParseUint(f[0], 10, 64)
		m[k] = n * 1024
	}
	mem := Mem{Total: m["MemTotal"], Avail: m["MemAvailable"], Free: m["MemFree"], Buffers: m["Buffers"], Cached: m["Cached"] + m["SReclaimable"]}
	if mem.Avail == 0 { // pre-3.14 kernels: approximate
		mem.Avail = m["MemFree"] + m["Buffers"] + m["Cached"]
	}
	mem.Used = mem.Total - mem.Avail
	swap := Mem{Total: m["SwapTotal"], Avail: m["SwapFree"]}
	swap.Used = swap.Total - swap.Avail
	return mem, swap
}

func uptime() float64 {
	b, err := os.ReadFile("/proc/uptime")
	if err != nil {
		return 0
	}
	if f := strings.Fields(string(b)); len(f) > 0 {
		v, _ := strconv.ParseFloat(f[0], 64)
		return v
	}
	return 0
}

// ---- temperatures and fans ----

var sensorRe = regexp.MustCompile(`^(temp|fan)(\d+)_input$`)

// tempLimit reads a *_max / *_crit file; values outside 0-200 °C are unset sentinels
// (NVMe reports ~65000 for absent thresholds).
func tempLimit(p string) float64 {
	if v, ok := readInt(p); ok && v > 0 && v < 200000 {
		return float64(v) / 1000
	}
	return 0
}

// hwmonDir reads every temp*/fan* sensor under one hwmon directory.
func hwmonDir(d string) (chip string, temps []Temp, fans []Fan) {
	chip = strings.TrimSpace(readStr(filepath.Join(d, "name")))
	ents, err := os.ReadDir(d)
	if err != nil {
		return
	}
	for _, e := range ents {
		m := sensorRe.FindStringSubmatch(e.Name())
		if m == nil {
			continue
		}
		base := m[1] + m[2]
		raw, ok := readInt(filepath.Join(d, e.Name()))
		if !ok {
			continue
		}
		label := strings.TrimSpace(readStr(filepath.Join(d, base+"_label")))
		if label == "" {
			label = base
		}
		if m[1] == "temp" {
			if raw < -60000 { // an unpowered sensor's sentinel (an idle audio amp reads -93 °C), not weather
				continue
			}
			temps = append(temps, Temp{Chip: chip, Label: label, C: float64(raw) / 1000,
				Max: tempLimit(filepath.Join(d, base+"_max")), Crit: tempLimit(filepath.Join(d, base+"_crit"))})
		} else if raw > 0 || fileExists(filepath.Join(d, base+"_enable")) {
			fans = append(fans, Fan{Chip: chip, Label: label, RPM: int(raw)})
		}
	}
	return
}

func hwmon() ([]Temp, []Fan) {
	temps, fans := []Temp{}, []Fan{}
	dirs, _ := filepath.Glob("/sys/class/hwmon/hwmon*")
	for _, d := range dirs {
		_, t, f := hwmonDir(d)
		temps = append(temps, t...)
		fans = append(fans, f...)
	}
	sort.SliceStable(temps, func(i, j int) bool {
		if temps[i].Chip != temps[j].Chip {
			return temps[i].Chip < temps[j].Chip
		}
		return temps[i].Label < temps[j].Label
	})
	return temps, fans
}

func thermalZones() []Temp {
	out := []Temp{}
	dirs, _ := filepath.Glob("/sys/class/thermal/thermal_zone*")
	for _, d := range dirs {
		raw, ok := readInt(filepath.Join(d, "temp"))
		if !ok || raw <= 0 {
			continue
		}
		typ := strings.TrimSpace(readStr(filepath.Join(d, "type")))
		if typ == "" {
			typ = filepath.Base(d)
		}
		t := Temp{Chip: "thermal", Label: typ, C: float64(raw) / 1000}
		// trip points: trip_point_N_type in {critical, hot, passive, active}
		trips, _ := filepath.Glob(filepath.Join(d, "trip_point_*_type"))
		for _, tp := range trips {
			kind := strings.TrimSpace(readStr(tp))
			v := tempLimit(strings.TrimSuffix(tp, "_type") + "_temp")
			switch kind {
			case "critical":
				t.Crit = v
			case "hot", "passive":
				if t.Max == 0 || v < t.Max {
					t.Max = v
				}
			}
		}
		out = append(out, t)
	}
	return out
}

// blockTemp finds the drive's own temperature via the hwmon node the kernel attaches
// to the device: the nvme driver and drivetemp (SATA/SAS) both do this.
func blockTemp(name string) float64 {
	for _, pat := range []string{"/sys/block/" + name + "/device/hwmon*", "/sys/block/" + name + "/device/hwmon/hwmon*"} {
		dirs, _ := filepath.Glob(pat)
		for _, d := range dirs {
			if v, ok := readInt(filepath.Join(d, "temp1_input")); ok {
				return float64(v) / 1000
			}
		}
	}
	return 0
}

// ---- network ----

// Virtual interfaces that only ever carry traffic already counted on a real one.
var skipIfacePrefix = []string{"veth", "br-", "docker", "virbr", "vnet", "tap", "lxc", "lxd", "cni", "flannel", "cali", "kube", "podman", "dummy"}

func skipIface(n string) bool {
	if n == "lo" {
		return true
	}
	for _, p := range skipIfacePrefix {
		if strings.HasPrefix(n, p) {
			return true
		}
	}
	return false
}

func (c *Collector) net(dt float64) []Net {
	out := []Net{}
	f, err := os.Open("/proc/net/dev")
	if err != nil {
		return out
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		name, rest, ok := strings.Cut(sc.Text(), ":")
		if !ok {
			continue
		}
		name = strings.TrimSpace(name)
		if skipIface(name) {
			continue
		}
		fs := strings.Fields(rest)
		if len(fs) < 16 {
			continue
		}
		rx, _ := strconv.ParseUint(fs[0], 10, 64)
		tx, _ := strconv.ParseUint(fs[8], 10, 64)
		n := Net{Iface: name, RxTotal: rx, TxTotal: tx}
		st := strings.TrimSpace(readStr("/sys/class/net/" + name + "/operstate"))
		n.Up = st == "up" || st == "unknown" // wireguard/tun report "unknown" while carrying traffic
		if p, ok := c.prevNet[name]; ok && dt > 0 {
			n.RxBps = round1(float64(rx-p[0]) / dt)
			n.TxBps = round1(float64(tx-p[1]) / dt)
		}
		c.prevNet[name] = [2]uint64{rx, tx}
		out = append(out, n)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Iface < out[j].Iface })
	return out
}

// ---- disks ----

// Pseudo block devices with nothing physical behind them.
var skipDiskPrefix = []string{"loop", "ram", "zram", "nbd", "sr", "fd", "dm-", "md", "drbd", "zd", "pmem"}

func skipDisk(n string) bool {
	// eMMC boot/RPMB/general-purpose areas: regions of the chip, not drives
	if strings.HasPrefix(n, "mmcblk") && (strings.Contains(n, "boot") || strings.HasSuffix(n, "rpmb") || strings.Contains(n, "gp")) {
		return true
	}
	for _, p := range skipDiskPrefix {
		if strings.HasPrefix(n, p) {
			return true
		}
	}
	return false
}

func diskModel(name string) string {
	dev := "/sys/block/" + name + "/device/"
	for _, f := range []string{"model", "name", "vendor"} { // SCSI/NVMe: model; MMC/SD: name
		if s := strings.TrimSpace(readStr(dev + f)); s != "" {
			return s
		}
	}
	return ""
}

func (c *Collector) disks(dt float64) []Disk {
	out := []Disk{}
	// /sys/block lists whole devices only (partitions live underneath), so this is
	// the authoritative "is a disk" test across sd/nvme/mmcblk/vd/xvd/hd/…
	ents, err := os.ReadDir("/sys/block")
	if err != nil {
		return out
	}
	whole := map[string]bool{}
	for _, e := range ents {
		if !skipDisk(e.Name()) {
			whole[e.Name()] = true
		}
	}
	f, err := os.Open("/proc/diskstats")
	if err != nil {
		return out
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		fs := strings.Fields(sc.Text())
		if len(fs) < 14 || !whole[fs[2]] {
			continue
		}
		name := fs[2]
		rd, _ := strconv.ParseUint(fs[5], 10, 64)
		wr, _ := strconv.ParseUint(fs[9], 10, 64)
		ticks, _ := strconv.ParseUint(fs[12], 10, 64)
		d := Disk{Name: name, Model: diskModel(name), Temp: blockTemp(name)}
		if sz, ok := readInt("/sys/block/" + name + "/size"); ok {
			d.Size = uint64(sz) * sectorSize
		}
		if d.Size == 0 {
			continue // empty card reader / detached media
		}
		if p, ok := c.prevDisk[name]; ok && dt > 0 {
			d.ReadBps = round1(float64(rd-p[0]) * sectorSize / dt)
			d.WriteBps = round1(float64(wr-p[1]) * sectorSize / dt)
			d.UtilPct = round1(float64(ticks-p[2]) / (dt * 1000) * 100)
			if d.UtilPct > 100 {
				d.UtilPct = 100
			}
		}
		c.prevDisk[name] = [3]uint64{rd, wr, ticks}
		out = append(out, d)
	}
	out = foldNVMe(out)
	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
	return out
}

var nsRe = regexp.MustCompile(`^(nvme\d+)n\d+$`)

// foldNVMe merges nvme0n1, nvme0n2, … into the drive they are: one controller presenting
// several namespaces, which the kernel lists as whole disks apiece. An Apple SSD shows three —
// the 466 GiB of storage and two of firmware, a few MiB each — and a page that lists disks was
// listing all three as separate drives. The Disks page has always folded them (internal/storage
// does the same); this is what the dashboard, the Top tab and the disk.* sensors read.
//
// Rates add up across namespaces. Busy does not: they share one controller, so the time it
// spent working is the longest of them, never the sum.
func foldNVMe(devs []Disk) []Disk {
	byCtrl := map[string]int{}
	out := devs[:0]
	for _, d := range devs {
		m := nsRe.FindStringSubmatch(d.Name)
		if m == nil {
			out = append(out, d)
			continue
		}
		ctrl := m[1]
		if i, ok := byCtrl[ctrl]; ok {
			out[i].Size += d.Size
			out[i].ReadBps = round1(out[i].ReadBps + d.ReadBps)
			out[i].WriteBps = round1(out[i].WriteBps + d.WriteBps)
			if d.UtilPct > out[i].UtilPct {
				out[i].UtilPct = d.UtilPct
			}
			if out[i].Temp == 0 {
				out[i].Temp = d.Temp
			}
			continue
		}
		d.Name = ctrl
		if mdl := strings.TrimSpace(readStr("/sys/class/nvme/" + ctrl + "/model")); mdl != "" {
			d.Model = mdl
		}
		byCtrl[ctrl] = len(out)
		out = append(out, d)
	}
	return out
}

// ---- filesystems ----

// Kernel/virtual filesystems that aren't storage. Anything else that's mounted is
// shown, so new real filesystems (bcachefs, exotic FUSE) work without a code change.
// remoteFS is every filesystem type whose storage is not in this machine.
var remoteFS = map[string]bool{
	"cifs": true, "smb3": true, "smbfs": true, "nfs": true, "nfs4": true, "afs": true,
	"ceph": true, "glusterfs": true, "9p": true, "afpfs": true, "davfs": true,
	"fuse.sshfs": true, "sshfs": true, "fuse.rclone": true, "fuse.s3fs": true,
	"fuse.davfs2": true, "fuse.curlftpfs": true, "fuse.gcsfuse": true, "beegfs": true,
	"lustre": true, "orangefs": true, "pvfs2": true, "coda": true, "ncpfs": true,
}

var pseudoFS = map[string]bool{
	"proc": true, "sysfs": true, "devtmpfs": true, "devpts": true, "tmpfs": true, "ramfs": true,
	"cgroup": true, "cgroup2": true, "pstore": true, "bpf": true, "securityfs": true, "debugfs": true,
	"tracefs": true, "configfs": true, "fusectl": true, "hugetlbfs": true, "mqueue": true, "efivarfs": true,
	"binfmt_misc": true, "autofs": true, "rpc_pipefs": true, "nsfs": true, "overlay": true, "squashfs": true,
	"fuse.portal": true, "fuse.gvfsd-fuse": true, "fuse.lxcfs": true, "selinuxfs": true, "none": true, "zfs_snap": true,
}

// Mount prefixes that are container/snap/runtime internals, not the host's storage.
var skipMountPrefix = []string{"/var/lib/docker/", "/var/lib/containers/", "/var/lib/kubelet/", "/run/", "/snap/", "/proc/", "/sys/", "/dev/"}

func skipMount(m string) bool {
	for _, p := range skipMountPrefix {
		if strings.HasPrefix(m, p) {
			return true
		}
	}
	return false
}

func filesystems() []FS {
	out := []FS{}
	b, err := os.ReadFile("/proc/mounts")
	if err != nil {
		return out
	}
	seen := map[string]bool{}
	for _, line := range strings.Split(string(b), "\n") {
		fs := strings.Fields(line)
		if len(fs) < 3 || pseudoFS[fs[2]] {
			continue
		}
		mount := strings.ReplaceAll(fs[1], `\040`, " ")
		if seen[mount] || skipMount(mount) {
			continue
		}
		seen[mount] = true
		// statfs on a dead network mount can block; bound it.
		done := make(chan FS, 1)
		go func() {
			var st syscall.Statfs_t
			if err := syscall.Statfs(mount, &st); err != nil {
				done <- FS{}
				return
			}
			bs := uint64(st.Bsize)
			done <- FS{Mount: mount, Type: fs[2], Size: st.Blocks * bs, Used: (st.Blocks - st.Bfree) * bs, Remote: remoteFS[fs[2]], Disk: diskOf(fs[0])}
		}()
		select {
		case r := <-done:
			if r.Size > 0 {
				out = append(out, r)
			}
		case <-time.After(3 * time.Second):
			out = append(out, FS{Mount: mount, Type: fs[2], Remote: remoteFS[fs[2]]}) // unreachable: size 0 signals it
		}
	}
	return out
}

// diskOf names the physical drive a mount's source device sits on — "/dev/nvme0n1p2" is on
// "nvme0n1". It follows one layer of device-mapper or md (LUKS on a partition, say) but
// gives up when a thing spans several drives, because then no single drive is the answer.
func diskOf(source string) string {
	if !strings.HasPrefix(source, "/dev/") {
		return "" // a ZFS pool, a network share, a tmpfs: not a block device at all
	}
	d := parentDisk(strings.TrimPrefix(source, "/dev/"), 0)
	// the sampler reports one drive per NVMe controller, so name it the same way
	if m := nsRe.FindStringSubmatch(d); m != nil {
		return m[1]
	}
	return d
}

func parentDisk(name string, depth int) string {
	if name == "" || depth > 4 || strings.Contains(name, "/") {
		return ""
	}
	base := "/sys/class/block/" + name
	if _, err := os.Stat(base); err != nil {
		return ""
	}
	// a partition: its parent directory in sysfs is the whole drive
	if _, err := os.Stat(base + "/partition"); err == nil {
		if target, err := os.Readlink(base); err == nil {
			return filepath.Base(filepath.Dir(target))
		}
		return ""
	}
	// device-mapper or md: one drive underneath is an answer, several is not
	if ents, err := os.ReadDir(base + "/slaves"); err == nil && len(ents) > 0 {
		if len(ents) > 1 {
			return ""
		}
		return parentDisk(ents[0].Name(), depth+1)
	}
	return name // a whole drive, used without partitioning
}

// ---- UPS ----

func runOut(args ...string) (string, bool) {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	b, err := exec.CommandContext(ctx, args[0], args[1:]...).Output()
	if err != nil {
		return "", false
	}
	return string(b), true
}

// leadingNum parses the number at the start of "119 V", "47 min.", "170 Watt(17 %)".
func leadingNum(s string) float64 {
	s = strings.TrimSpace(s)
	end := 0
	for end < len(s) && (s[end] >= '0' && s[end] <= '9' || s[end] == '.') {
		end++
	}
	v, _ := strconv.ParseFloat(s[:end], 64)
	return v
}

// upsPwrstat: CyberPower PowerPanel. pwrstat only talks to its daemon as root, so fall
// back to passwordless sudo when plain execution is refused.
func upsPwrstat() *UPS {
	out, ok := runOut("pwrstat", "-status")
	if !ok {
		if !priv.Available() {
			return nil
		}
		if out, ok = runOut(priv.Wrap("pwrstat", "-status")...); !ok {
			return nil
		}
	}
	u := &UPS{Driver: "pwrstat"}
	got := false
	for _, line := range strings.Split(out, "\n") {
		k, v, ok := strings.Cut(line, "....")
		if !ok {
			continue
		}
		k = strings.TrimSpace(k)
		v = strings.TrimSpace(strings.TrimLeft(v, "."))
		got = true
		switch k {
		case "Model Name":
			u.Model = v
		case "State":
			u.State = v
		case "Power Supply by":
			u.Source = v
			u.OnBattery = !strings.Contains(v, "Utility")
		case "Utility Voltage":
			u.InV = leadingNum(v)
		case "Output Voltage":
			u.OutV = leadingNum(v)
		case "Battery Capacity":
			u.BatteryPct = leadingNum(v)
		case "Remaining Runtime":
			u.RuntimeMin = leadingNum(v)
		case "Load":
			u.LoadW = leadingNum(v)
			if i := strings.Index(v, "("); i >= 0 {
				u.LoadPct = leadingNum(v[i+1:])
			}
		case "Last Power Event":
			u.LastEvent = v
		}
	}
	if !got {
		return nil
	}
	return u
}

// upsNUT: Network UPS Tools. `upsc -l` names the UPSes; `upsc <name>` dumps key: value.
func upsNUT() *UPS {
	list, ok := runOut("upsc", "-l")
	if !ok {
		return nil
	}
	name := ""
	for _, l := range strings.Split(list, "\n") {
		if l = strings.TrimSpace(l); l != "" {
			name = l
			break
		}
	}
	if name == "" {
		return nil
	}
	out, ok := runOut("upsc", name)
	if !ok {
		return nil
	}
	kv := map[string]string{}
	for _, l := range strings.Split(out, "\n") {
		if k, v, ok := strings.Cut(l, ":"); ok {
			kv[strings.TrimSpace(k)] = strings.TrimSpace(v)
		}
	}
	if len(kv) == 0 {
		return nil
	}
	u := &UPS{Driver: "nut", Model: strings.TrimSpace(kv["ups.mfr"] + " " + kv["ups.model"])}
	status := kv["ups.status"]
	u.OnBattery = strings.Contains(status, "OB")
	u.State = status
	u.Source = "Utility Power"
	if u.OnBattery {
		u.Source = "Battery"
	}
	u.InV = leadingNum(kv["input.voltage"])
	u.OutV = leadingNum(kv["output.voltage"])
	u.BatteryPct = leadingNum(kv["battery.charge"])
	u.RuntimeMin = round1(leadingNum(kv["battery.runtime"]) / 60)
	u.LoadPct = leadingNum(kv["ups.load"])
	u.LoadW = leadingNum(kv["ups.realpower"])
	if u.LoadW == 0 {
		u.LoadW = round1(leadingNum(kv["ups.realpower.nominal"]) * u.LoadPct / 100)
	}
	return u
}

// upsAPC: apcupsd. `apcaccess` prints "KEY : value" lines.
func upsAPC() *UPS {
	out, ok := runOut("apcaccess", "status")
	if !ok {
		return nil
	}
	kv := map[string]string{}
	for _, l := range strings.Split(out, "\n") {
		if k, v, ok := strings.Cut(l, ":"); ok {
			kv[strings.TrimSpace(k)] = strings.TrimSpace(v)
		}
	}
	if kv["STATUS"] == "" {
		return nil
	}
	u := &UPS{Driver: "apcupsd", Model: kv["MODEL"], State: kv["STATUS"]}
	u.OnBattery = strings.Contains(kv["STATUS"], "ONBATT")
	u.Source = "Utility Power"
	if u.OnBattery {
		u.Source = "Battery"
	}
	u.InV = leadingNum(kv["LINEV"])
	u.OutV = leadingNum(kv["OUTPUTV"])
	u.BatteryPct = leadingNum(kv["BCHARGE"])
	u.RuntimeMin = leadingNum(kv["TIMELEFT"])
	u.LoadPct = leadingNum(kv["LOADPCT"])
	u.LoadW = round1(leadingNum(kv["NOMPOWER"]) * u.LoadPct / 100)
	if kv["LASTXFER"] != "" {
		u.LastEvent = kv["LASTXFER"]
	}
	return u
}

// ---- processes (Processes tab; sampled only while someone is looking) ----

type procTimes struct {
	ticks uint64
	at    time.Time
}

type procStatic struct {
	user string
	cmd  string
}

var clkTck = float64(100) // Linux USER_HZ; sysconf(_SC_CLK_TCK) is 100 on every mainstream build
var uidNames = map[string]string{}

func uidName(uid string) string {
	if n, ok := uidNames[uid]; ok {
		return n
	}
	n := uid
	if u, err := user.LookupId(uid); err == nil {
		n = u.Username
	}
	uidNames[uid] = n
	return n
}

// coreIDs maps logical CPUs to physical cores from sysfs topology: on a chip with
// hyper-threading, CPUs 8–15 are the second threads of cores 0–7, and coretemp names its
// sensors by core. Read once; the topology does not change while we run.
var (
	coreIDsOnce sync.Once
	coreIDsVal  []int
)

func coreIDs(n int) []int {
	coreIDsOnce.Do(func() {
		for i := 0; ; i++ {
			b, err := os.ReadFile("/sys/devices/system/cpu/cpu" + strconv.Itoa(i) + "/topology/core_id")
			if err != nil {
				break
			}
			id, err := strconv.Atoi(strings.TrimSpace(string(b)))
			if err != nil {
				break
			}
			coreIDsVal = append(coreIDsVal, id)
		}
	})
	if len(coreIDsVal) < n {
		return nil
	}
	return coreIDsVal[:n]
}

// ProcTable is the Processes tab payload.
type ProcTable struct {
	T         int64      `json:"t"`
	CPU       float64    `json:"cpu"`
	Cores     []float64  `json:"cores"`
	CPUHist   []float64  `json:"cpu_hist"` // recent total-CPU samples, oldest first
	CPUModel  string     `json:"cpu_model,omitempty"`
	CPUMHz    float64    `json:"cpu_mhz,omitempty"`
	CPUMaxMHz float64    `json:"cpu_max_mhz,omitempty"`
	CPUTemp   float64    `json:"cpu_temp,omitempty"`
	CoreIDs   []int      `json:"core_ids,omitempty"` // for each logical CPU, the physical core it is a thread of (what the temperature sensors are named by)
	Load      [3]float64 `json:"load"`
	Mem       Mem        `json:"mem"`
	Swap      Mem        `json:"swap"`
	Net       []Net      `json:"net"`
	GPUs      []GPU      `json:"gpus"`
	Uptime    float64    `json:"uptime"`
	Procs     []Proc     `json:"procs"`
	Total     int        `json:"total"`
	Running   int        `json:"running"`
}

var cpuModel = func() string {
	for _, line := range strings.Split(readStr("/proc/cpuinfo"), "\n") {
		if strings.HasPrefix(line, "model name") || strings.HasPrefix(line, "Model") || strings.HasPrefix(line, "Hardware") {
			if _, v, ok := strings.Cut(line, ":"); ok {
				return strings.Join(strings.Fields(v), " ")
			}
		}
	}
	// ARM boards don't put a model name in cpuinfo; the device tree names the machine.
	for _, p := range []string{"/sys/firmware/devicetree/base/model", "/proc/device-tree/model", "/sys/devices/virtual/dmi/id/product_name"} {
		if m := strings.TrimRight(strings.TrimSpace(readStr(p)), "\x00"); m != "" {
			return m
		}
	}
	return ""
}()

var cpuMaxMHz = func() float64 {
	if v, ok := readInt("/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq"); ok {
		return float64(v) / 1000
	}
	return 0
}()

func cpuMHz() float64 {
	var sum float64
	n := 0
	for _, line := range strings.Split(readStr("/proc/cpuinfo"), "\n") {
		if strings.HasPrefix(line, "cpu MHz") {
			if _, v, ok := strings.Cut(line, ":"); ok {
				f, _ := strconv.ParseFloat(strings.TrimSpace(v), 64)
				sum += f
				n++
			}
		}
	}
	if n == 0 { // ARM etc.: scaling_cur_freq in kHz
		if v, ok := readInt("/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq"); ok {
			return float64(v) / 1000
		}
		return 0
	}
	return sum / float64(n)
}

// cpuTemp picks the sensor that best represents the CPU package.
func cpuTemp(temps []Temp) float64 {
	best, max := 0.0, 0.0
	for _, t := range temps {
		l := strings.ToLower(t.Label)
		if t.Chip == "coretemp" && strings.HasPrefix(l, "package") || t.Chip == "k10temp" && (l == "tctl" || l == "tdie") ||
			t.Chip == "thermal" && strings.Contains(l, "cpu") || t.Chip == "cpu_thermal" || t.Chip == "soc_thermal" {
			return t.C
		}
		if (t.Chip == "coretemp" || t.Chip == "k10temp") && t.C > max {
			max, best = t.C, t.C
		}
	}
	return best
}

var bootTime = func() int64 {
	for _, line := range strings.Split(readStr("/proc/stat"), "\n") {
		if strings.HasPrefix(line, "btime ") {
			v, _ := strconv.ParseInt(strings.TrimSpace(strings.TrimPrefix(line, "btime ")), 10, 64)
			return v
		}
	}
	return 0
}()

// Processes walks /proc and returns every process with CPU% since the previous call.
// The first call after a quiet spell reports 0% for everyone; the UI polls, so the
// next one is right. Static per-pid facts (user, cmdline) are cached.
func (c *Collector) Processes() ProcTable {
	now := time.Now()
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.prevProc == nil {
		c.prevProc = map[int]procTimes{}
		c.procInfo = map[int]procStatic{}
	}
	if now.Sub(c.procAt) < 100*time.Millisecond && c.lastTable != nil { // coalesce simultaneous clients only
		return *c.lastTable
	}
	ents, err := os.ReadDir("/proc")
	if err != nil {
		return ProcTable{Procs: []Proc{}}
	}
	cur := map[int]procTimes{}
	seen := map[int]bool{}
	pageSize := uint64(os.Getpagesize())
	procs := make([]Proc, 0, len(ents))
	for _, e := range ents {
		pid, err := strconv.Atoi(e.Name())
		if err != nil || !e.IsDir() {
			continue
		}
		stat := readStr("/proc/" + e.Name() + "/stat")
		close := strings.LastIndex(stat, ")")
		open := strings.Index(stat, "(")
		if open < 0 || close < 0 {
			continue
		}
		name := stat[open+1 : close]
		f := strings.Fields(stat[close+2:])
		if len(f) < 22 {
			continue
		}
		utime, _ := strconv.ParseUint(f[11], 10, 64)
		stime, _ := strconv.ParseUint(f[12], 10, 64)
		threads, _ := strconv.Atoi(f[17])
		startTicks, _ := strconv.ParseUint(f[19], 10, 64)
		rssPages, _ := strconv.ParseUint(f[21], 10, 64)
		ppid, _ := strconv.Atoi(f[1])
		nice, _ := strconv.Atoi(f[16])
		ticks := utime + stime
		cur[pid] = procTimes{ticks: ticks, at: now}
		seen[pid] = true
		p := Proc{PID: pid, PPID: ppid, Name: name, State: f[0], Threads: threads, Nice: nice, RSS: rssPages * pageSize,
			CPUTime: float64(ticks) / clkTck, Started: (bootTime + int64(float64(startTicks)/clkTck)) * 1000}
		if prev, ok := c.prevProc[pid]; ok && prev.ticks <= ticks {
			if dt := now.Sub(prev.at).Seconds(); dt > 0 {
				p.CPU = round1(float64(ticks-prev.ticks) / clkTck / dt * 100)
			}
		}
		if p.State == "Z" {
			p.RSS = 0
		}
		info, ok := c.procInfo[pid]
		if !ok {
			for _, line := range strings.Split(readStr("/proc/"+e.Name()+"/status"), "\n") {
				if strings.HasPrefix(line, "Uid:") {
					if fs := strings.Fields(line); len(fs) > 1 {
						info.user = uidName(fs[1])
					}
					break
				}
			}
			if cmd := strings.ReplaceAll(strings.TrimRight(readStr("/proc/"+e.Name()+"/cmdline"), "\x00"), "\x00", " "); cmd != "" {
				if len(cmd) > 300 {
					cmd = cmd[:300] + "…"
				}
				info.cmd = cmd
			}
			c.procInfo[pid] = info
		}
		p.User, p.Cmd = info.user, info.cmd
		procs = append(procs, p)
	}
	for pid := range c.procInfo {
		if !seen[pid] {
			delete(c.procInfo, pid)
		}
	}
	c.prevProc, c.procAt = cur, now
	// what each process is doing on the GPUs, from the newest sample's client lists
	gpuBy := map[int]float64{}
	gpus := []GPU{}
	if n := len(c.ring); n > 0 && c.ring[n-1].GPUs != nil {
		gpus = c.ring[n-1].GPUs
		for _, g := range gpus {
			for _, cl := range g.Clients {
				gpuBy[cl.PID] += cl.Usage
			}
		}
	}
	for i := range procs {
		procs[i].GPU = round1(gpuBy[procs[i].PID])
	}
	sort.Slice(procs, func(i, j int) bool {
		if procs[i].CPU != procs[j].CPU {
			return procs[i].CPU > procs[j].CPU
		}
		return procs[i].RSS > procs[j].RSS
	})
	t := ProcTable{T: now.UnixMilli(), Procs: procs, Total: len(procs), Uptime: uptime(), Load: loadavg(),
		CPUModel: cpuModel, CPUMHz: round1(cpuMHz()), CPUMaxMHz: cpuMaxMHz, CPUHist: []float64{}, Net: []Net{}, GPUs: gpus}
	for _, p := range procs {
		if p.State == "R" {
			t.Running++
		}
	}
	t.Mem, t.Swap = meminfo()
	if n := len(c.ring); n > 0 {
		last := c.ring[n-1]
		t.CPU, t.Cores, t.Net, t.CPUTemp = last.CPU, last.Cores, last.Net, cpuTemp(last.Temps)
		t.CoreIDs = coreIDs(len(last.Cores))
		from := n - 90
		if from < 0 {
			from = 0
		}
		for _, s := range c.ring[from:] {
			t.CPUHist = append(t.CPUHist, s.CPU)
		}
	}
	c.lastTable = &t
	return t
}

// Signal sends a signal to a pid; falls back to sudo when the process isn't ours.
func Signal(pid int, sig syscall.Signal) error {
	if pid <= 1 || pid == os.Getpid() {
		return errors.New("refusing to signal pid " + strconv.Itoa(pid))
	}
	if strings.TrimRight(readStr("/proc/"+strconv.Itoa(pid)+"/cmdline"), "\x00") == "" {
		if _, err := os.Stat("/proc/" + strconv.Itoa(pid)); err == nil {
			return errors.New("refusing to signal a kernel thread")
		}
	}
	if err := syscall.Kill(pid, sig); err == nil {
		return nil
	} else if !errors.Is(err, syscall.EPERM) {
		return err
	}
	if priv.Available() {
		k := priv.Wrap("kill", "-"+strconv.Itoa(int(sig)), strconv.Itoa(pid))
		if err := exec.Command(k[0], k[1:]...).Run(); err == nil {
			return nil
		}
	}
	return errors.New("permission denied")
}

// ---- small helpers ----

func readStr(p string) string {
	b, err := os.ReadFile(p)
	if err != nil {
		return ""
	}
	return string(b)
}

func readInt(p string) (int64, bool) {
	s := strings.TrimSpace(readStr(p))
	if s == "" {
		return 0, false
	}
	v, err := strconv.ParseInt(s, 10, 64)
	return v, err == nil
}

func fileExists(p string) bool { _, err := os.Stat(p); return err == nil }

func round1(v float64) float64 { return float64(int64(v*10+0.5)) / 10 }

func lookPath(bin string) bool { _, err := exec.LookPath(bin); return err == nil }