// Package metrics samples host telemetry (CPU, memory, temps, fans, network, disks, // filesystems, UPS) on a timer into an in-memory ring, for the Dashboard tab. // // Portability rules: everything comes from /proc and /sys, which every Linux kernel // exposes the same way regardless of distro or architecture; nothing here needs // lm-sensors, smartmontools or root. Sections whose source is absent (no fans, no // UPS daemon, no hwmon at all on a VM) simply come back empty rather than failing. // The only external commands are optional UPS daemons: pwrstat (CyberPower), upsc // (NUT) and apcaccess (apcupsd), tried in that order. Two readings are root-only, the GPU // client walk and the CPU energy counters; without root they come from small helpers of // this binary started once through sudo or doas (gpu.go, cpupower.go), or not at all. package metrics import ( "bufio" "context" "errors" "os" "os/exec" "os/user" "path/filepath" "regexp" "sort" "strconv" "strings" "sync" "syscall" "time" "hamlab/internal/priv" ) const ( DefaultInterval = 2 * time.Second MinInterval = 100 * time.Millisecond MaxInterval = 10 * time.Second History = 300 // points kept per curve; the window below sets how far apart they are // How far back the kept history reaches. The point count is fixed, so this sets the spacing: // at a minute every sample is its own point, at a day each point is the average of ~5 minutes. DefaultWindow = 10 * time.Minute MinWindow = 1 * time.Minute MaxWindow = 24 * time.Hour fsEvery = 30 * time.Second // filesystem usage changes slowly and statfs on network mounts isn't free upsEvery = 10 * time.Second // UPS daemons are an exec each; don't hammer them at fast intervals sectorSize = 512 // /proc/diskstats always counts 512-byte sectors ) type Mem struct { Total uint64 `json:"total"` Used uint64 `json:"used"` Avail uint64 `json:"avail"` Free uint64 `json:"free,omitempty"` Cached uint64 `json:"cached,omitempty"` Buffers uint64 `json:"buffers,omitempty"` } type Temp struct { Chip string `json:"chip"` Label string `json:"label"` C float64 `json:"c"` Max float64 `json:"max,omitempty"` Crit float64 `json:"crit,omitempty"` } type Fan struct { Chip string `json:"chip"` Label string `json:"label"` RPM int `json:"rpm"` } type Net struct { Iface string `json:"iface"` Up bool `json:"up"` RxBps float64 `json:"rx_bps"` TxBps float64 `json:"tx_bps"` RxTotal uint64 `json:"rx_total"` TxTotal uint64 `json:"tx_total"` } type Disk struct { Name string `json:"name"` Model string `json:"model,omitempty"` Size uint64 `json:"size"` ReadBps float64 `json:"read_bps"` WriteBps float64 `json:"write_bps"` UtilPct float64 `json:"util_pct"` Temp float64 `json:"temp,omitempty"` } type FS struct { Mount string `json:"mount"` Type string `json:"type"` Size uint64 `json:"size"` Used uint64 `json:"used"` // Remote is set when the filesystem lives on another machine — a CIFS or NFS share, // sshfs, and so on. Its bytes are somebody else's disk, so a panel about this machine's // hardware leaves it out, while the sensor stays for anyone who does want it. Remote bool `json:"remote,omitempty"` // Disk is the physical drive these bytes are on ("nvme0", "sda"), named as the sampler // names it; empty when that cannot be // said honestly: a ZFS pool or an md array spreads a filesystem over several drives, and // a remote mount is on none of them. Disk string `json:"disk,omitempty"` } type UPS struct { Driver string `json:"driver"` // pwrstat | nut | apcupsd Model string `json:"model"` State string `json:"state"` Source string `json:"source"` OnBattery bool `json:"on_battery"` InV float64 `json:"in_v"` OutV float64 `json:"out_v"` BatteryPct float64 `json:"battery_pct"` RuntimeMin float64 `json:"runtime_min"` LoadW float64 `json:"load_w"` LoadPct float64 `json:"load_pct"` LastEvent string `json:"last_event,omitempty"` } type Proc struct { PID int `json:"pid"` PPID int `json:"ppid"` Name string `json:"name"` Cmd string `json:"cmd,omitempty"` User string `json:"user"` CPU float64 `json:"cpu"` // percent of one core GPU float64 `json:"gpu,omitempty"` // its busiest GPU engine, percent; summed over the GPUs it has open CPUTime float64 `json:"cpu_time"` // seconds, lifetime RSS uint64 `json:"rss"` // bytes Threads int `json:"threads"` State string `json:"state"` Nice int `json:"nice"` Started int64 `json:"started"` // unix ms } type Sample struct { T int64 `json:"t"` // unix ms Interval int64 `json:"interval_ms"` // sampling interval in effect CPU float64 `json:"cpu"` Cores []float64 `json:"cores"` CPUPower float64 `json:"cpu_power_w,omitempty"` // the CPU packages' own energy counters; 0 = no reading Load [3]float64 `json:"load"` Mem Mem `json:"mem"` Swap Mem `json:"swap"` Temps []Temp `json:"temps"` Fans []Fan `json:"fans"` Net []Net `json:"net"` Disks []Disk `json:"disks"` FS []FS `json:"fs"` UPS *UPS `json:"ups"` GPUs []GPU `json:"gpus"` ARC *ARC `json:"arc"` // the ZFS read cache; nil without the module Uptime float64 `json:"uptime"` fsT, upsT int64 // when the slow sections were really read; sensors report these, not T } type Collector struct { mu sync.RWMutex ring []Sample // the kept history: one point per stride, each the average of its bucket last Sample // the newest raw sample, whatever the stride bucket []Sample // samples since the last point was kept lastKept int64 // when that was, unix ms interval time.Duration window time.Duration wake chan struct{} // nudges Run when the interval changes prevCPU []cpuTimes prevNet map[string][2]uint64 prevDisk map[string][3]uint64 // sectors read, sectors written, io ticks ms prevT time.Time lastFS []FS fsAt time.Time lastUPS *UPS upsAt time.Time prevProc map[int]procTimes procInfo map[int]procStatic procAt time.Time lastTable *ProcTable upsProbe func() *UPS // nil when no UPS daemon is installed gpu gpuState power cpuPowerState prevARC arcPrev subMu sync.Mutex subs map[chan struct{}]struct{} // told after every sample; see Subscribe } func NewCollector() *Collector { c := &Collector{prevNet: map[string][2]uint64{}, prevDisk: map[string][3]uint64{}, interval: DefaultInterval, window: DefaultWindow, wake: make(chan struct{}, 1)} switch { case lookPath("pwrstat"): c.upsProbe = upsPwrstat case lookPath("upsc"): c.upsProbe = upsNUT case lookPath("apcaccess"): c.upsProbe = upsAPC } return c } // Run samples until ctx is cancelled, at whatever interval is current. func (c *Collector) Run(ctx context.Context) { c.sample() // prime deltas for { // The wait is what's left of the interval after the work, so a machine whose // sample takes 30 ms still delivers 4 a second at 250 ms, not 3.6. wait := c.Interval() - time.Since(c.prevT) if wait < 0 { wait = 0 } t := time.NewTimer(wait) select { case <-ctx.Done(): t.Stop() return case <-c.wake: // interval changed: restart the wait t.Stop() case <-t.C: c.sample() } } } // SetInterval changes the sampling period, clamped to [MinInterval, MaxInterval]. func (c *Collector) SetInterval(d time.Duration) { if d < MinInterval { d = MinInterval } if d > MaxInterval { d = MaxInterval } c.mu.Lock() changed := d != c.interval c.interval = d c.mu.Unlock() if changed { select { case c.wake <- struct{}{}: default: } } } func (c *Collector) Interval() time.Duration { c.mu.RLock() defer c.mu.RUnlock() return c.interval } // SetWindow sets how far back the kept history reaches, clamped to [MinWindow, MaxWindow]. func (c *Collector) SetWindow(d time.Duration) { if d < MinWindow { d = MinWindow } if d > MaxWindow { d = MaxWindow } c.mu.Lock() defer c.mu.Unlock() c.window = d c.trim() } func (c *Collector) Window() time.Duration { c.mu.RLock() defer c.mu.RUnlock() return c.window } // Stride is how far apart the kept points are: the sampling interval until the window is // wider than History samples of it, then the window split History ways. func (c *Collector) Stride() time.Duration { c.mu.RLock() defer c.mu.RUnlock() return c.stride() } func (c *Collector) stride() time.Duration { if s := c.window / History; s > c.interval { return s } return c.interval } // keep folds samples into the history, one point per stride. Averaged, not sampled: a CPU // graph of the last six hours should show the load through each five minutes, not whichever // two-second instant happened to land on the mark. func (c *Collector) keep(s Sample) { c.bucket = append(c.bucket, s) // A point is due a stride after the last one (for the first, a stride after the interval its // first sample stands for). Half an interval of slack, because a timer never fires exactly on // time: without it a tick that lands 5 ms early is "not yet", and every point runs one sample // long. Measured from the last point kept, not by counting samples, so a machine that stalled // or slept simply keeps the next sample it gets. iv := c.interval.Milliseconds() since := c.lastKept if since == 0 { since = c.bucket[0].T - iv } if s.T-since < c.stride().Milliseconds()-iv/2 { return } c.lastKept = s.T c.ring = append(c.ring, fold(c.bucket)) c.bucket = c.bucket[:0] c.trim() } // trim drops points past the count budget and past the window. Caller holds c.mu. func (c *Collector) trim() { if len(c.ring) > History { c.ring = c.ring[len(c.ring)-History:] } n := len(c.ring) if n == 0 { return } cut := c.ring[n-1].T - c.window.Milliseconds() i := 0 for i < n && c.ring[i].T < cut { i++ } if i > 0 { c.ring = c.ring[i:] } } func (c *Collector) Latest() (Sample, bool) { c.mu.RLock() defer c.mu.RUnlock() if c.last.T == 0 { return Sample{}, false } return c.last, true } // ---- sampling ---- func (c *Collector) sample() { now := time.Now() dt := now.Sub(c.prevT).Seconds() first := c.prevT.IsZero() c.prevT = now s := Sample{T: now.UnixMilli(), Interval: c.Interval().Milliseconds(), Temps: []Temp{}, Fans: []Fan{}, Net: []Net{}, Disks: []Disk{}} s.CPU, s.Cores = c.cpu() s.CPUPower = c.cpuPower(now) s.Load = loadavg() s.Mem, s.Swap = meminfo() s.Uptime = uptime() s.Temps, s.Fans = hwmon() if len(s.Temps) == 0 { s.Temps = thermalZones() // ARM SoCs and older kernels often expose only these } s.Net = c.net(dt) s.Disks = c.disks(dt) if c.lastFS == nil || now.Sub(c.fsAt) >= fsEvery { c.lastFS, c.fsAt = filesystems(), now } s.FS, s.fsT = c.lastFS, c.fsAt.UnixMilli() if c.upsProbe != nil && (c.upsAt.IsZero() || now.Sub(c.upsAt) >= upsEvery) { c.lastUPS, c.upsAt = c.upsProbe(), now } s.UPS, s.upsT = c.lastUPS, c.upsAt.UnixMilli() s.GPUs = c.gpus(now, cpuTemp(s.Temps)) s.ARC = c.arc(now) if first { return // deltas are meaningless on the priming pass } c.mu.Lock() c.last = s c.keep(s) c.mu.Unlock() c.subMu.Lock() for ch := range c.subs { select { case ch <- struct{}{}: default: // a slow listener misses a tick; it reads the latest on the next one } } c.subMu.Unlock() } // Subscribe returns a channel that gets a nudge after every new sample, and the func // that ends the subscription. func (c *Collector) Subscribe() (<-chan struct{}, func()) { ch := make(chan struct{}, 1) c.subMu.Lock() if c.subs == nil { c.subs = map[chan struct{}]struct{}{} } c.subs[ch] = struct{}{} c.subMu.Unlock() return ch, func() { c.subMu.Lock(); delete(c.subs, ch); c.subMu.Unlock() } } type cpuTimes struct{ busy, total uint64 } func (c *Collector) cpu() (float64, []float64) { f, err := os.Open("/proc/stat") if err != nil { return 0, nil } defer f.Close() var cur []cpuTimes sc := bufio.NewScanner(f) for sc.Scan() { fs := strings.Fields(sc.Text()) if len(fs) < 8 || !strings.HasPrefix(fs[0], "cpu") { continue } var vals [10]uint64 for i := 1; i < len(fs) && i <= 10; i++ { vals[i-1], _ = strconv.ParseUint(fs[i], 10, 64) } idle := vals[3] + vals[4] var total uint64 for _, v := range vals { total += v } cur = append(cur, cpuTimes{busy: total - idle, total: total}) } prev := c.prevCPU c.prevCPU = cur if len(prev) != len(cur) || len(cur) == 0 { return 0, nil } pct := func(i int) float64 { dt := float64(cur[i].total - prev[i].total) if dt <= 0 { return 0 } return 100 * float64(cur[i].busy-prev[i].busy) / dt } cores := make([]float64, 0, len(cur)-1) for i := 1; i < len(cur); i++ { cores = append(cores, round1(pct(i))) } return round1(pct(0)), cores } func loadavg() (l [3]float64) { b, err := os.ReadFile("/proc/loadavg") if err != nil { return } f := strings.Fields(string(b)) for i := 0; i < 3 && i < len(f); i++ { l[i], _ = strconv.ParseFloat(f[i], 64) } return } func meminfo() (Mem, Mem) { m := map[string]uint64{} b, err := os.ReadFile("/proc/meminfo") if err != nil { return Mem{}, Mem{} } for _, line := range strings.Split(string(b), "\n") { k, v, ok := strings.Cut(line, ":") if !ok { continue } f := strings.Fields(v) if len(f) == 0 { continue } n, _ := strconv.ParseUint(f[0], 10, 64) m[k] = n * 1024 } mem := Mem{Total: m["MemTotal"], Avail: m["MemAvailable"], Free: m["MemFree"], Buffers: m["Buffers"], Cached: m["Cached"] + m["SReclaimable"]} if mem.Avail == 0 { // pre-3.14 kernels: approximate mem.Avail = m["MemFree"] + m["Buffers"] + m["Cached"] } mem.Used = mem.Total - mem.Avail swap := Mem{Total: m["SwapTotal"], Avail: m["SwapFree"]} swap.Used = swap.Total - swap.Avail return mem, swap } func uptime() float64 { b, err := os.ReadFile("/proc/uptime") if err != nil { return 0 } if f := strings.Fields(string(b)); len(f) > 0 { v, _ := strconv.ParseFloat(f[0], 64) return v } return 0 } // ---- temperatures and fans ---- var sensorRe = regexp.MustCompile(`^(temp|fan)(\d+)_input$`) // tempLimit reads a *_max / *_crit file; values outside 0-200 °C are unset sentinels // (NVMe reports ~65000 for absent thresholds). func tempLimit(p string) float64 { if v, ok := readInt(p); ok && v > 0 && v < 200000 { return float64(v) / 1000 } return 0 } // hwmonDir reads every temp*/fan* sensor under one hwmon directory. func hwmonDir(d string) (chip string, temps []Temp, fans []Fan) { chip = strings.TrimSpace(readStr(filepath.Join(d, "name"))) ents, err := os.ReadDir(d) if err != nil { return } for _, e := range ents { m := sensorRe.FindStringSubmatch(e.Name()) if m == nil { continue } base := m[1] + m[2] raw, ok := readInt(filepath.Join(d, e.Name())) if !ok { continue } label := strings.TrimSpace(readStr(filepath.Join(d, base+"_label"))) if label == "" { label = base } if m[1] == "temp" { if raw < -60000 { // an unpowered sensor's sentinel (an idle audio amp reads -93 °C), not weather continue } temps = append(temps, Temp{Chip: chip, Label: label, C: float64(raw) / 1000, Max: tempLimit(filepath.Join(d, base+"_max")), Crit: tempLimit(filepath.Join(d, base+"_crit"))}) } else if raw > 0 || fileExists(filepath.Join(d, base+"_enable")) { fans = append(fans, Fan{Chip: chip, Label: label, RPM: int(raw)}) } } return } func hwmon() ([]Temp, []Fan) { temps, fans := []Temp{}, []Fan{} dirs, _ := filepath.Glob("/sys/class/hwmon/hwmon*") for _, d := range dirs { _, t, f := hwmonDir(d) temps = append(temps, t...) fans = append(fans, f...) } sort.SliceStable(temps, func(i, j int) bool { if temps[i].Chip != temps[j].Chip { return temps[i].Chip < temps[j].Chip } return temps[i].Label < temps[j].Label }) return temps, fans } func thermalZones() []Temp { out := []Temp{} dirs, _ := filepath.Glob("/sys/class/thermal/thermal_zone*") for _, d := range dirs { raw, ok := readInt(filepath.Join(d, "temp")) if !ok || raw <= 0 { continue } typ := strings.TrimSpace(readStr(filepath.Join(d, "type"))) if typ == "" { typ = filepath.Base(d) } t := Temp{Chip: "thermal", Label: typ, C: float64(raw) / 1000} // trip points: trip_point_N_type in {critical, hot, passive, active} trips, _ := filepath.Glob(filepath.Join(d, "trip_point_*_type")) for _, tp := range trips { kind := strings.TrimSpace(readStr(tp)) v := tempLimit(strings.TrimSuffix(tp, "_type") + "_temp") switch kind { case "critical": t.Crit = v case "hot", "passive": if t.Max == 0 || v < t.Max { t.Max = v } } } out = append(out, t) } return out } // blockTemp finds the drive's own temperature via the hwmon node the kernel attaches // to the device: the nvme driver and drivetemp (SATA/SAS) both do this. func blockTemp(name string) float64 { for _, pat := range []string{"/sys/block/" + name + "/device/hwmon*", "/sys/block/" + name + "/device/hwmon/hwmon*"} { dirs, _ := filepath.Glob(pat) for _, d := range dirs { if v, ok := readInt(filepath.Join(d, "temp1_input")); ok { return float64(v) / 1000 } } } return 0 } // ---- network ---- // Virtual interfaces that only ever carry traffic already counted on a real one. var skipIfacePrefix = []string{"veth", "br-", "docker", "virbr", "vnet", "tap", "lxc", "lxd", "cni", "flannel", "cali", "kube", "podman", "dummy"} func skipIface(n string) bool { if n == "lo" { return true } for _, p := range skipIfacePrefix { if strings.HasPrefix(n, p) { return true } } return false } func (c *Collector) net(dt float64) []Net { out := []Net{} f, err := os.Open("/proc/net/dev") if err != nil { return out } defer f.Close() sc := bufio.NewScanner(f) for sc.Scan() { name, rest, ok := strings.Cut(sc.Text(), ":") if !ok { continue } name = strings.TrimSpace(name) if skipIface(name) { continue } fs := strings.Fields(rest) if len(fs) < 16 { continue } rx, _ := strconv.ParseUint(fs[0], 10, 64) tx, _ := strconv.ParseUint(fs[8], 10, 64) n := Net{Iface: name, RxTotal: rx, TxTotal: tx} st := strings.TrimSpace(readStr("/sys/class/net/" + name + "/operstate")) n.Up = st == "up" || st == "unknown" // wireguard/tun report "unknown" while carrying traffic if p, ok := c.prevNet[name]; ok && dt > 0 { n.RxBps = round1(float64(rx-p[0]) / dt) n.TxBps = round1(float64(tx-p[1]) / dt) } c.prevNet[name] = [2]uint64{rx, tx} out = append(out, n) } sort.Slice(out, func(i, j int) bool { return out[i].Iface < out[j].Iface }) return out } // ---- disks ---- // Pseudo block devices with nothing physical behind them. var skipDiskPrefix = []string{"loop", "ram", "zram", "nbd", "sr", "fd", "dm-", "md", "drbd", "zd", "pmem"} func skipDisk(n string) bool { // eMMC boot/RPMB/general-purpose areas: regions of the chip, not drives if strings.HasPrefix(n, "mmcblk") && (strings.Contains(n, "boot") || strings.HasSuffix(n, "rpmb") || strings.Contains(n, "gp")) { return true } for _, p := range skipDiskPrefix { if strings.HasPrefix(n, p) { return true } } return false } func diskModel(name string) string { dev := "/sys/block/" + name + "/device/" for _, f := range []string{"model", "name", "vendor"} { // SCSI/NVMe: model; MMC/SD: name if s := strings.TrimSpace(readStr(dev + f)); s != "" { return s } } return "" } func (c *Collector) disks(dt float64) []Disk { out := []Disk{} // /sys/block lists whole devices only (partitions live underneath), so this is // the authoritative "is a disk" test across sd/nvme/mmcblk/vd/xvd/hd/… ents, err := os.ReadDir("/sys/block") if err != nil { return out } whole := map[string]bool{} for _, e := range ents { if !skipDisk(e.Name()) { whole[e.Name()] = true } } f, err := os.Open("/proc/diskstats") if err != nil { return out } defer f.Close() sc := bufio.NewScanner(f) for sc.Scan() { fs := strings.Fields(sc.Text()) if len(fs) < 14 || !whole[fs[2]] { continue } name := fs[2] rd, _ := strconv.ParseUint(fs[5], 10, 64) wr, _ := strconv.ParseUint(fs[9], 10, 64) ticks, _ := strconv.ParseUint(fs[12], 10, 64) d := Disk{Name: name, Model: diskModel(name), Temp: blockTemp(name)} if sz, ok := readInt("/sys/block/" + name + "/size"); ok { d.Size = uint64(sz) * sectorSize } if d.Size == 0 { continue // empty card reader / detached media } if p, ok := c.prevDisk[name]; ok && dt > 0 { d.ReadBps = round1(float64(rd-p[0]) * sectorSize / dt) d.WriteBps = round1(float64(wr-p[1]) * sectorSize / dt) d.UtilPct = round1(float64(ticks-p[2]) / (dt * 1000) * 100) if d.UtilPct > 100 { d.UtilPct = 100 } } c.prevDisk[name] = [3]uint64{rd, wr, ticks} out = append(out, d) } out = foldNVMe(out) sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) return out } var nsRe = regexp.MustCompile(`^(nvme\d+)n\d+$`) // foldNVMe merges nvme0n1, nvme0n2, … into the drive they are: one controller presenting // several namespaces, which the kernel lists as whole disks apiece. An Apple SSD shows three — // the 466 GiB of storage and two of firmware, a few MiB each — and a page that lists disks was // listing all three as separate drives. The Disks page has always folded them (internal/storage // does the same); this is what the dashboard, the Top tab and the disk.* sensors read. // // Rates add up across namespaces. Busy does not: they share one controller, so the time it // spent working is the longest of them, never the sum. func foldNVMe(devs []Disk) []Disk { byCtrl := map[string]int{} out := devs[:0] for _, d := range devs { m := nsRe.FindStringSubmatch(d.Name) if m == nil { out = append(out, d) continue } ctrl := m[1] if i, ok := byCtrl[ctrl]; ok { out[i].Size += d.Size out[i].ReadBps = round1(out[i].ReadBps + d.ReadBps) out[i].WriteBps = round1(out[i].WriteBps + d.WriteBps) if d.UtilPct > out[i].UtilPct { out[i].UtilPct = d.UtilPct } if out[i].Temp == 0 { out[i].Temp = d.Temp } continue } d.Name = ctrl if mdl := strings.TrimSpace(readStr("/sys/class/nvme/" + ctrl + "/model")); mdl != "" { d.Model = mdl } byCtrl[ctrl] = len(out) out = append(out, d) } return out } // ---- filesystems ---- // Kernel/virtual filesystems that aren't storage. Anything else that's mounted is // shown, so new real filesystems (bcachefs, exotic FUSE) work without a code change. // remoteFS is every filesystem type whose storage is not in this machine. var remoteFS = map[string]bool{ "cifs": true, "smb3": true, "smbfs": true, "nfs": true, "nfs4": true, "afs": true, "ceph": true, "glusterfs": true, "9p": true, "afpfs": true, "davfs": true, "fuse.sshfs": true, "sshfs": true, "fuse.rclone": true, "fuse.s3fs": true, "fuse.davfs2": true, "fuse.curlftpfs": true, "fuse.gcsfuse": true, "beegfs": true, "lustre": true, "orangefs": true, "pvfs2": true, "coda": true, "ncpfs": true, } var pseudoFS = map[string]bool{ "proc": true, "sysfs": true, "devtmpfs": true, "devpts": true, "tmpfs": true, "ramfs": true, "cgroup": true, "cgroup2": true, "pstore": true, "bpf": true, "securityfs": true, "debugfs": true, "tracefs": true, "configfs": true, "fusectl": true, "hugetlbfs": true, "mqueue": true, "efivarfs": true, "binfmt_misc": true, "autofs": true, "rpc_pipefs": true, "nsfs": true, "overlay": true, "squashfs": true, "fuse.portal": true, "fuse.gvfsd-fuse": true, "fuse.lxcfs": true, "selinuxfs": true, "none": true, "zfs_snap": true, } // Mount prefixes that are container/snap/runtime internals, not the host's storage. var skipMountPrefix = []string{"/var/lib/docker/", "/var/lib/containers/", "/var/lib/kubelet/", "/run/", "/snap/", "/proc/", "/sys/", "/dev/"} func skipMount(m string) bool { for _, p := range skipMountPrefix { if strings.HasPrefix(m, p) { return true } } return false } func filesystems() []FS { out := []FS{} b, err := os.ReadFile("/proc/mounts") if err != nil { return out } seen := map[string]bool{} for _, line := range strings.Split(string(b), "\n") { fs := strings.Fields(line) if len(fs) < 3 || pseudoFS[fs[2]] { continue } mount := strings.ReplaceAll(fs[1], `\040`, " ") if seen[mount] || skipMount(mount) { continue } seen[mount] = true // statfs on a dead network mount can block; bound it. done := make(chan FS, 1) go func() { var st syscall.Statfs_t if err := syscall.Statfs(mount, &st); err != nil { done <- FS{} return } bs := uint64(st.Bsize) done <- FS{Mount: mount, Type: fs[2], Size: st.Blocks * bs, Used: (st.Blocks - st.Bfree) * bs, Remote: remoteFS[fs[2]], Disk: diskOf(fs[0])} }() select { case r := <-done: if r.Size > 0 { out = append(out, r) } case <-time.After(3 * time.Second): out = append(out, FS{Mount: mount, Type: fs[2], Remote: remoteFS[fs[2]]}) // unreachable: size 0 signals it } } return out } // diskOf names the physical drive a mount's source device sits on — "/dev/nvme0n1p2" is on // "nvme0n1". It follows one layer of device-mapper or md (LUKS on a partition, say) but // gives up when a thing spans several drives, because then no single drive is the answer. func diskOf(source string) string { if !strings.HasPrefix(source, "/dev/") { return "" // a ZFS pool, a network share, a tmpfs: not a block device at all } d := parentDisk(strings.TrimPrefix(source, "/dev/"), 0) // the sampler reports one drive per NVMe controller, so name it the same way if m := nsRe.FindStringSubmatch(d); m != nil { return m[1] } return d } func parentDisk(name string, depth int) string { if name == "" || depth > 4 || strings.Contains(name, "/") { return "" } base := "/sys/class/block/" + name if _, err := os.Stat(base); err != nil { return "" } // a partition: its parent directory in sysfs is the whole drive if _, err := os.Stat(base + "/partition"); err == nil { if target, err := os.Readlink(base); err == nil { return filepath.Base(filepath.Dir(target)) } return "" } // device-mapper or md: one drive underneath is an answer, several is not if ents, err := os.ReadDir(base + "/slaves"); err == nil && len(ents) > 0 { if len(ents) > 1 { return "" } return parentDisk(ents[0].Name(), depth+1) } return name // a whole drive, used without partitioning } // ---- UPS ---- func runOut(args ...string) (string, bool) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() b, err := exec.CommandContext(ctx, args[0], args[1:]...).Output() if err != nil { return "", false } return string(b), true } // leadingNum parses the number at the start of "119 V", "47 min.", "170 Watt(17 %)". func leadingNum(s string) float64 { s = strings.TrimSpace(s) end := 0 for end < len(s) && (s[end] >= '0' && s[end] <= '9' || s[end] == '.') { end++ } v, _ := strconv.ParseFloat(s[:end], 64) return v } // upsPwrstat: CyberPower PowerPanel. pwrstat only talks to its daemon as root, so fall // back to passwordless sudo when plain execution is refused. func upsPwrstat() *UPS { out, ok := runOut("pwrstat", "-status") if !ok { if !priv.Available() { return nil } if out, ok = runOut(priv.Wrap("pwrstat", "-status")...); !ok { return nil } } u := &UPS{Driver: "pwrstat"} got := false for _, line := range strings.Split(out, "\n") { k, v, ok := strings.Cut(line, "....") if !ok { continue } k = strings.TrimSpace(k) v = strings.TrimSpace(strings.TrimLeft(v, ".")) got = true switch k { case "Model Name": u.Model = v case "State": u.State = v case "Power Supply by": u.Source = v u.OnBattery = !strings.Contains(v, "Utility") case "Utility Voltage": u.InV = leadingNum(v) case "Output Voltage": u.OutV = leadingNum(v) case "Battery Capacity": u.BatteryPct = leadingNum(v) case "Remaining Runtime": u.RuntimeMin = leadingNum(v) case "Load": u.LoadW = leadingNum(v) if i := strings.Index(v, "("); i >= 0 { u.LoadPct = leadingNum(v[i+1:]) } case "Last Power Event": u.LastEvent = v } } if !got { return nil } return u } // upsNUT: Network UPS Tools. `upsc -l` names the UPSes; `upsc ` dumps key: value. func upsNUT() *UPS { list, ok := runOut("upsc", "-l") if !ok { return nil } name := "" for _, l := range strings.Split(list, "\n") { if l = strings.TrimSpace(l); l != "" { name = l break } } if name == "" { return nil } out, ok := runOut("upsc", name) if !ok { return nil } kv := map[string]string{} for _, l := range strings.Split(out, "\n") { if k, v, ok := strings.Cut(l, ":"); ok { kv[strings.TrimSpace(k)] = strings.TrimSpace(v) } } if len(kv) == 0 { return nil } u := &UPS{Driver: "nut", Model: strings.TrimSpace(kv["ups.mfr"] + " " + kv["ups.model"])} status := kv["ups.status"] u.OnBattery = strings.Contains(status, "OB") u.State = status u.Source = "Utility Power" if u.OnBattery { u.Source = "Battery" } u.InV = leadingNum(kv["input.voltage"]) u.OutV = leadingNum(kv["output.voltage"]) u.BatteryPct = leadingNum(kv["battery.charge"]) u.RuntimeMin = round1(leadingNum(kv["battery.runtime"]) / 60) u.LoadPct = leadingNum(kv["ups.load"]) u.LoadW = leadingNum(kv["ups.realpower"]) if u.LoadW == 0 { u.LoadW = round1(leadingNum(kv["ups.realpower.nominal"]) * u.LoadPct / 100) } return u } // upsAPC: apcupsd. `apcaccess` prints "KEY : value" lines. func upsAPC() *UPS { out, ok := runOut("apcaccess", "status") if !ok { return nil } kv := map[string]string{} for _, l := range strings.Split(out, "\n") { if k, v, ok := strings.Cut(l, ":"); ok { kv[strings.TrimSpace(k)] = strings.TrimSpace(v) } } if kv["STATUS"] == "" { return nil } u := &UPS{Driver: "apcupsd", Model: kv["MODEL"], State: kv["STATUS"]} u.OnBattery = strings.Contains(kv["STATUS"], "ONBATT") u.Source = "Utility Power" if u.OnBattery { u.Source = "Battery" } u.InV = leadingNum(kv["LINEV"]) u.OutV = leadingNum(kv["OUTPUTV"]) u.BatteryPct = leadingNum(kv["BCHARGE"]) u.RuntimeMin = leadingNum(kv["TIMELEFT"]) u.LoadPct = leadingNum(kv["LOADPCT"]) u.LoadW = round1(leadingNum(kv["NOMPOWER"]) * u.LoadPct / 100) if kv["LASTXFER"] != "" { u.LastEvent = kv["LASTXFER"] } return u } // ---- processes (Processes tab; sampled only while someone is looking) ---- type procTimes struct { ticks uint64 at time.Time } type procStatic struct { user string cmd string } var clkTck = float64(100) // Linux USER_HZ; sysconf(_SC_CLK_TCK) is 100 on every mainstream build var uidNames = map[string]string{} func uidName(uid string) string { if n, ok := uidNames[uid]; ok { return n } n := uid if u, err := user.LookupId(uid); err == nil { n = u.Username } uidNames[uid] = n return n } // coreIDs maps logical CPUs to physical cores from sysfs topology: on a chip with // hyper-threading, CPUs 8–15 are the second threads of cores 0–7, and coretemp names its // sensors by core. Read once; the topology does not change while we run. var ( coreIDsOnce sync.Once coreIDsVal []int ) func coreIDs(n int) []int { coreIDsOnce.Do(func() { for i := 0; ; i++ { b, err := os.ReadFile("/sys/devices/system/cpu/cpu" + strconv.Itoa(i) + "/topology/core_id") if err != nil { break } id, err := strconv.Atoi(strings.TrimSpace(string(b))) if err != nil { break } coreIDsVal = append(coreIDsVal, id) } }) if len(coreIDsVal) < n { return nil } return coreIDsVal[:n] } // ProcTable is the Processes tab payload. type ProcTable struct { T int64 `json:"t"` CPU float64 `json:"cpu"` Cores []float64 `json:"cores"` CPUHist []float64 `json:"cpu_hist"` // recent total-CPU samples, oldest first CPUModel string `json:"cpu_model,omitempty"` CPUMHz float64 `json:"cpu_mhz,omitempty"` CPUMaxMHz float64 `json:"cpu_max_mhz,omitempty"` CPUPowerW float64 `json:"cpu_power_w,omitempty"` CPUTemp float64 `json:"cpu_temp,omitempty"` CoreIDs []int `json:"core_ids,omitempty"` // for each logical CPU, the physical core it is a thread of (what the temperature sensors are named by) Load [3]float64 `json:"load"` Mem Mem `json:"mem"` Swap Mem `json:"swap"` Net []Net `json:"net"` GPUs []GPU `json:"gpus"` Uptime float64 `json:"uptime"` Procs []Proc `json:"procs"` Total int `json:"total"` Running int `json:"running"` } var cpuModel = func() string { for _, line := range strings.Split(readStr("/proc/cpuinfo"), "\n") { if strings.HasPrefix(line, "model name") || strings.HasPrefix(line, "Model") || strings.HasPrefix(line, "Hardware") { if _, v, ok := strings.Cut(line, ":"); ok { return strings.Join(strings.Fields(v), " ") } } } // ARM boards don't put a model name in cpuinfo; the device tree names the machine. for _, p := range []string{"/sys/firmware/devicetree/base/model", "/proc/device-tree/model", "/sys/devices/virtual/dmi/id/product_name"} { if m := strings.TrimRight(strings.TrimSpace(readStr(p)), "\x00"); m != "" { return m } } return "" }() var cpuMaxMHz = func() float64 { if v, ok := readInt("/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq"); ok { return float64(v) / 1000 } return 0 }() func cpuMHz() float64 { var sum float64 n := 0 for _, line := range strings.Split(readStr("/proc/cpuinfo"), "\n") { if strings.HasPrefix(line, "cpu MHz") { if _, v, ok := strings.Cut(line, ":"); ok { f, _ := strconv.ParseFloat(strings.TrimSpace(v), 64) sum += f n++ } } } if n == 0 { // ARM etc.: scaling_cur_freq in kHz if v, ok := readInt("/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq"); ok { return float64(v) / 1000 } return 0 } return sum / float64(n) } // cpuTemp picks the sensor that best represents the CPU package. func cpuTemp(temps []Temp) float64 { best, max := 0.0, 0.0 for _, t := range temps { l := strings.ToLower(t.Label) if t.Chip == "coretemp" && strings.HasPrefix(l, "package") || t.Chip == "k10temp" && (l == "tctl" || l == "tdie") || t.Chip == "thermal" && strings.Contains(l, "cpu") || t.Chip == "cpu_thermal" || t.Chip == "soc_thermal" { return t.C } if (t.Chip == "coretemp" || t.Chip == "k10temp") && t.C > max { max, best = t.C, t.C } } return best } var bootTime = func() int64 { for _, line := range strings.Split(readStr("/proc/stat"), "\n") { if strings.HasPrefix(line, "btime ") { v, _ := strconv.ParseInt(strings.TrimSpace(strings.TrimPrefix(line, "btime ")), 10, 64) return v } } return 0 }() // Processes walks /proc and returns every process with CPU% since the previous call. // The first call after a quiet spell reports 0% for everyone; the UI polls, so the // next one is right. Static per-pid facts (user, cmdline) are cached. func (c *Collector) Processes() ProcTable { now := time.Now() c.mu.Lock() defer c.mu.Unlock() if c.prevProc == nil { c.prevProc = map[int]procTimes{} c.procInfo = map[int]procStatic{} } if now.Sub(c.procAt) < 100*time.Millisecond && c.lastTable != nil { // coalesce simultaneous clients only return *c.lastTable } ents, err := os.ReadDir("/proc") if err != nil { return ProcTable{Procs: []Proc{}} } cur := map[int]procTimes{} seen := map[int]bool{} pageSize := uint64(os.Getpagesize()) procs := make([]Proc, 0, len(ents)) for _, e := range ents { pid, err := strconv.Atoi(e.Name()) if err != nil || !e.IsDir() { continue } stat := readStr("/proc/" + e.Name() + "/stat") close := strings.LastIndex(stat, ")") open := strings.Index(stat, "(") if open < 0 || close < 0 { continue } name := stat[open+1 : close] f := strings.Fields(stat[close+2:]) if len(f) < 22 { continue } utime, _ := strconv.ParseUint(f[11], 10, 64) stime, _ := strconv.ParseUint(f[12], 10, 64) threads, _ := strconv.Atoi(f[17]) startTicks, _ := strconv.ParseUint(f[19], 10, 64) rssPages, _ := strconv.ParseUint(f[21], 10, 64) ppid, _ := strconv.Atoi(f[1]) nice, _ := strconv.Atoi(f[16]) ticks := utime + stime cur[pid] = procTimes{ticks: ticks, at: now} seen[pid] = true p := Proc{PID: pid, PPID: ppid, Name: name, State: f[0], Threads: threads, Nice: nice, RSS: rssPages * pageSize, CPUTime: float64(ticks) / clkTck, Started: (bootTime + int64(float64(startTicks)/clkTck)) * 1000} if prev, ok := c.prevProc[pid]; ok && prev.ticks <= ticks { if dt := now.Sub(prev.at).Seconds(); dt > 0 { p.CPU = round1(float64(ticks-prev.ticks) / clkTck / dt * 100) } } if p.State == "Z" { p.RSS = 0 } info, ok := c.procInfo[pid] if !ok { for _, line := range strings.Split(readStr("/proc/"+e.Name()+"/status"), "\n") { if strings.HasPrefix(line, "Uid:") { if fs := strings.Fields(line); len(fs) > 1 { info.user = uidName(fs[1]) } break } } if cmd := strings.ReplaceAll(strings.TrimRight(readStr("/proc/"+e.Name()+"/cmdline"), "\x00"), "\x00", " "); cmd != "" { if len(cmd) > 300 { cmd = cmd[:300] + "…" } info.cmd = cmd } c.procInfo[pid] = info } p.User, p.Cmd = info.user, info.cmd procs = append(procs, p) } for pid := range c.procInfo { if !seen[pid] { delete(c.procInfo, pid) } } c.prevProc, c.procAt = cur, now // what each process is doing on the GPUs, from the newest sample's client lists gpuBy := map[int]float64{} gpus := []GPU{} if n := len(c.ring); n > 0 && c.ring[n-1].GPUs != nil { gpus = c.ring[n-1].GPUs for _, g := range gpus { for _, cl := range g.Clients { gpuBy[cl.PID] += cl.Usage } } } for i := range procs { procs[i].GPU = round1(gpuBy[procs[i].PID]) } sort.Slice(procs, func(i, j int) bool { if procs[i].CPU != procs[j].CPU { return procs[i].CPU > procs[j].CPU } return procs[i].RSS > procs[j].RSS }) t := ProcTable{T: now.UnixMilli(), Procs: procs, Total: len(procs), Uptime: uptime(), Load: loadavg(), CPUModel: cpuModel, CPUMHz: round1(cpuMHz()), CPUMaxMHz: cpuMaxMHz, CPUHist: []float64{}, Net: []Net{}, GPUs: gpus} for _, p := range procs { if p.State == "R" { t.Running++ } } t.Mem, t.Swap = meminfo() if n := len(c.ring); n > 0 { last := c.ring[n-1] t.CPU, t.Cores, t.Net, t.CPUTemp, t.CPUPowerW = last.CPU, last.Cores, last.Net, cpuTemp(last.Temps), last.CPUPower t.CoreIDs = coreIDs(len(last.Cores)) from := n - 90 if from < 0 { from = 0 } for _, s := range c.ring[from:] { t.CPUHist = append(t.CPUHist, s.CPU) } } c.lastTable = &t return t } // Signal sends a signal to a pid; falls back to sudo when the process isn't ours. func Signal(pid int, sig syscall.Signal) error { if pid <= 1 || pid == os.Getpid() { return errors.New("refusing to signal pid " + strconv.Itoa(pid)) } if strings.TrimRight(readStr("/proc/"+strconv.Itoa(pid)+"/cmdline"), "\x00") == "" { if _, err := os.Stat("/proc/" + strconv.Itoa(pid)); err == nil { return errors.New("refusing to signal a kernel thread") } } if err := syscall.Kill(pid, sig); err == nil { return nil } else if !errors.Is(err, syscall.EPERM) { return err } if priv.Available() { k := priv.Wrap("kill", "-"+strconv.Itoa(int(sig)), strconv.Itoa(pid)) if err := exec.Command(k[0], k[1:]...).Run(); err == nil { return nil } } return errors.New("permission denied") } // ---- small helpers ---- // readStr reads a /proc or /sys file whole, with plain blocking syscalls and never through // the runtime poller. os.ReadFile would hand a sysfs file to the poller — kernfs accepts // epoll — and a driver that answers a non-blocking read with EAGAIN (i915's fan attribute // did, on grid, 2026-10-02) then leaves the reader waiting for a readiness kernfs only // signals on sysfs_notify: never. The sampler sat in that wait for 42 minutes. func readStr(p string) string { fd, err := syscall.Open(p, syscall.O_RDONLY|syscall.O_CLOEXEC, 0) if err != nil { return "" } defer syscall.Close(fd) var out []byte buf := make([]byte, 4096) for { n, err := syscall.Read(fd, buf) if n > 0 { out = append(out, buf[:n]...) } if err == syscall.EINTR { continue } if err != nil || n == 0 { break } } return string(out) } func readInt(p string) (int64, bool) { s := strings.TrimSpace(readStr(p)) if s == "" { return 0, false } v, err := strconv.ParseInt(s, 10, 64) return v, err == nil } func fileExists(p string) bool { _, err := os.Stat(p); return err == nil } func round1(v float64) float64 { return float64(int64(v*10+0.5)) / 10 } func lookPath(bin string) bool { _, err := exec.LookPath(bin); return err == nil }