Processes: CPU power beside the clock, from the packages' energy counters

New sensor cpu.power: watts from the RAPL energy counters (Intel, and AMD
Zen through the same driver), summed over every package and averaged into
history. Parts of a package, psys and the MMIO duplicate are left out.

The counters are root-only, so a daemon that isn't root reads them through
`hamlab cpu-energy`, a root helper started once with sudo or doas and asked
over a pipe each sample, the same shape as the GPU client walk's helper.
It reads only the counters it finds itself. No counters or no root means
no reading, and the CPU box shows nothing extra.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/metrics/cpupower.go

@@ -0,0 +1,259 @@
+package metrics
+
+// CPU power, from the RAPL energy counters the kernel keeps under /sys/class/powercap: Intel
+// since Sandy Bridge, and AMD Zen through the same driver. Each counter is a running total in
+// µJ, so power is the difference between two readings over the time between them. Every
+// package is summed (`package-0`, and `package-1` on a two-socket board). The other top-level
+// domains are left out. `psys` claims the whole platform but is only as good as the firmware's
+// calibration, and on one machine here it reads lower than the package inside it. The MMIO
+// copy of a package (`intel-rapl-mmio:0`) measures the same thing again. This is the CPU's own
+// figure, shown beside its clock: cores, built-in graphics and memory controller, not disks,
+// fans or the power supply.
+//
+// The counters are root-only since the PLATYPUS side channel (2020). So, like the GPU client
+// walk, they are read in-process when the daemon is root, and otherwise by a root helper,
+// `hamlab cpu-energy`, started once through sudo or doas and asked over a pipe at every
+// sample: one privilege escalation per daemon, not one a second in the auth log. The helper
+// takes nothing from the daemon but the newline that asks; it finds and reads the counters
+// itself. Without root there is no reading, and nothing is shown.
+
+import (
+	"bufio"
+	"encoding/json"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"hamlab/internal/priv"
+)
+
+const raplRoot = "/sys/class/powercap"
+
+var raplPkgRe = regexp.MustCompile(`^package-\d+$`)
+
+// raplCount is one package's counter at one moment.
+type raplCount struct {
+	Dir string `json:"d"`
+	UJ  uint64 `json:"uj"`
+}
+
+type cpuPowerState struct {
+	probed    bool
+	dirs      []string          // the package domains, found at the first sample
+	max       map[string]uint64 // each counter's range: where it wraps to zero
+	useHelper bool              // the counters are not readable in-process
+	helper    *energyHelper
+	prev      []raplCount
+	prevAt    time.Time
+}
+
+// raplPackages lists the package domains under root, sorted. intel-rapl:0 is a package;
+// intel-rapl:0:0 is a part of one (core, uncore, dram).
+func raplPackages(root string) []string {
+	ents, err := os.ReadDir(root)
+	if err != nil {
+		return nil
+	}
+	var out []string
+	for _, e := range ents {
+		n := e.Name()
+		if !strings.HasPrefix(n, "intel-rapl:") || strings.Count(n, ":") != 1 {
+			continue
+		}
+		if raplPkgRe.MatchString(strings.TrimSpace(readStr(filepath.Join(root, n, "name")))) {
+			out = append(out, n)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+func readRAPL(root string, dirs []string) ([]raplCount, bool) {
+	out := make([]raplCount, 0, len(dirs))
+	for _, d := range dirs {
+		b, err := os.ReadFile(filepath.Join(root, d, "energy_uj"))
+		if err != nil {
+			return nil, false
+		}
+		v, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64)
+		if err != nil {
+			return nil, false
+		}
+		out = append(out, raplCount{d, v})
+	}
+	return out, len(out) > 0
+}
+
+// raplWatts is the power between two readings of the same counters. A counter that wrapped
+// is carried across its range. One that went backwards with no known range, or a set of
+// counters that changed between readings, gives no answer rather than a wrong one.
+func raplWatts(prev, cur []raplCount, max map[string]uint64, dt float64) (float64, bool) {
+	if dt <= 0 || len(prev) == 0 || len(prev) != len(cur) {
+		return 0, false
+	}
+	var uj float64
+	for i := range cur {
+		if cur[i].Dir != prev[i].Dir {
+			return 0, false
+		}
+		if cur[i].UJ >= prev[i].UJ {
+			uj += float64(cur[i].UJ - prev[i].UJ)
+			continue
+		}
+		m := max[cur[i].Dir]
+		if m == 0 || prev[i].UJ > m {
+			return 0, false
+		}
+		uj += float64(m - prev[i].UJ + cur[i].UJ)
+	}
+	return uj / 1e6 / dt, true
+}
+
+// cpuPower is the CPU packages' power since the previous sample, in watts; 0 when this
+// machine has no counters or no way to read them.
+func (c *Collector) cpuPower(now time.Time) float64 {
+	p := &c.power
+	if !p.probed {
+		p.probed = true
+		p.dirs = raplPackages(raplRoot)
+		p.max = map[string]uint64{}
+		for _, d := range p.dirs {
+			if v, ok := readInt(filepath.Join(raplRoot, d, "max_energy_range_uj")); ok && v > 0 {
+				p.max[d] = uint64(v)
+			}
+		}
+	}
+	if len(p.dirs) == 0 {
+		return 0
+	}
+	var cur []raplCount
+	ok := false
+	if !p.useHelper {
+		if cur, ok = readRAPL(raplRoot, p.dirs); !ok && priv.Mode() != "root" && priv.Available() {
+			p.useHelper = true
+		}
+	}
+	if p.useHelper {
+		if p.helper == nil {
+			p.helper = &energyHelper{}
+		}
+		cur, ok = p.helper.read()
+	}
+	if !ok {
+		p.prev = nil
+		return 0
+	}
+	w, good := raplWatts(p.prev, cur, p.max, now.Sub(p.prevAt).Seconds())
+	p.prev, p.prevAt = cur, now
+	if !good {
+		return 0
+	}
+	return round1(w)
+}
+
+// energyHelper is the root child that answers each newline with the package counters, one
+// JSON line. The same shape as the GPU client walk's helper, kept separate from it.
+type energyHelper struct {
+	mu       sync.Mutex
+	cmd      *exec.Cmd
+	in       io.WriteCloser
+	out      *bufio.Reader
+	failedAt time.Time
+}
+
+func (h *energyHelper) read() ([]raplCount, bool) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	if h.cmd == nil {
+		if !h.failedAt.IsZero() && time.Since(h.failedAt) < helperRetry {
+			return nil, false
+		}
+		if !h.start() {
+			h.failedAt = time.Now()
+			return nil, false
+		}
+	}
+	// a stuck child must not stall the sampler: it is killed, and tried again later
+	cmd := h.cmd
+	stop := time.AfterFunc(5*time.Second, func() { cmd.Process.Kill() })
+	defer stop.Stop()
+	if _, err := io.WriteString(h.in, "\n"); err != nil {
+		h.fail()
+		return nil, false
+	}
+	line, err := h.out.ReadBytes('\n')
+	if err != nil {
+		h.fail()
+		return nil, false
+	}
+	var cs []raplCount
+	if json.Unmarshal(line, &cs) != nil {
+		h.fail()
+		return nil, false
+	}
+	return cs, len(cs) > 0
+}
+
+func (h *energyHelper) start() bool {
+	self, err := os.Executable()
+	if err != nil {
+		return false
+	}
+	full := priv.Wrap(self, "cpu-energy")
+	cmd := exec.Command(full[0], full[1:]...)
+	in, err := cmd.StdinPipe()
+	if err != nil {
+		return false
+	}
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return false
+	}
+	if err := cmd.Start(); err != nil {
+		return false
+	}
+	h.cmd, h.in, h.out = cmd, in, bufio.NewReader(out)
+	return true
+}
+
+func (h *energyHelper) fail() {
+	if h.cmd != nil {
+		h.in.Close()
+		h.cmd.Process.Kill()
+		go h.cmd.Wait()
+	}
+	h.cmd, h.in, h.out = nil, nil, nil
+	h.failedAt = time.Now()
+}
+
+// CPUEnergyHelper is `hamlab cpu-energy`: for every line read on stdin, one JSON line of the
+// package counters on stdout. Ends when stdin does — when the daemon that started it has gone.
+func CPUEnergyHelper(stdin io.Reader, stdout io.Writer) int {
+	return cpuEnergyHelper(raplRoot, stdin, stdout)
+}
+
+func cpuEnergyHelper(root string, stdin io.Reader, stdout io.Writer) int {
+	dirs := raplPackages(root)
+	sc := bufio.NewScanner(stdin)
+	w := bufio.NewWriter(stdout)
+	for sc.Scan() {
+		cs, ok := readRAPL(root, dirs)
+		if !ok {
+			cs = []raplCount{}
+		}
+		b, _ := json.Marshal(cs)
+		w.Write(b)
+		w.WriteByte('\n')
+		if w.Flush() != nil {
+			return 1
+		}
+	}
+	return 0
+}

internal/metrics/cpupower_test.go

@@ -0,0 +1,106 @@
+package metrics
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// A fake /sys/class/powercap: a package with its parts, the platform domain, and the MMIO
+// copy of the same package. Only the package counts.
+func fakeRAPL(t *testing.T, domains map[string][2]string) string {
+	t.Helper()
+	root := t.TempDir()
+	for dir, v := range domains {
+		d := filepath.Join(root, dir)
+		if err := os.MkdirAll(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		os.WriteFile(filepath.Join(d, "name"), []byte(v[0]+"\n"), 0o644)
+		os.WriteFile(filepath.Join(d, "energy_uj"), []byte(v[1]+"\n"), 0o644)
+	}
+	return root
+}
+
+func TestRAPLPackagesOnlyCountsPackages(t *testing.T) {
+	root := fakeRAPL(t, map[string][2]string{
+		"intel-rapl:0":      {"package-0", "1000"},
+		"intel-rapl:0:0":    {"core", "600"},
+		"intel-rapl:0:1":    {"uncore", "10"},
+		"intel-rapl:1":      {"psys", "300"},
+		"intel-rapl-mmio:0": {"package-0", "1000"},
+	})
+	if got := raplPackages(root); len(got) != 1 || got[0] != "intel-rapl:0" {
+		t.Fatalf("packages = %v, want [intel-rapl:0]", got)
+	}
+
+	two := fakeRAPL(t, map[string][2]string{"intel-rapl:0": {"package-0", "5"}, "intel-rapl:1": {"package-1", "7"}})
+	got := raplPackages(two)
+	if strings.Join(got, ",") != "intel-rapl:0,intel-rapl:1" {
+		t.Fatalf("two sockets: packages = %v", got)
+	}
+	cs, ok := readRAPL(two, got)
+	if !ok || len(cs) != 2 || cs[0].UJ != 5 || cs[1].UJ != 7 {
+		t.Fatalf("readRAPL = %+v, %v", cs, ok)
+	}
+	if got := raplPackages(filepath.Join(two, "nope")); len(got) != 0 {
+		t.Fatalf("no powercap: packages = %v, want none", got)
+	}
+}
+
+func TestRAPLWatts(t *testing.T) {
+	c := func(d string, uj uint64) raplCount { return raplCount{d, uj} }
+	max := map[string]uint64{"intel-rapl:0": 1_000_000_000}
+	cases := []struct {
+		name      string
+		prev, cur []raplCount
+		dt        float64
+		want      float64
+		ok        bool
+	}{
+		{"steady", []raplCount{c("intel-rapl:0", 0)}, []raplCount{c("intel-rapl:0", 10_000_000)}, 2, 5, true},
+		{"two packages add", []raplCount{c("intel-rapl:0", 0), c("intel-rapl:1", 0)}, []raplCount{c("intel-rapl:0", 4_000_000), c("intel-rapl:1", 6_000_000)}, 1, 10, true},
+		{"wrapped", []raplCount{c("intel-rapl:0", 999_000_000)}, []raplCount{c("intel-rapl:0", 1_000_000)}, 1, 2, true},
+		{"backwards, range unknown", []raplCount{c("intel-rapl:1", 5_000_000)}, []raplCount{c("intel-rapl:1", 1)}, 1, 0, false},
+		{"counters changed", []raplCount{c("intel-rapl:0", 0)}, []raplCount{c("intel-rapl:1", 9)}, 1, 0, false},
+		{"first reading", nil, []raplCount{c("intel-rapl:0", 9)}, 1, 0, false},
+		{"no time passed", []raplCount{c("intel-rapl:0", 0)}, []raplCount{c("intel-rapl:0", 9)}, 0, 0, false},
+	}
+	for _, tc := range cases {
+		got, ok := raplWatts(tc.prev, tc.cur, max, tc.dt)
+		if ok != tc.ok || (ok && (got < tc.want-1e-9 || got > tc.want+1e-9)) {
+			t.Errorf("%s: got %v, %v; want %v, %v", tc.name, got, ok, tc.want, tc.ok)
+		}
+	}
+}
+
+// The helper answers every newline with the counters it found itself, and stops at EOF.
+func TestCPUEnergyHelper(t *testing.T) {
+	root := fakeRAPL(t, map[string][2]string{"intel-rapl:0": {"package-0", "42"}, "intel-rapl:0:0": {"core", "7"}})
+	var out bytes.Buffer
+	if code := cpuEnergyHelper(root, strings.NewReader("\n\n"), &out); code != 0 {
+		t.Fatalf("exit %d", code)
+	}
+	lines := strings.Split(strings.TrimSpace(out.String()), "\n")
+	if len(lines) != 2 {
+		t.Fatalf("got %d lines, want 2: %q", len(lines), out.String())
+	}
+	var cs []raplCount
+	if err := json.Unmarshal([]byte(lines[1]), &cs); err != nil || len(cs) != 1 || cs[0] != (raplCount{"intel-rapl:0", 42}) {
+		t.Fatalf("line = %s (%v)", lines[1], err)
+	}
+}
+
+// History keeps the average of the samples that had a reading.
+func TestFoldCPUPower(t *testing.T) {
+	b := []Sample{{T: 1, CPUPower: 4}, {T: 2}, {T: 3, CPUPower: 8}}
+	if got := fold(b).CPUPower; got != 6 {
+		t.Fatalf("folded CPU power = %v, want 6", got)
+	}
+	if got := fold([]Sample{{T: 1}, {T: 2}}).CPUPower; got != 0 {
+		t.Fatalf("no readings folded to %v, want 0", got)
+	}
+}

internal/metrics/fold.go

@@ -32,6 +32,17 @@ func fold(b []Sample) Sample {
 		return t / uint64(len(b))
 	}
 	out.CPU = sum(func(s Sample) float64 { return s.CPU })
+	// CPU power: over the samples that have a reading, so a helper restart is not a dip to zero
+	var pw, pn float64
+	for _, s := range b {
+		if s.CPUPower > 0 {
+			pw, pn = pw+s.CPUPower, pn+1
+		}
+	}
+	out.CPUPower = 0
+	if pn > 0 {
+		out.CPUPower = round1(pw / pn)
+	}
 	out.Load[0] = sum(func(s Sample) float64 { return s.Load[0] })
 	out.Load[1] = sum(func(s Sample) float64 { return s.Load[1] })
 	out.Load[2] = sum(func(s Sample) float64 { return s.Load[2] })

internal/metrics/metrics.go

@@ -6,7 +6,9 @@
 // lm-sensors, smartmontools or root. Sections whose source is absent (no fans, no
 // UPS daemon, no hwmon at all on a VM) simply come back empty rather than failing.
 // The only external commands are optional UPS daemons: pwrstat (CyberPower), upsc
-// (NUT) and apcaccess (apcupsd), tried in that order.
+// (NUT) and apcaccess (apcupsd), tried in that order. Two readings are root-only, the GPU
+// client walk and the CPU energy counters; without root they come from small helpers of
+// this binary started once through sudo or doas (gpu.go, cpupower.go), or not at all.
 package metrics
 
 import (
@@ -137,6 +139,7 @@ type Sample struct {
 	Interval int64      `json:"interval_ms"` // sampling interval in effect
 	CPU      float64    `json:"cpu"`
 	Cores    []float64  `json:"cores"`
+	CPUPower float64    `json:"cpu_power_w,omitempty"` // the CPU packages' own energy counters; 0 = no reading
 	Load     [3]float64 `json:"load"`
 	Mem      Mem        `json:"mem"`
 	Swap     Mem        `json:"swap"`
@@ -177,6 +180,7 @@ type Collector struct {
 	lastTable *ProcTable
 	upsProbe  func() *UPS // nil when no UPS daemon is installed
 	gpu       gpuState
+	power     cpuPowerState
 	prevARC   arcPrev
 
 	subMu sync.Mutex
@@ -343,6 +347,7 @@ func (c *Collector) sample() {
 
 	s := Sample{T: now.UnixMilli(), Interval: c.Interval().Milliseconds(), Temps: []Temp{}, Fans: []Fan{}, Net: []Net{}, Disks: []Disk{}}
 	s.CPU, s.Cores = c.cpu()
+	s.CPUPower = c.cpuPower(now)
 	s.Load = loadavg()
 	s.Mem, s.Swap = meminfo()
 	s.Uptime = uptime()
@@ -1113,6 +1118,7 @@ type ProcTable struct {
 	CPUModel  string     `json:"cpu_model,omitempty"`
 	CPUMHz    float64    `json:"cpu_mhz,omitempty"`
 	CPUMaxMHz float64    `json:"cpu_max_mhz,omitempty"`
+	CPUPowerW float64    `json:"cpu_power_w,omitempty"`
 	CPUTemp   float64    `json:"cpu_temp,omitempty"`
 	CoreIDs   []int      `json:"core_ids,omitempty"` // for each logical CPU, the physical core it is a thread of (what the temperature sensors are named by)
 	Load      [3]float64 `json:"load"`
@@ -1312,7 +1318,7 @@ func (c *Collector) Processes() ProcTable {
 	t.Mem, t.Swap = meminfo()
 	if n := len(c.ring); n > 0 {
 		last := c.ring[n-1]
-		t.CPU, t.Cores, t.Net, t.CPUTemp = last.CPU, last.Cores, last.Net, cpuTemp(last.Temps)
+		t.CPU, t.Cores, t.Net, t.CPUTemp, t.CPUPowerW = last.CPU, last.Cores, last.Net, cpuTemp(last.Temps), last.CPUPower
 		t.CoreIDs = coreIDs(len(last.Cores))
 		from := n - 90
 		if from < 0 {

internal/metrics/sensors.go

@@ -62,6 +62,9 @@ func flatten(s Sample, want func(id string) bool) []sensors.Sensor {
 	if c := cpuTemp(s.Temps); c > 0 {
 		f.add("cpu.temp", "CPU temperature", "temperature", "°C", c, t, none)
 	}
+	if s.CPUPower > 0 {
+		f.add("cpu.power", "CPU power", "power", "W", s.CPUPower, t, none)
+	}
 	for i, w := range []string{"1", "5", "15"} {
 		f.add("load."+w, "Load "+w+" min", "load", "", s.Load[i], t, none)
 	}