Sensors: per-service and container sensors, pattern-limited tokens, fleet query, steady sampler

- service.<unit>.active|state for services that are enabled, up or failed - containers.* and container.<name>.running|state|health|mem|cpu (cgroup reads, no docker stats) - sensors:read tokens can be limited to ID patterns, enforced on every endpoint and, through the hub's filter header, on managed machines; the header can only narrow - /api/fleet/sensors/query: one shaped query, every machine, one request - the sampler waits what's left of the interval, so 250 ms means 4 a second - Monitors asks for the families it draws instead of * - unpowered hwmon sentinels (-93 °C) are not temperatures Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-19 08:08:44 -07:00
parent 4ddf10be51
commit 4c2eb57ea6
11 changed files with 383 additions and 51 deletions
@@ -2,10 +2,15 @@ package api

 import (
 	"context"
+	"encoding/json"
+	"os"
+	"os/exec"
+	"strconv"
 	"strings"
 	"sync"
 	"time"

+	"hamlab/internal/priv"
 	"hamlab/internal/sensors"
 	"hamlab/internal/storage"
 )
@@ -18,18 +23,22 @@ type statusSensors struct {
 	s  *Server
 	mu sync.RWMutex
 	by map[string][]sensors.Sensor // source → its sensors
+
+	cpuPrev map[string]cpuMark // container ID → CPU time at the previous read
 }

 const (
-	servicesEvery = 30 * time.Second
-	updatesEvery  = 5 * time.Minute
-	zfsEvery      = time.Minute
-	smartEvery    = 10 * time.Minute // storage.Manager caches SMART that long anyway, and never wakes a sleeping disk
+	servicesEvery   = 30 * time.Second
+	containersEvery = 15 * time.Second
+	updatesEvery    = 5 * time.Minute
+	zfsEvery        = time.Minute
+	smartEvery      = 10 * time.Minute // storage.Manager caches SMART that long anyway, and never wakes a sleeping disk
 )

 func newStatusSensors(s *Server) *statusSensors {
 	p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}}
 	go p.loop("services", servicesEvery, p.services)
+	go p.loop("containers", containersEvery, p.containers)
 	go p.loop("updates", updatesEvery, p.updates)
 	go p.loop("zfs", zfsEvery, p.zfs)
 	go p.loop("smart", smartEvery, p.smart)
@@ -65,6 +74,7 @@ func (p *statusSensors) services(ctx context.Context, t int64) []sensors.Sensor
 	}
 	var failed, running float64
 	var names []string
+	var per []sensors.Sensor
 	for _, u := range units {
 		if u.Failed {
 			failed++
@@ -73,12 +83,115 @@ func (p *statusSensors) services(ctx context.Context, t int64) []sensors.Sensor
 		if u.Sub == "running" {
 			running++
 		}
+		// One pair per service that is meant to be doing something: enabled, up, or broken.
+		// (Every static and one-shot unit on the box would triple the catalog for nothing.)
+		short, isService := strings.CutSuffix(u.Name, ".service")
+		if !isService || !(u.Enabled == "enabled" || u.Active == "active" || u.Failed) {
+			continue
+		}
+		id, l := "service."+sensors.Seg(short), map[string]string{"unit": u.Name, "description": u.Description}
+		per = append(per,
+			sensors.Sensor{ID: id + ".active", Name: short, Kind: "state", Value: u.Active == "active", T: t, Labels: l},
+			sensors.Sensor{ID: id + ".state", Name: short + " state", Kind: "state", Value: u.Sub, T: t, Labels: l})
 	}
-	return []sensors.Sensor{
+	return append(per, []sensors.Sensor{
 		{ID: "services.failed", Name: "Failed services", Kind: "count", Value: failed, T: t, Warn: sensors.F(1)},
 		{ID: "services.failed_names", Name: "Failed services", Kind: "state", Value: strings.Join(names, ", "), T: t},
 		{ID: "services.running", Name: "Running services", Kind: "count", Value: running, T: t},
+	}...)
+}
+
+// containers reads Docker (or Podman) with one `ps`, then each running container's memory
+// and CPU straight from its cgroup — `docker stats` takes seconds and a daemon round-trip
+// per call; two small files per container take microseconds.
+func (p *statusSensors) containers(ctx context.Context, t int64) []sensors.Sensor {
+	var out []byte
+	var err error
+	for _, tool := range []string{"docker", "podman"} {
+		if _, e := exec.LookPath(tool); e != nil {
+			continue
+		}
+		args := []string{tool, "ps", "-a", "--no-trunc", "--format", "{{json .}}"}
+		if out, err = exec.CommandContext(ctx, args[0], args[1:]...).Output(); err != nil && priv.Available() {
+			full := priv.Wrap(args...) // not in the docker group: ask as root
+			out, err = exec.CommandContext(ctx, full[0], full[1:]...).Output()
+		}
+		break
 	}
+	if err != nil || len(out) == 0 {
+		return nil
+	}
+	var res []sensors.Sensor
+	var running, unhealthy, total float64
+	seenCPU := map[string]cpuMark{}
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		var c struct{ ID, Names, Image, State, Status string }
+		if json.Unmarshal([]byte(line), &c) != nil || c.Names == "" {
+			continue
+		}
+		total++
+		name := strings.Split(c.Names, ",")[0]
+		id, l := "container."+sensors.Seg(name), map[string]string{"container": name, "image": c.Image}
+		up := c.State == "running"
+		health := "none"
+		for _, h := range []string{"unhealthy", "healthy", "starting"} { // "unhealthy" first: it contains "healthy"
+			if strings.Contains(c.Status, "("+h+")") || strings.Contains(c.Status, "(health: "+h+")") {
+				health = h
+				break
+			}
+		}
+		if up {
+			running++
+		}
+		if health == "unhealthy" {
+			unhealthy++
+		}
+		res = append(res,
+			sensors.Sensor{ID: id + ".running", Name: name, Kind: "state", Value: up, T: t, Labels: l},
+			sensors.Sensor{ID: id + ".state", Name: name + " state", Kind: "state", Value: c.State, T: t, Labels: l},
+			sensors.Sensor{ID: id + ".health", Name: name + " health", Kind: "state", Value: health, T: t, Labels: l})
+		if !up {
+			continue
+		}
+		cg := ""
+		for _, dir := range []string{"/sys/fs/cgroup/system.slice/docker-" + c.ID + ".scope", "/sys/fs/cgroup/docker/" + c.ID, "/sys/fs/cgroup/machine.slice/libpod-" + c.ID + ".scope"} {
+			if _, e := os.Stat(dir + "/memory.current"); e == nil {
+				cg = dir
+				break
+			}
+		}
+		if cg == "" {
+			continue
+		}
+		if b, e := os.ReadFile(cg + "/memory.current"); e == nil {
+			if v, e := strconv.ParseFloat(strings.TrimSpace(string(b)), 64); e == nil {
+				res = append(res, sensors.Sensor{ID: id + ".mem", Name: name + " memory", Kind: "bytes", Unit: "B", Value: v, T: t, Labels: l})
+			}
+		}
+		if b, e := os.ReadFile(cg + "/cpu.stat"); e == nil {
+			for _, ln := range strings.Split(string(b), "\n") {
+				if f := strings.Fields(ln); len(f) == 2 && f[0] == "usage_usec" {
+					usec, _ := strconv.ParseFloat(f[1], 64)
+					seenCPU[c.ID] = cpuMark{usec, t}
+					if prev, ok := p.cpuPrev[c.ID]; ok && t > prev.t && usec >= prev.usec {
+						// percent of one core, like the Processes tab
+						pctv := (usec - prev.usec) / 1000 / float64(t-prev.t) * 100
+						res = append(res, sensors.Sensor{ID: id + ".cpu", Name: name + " CPU", Kind: "usage", Unit: "%", Value: float64(int(pctv*10+0.5)) / 10, T: t, Min: sensors.F(0), Labels: l})
+					}
+				}
+			}
+		}
+	}
+	p.cpuPrev = seenCPU // only this loop touches it
+	return append(res,
+		sensors.Sensor{ID: "containers.total", Name: "Containers", Kind: "count", Value: total, T: t},
+		sensors.Sensor{ID: "containers.running", Name: "Running containers", Kind: "count", Value: running, T: t},
+		sensors.Sensor{ID: "containers.unhealthy", Name: "Unhealthy containers", Kind: "count", Value: unhealthy, T: t, Warn: sensors.F(1)})
+}
+
+type cpuMark struct {
+	usec float64
+	t    int64
 }

 func (p *statusSensors) updates(ctx context.Context, t int64) []sensors.Sensor {