Sensors: per-service and container sensors, pattern-limited tokens, fleet query, steady sampler

- service.<unit>.active|state for services that are enabled, up or failed
- containers.* and container.<name>.running|state|health|mem|cpu (cgroup reads, no docker stats)
- sensors:read tokens can be limited to ID patterns, enforced on every endpoint and,
  through the hub's filter header, on managed machines; the header can only narrow
- /api/fleet/sensors/query: one shaped query, every machine, one request
- the sampler waits what's left of the interval, so 250 ms means 4 a second
- Monitors asks for the families it draws instead of *
- unpowered hwmon sentinels (-93 °C) are not temperatures

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

internal/api/server.go

@@ -4,6 +4,7 @@ package api
 import (
 	"bytes"
 	"context"
+	"crypto/subtle"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -22,6 +23,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -123,6 +125,8 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
 	m.HandleFunc("GET /api/nodes/{name}/sensors/{rest...}", s.sensorAuth(s.proxyNodeSensors))
 	m.HandleFunc("POST /api/nodes/{name}/sensors/query", s.sensorAuth(s.proxyNodeSensors))
 	m.HandleFunc("POST /api/tokens/read", s.auth(s.issueReadToken))
+	m.HandleFunc("GET /api/fleet/sensors/query", s.sensorAuth(s.fleetQuery))
+	m.HandleFunc("POST /api/fleet/sensors/query", s.sensorAuth(s.fleetQuery))
 	m.HandleFunc("GET /api/services", s.auth(s.listServices))
 	m.HandleFunc("GET /api/services/{unit}", s.auth(s.showService))
 	m.HandleFunc("GET /api/services/{unit}/file", s.auth(s.serviceFile))
@@ -250,34 +254,81 @@ func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc {
 // Authorization header, or a sensors:read token as ?token= — an iframe or a browser
 // EventSource can't send headers, and a token that can only read fan speeds is one we
 // can afford to see in a URL. Full-control tokens are never accepted from the URL.
+//
+// A sensors:read token may be narrowed to ID patterns; they ride the request context
+// and every handler reads sensors through s.visible. A hub forwarding such a caller to
+// a node states the patterns in filterHeader, which a node believes only from a caller
+// holding full control (the hub) — so the header can narrow a request, never widen one.
 func (s *Server) sensorAuth(next http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		ok, trusted := false, false
+		var patterns []string
 		if c, err := r.Cookie(cookieName); err == nil && s.sessions.Valid(c.Value) {
-			next(w, r)
-			return
-		}
-		presented, fromURL := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), false
-		if presented == "" {
-			presented, fromURL = r.URL.Query().Get("token"), true
-		}
-		if presented != "" {
-			var hashes []string
-			s.cfg.View(func(c *config.Config) {
-				for _, t := range c.Tokens {
-					if t.Scope == config.ScopeSensorsRead || (t.Scope == "" && !fromURL) {
-						hashes = append(hashes, t.Hash)
+			ok, trusted = true, true
+		} else {
+			presented, fromURL := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), false
+			if presented == "" {
+				presented, fromURL = r.URL.Query().Get("token"), true
+			}
+			if presented != "" {
+				h := []byte(auth.HashToken(presented))
+				s.cfg.View(func(c *config.Config) {
+					for _, t := range c.Tokens {
+						if subtle.ConstantTimeCompare(h, []byte(t.Hash)) != 1 {
+							continue
+						}
+						if t.Scope == config.ScopeSensorsRead {
+							ok, patterns = true, t.Sensors
+						} else if t.Scope == "" && !fromURL {
+							ok, trusted = true, true
+						}
 					}
-				}
-			})
-			if auth.TokenMatches(presented, hashes) {
-				next(w, r)
-				return
+				})
 			}
 		}
-		jsonErr(w, 401, "unauthenticated")
+		if !ok {
+			jsonErr(w, 401, "unauthenticated")
+			return
+		}
+		if f := r.Header.Get(filterHeader); trusted && f != "" {
+			patterns = strings.Split(f, ",")
+		}
+		next(w, r.WithContext(context.WithValue(r.Context(), filterKey{}, patterns)))
 	}
 }
 
+const filterHeader = "X-Hamlab-Sensor-Filter"
+
+type filterKey struct{}
+
+// allowed is the caller's ID filter: nil when it may read everything.
+func allowed(r *http.Request) func(id string) bool {
+	pats, _ := r.Context().Value(filterKey{}).([]string)
+	if len(pats) == 0 {
+		return nil
+	}
+	match, err := sensors.Matcher(strings.Join(pats, ","))
+	if err != nil {
+		return func(string) bool { return false } // a broken restriction opens nothing
+	}
+	return match
+}
+
+// visible is every sensor this caller may read.
+func (s *Server) visible(r *http.Request) []sensors.Sensor {
+	all, ok := s.sensors.All(), allowed(r)
+	if ok == nil {
+		return all
+	}
+	out := all[:0:0]
+	for _, x := range all {
+		if ok(x.ID) {
+			out = append(out, x)
+		}
+	}
+	return out
+}
+
 // proxyNodeSensors lets a sensors:read token reach a managed machine's sensors through
 // this hub. The hub authenticates to the node as itself; the caller's token stops here.
 func (s *Server) proxyNodeSensors(w http.ResponseWriter, r *http.Request) {
@@ -285,18 +336,33 @@ func (s *Server) proxyNodeSensors(w http.ResponseWriter, r *http.Request) {
 		q.Del("token")
 		r.URL.RawQuery = q.Encode()
 	}
+	r.Header.Del(filterHeader)
+	if pats, _ := r.Context().Value(filterKey{}).([]string); len(pats) > 0 {
+		r.Header.Set(filterHeader, strings.Join(pats, ","))
+	}
 	s.proxyNode(w, r)
 }
 
 // issueReadToken makes a sensors:read token for a dashboard. Shown once, stored hashed.
 func (s *Server) issueReadToken(w http.ResponseWriter, r *http.Request) {
 	var body struct {
-		Name string `json:"name"`
+		Name    string   `json:"name"`
+		Sensors []string `json:"sensors"` // optional ID patterns the token is limited to
 	}
 	if err := json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&body); err != nil || strings.TrimSpace(body.Name) == "" {
 		jsonErr(w, 400, "name required")
 		return
 	}
+	var pats []string
+	for _, p := range body.Sensors {
+		if p = strings.TrimSpace(p); p != "" && !strings.Contains(p, ",") {
+			pats = append(pats, p)
+		}
+	}
+	if _, err := sensors.Matcher(strings.Join(pats, ",")); err != nil {
+		jsonErr(w, 400, err.Error())
+		return
+	}
 	name := strings.TrimSpace(body.Name)
 	tok, hash, err := auth.NewToken()
 	if err != nil {
@@ -311,7 +377,7 @@ func (s *Server) issueReadToken(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		}
-		c.Tokens = append(c.Tokens, config.Token{Name: name, Hash: hash, Created: time.Now(), Scope: config.ScopeSensorsRead})
+		c.Tokens = append(c.Tokens, config.Token{Name: name, Hash: hash, Created: time.Now(), Scope: config.ScopeSensorsRead, Sensors: pats})
 	}); err != nil {
 		jsonErr(w, 500, err.Error())
 		return
@@ -374,7 +440,7 @@ func (s *Server) listTokens(w http.ResponseWriter, _ *http.Request) {
 	var out []map[string]any
 	s.cfg.View(func(c *config.Config) {
 		for _, t := range c.Tokens {
-			out = append(out, map[string]any{"name": t.Name, "created": t.Created, "scope": t.Scope})
+			out = append(out, map[string]any{"name": t.Name, "created": t.Created, "scope": t.Scope, "sensors": t.Sensors})
 		}
 	})
 	if out == nil {
@@ -656,7 +722,7 @@ func (s *Server) me(w http.ResponseWriter, _ *http.Request) {
 // sensorCatalog lists every sensor with its current value; ?ids= narrows it with
 // comma-separated IDs or globs.
 func (s *Server) sensorCatalog(w http.ResponseWriter, r *http.Request) {
-	all := s.sensors.All()
+	all := s.visible(r)
 	if ids := r.URL.Query().Get("ids"); ids != "" {
 		picked := []sensors.Sensor{}
 		for _, sel := range strings.Split(ids, ",") {
@@ -675,7 +741,7 @@ func (s *Server) sensorCatalog(w http.ResponseWriter, r *http.Request) {
 // sensorOne is a single sensor; ?format=text is the bare value, for shell scripts and
 // anything else that can only fetch a URL and read a number.
 func (s *Server) sensorOne(w http.ResponseWriter, r *http.Request) {
-	got, _ := sensors.Select(s.sensors.All(), r.PathValue("id"))
+	got, _ := sensors.Select(s.visible(r), r.PathValue("id"))
 	if len(got) != 1 {
 		jsonErr(w, 404, "no such sensor")
 		return
@@ -690,6 +756,9 @@ func (s *Server) sensorOne(w http.ResponseWriter, r *http.Request) {
 
 func (s *Server) sensorHistory(w http.ResponseWriter, r *http.Request) {
 	h, ok := s.sensors.History(r.PathValue("id"))
+	if may := allowed(r); may != nil && !may(r.PathValue("id")) {
+		ok = false
+	}
 	if !ok {
 		jsonErr(w, 404, "no history for that sensor")
 		return
@@ -704,13 +773,17 @@ func (s *Server) sensorHistories(w http.ResponseWriter, r *http.Request) {
 		jsonErr(w, 400, err.Error())
 		return
 	}
+	if may := allowed(r); may != nil {
+		asked := match
+		match = func(id string) bool { return asked(id) && may(id) }
+	}
 	jsonOK(w, 200, map[string]any{"series": s.sensors.Histories(match)})
 }
 
 // sensorProm is every numeric sensor in the Prometheus text format.
-func (s *Server) sensorProm(w http.ResponseWriter, _ *http.Request) {
+func (s *Server) sensorProm(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
-	sensors.WriteProm(w, s.sensors.All())
+	sensors.WriteProm(w, s.visible(r))
 }
 
 // queryOf reads a shaped query from a request: the POST body, or on a GET either ?q= (the
@@ -739,7 +812,7 @@ func queryOf(w http.ResponseWriter, r *http.Request) (q json.RawMessage, full bo
 	return q, full, nil
 }
 
-var reservedParam = map[string]bool{"full": true, "token": true, "q": true, "every": true}
+var reservedParam = map[string]bool{"full": true, "token": true, "q": true, "every": true, "machines": true}
 
 // sensorStream is a shaped query that keeps answering: the same object, pushed as a
 // server-sent event after every sample. ?every=<ms> spaces the events out for a consumer
@@ -747,7 +820,7 @@ var reservedParam = map[string]bool{"full": true, "token": true, "q": true, "eve
 func (s *Server) sensorStream(w http.ResponseWriter, r *http.Request) {
 	q, full, err := queryOf(w, r)
 	if err == nil {
-		_, err = sensors.Shape(s.sensors.All(), q, full) // refuse a bad query before the stream starts
+		_, err = sensors.Shape(s.visible(r), q, full) // refuse a bad query before the stream starts
 	}
 	if err != nil {
 		jsonErr(w, 400, err.Error())
@@ -774,7 +847,7 @@ func (s *Server) sensorStream(w http.ResponseWriter, r *http.Request) {
 	h.Set("Cache-Control", "no-store")
 	h.Set("X-Accel-Buffering", "no") // nginx (NPM) would otherwise hold events back
 	send := func() bool {
-		out, _ := sensors.Shape(s.sensors.All(), q, full)
+		out, _ := sensors.Shape(s.visible(r), q, full)
 		b, err := json.Marshal(out)
 		if err != nil {
 			return false
@@ -821,7 +894,7 @@ func (s *Server) sensorQuery(w http.ResponseWriter, r *http.Request) {
 		jsonErr(w, 413, err.Error())
 		return
 	}
-	out, err := sensors.Shape(s.sensors.All(), q, full)
+	out, err := sensors.Shape(s.visible(r), q, full)
 	if err != nil {
 		jsonErr(w, 400, err.Error())
 		return
@@ -829,6 +902,81 @@ func (s *Server) sensorQuery(w http.ResponseWriter, r *http.Request) {
 	jsonOK(w, 200, out)
 }
 
+// fleetQuery asks several machines one shaped query in one request and answers
+// {machine: result}. POST {"machines": "*" | ["grid", …], "query": <shape>}, or GET with
+// the flat form plus ?machines=grid,wyse (default: all). This machine answers under its
+// own hostname. One that can't be reached answers {"error": …}; the rest still do.
+func (s *Server) fleetQuery(w http.ResponseWriter, r *http.Request) {
+	var q json.RawMessage
+	var want []string
+	full := r.URL.Query().Get("full") == "1"
+	if r.Method == http.MethodPost {
+		var body struct {
+			Machines json.RawMessage `json:"machines"`
+			Query    json.RawMessage `json:"query"`
+		}
+		if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&body); err != nil || len(body.Query) == 0 {
+			jsonErr(w, 400, `expected {"machines": "*" or [names], "query": <shape>}`)
+			return
+		}
+		q = body.Query
+		_ = json.Unmarshal(body.Machines, &want) // "*" or absent leaves it empty: everyone
+	} else {
+		q, full, _ = queryOf(w, r)
+		if m := r.URL.Query().Get("machines"); m != "" && m != "*" {
+			want = strings.Split(m, ",")
+		}
+	}
+	local, err := sensors.Shape(s.visible(r), q, full)
+	if err != nil {
+		jsonErr(w, 400, err.Error())
+		return
+	}
+	names := append([]string{s.host}, s.nodes.Names()...)
+	if len(want) > 0 {
+		names = names[:0]
+		for _, n := range want {
+			names = append(names, strings.TrimSpace(n))
+		}
+	}
+	hdr := http.Header{"Content-Type": {"application/json"}}
+	if pats, _ := r.Context().Value(filterKey{}).([]string); len(pats) > 0 {
+		hdr.Set(filterHeader, strings.Join(pats, ","))
+	}
+	path := "/api/sensors/query"
+	if full {
+		path += "?full=1"
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+	defer cancel()
+	out := make(map[string]any, len(names))
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+	for _, name := range names {
+		if name == s.host {
+			out[name] = local
+			continue
+		}
+		wg.Add(1)
+		go func(name string) {
+			defer wg.Done()
+			var res any
+			code, b, err := s.nodes.Do(ctx, name, "POST", path, q, hdr)
+			switch {
+			case err != nil:
+				res = map[string]string{"error": err.Error()}
+			case code != 200 || json.Unmarshal(b, &res) != nil:
+				res = map[string]string{"error": fmt.Sprintf("HTTP %d", code)}
+			}
+			mu.Lock()
+			out[name] = res
+			mu.Unlock()
+		}(name)
+	}
+	wg.Wait()
+	jsonOK(w, 200, out)
+}
+
 // ---- services (systemd) ----
 
 func (s *Server) listServices(w http.ResponseWriter, r *http.Request) {

internal/api/status_sensors.go

@@ -2,10 +2,15 @@ package api
 
 import (
 	"context"
+	"encoding/json"
+	"os"
+	"os/exec"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"hamlab/internal/priv"
 	"hamlab/internal/sensors"
 	"hamlab/internal/storage"
 )
@@ -18,18 +23,22 @@ type statusSensors struct {
 	s  *Server
 	mu sync.RWMutex
 	by map[string][]sensors.Sensor // source → its sensors
+
+	cpuPrev map[string]cpuMark // container ID → CPU time at the previous read
 }
 
 const (
-	servicesEvery = 30 * time.Second
-	updatesEvery  = 5 * time.Minute
-	zfsEvery      = time.Minute
-	smartEvery    = 10 * time.Minute // storage.Manager caches SMART that long anyway, and never wakes a sleeping disk
+	servicesEvery   = 30 * time.Second
+	containersEvery = 15 * time.Second
+	updatesEvery    = 5 * time.Minute
+	zfsEvery        = time.Minute
+	smartEvery      = 10 * time.Minute // storage.Manager caches SMART that long anyway, and never wakes a sleeping disk
 )
 
 func newStatusSensors(s *Server) *statusSensors {
 	p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}}
 	go p.loop("services", servicesEvery, p.services)
+	go p.loop("containers", containersEvery, p.containers)
 	go p.loop("updates", updatesEvery, p.updates)
 	go p.loop("zfs", zfsEvery, p.zfs)
 	go p.loop("smart", smartEvery, p.smart)
@@ -65,6 +74,7 @@ func (p *statusSensors) services(ctx context.Context, t int64) []sensors.Sensor
 	}
 	var failed, running float64
 	var names []string
+	var per []sensors.Sensor
 	for _, u := range units {
 		if u.Failed {
 			failed++
@@ -73,12 +83,115 @@ func (p *statusSensors) services(ctx context.Context, t int64) []sensors.Sensor
 		if u.Sub == "running" {
 			running++
 		}
+		// One pair per service that is meant to be doing something: enabled, up, or broken.
+		// (Every static and one-shot unit on the box would triple the catalog for nothing.)
+		short, isService := strings.CutSuffix(u.Name, ".service")
+		if !isService || !(u.Enabled == "enabled" || u.Active == "active" || u.Failed) {
+			continue
+		}
+		id, l := "service."+sensors.Seg(short), map[string]string{"unit": u.Name, "description": u.Description}
+		per = append(per,
+			sensors.Sensor{ID: id + ".active", Name: short, Kind: "state", Value: u.Active == "active", T: t, Labels: l},
+			sensors.Sensor{ID: id + ".state", Name: short + " state", Kind: "state", Value: u.Sub, T: t, Labels: l})
 	}
-	return []sensors.Sensor{
+	return append(per, []sensors.Sensor{
 		{ID: "services.failed", Name: "Failed services", Kind: "count", Value: failed, T: t, Warn: sensors.F(1)},
 		{ID: "services.failed_names", Name: "Failed services", Kind: "state", Value: strings.Join(names, ", "), T: t},
 		{ID: "services.running", Name: "Running services", Kind: "count", Value: running, T: t},
+	}...)
+}
+
+// containers reads Docker (or Podman) with one `ps`, then each running container's memory
+// and CPU straight from its cgroup — `docker stats` takes seconds and a daemon round-trip
+// per call; two small files per container take microseconds.
+func (p *statusSensors) containers(ctx context.Context, t int64) []sensors.Sensor {
+	var out []byte
+	var err error
+	for _, tool := range []string{"docker", "podman"} {
+		if _, e := exec.LookPath(tool); e != nil {
+			continue
+		}
+		args := []string{tool, "ps", "-a", "--no-trunc", "--format", "{{json .}}"}
+		if out, err = exec.CommandContext(ctx, args[0], args[1:]...).Output(); err != nil && priv.Available() {
+			full := priv.Wrap(args...) // not in the docker group: ask as root
+			out, err = exec.CommandContext(ctx, full[0], full[1:]...).Output()
+		}
+		break
 	}
+	if err != nil || len(out) == 0 {
+		return nil
+	}
+	var res []sensors.Sensor
+	var running, unhealthy, total float64
+	seenCPU := map[string]cpuMark{}
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		var c struct{ ID, Names, Image, State, Status string }
+		if json.Unmarshal([]byte(line), &c) != nil || c.Names == "" {
+			continue
+		}
+		total++
+		name := strings.Split(c.Names, ",")[0]
+		id, l := "container."+sensors.Seg(name), map[string]string{"container": name, "image": c.Image}
+		up := c.State == "running"
+		health := "none"
+		for _, h := range []string{"unhealthy", "healthy", "starting"} { // "unhealthy" first: it contains "healthy"
+			if strings.Contains(c.Status, "("+h+")") || strings.Contains(c.Status, "(health: "+h+")") {
+				health = h
+				break
+			}
+		}
+		if up {
+			running++
+		}
+		if health == "unhealthy" {
+			unhealthy++
+		}
+		res = append(res,
+			sensors.Sensor{ID: id + ".running", Name: name, Kind: "state", Value: up, T: t, Labels: l},
+			sensors.Sensor{ID: id + ".state", Name: name + " state", Kind: "state", Value: c.State, T: t, Labels: l},
+			sensors.Sensor{ID: id + ".health", Name: name + " health", Kind: "state", Value: health, T: t, Labels: l})
+		if !up {
+			continue
+		}
+		cg := ""
+		for _, dir := range []string{"/sys/fs/cgroup/system.slice/docker-" + c.ID + ".scope", "/sys/fs/cgroup/docker/" + c.ID, "/sys/fs/cgroup/machine.slice/libpod-" + c.ID + ".scope"} {
+			if _, e := os.Stat(dir + "/memory.current"); e == nil {
+				cg = dir
+				break
+			}
+		}
+		if cg == "" {
+			continue
+		}
+		if b, e := os.ReadFile(cg + "/memory.current"); e == nil {
+			if v, e := strconv.ParseFloat(strings.TrimSpace(string(b)), 64); e == nil {
+				res = append(res, sensors.Sensor{ID: id + ".mem", Name: name + " memory", Kind: "bytes", Unit: "B", Value: v, T: t, Labels: l})
+			}
+		}
+		if b, e := os.ReadFile(cg + "/cpu.stat"); e == nil {
+			for _, ln := range strings.Split(string(b), "\n") {
+				if f := strings.Fields(ln); len(f) == 2 && f[0] == "usage_usec" {
+					usec, _ := strconv.ParseFloat(f[1], 64)
+					seenCPU[c.ID] = cpuMark{usec, t}
+					if prev, ok := p.cpuPrev[c.ID]; ok && t > prev.t && usec >= prev.usec {
+						// percent of one core, like the Processes tab
+						pctv := (usec - prev.usec) / 1000 / float64(t-prev.t) * 100
+						res = append(res, sensors.Sensor{ID: id + ".cpu", Name: name + " CPU", Kind: "usage", Unit: "%", Value: float64(int(pctv*10+0.5)) / 10, T: t, Min: sensors.F(0), Labels: l})
+					}
+				}
+			}
+		}
+	}
+	p.cpuPrev = seenCPU // only this loop touches it
+	return append(res,
+		sensors.Sensor{ID: "containers.total", Name: "Containers", Kind: "count", Value: total, T: t},
+		sensors.Sensor{ID: "containers.running", Name: "Running containers", Kind: "count", Value: running, T: t},
+		sensors.Sensor{ID: "containers.unhealthy", Name: "Unhealthy containers", Kind: "count", Value: unhealthy, T: t, Warn: sensors.F(1)})
+}
+
+type cpuMark struct {
+	usec float64
+	t    int64
 }
 
 func (p *statusSensors) updates(ctx context.Context, t int64) []sensors.Sensor {

internal/config/config.go

@@ -22,6 +22,8 @@ type Token struct {
 	// Scope is what the token may do. "" is everything (a hub's token: full control of
 	// this machine). ScopeSensorsRead only reads sensors — what a dashboard gets.
 	Scope string `json:"scope,omitempty"`
+	// Sensors narrows a sensors:read token to the IDs these patterns match. Empty is all.
+	Sensors []string `json:"sensors,omitempty"`
 }
 
 const ScopeSensorsRead = "sensors:read"

internal/metrics/metrics.go

@@ -178,7 +178,13 @@ func NewCollector() *Collector {
 func (c *Collector) Run(ctx context.Context) {
 	c.sample() // prime deltas
 	for {
-		t := time.NewTimer(c.Interval())
+		// The wait is what's left of the interval after the work, so a machine whose
+		// sample takes 30 ms still delivers 4 a second at 250 ms, not 3.6.
+		wait := c.Interval() - time.Since(c.prevT)
+		if wait < 0 {
+			wait = 0
+		}
+		t := time.NewTimer(wait)
 		select {
 		case <-ctx.Done():
 			t.Stop()
@@ -420,6 +426,9 @@ func hwmonDir(d string) (chip string, temps []Temp, fans []Fan) {
 			label = base
 		}
 		if m[1] == "temp" {
+			if raw < -60000 { // an unpowered sensor's sentinel (an idle audio amp reads -93 °C), not weather
+				continue
+			}
 			temps = append(temps, Temp{Chip: chip, Label: label, C: float64(raw) / 1000,
 				Max: tempLimit(filepath.Join(d, base+"_max")), Crit: tempLimit(filepath.Join(d, base+"_crit"))})
 		} else if raw > 0 || fileExists(filepath.Join(d, base+"_enable")) {

internal/nodes/nodes.go

@@ -381,6 +381,40 @@ func (m *Manager) PushAll(ctx context.Context, binPath, ourArch string, force bo
 // Proxy returns the handler for /api/nodes/<name>/<rest>: it rewrites to <node>/api/<rest>,
 // swaps the hub's cookie for the node's bearer token, and drops Origin so the node's
 // WebSocket origin check (meant for browsers) doesn't reject the hub.
+// Do makes one request to a managed machine as this hub and returns the whole response.
+func (m *Manager) Do(ctx context.Context, name, method, path string, body []byte, hdr http.Header) (int, []byte, error) {
+	n, ok := m.node(name)
+	if !ok {
+		return 0, nil, errors.New("no such machine")
+	}
+	req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(n.URL, "/")+path, bytes.NewReader(body))
+	if err != nil {
+		return 0, nil, err
+	}
+	for k, v := range hdr {
+		req.Header[k] = v
+	}
+	req.Header.Set("Authorization", "Bearer "+n.Token)
+	resp, err := m.client(n).Do(req)
+	if err != nil {
+		return 0, nil, errors.New(shortErr(err))
+	}
+	defer resp.Body.Close()
+	b, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+	return resp.StatusCode, b, err
+}
+
+// Names lists the managed machines.
+func (m *Manager) Names() []string {
+	var out []string
+	m.cfg.View(func(c *config.Config) {
+		for _, n := range c.Nodes {
+			out = append(out, n.Name)
+		}
+	})
+	return out
+}
+
 func (m *Manager) Proxy(name string) (http.Handler, bool) {
 	n, ok := m.node(name)
 	if !ok {