Sensors: read-only tokens, live stream, Prometheus, status sensors, widgets, embeds

- sensors:read tokens open the sensor endpoints (here and on managed machines through the hub) and nothing else; full tokens are never accepted from a URL - /api/sensors/stream: a shaped query that keeps answering, over SSE - /metrics: Prometheus text format - services, updates, ZFS and SMART as background status sensors - widget library over the sensor API; custom widgets on the dashboard - /embed/widget: one live widget in its own bundle, the only frameable path - Dashboard tab is now Monitors Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-19 07:37:10 -07:00
parent 468f45251d
commit b05a053920
20 changed files with 950 additions and 56 deletions
@@ -84,6 +84,7 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
 	if s.user == "" {
 		s.user = "?"
 	}
+	s.sensors.Add(newStatusSensors(s))
 	// The saved dashboard layout carries the sampling interval; apply it on boot.
 	var raw json.RawMessage
 	st.View(func(x *store.State) { raw = x.Dashboard })
@@ -111,11 +112,18 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
 	m.HandleFunc("GET /api/terminals/{id}/attach", s.auth(s.attach))
 	m.HandleFunc("GET /api/metrics", s.auth(s.metricsLatest))
 	m.HandleFunc("GET /api/metrics/history", s.auth(s.metricsHistory))
-	m.HandleFunc("GET /api/sensors", s.auth(s.sensorCatalog))
-	m.HandleFunc("GET /api/sensors/query", s.auth(s.sensorQuery))
-	m.HandleFunc("POST /api/sensors/query", s.auth(s.sensorQuery))
-	m.HandleFunc("GET /api/sensors/{id}", s.auth(s.sensorOne))
-	m.HandleFunc("GET /api/sensors/{id}/history", s.auth(s.sensorHistory))
+	m.HandleFunc("GET /api/sensors", s.sensorAuth(s.sensorCatalog))
+	m.HandleFunc("GET /api/sensors/query", s.sensorAuth(s.sensorQuery))
+	m.HandleFunc("POST /api/sensors/query", s.sensorAuth(s.sensorQuery))
+	m.HandleFunc("GET /api/sensors/stream", s.sensorAuth(s.sensorStream))
+	m.HandleFunc("GET /api/sensors/prometheus", s.sensorAuth(s.sensorProm))
+	m.HandleFunc("GET /metrics", s.sensorAuth(s.sensorProm))
+	m.HandleFunc("GET /api/sensors/{id}", s.sensorAuth(s.sensorOne))
+	m.HandleFunc("GET /api/sensors/{id}/history", s.sensorAuth(s.sensorHistory))
+	m.HandleFunc("GET /api/nodes/{name}/sensors", s.sensorAuth(s.proxyNodeSensors))
+	m.HandleFunc("GET /api/nodes/{name}/sensors/{rest...}", s.sensorAuth(s.proxyNodeSensors))
+	m.HandleFunc("POST /api/nodes/{name}/sensors/query", s.sensorAuth(s.proxyNodeSensors))
+	m.HandleFunc("POST /api/tokens/read", s.auth(s.issueReadToken))
 	m.HandleFunc("GET /api/services", s.auth(s.listServices))
 	m.HandleFunc("GET /api/services/{unit}", s.auth(s.showService))
 	m.HandleFunc("GET /api/services/{unit}/file", s.auth(s.serviceFile))
@@ -179,8 +187,12 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
 func (s *Server) Handler() http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		h := w.Header()
-		h.Set("X-Frame-Options", "DENY")
-		h.Set("Content-Security-Policy", "frame-ancestors 'none'")
+		// The console must never be framed. An embedded widget exists to be: it is a page
+		// that can only read sensors, and only with a read-only token in its URL.
+		if !strings.HasPrefix(r.URL.Path, "/embed/") {
+			h.Set("X-Frame-Options", "DENY")
+			h.Set("Content-Security-Policy", "frame-ancestors 'none'")
+		}
 		h.Set("X-Content-Type-Options", "nosniff")
 		h.Set("Referrer-Policy", "no-referrer")
 		s.mux.ServeHTTP(w, r)
@@ -221,7 +233,9 @@ func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc {
 			var hashes []string
 			s.cfg.View(func(c *config.Config) {
 				for _, t := range c.Tokens {
-					hashes = append(hashes, t.Hash)
+					if t.Scope == "" { // a scoped token is not a key to the machine
+						hashes = append(hashes, t.Hash)
+					}
 				}
 			})
 			if auth.TokenMatches(strings.TrimPrefix(h, "Bearer "), hashes) {
@@ -233,6 +247,84 @@ func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc {
 	}
 }

+// sensorAuth guards the read-only sensor endpoints: a session, any token in the
+// Authorization header, or a sensors:read token as ?token= — an iframe or a browser
+// EventSource can't send headers, and a token that can only read fan speeds is one we
+// can afford to see in a URL. Full-control tokens are never accepted from the URL.
+func (s *Server) sensorAuth(next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if c, err := r.Cookie(cookieName); err == nil && s.sessions.Valid(c.Value) {
+			next(w, r)
+			return
+		}
+		presented, fromURL := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), false
+		if presented == "" {
+			presented, fromURL = r.URL.Query().Get("token"), true
+		}
+		if presented != "" {
+			var hashes []string
+			s.cfg.View(func(c *config.Config) {
+				for _, t := range c.Tokens {
+					if t.Scope == config.ScopeSensorsRead || (t.Scope == "" && !fromURL) {
+						hashes = append(hashes, t.Hash)
+					}
+				}
+			})
+			if auth.TokenMatches(presented, hashes) {
+				next(w, r)
+				return
+			}
+		}
+		jsonErr(w, 401, "unauthenticated")
+	}
+}
+
+// proxyNodeSensors lets a sensors:read token reach a managed machine's sensors through
+// this hub. The hub authenticates to the node as itself; the caller's token stops here.
+func (s *Server) proxyNodeSensors(w http.ResponseWriter, r *http.Request) {
+	if q := r.URL.Query(); q.Has("token") {
+		q.Del("token")
+		r.URL.RawQuery = q.Encode()
+	}
+	s.proxyNode(w, r)
+}
+
+// issueReadToken makes a sensors:read token for a dashboard. Shown once, stored hashed.
+func (s *Server) issueReadToken(w http.ResponseWriter, r *http.Request) {
+	var body struct {
+		Name string `json:"name"`
+	}
+	if err := json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&body); err != nil || strings.TrimSpace(body.Name) == "" {
+		jsonErr(w, 400, "name required")
+		return
+	}
+	name := strings.TrimSpace(body.Name)
+	tok, hash, err := auth.NewToken()
+	if err != nil {
+		jsonErr(w, 500, "token error")
+		return
+	}
+	clash := false
+	if err := s.cfg.Update(func(c *config.Config) {
+		for _, t := range c.Tokens {
+			if t.Name == name {
+				clash = true
+				return
+			}
+		}
+		c.Tokens = append(c.Tokens, config.Token{Name: name, Hash: hash, Created: time.Now(), Scope: config.ScopeSensorsRead})
+	}); err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	if clash {
+		jsonErr(w, 409, "a token with that name exists; revoke it first")
+		return
+	}
+	log.Printf("read-only sensor token %q created (from %s)", name, clientIP(r))
+	jsonOK(w, 200, map[string]string{"token": tok, "name": name, "scope": config.ScopeSensorsRead})
+}
+
 // ---- API tokens: how a hub authenticates to this node ----

 // issueToken trades the password for a long-lived token, named by the caller (a hub).
@@ -283,7 +375,7 @@ func (s *Server) listTokens(w http.ResponseWriter, _ *http.Request) {
 	var out []map[string]any
 	s.cfg.View(func(c *config.Config) {
 		for _, t := range c.Tokens {
-			out = append(out, map[string]any{"name": t.Name, "created": t.Created})
+			out = append(out, map[string]any{"name": t.Name, "created": t.Created, "scope": t.Scope})
 		}
 	})
 	if out == nil {
@@ -621,27 +713,114 @@ func (s *Server) sensorHistory(w http.ResponseWriter, r *http.Request) {
 	jsonOK(w, 200, h)
 }

+// sensorProm is every numeric sensor in the Prometheus text format.
+func (s *Server) sensorProm(w http.ResponseWriter, _ *http.Request) {
+	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+	sensors.WriteProm(w, s.sensors.All())
+}
+
+// queryOf reads a shaped query from a request: the POST body, or on a GET either ?q= (the
+// JSON shape, URL-encoded — a browser EventSource can only GET) or the flat form, where
+// every other parameter is a key and its selector.
+func queryOf(w http.ResponseWriter, r *http.Request) (q json.RawMessage, full bool, err error) {
+	params := r.URL.Query()
+	full = params.Get("full") == "1"
+	if r.Method == http.MethodPost {
+		b, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10))
+		if err != nil {
+			return nil, full, errors.New("query larger than 64 KiB")
+		}
+		return b, full, nil
+	}
+	if raw := params.Get("q"); raw != "" {
+		return json.RawMessage(raw), full, nil
+	}
+	flat := map[string]string{}
+	for k, v := range params {
+		if !reservedParam[k] && len(v) > 0 {
+			flat[k] = v[0]
+		}
+	}
+	q, _ = json.Marshal(flat)
+	return q, full, nil
+}
+
+var reservedParam = map[string]bool{"full": true, "token": true, "q": true, "every": true}
+
+// sensorStream is a shaped query that keeps answering: the same object, pushed as a
+// server-sent event after every sample. ?every=<ms> spaces the events out for a consumer
+// that doesn't want them at the sampling rate.
+func (s *Server) sensorStream(w http.ResponseWriter, r *http.Request) {
+	q, full, err := queryOf(w, r)
+	if err == nil {
+		_, err = sensors.Shape(s.sensors.All(), q, full) // refuse a bad query before the stream starts
+	}
+	if err != nil {
+		jsonErr(w, 400, err.Error())
+		return
+	}
+	tick, cancel, ok := s.sensors.Subscribe()
+	fl, canFlush := w.(http.Flusher)
+	if !ok || !canFlush {
+		jsonErr(w, 501, "streaming isn't available here")
+		return
+	}
+	defer cancel()
+	every := time.Duration(0)
+	if ms, err := strconv.Atoi(r.URL.Query().Get("every")); err == nil && ms > 0 {
+		every = time.Duration(ms) * time.Millisecond
+	}
+	h := w.Header()
+	h.Set("Content-Type", "text/event-stream")
+	h.Set("Cache-Control", "no-store")
+	h.Set("X-Accel-Buffering", "no") // nginx (NPM) would otherwise hold events back
+	send := func() bool {
+		out, _ := sensors.Shape(s.sensors.All(), q, full)
+		b, err := json.Marshal(out)
+		if err != nil {
+			return false
+		}
+		if _, err := fmt.Fprintf(w, "data: %s\n\n", b); err != nil {
+			return false
+		}
+		fl.Flush()
+		return true
+	}
+	if !send() {
+		return
+	}
+	last := time.Now()
+	keepalive := time.NewTicker(25 * time.Second)
+	defer keepalive.Stop()
+	for {
+		select {
+		case <-r.Context().Done():
+			return
+		case <-keepalive.C:
+			if _, err := io.WriteString(w, ": keepalive\n\n"); err != nil {
+				return
+			}
+			fl.Flush()
+		case <-tick:
+			if every > 0 && time.Since(last) < every-5*time.Millisecond {
+				continue
+			}
+			last = time.Now()
+			if !send() {
+				return
+			}
+		}
+	}
+}
+
 // sensorQuery answers a shaped query. POST sends the shape as the body; GET builds a
 // flat one from the query string (?cpu=cpu.usage&fans=fan.*). ?full=1 returns whole
 // sensors instead of bare values.
 func (s *Server) sensorQuery(w http.ResponseWriter, r *http.Request) {
-	full := r.URL.Query().Get("full") == "1"
-	var q json.RawMessage
-	if r.Method == http.MethodPost {
-		b, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10))
-		if err != nil {
-			jsonErr(w, 413, "query larger than 64 KiB")
-			return
-		}
-		q = b
-	} else {
-		flat := map[string]string{}
-		for k, v := range r.URL.Query() {
-			if k != "full" && len(v) > 0 {
-				flat[k] = v[0]
-			}
-		}
-		q, _ = json.Marshal(flat)
+	q, full, err := queryOf(w, r)
+	if err != nil {
+		jsonErr(w, 413, err.Error())
+		return
 	}
 	out, err := sensors.Shape(s.sensors.All(), q, full)
 	if err != nil {
@@ -0,0 +1,137 @@
+package api
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+
+	"hamlab/internal/sensors"
+	"hamlab/internal/storage"
+)
+
+// statusSensors is the sensors.Provider for what isn't a /proc number: failed services,
+// pending updates, pool health, drive health. Each source costs an exec or several, so
+// they are read in the background on their own clocks and served from memory; every
+// sensor's t says how old its reading really is.
+type statusSensors struct {
+	s  *Server
+	mu sync.RWMutex
+	by map[string][]sensors.Sensor // source → its sensors
+}
+
+const (
+	servicesEvery = 30 * time.Second
+	updatesEvery  = 5 * time.Minute
+	zfsEvery      = time.Minute
+	smartEvery    = 10 * time.Minute // storage.Manager caches SMART that long anyway, and never wakes a sleeping disk
+)
+
+func newStatusSensors(s *Server) *statusSensors {
+	p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}}
+	go p.loop("services", servicesEvery, p.services)
+	go p.loop("updates", updatesEvery, p.updates)
+	go p.loop("zfs", zfsEvery, p.zfs)
+	go p.loop("smart", smartEvery, p.smart)
+	return p
+}
+
+func (p *statusSensors) loop(name string, every time.Duration, read func(context.Context, int64) []sensors.Sensor) {
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+		got := read(ctx, time.Now().UnixMilli())
+		cancel()
+		p.mu.Lock()
+		p.by[name] = got
+		p.mu.Unlock()
+		time.Sleep(every)
+	}
+}
+
+func (p *statusSensors) Sensors() []sensors.Sensor {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	var out []sensors.Sensor
+	for _, v := range p.by {
+		out = append(out, v...)
+	}
+	return out
+}
+
+func (p *statusSensors) services(ctx context.Context, t int64) []sensors.Sensor {
+	units, err := p.s.svc.List(ctx)
+	if err != nil {
+		return nil
+	}
+	var failed, running float64
+	var names []string
+	for _, u := range units {
+		if u.Failed {
+			failed++
+			names = append(names, u.Name)
+		}
+		if u.Sub == "running" {
+			running++
+		}
+	}
+	return []sensors.Sensor{
+		{ID: "services.failed", Name: "Failed services", Kind: "count", Value: failed, T: t, Warn: sensors.F(1)},
+		{ID: "services.failed_names", Name: "Failed services", Kind: "state", Value: strings.Join(names, ", "), T: t},
+		{ID: "services.running", Name: "Running services", Kind: "count", Value: running, T: t},
+	}
+}
+
+func (p *statusSensors) updates(ctx context.Context, t int64) []sensors.Sensor {
+	st := p.s.updates.Status(ctx)
+	if st.Manager == "" {
+		return nil
+	}
+	return []sensors.Sensor{
+		{ID: "updates.pending", Name: "Pending updates", Kind: "count", Value: float64(len(st.Packages)), T: t, Labels: map[string]string{"manager": st.Manager}},
+		{ID: "updates.security", Name: "Pending security updates", Kind: "count", Value: float64(st.SecurityCount), T: t, Warn: sensors.F(1)},
+		{ID: "updates.reboot_required", Name: "Reboot required", Kind: "state", Value: st.RebootNeeded, T: t},
+	}
+}
+
+func (p *statusSensors) zfs(ctx context.Context, t int64) []sensors.Sensor {
+	z := p.s.storage.ZFS(ctx)
+	var out []sensors.Sensor
+	for _, pool := range z.Pools {
+		id, l := "zfs."+sensors.Seg(pool.Name), map[string]string{"pool": pool.Name}
+		out = append(out,
+			sensors.Sensor{ID: id + ".health", Name: pool.Name + " health", Kind: "state", Value: pool.Health, T: t, Labels: l},
+			sensors.Sensor{ID: id + ".healthy", Name: pool.Name + " healthy", Kind: "state", Value: pool.Health == "ONLINE", T: t, Labels: l},
+			sensors.Sensor{ID: id + ".size", Name: pool.Name + " size", Kind: "bytes", Unit: "B", Value: float64(pool.Size), T: t, Labels: l},
+			sensors.Sensor{ID: id + ".used", Name: pool.Name + " used", Kind: "bytes", Unit: "B", Value: float64(pool.Alloc), T: t, Min: sensors.F(0), Max: sensors.F(float64(pool.Size)), Labels: l},
+			sensors.Sensor{ID: id + ".used_pct", Name: pool.Name + " used", Kind: "usage", Unit: "%", Value: float64(pool.Cap), T: t, Min: sensors.F(0), Max: sensors.F(100), Labels: l},
+			sensors.Sensor{ID: id + ".frag", Name: pool.Name + " fragmentation", Kind: "fragmentation", Unit: "%", Value: float64(pool.Frag), T: t, Min: sensors.F(0), Max: sensors.F(100), Labels: l},
+		)
+	}
+	return out
+}
+
+func (p *statusSensors) smart(ctx context.Context, t int64) []sensors.Sensor {
+	devs, err := storage.Devices(ctx)
+	if err != nil {
+		return nil
+	}
+	var out []sensors.Sensor
+	for _, d := range devs {
+		sm := p.s.storage.Smart(ctx, d.Name, false)
+		if sm.Passed == nil || sm.Standby || sm.Error != "" {
+			continue // no verdict to report: no smartctl, asleep, or not a SMART device
+		}
+		id, l := "smart."+sensors.Seg(d.Name), map[string]string{"device": d.Name, "model": d.Model}
+		out = append(out,
+			sensors.Sensor{ID: id + ".passed", Name: d.Name + " health", Kind: "state", Value: *sm.Passed, T: sm.FetchedAt, Labels: l},
+			sensors.Sensor{ID: id + ".warnings", Name: d.Name + " warnings", Kind: "count", Value: float64(len(sm.Warnings)), T: sm.FetchedAt, Warn: sensors.F(1), Labels: l},
+		)
+		if sm.PowerOnHours > 0 {
+			out = append(out, sensors.Sensor{ID: id + ".power_on", Name: d.Name + " powered on", Kind: "duration", Unit: "s", Value: float64(sm.PowerOnHours) * 3600, T: sm.FetchedAt, Labels: l})
+		}
+		if sm.NVMe != nil {
+			out = append(out, sensors.Sensor{ID: id + ".wear", Name: d.Name + " endurance used", Kind: "usage", Unit: "%", Value: float64(sm.NVMe.PercentageUsed), T: sm.FetchedAt, Min: sensors.F(0), Max: sensors.F(100), Labels: l})
+		}
+	}
+	return out
+}