Sensors: read-only tokens, live stream, Prometheus, status sensors, widgets, embeds

- sensors:read tokens open the sensor endpoints (here and on managed machines through the hub) and nothing else; full tokens are never accepted from a URL - /api/sensors/stream: a shaped query that keeps answering, over SSE - /metrics: Prometheus text format - services, updates, ZFS and SMART as background status sensors - widget library over the sensor API; custom widgets on the dashboard - /embed/widget: one live widget in its own bundle, the only frameable path - Dashboard tab is now Monitors Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-19 07:37:10 -07:00
parent 468f45251d
commit b05a053920
20 changed files with 950 additions and 56 deletions
@@ -0,0 +1,137 @@
+package api
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+
+	"hamlab/internal/sensors"
+	"hamlab/internal/storage"
+)
+
+// statusSensors is the sensors.Provider for what isn't a /proc number: failed services,
+// pending updates, pool health, drive health. Each source costs an exec or several, so
+// they are read in the background on their own clocks and served from memory; every
+// sensor's t says how old its reading really is.
+type statusSensors struct {
+	s  *Server
+	mu sync.RWMutex
+	by map[string][]sensors.Sensor // source → its sensors
+}
+
+const (
+	servicesEvery = 30 * time.Second
+	updatesEvery  = 5 * time.Minute
+	zfsEvery      = time.Minute
+	smartEvery    = 10 * time.Minute // storage.Manager caches SMART that long anyway, and never wakes a sleeping disk
+)
+
+func newStatusSensors(s *Server) *statusSensors {
+	p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}}
+	go p.loop("services", servicesEvery, p.services)
+	go p.loop("updates", updatesEvery, p.updates)
+	go p.loop("zfs", zfsEvery, p.zfs)
+	go p.loop("smart", smartEvery, p.smart)
+	return p
+}
+
+func (p *statusSensors) loop(name string, every time.Duration, read func(context.Context, int64) []sensors.Sensor) {
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+		got := read(ctx, time.Now().UnixMilli())
+		cancel()
+		p.mu.Lock()
+		p.by[name] = got
+		p.mu.Unlock()
+		time.Sleep(every)
+	}
+}
+
+func (p *statusSensors) Sensors() []sensors.Sensor {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	var out []sensors.Sensor
+	for _, v := range p.by {
+		out = append(out, v...)
+	}
+	return out
+}
+
+func (p *statusSensors) services(ctx context.Context, t int64) []sensors.Sensor {
+	units, err := p.s.svc.List(ctx)
+	if err != nil {
+		return nil
+	}
+	var failed, running float64
+	var names []string
+	for _, u := range units {
+		if u.Failed {
+			failed++
+			names = append(names, u.Name)
+		}
+		if u.Sub == "running" {
+			running++
+		}
+	}
+	return []sensors.Sensor{
+		{ID: "services.failed", Name: "Failed services", Kind: "count", Value: failed, T: t, Warn: sensors.F(1)},
+		{ID: "services.failed_names", Name: "Failed services", Kind: "state", Value: strings.Join(names, ", "), T: t},
+		{ID: "services.running", Name: "Running services", Kind: "count", Value: running, T: t},
+	}
+}
+
+func (p *statusSensors) updates(ctx context.Context, t int64) []sensors.Sensor {
+	st := p.s.updates.Status(ctx)
+	if st.Manager == "" {
+		return nil
+	}
+	return []sensors.Sensor{
+		{ID: "updates.pending", Name: "Pending updates", Kind: "count", Value: float64(len(st.Packages)), T: t, Labels: map[string]string{"manager": st.Manager}},
+		{ID: "updates.security", Name: "Pending security updates", Kind: "count", Value: float64(st.SecurityCount), T: t, Warn: sensors.F(1)},
+		{ID: "updates.reboot_required", Name: "Reboot required", Kind: "state", Value: st.RebootNeeded, T: t},
+	}
+}
+
+func (p *statusSensors) zfs(ctx context.Context, t int64) []sensors.Sensor {
+	z := p.s.storage.ZFS(ctx)
+	var out []sensors.Sensor
+	for _, pool := range z.Pools {
+		id, l := "zfs."+sensors.Seg(pool.Name), map[string]string{"pool": pool.Name}
+		out = append(out,
+			sensors.Sensor{ID: id + ".health", Name: pool.Name + " health", Kind: "state", Value: pool.Health, T: t, Labels: l},
+			sensors.Sensor{ID: id + ".healthy", Name: pool.Name + " healthy", Kind: "state", Value: pool.Health == "ONLINE", T: t, Labels: l},
+			sensors.Sensor{ID: id + ".size", Name: pool.Name + " size", Kind: "bytes", Unit: "B", Value: float64(pool.Size), T: t, Labels: l},
+			sensors.Sensor{ID: id + ".used", Name: pool.Name + " used", Kind: "bytes", Unit: "B", Value: float64(pool.Alloc), T: t, Min: sensors.F(0), Max: sensors.F(float64(pool.Size)), Labels: l},
+			sensors.Sensor{ID: id + ".used_pct", Name: pool.Name + " used", Kind: "usage", Unit: "%", Value: float64(pool.Cap), T: t, Min: sensors.F(0), Max: sensors.F(100), Labels: l},
+			sensors.Sensor{ID: id + ".frag", Name: pool.Name + " fragmentation", Kind: "fragmentation", Unit: "%", Value: float64(pool.Frag), T: t, Min: sensors.F(0), Max: sensors.F(100), Labels: l},
+		)
+	}
+	return out
+}
+
+func (p *statusSensors) smart(ctx context.Context, t int64) []sensors.Sensor {
+	devs, err := storage.Devices(ctx)
+	if err != nil {
+		return nil
+	}
+	var out []sensors.Sensor
+	for _, d := range devs {
+		sm := p.s.storage.Smart(ctx, d.Name, false)
+		if sm.Passed == nil || sm.Standby || sm.Error != "" {
+			continue // no verdict to report: no smartctl, asleep, or not a SMART device
+		}
+		id, l := "smart."+sensors.Seg(d.Name), map[string]string{"device": d.Name, "model": d.Model}
+		out = append(out,
+			sensors.Sensor{ID: id + ".passed", Name: d.Name + " health", Kind: "state", Value: *sm.Passed, T: sm.FetchedAt, Labels: l},
+			sensors.Sensor{ID: id + ".warnings", Name: d.Name + " warnings", Kind: "count", Value: float64(len(sm.Warnings)), T: sm.FetchedAt, Warn: sensors.F(1), Labels: l},
+		)
+		if sm.PowerOnHours > 0 {
+			out = append(out, sensors.Sensor{ID: id + ".power_on", Name: d.Name + " powered on", Kind: "duration", Unit: "s", Value: float64(sm.PowerOnHours) * 3600, T: sm.FetchedAt, Labels: l})
+		}
+		if sm.NVMe != nil {
+			out = append(out, sensors.Sensor{ID: id + ".wear", Name: d.Name + " endurance used", Kind: "usage", Unit: "%", Value: float64(sm.NVMe.PercentageUsed), T: sm.FetchedAt, Min: sensors.F(0), Max: sensors.F(100), Labels: l})
+		}
+	}
+	return out
+}