Sensors: read-only tokens, live stream, Prometheus, status sensors, widgets, embeds

- sensors:read tokens open the sensor endpoints (here and on managed machines
  through the hub) and nothing else; full tokens are never accepted from a URL
- /api/sensors/stream: a shaped query that keeps answering, over SSE
- /metrics: Prometheus text format
- services, updates, ZFS and SMART as background status sensors
- widget library over the sensor API; custom widgets on the dashboard
- /embed/widget: one live widget in its own bundle, the only frameable path
- Dashboard tab is now Monitors

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
2026-09-19 07:37:10 -07:00
co-authored by Claude Fable 5.1
parent 468f45251d
commit b05a053920
20 changed files with 950 additions and 56 deletions
+205 -26
View File
@@ -84,6 +84,7 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
if s.user == "" {
s.user = "?"
}
s.sensors.Add(newStatusSensors(s))
// The saved dashboard layout carries the sampling interval; apply it on boot.
var raw json.RawMessage
st.View(func(x *store.State) { raw = x.Dashboard })
@@ -111,11 +112,18 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
m.HandleFunc("GET /api/terminals/{id}/attach", s.auth(s.attach))
m.HandleFunc("GET /api/metrics", s.auth(s.metricsLatest))
m.HandleFunc("GET /api/metrics/history", s.auth(s.metricsHistory))
m.HandleFunc("GET /api/sensors", s.auth(s.sensorCatalog))
m.HandleFunc("GET /api/sensors/query", s.auth(s.sensorQuery))
m.HandleFunc("POST /api/sensors/query", s.auth(s.sensorQuery))
m.HandleFunc("GET /api/sensors/{id}", s.auth(s.sensorOne))
m.HandleFunc("GET /api/sensors/{id}/history", s.auth(s.sensorHistory))
m.HandleFunc("GET /api/sensors", s.sensorAuth(s.sensorCatalog))
m.HandleFunc("GET /api/sensors/query", s.sensorAuth(s.sensorQuery))
m.HandleFunc("POST /api/sensors/query", s.sensorAuth(s.sensorQuery))
m.HandleFunc("GET /api/sensors/stream", s.sensorAuth(s.sensorStream))
m.HandleFunc("GET /api/sensors/prometheus", s.sensorAuth(s.sensorProm))
m.HandleFunc("GET /metrics", s.sensorAuth(s.sensorProm))
m.HandleFunc("GET /api/sensors/{id}", s.sensorAuth(s.sensorOne))
m.HandleFunc("GET /api/sensors/{id}/history", s.sensorAuth(s.sensorHistory))
m.HandleFunc("GET /api/nodes/{name}/sensors", s.sensorAuth(s.proxyNodeSensors))
m.HandleFunc("GET /api/nodes/{name}/sensors/{rest...}", s.sensorAuth(s.proxyNodeSensors))
m.HandleFunc("POST /api/nodes/{name}/sensors/query", s.sensorAuth(s.proxyNodeSensors))
m.HandleFunc("POST /api/tokens/read", s.auth(s.issueReadToken))
m.HandleFunc("GET /api/services", s.auth(s.listServices))
m.HandleFunc("GET /api/services/{unit}", s.auth(s.showService))
m.HandleFunc("GET /api/services/{unit}/file", s.auth(s.serviceFile))
@@ -179,8 +187,12 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
func (s *Server) Handler() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h := w.Header()
h.Set("X-Frame-Options", "DENY")
h.Set("Content-Security-Policy", "frame-ancestors 'none'")
// The console must never be framed. An embedded widget exists to be: it is a page
// that can only read sensors, and only with a read-only token in its URL.
if !strings.HasPrefix(r.URL.Path, "/embed/") {
h.Set("X-Frame-Options", "DENY")
h.Set("Content-Security-Policy", "frame-ancestors 'none'")
}
h.Set("X-Content-Type-Options", "nosniff")
h.Set("Referrer-Policy", "no-referrer")
s.mux.ServeHTTP(w, r)
@@ -221,7 +233,9 @@ func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc {
var hashes []string
s.cfg.View(func(c *config.Config) {
for _, t := range c.Tokens {
hashes = append(hashes, t.Hash)
if t.Scope == "" { // a scoped token is not a key to the machine
hashes = append(hashes, t.Hash)
}
}
})
if auth.TokenMatches(strings.TrimPrefix(h, "Bearer "), hashes) {
@@ -233,6 +247,84 @@ func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc {
}
}
// sensorAuth guards the read-only sensor endpoints: a session, any token in the
// Authorization header, or a sensors:read token as ?token= — an iframe or a browser
// EventSource can't send headers, and a token that can only read fan speeds is one we
// can afford to see in a URL. Full-control tokens are never accepted from the URL.
func (s *Server) sensorAuth(next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if c, err := r.Cookie(cookieName); err == nil && s.sessions.Valid(c.Value) {
next(w, r)
return
}
presented, fromURL := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), false
if presented == "" {
presented, fromURL = r.URL.Query().Get("token"), true
}
if presented != "" {
var hashes []string
s.cfg.View(func(c *config.Config) {
for _, t := range c.Tokens {
if t.Scope == config.ScopeSensorsRead || (t.Scope == "" && !fromURL) {
hashes = append(hashes, t.Hash)
}
}
})
if auth.TokenMatches(presented, hashes) {
next(w, r)
return
}
}
jsonErr(w, 401, "unauthenticated")
}
}
// proxyNodeSensors lets a sensors:read token reach a managed machine's sensors through
// this hub. The hub authenticates to the node as itself; the caller's token stops here.
func (s *Server) proxyNodeSensors(w http.ResponseWriter, r *http.Request) {
if q := r.URL.Query(); q.Has("token") {
q.Del("token")
r.URL.RawQuery = q.Encode()
}
s.proxyNode(w, r)
}
// issueReadToken makes a sensors:read token for a dashboard. Shown once, stored hashed.
func (s *Server) issueReadToken(w http.ResponseWriter, r *http.Request) {
var body struct {
Name string `json:"name"`
}
if err := json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&body); err != nil || strings.TrimSpace(body.Name) == "" {
jsonErr(w, 400, "name required")
return
}
name := strings.TrimSpace(body.Name)
tok, hash, err := auth.NewToken()
if err != nil {
jsonErr(w, 500, "token error")
return
}
clash := false
if err := s.cfg.Update(func(c *config.Config) {
for _, t := range c.Tokens {
if t.Name == name {
clash = true
return
}
}
c.Tokens = append(c.Tokens, config.Token{Name: name, Hash: hash, Created: time.Now(), Scope: config.ScopeSensorsRead})
}); err != nil {
jsonErr(w, 500, err.Error())
return
}
if clash {
jsonErr(w, 409, "a token with that name exists; revoke it first")
return
}
log.Printf("read-only sensor token %q created (from %s)", name, clientIP(r))
jsonOK(w, 200, map[string]string{"token": tok, "name": name, "scope": config.ScopeSensorsRead})
}
// ---- API tokens: how a hub authenticates to this node ----
// issueToken trades the password for a long-lived token, named by the caller (a hub).
@@ -283,7 +375,7 @@ func (s *Server) listTokens(w http.ResponseWriter, _ *http.Request) {
var out []map[string]any
s.cfg.View(func(c *config.Config) {
for _, t := range c.Tokens {
out = append(out, map[string]any{"name": t.Name, "created": t.Created})
out = append(out, map[string]any{"name": t.Name, "created": t.Created, "scope": t.Scope})
}
})
if out == nil {
@@ -621,27 +713,114 @@ func (s *Server) sensorHistory(w http.ResponseWriter, r *http.Request) {
jsonOK(w, 200, h)
}
// sensorProm is every numeric sensor in the Prometheus text format.
func (s *Server) sensorProm(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
sensors.WriteProm(w, s.sensors.All())
}
// queryOf reads a shaped query from a request: the POST body, or on a GET either ?q= (the
// JSON shape, URL-encoded — a browser EventSource can only GET) or the flat form, where
// every other parameter is a key and its selector.
func queryOf(w http.ResponseWriter, r *http.Request) (q json.RawMessage, full bool, err error) {
params := r.URL.Query()
full = params.Get("full") == "1"
if r.Method == http.MethodPost {
b, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10))
if err != nil {
return nil, full, errors.New("query larger than 64 KiB")
}
return b, full, nil
}
if raw := params.Get("q"); raw != "" {
return json.RawMessage(raw), full, nil
}
flat := map[string]string{}
for k, v := range params {
if !reservedParam[k] && len(v) > 0 {
flat[k] = v[0]
}
}
q, _ = json.Marshal(flat)
return q, full, nil
}
var reservedParam = map[string]bool{"full": true, "token": true, "q": true, "every": true}
// sensorStream is a shaped query that keeps answering: the same object, pushed as a
// server-sent event after every sample. ?every=<ms> spaces the events out for a consumer
// that doesn't want them at the sampling rate.
func (s *Server) sensorStream(w http.ResponseWriter, r *http.Request) {
q, full, err := queryOf(w, r)
if err == nil {
_, err = sensors.Shape(s.sensors.All(), q, full) // refuse a bad query before the stream starts
}
if err != nil {
jsonErr(w, 400, err.Error())
return
}
tick, cancel, ok := s.sensors.Subscribe()
fl, canFlush := w.(http.Flusher)
if !ok || !canFlush {
jsonErr(w, 501, "streaming isn't available here")
return
}
defer cancel()
every := time.Duration(0)
if ms, err := strconv.Atoi(r.URL.Query().Get("every")); err == nil && ms > 0 {
every = time.Duration(ms) * time.Millisecond
}
h := w.Header()
h.Set("Content-Type", "text/event-stream")
h.Set("Cache-Control", "no-store")
h.Set("X-Accel-Buffering", "no") // nginx (NPM) would otherwise hold events back
send := func() bool {
out, _ := sensors.Shape(s.sensors.All(), q, full)
b, err := json.Marshal(out)
if err != nil {
return false
}
if _, err := fmt.Fprintf(w, "data: %s\n\n", b); err != nil {
return false
}
fl.Flush()
return true
}
if !send() {
return
}
last := time.Now()
keepalive := time.NewTicker(25 * time.Second)
defer keepalive.Stop()
for {
select {
case <-r.Context().Done():
return
case <-keepalive.C:
if _, err := io.WriteString(w, ": keepalive\n\n"); err != nil {
return
}
fl.Flush()
case <-tick:
if every > 0 && time.Since(last) < every-5*time.Millisecond {
continue
}
last = time.Now()
if !send() {
return
}
}
}
}
// sensorQuery answers a shaped query. POST sends the shape as the body; GET builds a
// flat one from the query string (?cpu=cpu.usage&fans=fan.*). ?full=1 returns whole
// sensors instead of bare values.
func (s *Server) sensorQuery(w http.ResponseWriter, r *http.Request) {
full := r.URL.Query().Get("full") == "1"
var q json.RawMessage
if r.Method == http.MethodPost {
b, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10))
if err != nil {
jsonErr(w, 413, "query larger than 64 KiB")
return
}
q = b
} else {
flat := map[string]string{}
for k, v := range r.URL.Query() {
if k != "full" && len(v) > 0 {
flat[k] = v[0]
}
}
q, _ = json.Marshal(flat)
q, full, err := queryOf(w, r)
if err != nil {
jsonErr(w, 413, err.Error())
return
}
out, err := sensors.Shape(s.sensors.All(), q, full)
if err != nil {
+137
View File
@@ -0,0 +1,137 @@
package api
import (
"context"
"strings"
"sync"
"time"
"hamlab/internal/sensors"
"hamlab/internal/storage"
)
// statusSensors is the sensors.Provider for what isn't a /proc number: failed services,
// pending updates, pool health, drive health. Each source costs an exec or several, so
// they are read in the background on their own clocks and served from memory; every
// sensor's t says how old its reading really is.
type statusSensors struct {
s *Server
mu sync.RWMutex
by map[string][]sensors.Sensor // source → its sensors
}
const (
servicesEvery = 30 * time.Second
updatesEvery = 5 * time.Minute
zfsEvery = time.Minute
smartEvery = 10 * time.Minute // storage.Manager caches SMART that long anyway, and never wakes a sleeping disk
)
func newStatusSensors(s *Server) *statusSensors {
p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}}
go p.loop("services", servicesEvery, p.services)
go p.loop("updates", updatesEvery, p.updates)
go p.loop("zfs", zfsEvery, p.zfs)
go p.loop("smart", smartEvery, p.smart)
return p
}
func (p *statusSensors) loop(name string, every time.Duration, read func(context.Context, int64) []sensors.Sensor) {
for {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
got := read(ctx, time.Now().UnixMilli())
cancel()
p.mu.Lock()
p.by[name] = got
p.mu.Unlock()
time.Sleep(every)
}
}
func (p *statusSensors) Sensors() []sensors.Sensor {
p.mu.RLock()
defer p.mu.RUnlock()
var out []sensors.Sensor
for _, v := range p.by {
out = append(out, v...)
}
return out
}
func (p *statusSensors) services(ctx context.Context, t int64) []sensors.Sensor {
units, err := p.s.svc.List(ctx)
if err != nil {
return nil
}
var failed, running float64
var names []string
for _, u := range units {
if u.Failed {
failed++
names = append(names, u.Name)
}
if u.Sub == "running" {
running++
}
}
return []sensors.Sensor{
{ID: "services.failed", Name: "Failed services", Kind: "count", Value: failed, T: t, Warn: sensors.F(1)},
{ID: "services.failed_names", Name: "Failed services", Kind: "state", Value: strings.Join(names, ", "), T: t},
{ID: "services.running", Name: "Running services", Kind: "count", Value: running, T: t},
}
}
func (p *statusSensors) updates(ctx context.Context, t int64) []sensors.Sensor {
st := p.s.updates.Status(ctx)
if st.Manager == "" {
return nil
}
return []sensors.Sensor{
{ID: "updates.pending", Name: "Pending updates", Kind: "count", Value: float64(len(st.Packages)), T: t, Labels: map[string]string{"manager": st.Manager}},
{ID: "updates.security", Name: "Pending security updates", Kind: "count", Value: float64(st.SecurityCount), T: t, Warn: sensors.F(1)},
{ID: "updates.reboot_required", Name: "Reboot required", Kind: "state", Value: st.RebootNeeded, T: t},
}
}
func (p *statusSensors) zfs(ctx context.Context, t int64) []sensors.Sensor {
z := p.s.storage.ZFS(ctx)
var out []sensors.Sensor
for _, pool := range z.Pools {
id, l := "zfs."+sensors.Seg(pool.Name), map[string]string{"pool": pool.Name}
out = append(out,
sensors.Sensor{ID: id + ".health", Name: pool.Name + " health", Kind: "state", Value: pool.Health, T: t, Labels: l},
sensors.Sensor{ID: id + ".healthy", Name: pool.Name + " healthy", Kind: "state", Value: pool.Health == "ONLINE", T: t, Labels: l},
sensors.Sensor{ID: id + ".size", Name: pool.Name + " size", Kind: "bytes", Unit: "B", Value: float64(pool.Size), T: t, Labels: l},
sensors.Sensor{ID: id + ".used", Name: pool.Name + " used", Kind: "bytes", Unit: "B", Value: float64(pool.Alloc), T: t, Min: sensors.F(0), Max: sensors.F(float64(pool.Size)), Labels: l},
sensors.Sensor{ID: id + ".used_pct", Name: pool.Name + " used", Kind: "usage", Unit: "%", Value: float64(pool.Cap), T: t, Min: sensors.F(0), Max: sensors.F(100), Labels: l},
sensors.Sensor{ID: id + ".frag", Name: pool.Name + " fragmentation", Kind: "fragmentation", Unit: "%", Value: float64(pool.Frag), T: t, Min: sensors.F(0), Max: sensors.F(100), Labels: l},
)
}
return out
}
func (p *statusSensors) smart(ctx context.Context, t int64) []sensors.Sensor {
devs, err := storage.Devices(ctx)
if err != nil {
return nil
}
var out []sensors.Sensor
for _, d := range devs {
sm := p.s.storage.Smart(ctx, d.Name, false)
if sm.Passed == nil || sm.Standby || sm.Error != "" {
continue // no verdict to report: no smartctl, asleep, or not a SMART device
}
id, l := "smart."+sensors.Seg(d.Name), map[string]string{"device": d.Name, "model": d.Model}
out = append(out,
sensors.Sensor{ID: id + ".passed", Name: d.Name + " health", Kind: "state", Value: *sm.Passed, T: sm.FetchedAt, Labels: l},
sensors.Sensor{ID: id + ".warnings", Name: d.Name + " warnings", Kind: "count", Value: float64(len(sm.Warnings)), T: sm.FetchedAt, Warn: sensors.F(1), Labels: l},
)
if sm.PowerOnHours > 0 {
out = append(out, sensors.Sensor{ID: id + ".power_on", Name: d.Name + " powered on", Kind: "duration", Unit: "s", Value: float64(sm.PowerOnHours) * 3600, T: sm.FetchedAt, Labels: l})
}
if sm.NVMe != nil {
out = append(out, sensors.Sensor{ID: id + ".wear", Name: d.Name + " endurance used", Kind: "usage", Unit: "%", Value: float64(sm.NVMe.PercentageUsed), T: sm.FetchedAt, Min: sensors.F(0), Max: sensors.F(100), Labels: l})
}
}
return out
}
+5
View File
@@ -19,8 +19,13 @@ type Token struct {
Name string `json:"name"`
Hash string `json:"hash"`
Created time.Time `json:"created"`
// Scope is what the token may do. "" is everything (a hub's token: full control of
// this machine). ScopeSensorsRead only reads sensors — what a dashboard gets.
Scope string `json:"scope,omitempty"`
}
const ScopeSensorsRead = "sensors:read"
// Node is a remote Ham Lab this host (as hub) manages. The token is the one that
// node issued to us; the fingerprint pins its self-signed certificate.
type Node struct {
+25
View File
@@ -165,6 +165,9 @@ type Collector struct {
procAt time.Time
lastTable *ProcTable
upsProbe func() *UPS // nil when no UPS daemon is installed
subMu sync.Mutex
subs map[chan struct{}]struct{} // told after every sample; see Subscribe
}
func NewCollector() *Collector {
@@ -309,6 +312,28 @@ func (c *Collector) sample() {
c.ring = c.ring[len(c.ring)-History:]
}
c.mu.Unlock()
c.subMu.Lock()
for ch := range c.subs {
select {
case ch <- struct{}{}:
default: // a slow listener misses a tick; it reads the latest on the next one
}
}
c.subMu.Unlock()
}
// Subscribe returns a channel that gets a nudge after every new sample, and the func
// that ends the subscription.
func (c *Collector) Subscribe() (<-chan struct{}, func()) {
ch := make(chan struct{}, 1)
c.subMu.Lock()
if c.subs == nil {
c.subs = map[chan struct{}]struct{}{}
}
c.subs[ch] = struct{}{}
c.subMu.Unlock()
return ch, func() { c.subMu.Lock(); delete(c.subs, ch); c.subMu.Unlock() }
}
type cpuTimes struct{ busy, total uint64 }
+76
View File
@@ -0,0 +1,76 @@
package sensors
import (
"fmt"
"io"
"sort"
"strings"
)
// promFamily names the metric family for a sensor and scales its value to the base unit
// Prometheus convention wants (seconds, not minutes).
func promFamily(s Sensor) (name string, scale float64) {
scale = 1
unit := map[string]string{"%": "percent", "°C": "celsius", "rpm": "rpm", "B/s": "bytes_per_second", "B": "bytes",
"s": "seconds", "V": "volts", "W": "watts", "MHz": "megahertz"}[s.Unit]
if s.Unit == "min" {
unit, scale = "seconds", 60
}
if s.Kind == "count" { // hamlab_count{id=…} would say nothing; name it after the thing counted
return "hamlab_" + strings.ReplaceAll(Seg(strings.ReplaceAll(s.ID, ".", "_")), "-", "_"), 1
}
name = "hamlab_" + s.Kind
if unit != "" {
name += "_" + unit
}
return
}
var promEsc = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`)
// WriteProm writes sensors in the Prometheus text exposition format: one family per kind
// of measurement, the sensor's ID and its real-world labels as labels. Booleans are 0/1;
// a text state is a 1 carrying the text as a label, the usual way to export an enum.
func WriteProm(w io.Writer, all []Sensor) {
fams := map[string][]string{}
for _, s := range all {
name, scale := promFamily(s)
labels := []string{`id="` + promEsc.Replace(s.ID) + `"`}
keys := make([]string, 0, len(s.Labels))
for k := range s.Labels {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if s.Labels[k] != "" {
labels = append(labels, Seg(k)+`="`+promEsc.Replace(s.Labels[k])+`"`)
}
}
var v float64
switch x := s.Value.(type) {
case float64:
v = x * scale
case bool:
if x {
v = 1
}
case string:
v = 1
labels = append(labels, `value="`+promEsc.Replace(x)+`"`)
default:
continue
}
fams[name] = append(fams[name], fmt.Sprintf("%s{%s} %v", name, strings.Join(labels, ","), v))
}
names := make([]string, 0, len(fams))
for n := range fams {
names = append(names, n)
}
sort.Strings(names)
for _, n := range names {
fmt.Fprintf(w, "# TYPE %s gauge\n", n)
for _, line := range fams[n] {
fmt.Fprintln(w, line)
}
}
}
+16
View File
@@ -51,6 +51,22 @@ type Historian interface {
SensorHistory(id string) (s Series, ok bool)
}
// Ticker is a Provider that can say when its values have changed.
type Ticker interface {
Subscribe() (tick <-chan struct{}, cancel func())
}
// Subscribe follows the first provider that ticks. ok is false when none does.
func (r *Registry) Subscribe() (tick <-chan struct{}, cancel func(), ok bool) {
for _, p := range r.providers {
if t, is := p.(Ticker); is {
tick, cancel = t.Subscribe()
return tick, cancel, true
}
}
return nil, nil, false
}
// F is a pointer to a float, for the optional bounds.
func F(v float64) *float64 { return &v }
+24
View File
@@ -3,6 +3,7 @@ package sensors
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
@@ -51,3 +52,26 @@ func TestShape(t *testing.T) {
}
}
}
func TestWriteProm(t *testing.T) {
var b strings.Builder
WriteProm(&b, []Sensor{
{ID: "cpu.usage", Kind: "usage", Unit: "%", Value: 12.5},
{ID: "ups.runtime", Kind: "duration", Unit: "min", Value: 2.0},
{ID: "net.eth0.up", Kind: "state", Value: true, Labels: map[string]string{"iface": "eth0"}},
{ID: "ups.state", Kind: "state", Value: `Nor"mal`},
{ID: "updates.pending", Kind: "count", Value: 3.0},
})
got := b.String()
for _, want := range []string{
"# TYPE hamlab_usage_percent gauge\n" + `hamlab_usage_percent{id="cpu.usage"} 12.5`,
`hamlab_duration_seconds{id="ups.runtime"} 120`,
`hamlab_state{id="net.eth0.up",iface="eth0"} 1`,
`hamlab_state{id="ups.state",value="Nor\"mal"} 1`,
`hamlab_updates_pending{id="updates.pending"} 3`,
} {
if !strings.Contains(got, want) {
t.Errorf("missing %q in:\n%s", want, got)
}
}
}