diff --git a/SENSORS.md b/SENSORS.md index d48877f..0713405 100644 --- a/SENSORS.md +++ b/SENSORS.md @@ -49,8 +49,10 @@ On a hub, every one is also at `/api/nodes//sensors…` for each manage |---|---| | `GET /api/sensors` | Catalog, with current values. `?ids=fan.*,cpu.usage` narrows it. | | `GET /api/sensors/{id}` | One sensor. `?format=text` → the bare value as text/plain. | -| `GET /api/sensors/{id}/history` | `{id, unit, t[], v[]}` — what the ring holds (300 samples at the sampling interval). Numbers only. | +| `GET /api/sensors/{id}/history` | `{id, unit, t[], v[]}` — what the ring holds (300 points, spaced by the window below). Numbers only. | | `GET /api/sensors/history?ids=cpu.usage,net.*.rx` | Many histories in one request: `{series: [{id, unit, t[], v[]}]}`. A sensor younger than the ring has a shorter series. | +| `GET /api/prefs/history` | How far back the history reaches: `{seconds, points, stride_sec, min_seconds, max_seconds}`. | +| `PUT /api/prefs/history` | `{seconds}` — 60 to 86400. Set on the hub it is passed to every machine, so the fleet's graphs reach equally far back. | | `POST /api/sensors/query` | Shaped query, below. | | `GET /api/sensors/query?k=selector&…` | The same, flat, for consumers that can only fetch a URL. | @@ -197,6 +199,19 @@ GET /api/fleet/sensors/query?machines=grid,wyse&cpu=cpu.usage&failed=services.f → {"core": {"cpu": 4.8, "up": 50211}, "grid": {"cpu": 0.4, "up": 437177}, "mac": {"error": "timeout"}} ``` +## How far back a history reaches + +Every machine keeps **300 points per curve**, whatever the window — so the window sets how +far apart they are, not how much memory monitoring costs. At the default ten minutes each +point is one sample (2 s apart at the default rate); at a day each is the **average of the +five minutes behind it**, not whichever instant landed on the mark, so a long graph shows +the load that really happened. Ham Lab → Settings → **Graph history** is the one control, +1 minute to 24 hours; the hub passes it to every machine it manages. + +The window can't be finer than the sampling interval — 300 points of a 1-minute window at a +2-second rate is 30 points, and that is what you get. It is also all in memory: a machine +that restarts starts its graphs again. + ## What Ham Lab itself runs on Since 2026-09-19 the Dashboard tab (stock modules and custom widgets) and the Disks tab's diff --git a/internal/api/server.go b/internal/api/server.go index 2250476..24647a1 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -92,8 +92,16 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me s.alerts = newAlertEngine(s) // The saved dashboard layout carries the sampling interval; apply it on boot. var raw json.RawMessage - st.View(func(x *store.State) { raw = x.Dashboard }) + var histSec int + st.View(func(x *store.State) { raw, histSec = x.Dashboard, x.HistorySec }) s.applyPrefs(raw) + if histSec > 0 { + col.SetWindow(time.Duration(histSec) * time.Second) + } + // A node that restarted, or was added since, has its own idea of the window: tell them all. + if histSec > 0 && nm != nil { + go s.pushHistoryWindow(histSec) + } m := http.NewServeMux() m.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(204) }) @@ -215,6 +223,8 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api") pprof.Index(w, r) })) + m.HandleFunc("GET /api/prefs/history", s.auth(s.getHistoryWindow)) + m.HandleFunc("PUT /api/prefs/history", s.auth(s.putHistoryWindow)) m.HandleFunc("GET /api/prefs/dashboard", s.auth(s.getDashPrefs)) m.HandleFunc("PUT /api/prefs/dashboard", s.auth(s.putDashPrefs)) m.HandleFunc("/api/", func(w http.ResponseWriter, _ *http.Request) { jsonErr(w, 404, "not found") }) @@ -1888,6 +1898,61 @@ func (s *Server) putUIPrefs(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) } +// ---- how far back the graphs reach ---- +// +// One window for the whole fleet: the hub keeps it and pushes it to every machine, so a +// graph of `grid` opened from here reaches as far back as one of the hub. It is the +// machine's own setting either way — a node asked directly answers with what it was told. + +func (s *Server) getHistoryWindow(w http.ResponseWriter, _ *http.Request) { + jsonOK(w, 200, map[string]any{ + "seconds": int(s.metrics.Window().Seconds()), + "points": metrics.History, + "stride_sec": s.metrics.Stride().Seconds(), + "min_seconds": int(metrics.MinWindow.Seconds()), + "max_seconds": int(metrics.MaxWindow.Seconds()), + }) +} + +func (s *Server) putHistoryWindow(w http.ResponseWriter, r *http.Request) { + var body struct { + Seconds int `json:"seconds"` + } + if json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&body) != nil { + jsonErr(w, 400, "expected {\"seconds\": }") + return + } + lo, hi := int(metrics.MinWindow.Seconds()), int(metrics.MaxWindow.Seconds()) + if body.Seconds < lo || body.Seconds > hi { + jsonErr(w, 400, fmt.Sprintf("a window is between %d and %d seconds", lo, hi)) + return + } + s.metrics.SetWindow(time.Duration(body.Seconds) * time.Second) + if err := s.store.Update(func(st *store.State) { st.HistorySec = body.Seconds }); err != nil { + jsonErr(w, 500, err.Error()) + return + } + if s.nodes != nil && len(s.nodes.Names()) > 0 && r.Header.Get("X-Hamlab-From-Hub") == "" { + go s.pushHistoryWindow(body.Seconds) + } + s.getHistoryWindow(w, r) +} + +// pushHistoryWindow tells every managed machine the window. Best effort: a machine that is +// off or older simply keeps its own, and says so on its own pages. +func (s *Server) pushHistoryWindow(seconds int) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + body, _ := json.Marshal(map[string]int{"seconds": seconds}) + hdr := http.Header{"X-Hamlab-From-Hub": []string{"1"}, "Content-Type": []string{"application/json"}} + for _, name := range s.nodes.Names() { + code, _, err := s.nodes.Do(ctx, name, "PUT", "/api/prefs/history", body, hdr) + if err != nil || code >= 400 { + log.Printf("history window: %s did not take %ds (%d %v)", name, seconds, code, err) + } + } +} + // ---- dashboard layout prefs (opaque JSON object, owned by the UI) ---- func (s *Server) getDashPrefs(w http.ResponseWriter, _ *http.Request) { diff --git a/internal/api/status_sensors.go b/internal/api/status_sensors.go index 0267695..2710690 100644 --- a/internal/api/status_sensors.go +++ b/internal/api/status_sensors.go @@ -27,12 +27,21 @@ type statusSensors struct { cpuPrev map[string]cpuMark // container ID → CPU time at the previous read hist map[string]*sensors.Series // numeric sensors only, statusHistory points each + acc map[string]*bucket // the point being built for each, when the window is wider than the read rate } // statusHistory is how many readings each status sensor keeps: at 15 s that is over an // hour of container memory; at 10 min, two days of drive wear. const statusHistory = 300 +// A point being averaged, the same idea as the sampler's: these sources are read every 15 to +// 300 seconds, so a wide window keeps one point per stride rather than one per read. +type bucket struct { + start int64 + sum float64 + n float64 +} + const ( servicesEvery = 30 * time.Second containersEvery = 15 * time.Second @@ -42,7 +51,7 @@ const ( ) func newStatusSensors(s *Server) *statusSensors { - p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}, hist: map[string]*sensors.Series{}} + p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}, hist: map[string]*sensors.Series{}, acc: map[string]*bucket{}} go p.loop("services", servicesEvery, p.services) go p.loop("containers", containersEvery, p.containers) go p.loop("updates", updatesEvery, p.updates) @@ -58,6 +67,8 @@ func (p *statusSensors) loop(name string, every time.Duration, read func(context cancel() p.mu.Lock() p.by[name] = got + window := p.s.metrics.Window() + stride := window.Milliseconds() / statusHistory live := map[string]bool{} for _, x := range got { v, ok := x.Value.(float64) @@ -70,14 +81,32 @@ func (p *statusSensors) loop(name string, every time.Duration, read func(context h = &sensors.Series{ID: x.ID, Unit: x.Unit} p.hist[x.ID] = h } - h.T, h.V = append(h.T, x.T), append(h.V, v) + b := p.acc[x.ID] + if b == nil { + b = &bucket{start: x.T} + p.acc[x.ID] = b + } + b.sum, b.n = b.sum+v, b.n+1 + if x.T-b.start < stride && b.n > 1 { + continue // still filling this point + } + h.T, h.V = append(h.T, x.T), append(h.V, b.sum/b.n) + *b = bucket{start: x.T + 1} if len(h.T) > statusHistory { h.T, h.V = h.T[len(h.T)-statusHistory:], h.V[len(h.V)-statusHistory:] } + if cut := x.T - window.Milliseconds(); h.T[0] < cut { // and nothing older than the window + i := 0 + for i < len(h.T) && h.T[i] < cut { + i++ + } + h.T, h.V = h.T[i:], h.V[i:] + } } for id := range p.hist { // a container that is gone takes its history with it if strings.HasPrefix(id, prefixOf[name]) && !live[id] && prefixOf[name] != "" { delete(p.hist, id) + delete(p.acc, id) } } p.mu.Unlock() diff --git a/internal/metrics/fold.go b/internal/metrics/fold.go new file mode 100644 index 0000000..e01c09e --- /dev/null +++ b/internal/metrics/fold.go @@ -0,0 +1,141 @@ +package metrics + +// One kept history point is the average of the samples behind it. At the default ten-minute +// window that bucket is a single sample and nothing is averaged; at a day it is five minutes +// of them, and the difference matters — a graph of the last day drawn from whichever instant +// landed on each five-minute mark would show spikes that never happened and miss the ones +// that did. +// +// The newest sample of the bucket is the template: its shape, labels, totals and slow +// sections (filesystems, UPS) are the current ones. Only the moving numbers are averaged, +// and only where an entry is present in that sample — a disk plugged in halfway through the +// bucket is averaged over the samples that have it. + +func fold(b []Sample) Sample { + out := b[len(b)-1] + if len(b) == 1 { + return out + } + n := float64(len(b)) + sum := func(get func(Sample) float64) float64 { + var t float64 + for _, s := range b { + t += get(s) + } + return t / n + } + sumU := func(get func(Sample) uint64) uint64 { + var t uint64 + for _, s := range b { + t += get(s) + } + return t / uint64(len(b)) + } + out.CPU = sum(func(s Sample) float64 { return s.CPU }) + out.Load[0] = sum(func(s Sample) float64 { return s.Load[0] }) + out.Load[1] = sum(func(s Sample) float64 { return s.Load[1] }) + out.Load[2] = sum(func(s Sample) float64 { return s.Load[2] }) + + out.Cores = append([]float64(nil), out.Cores...) + for i := range out.Cores { + var t, c float64 + for _, s := range b { + if i < len(s.Cores) { + t, c = t+s.Cores[i], c+1 + } + } + if c > 0 { + out.Cores[i] = t / c + } + } + + out.Mem.Used = sumU(func(s Sample) uint64 { return s.Mem.Used }) + out.Mem.Avail = sumU(func(s Sample) uint64 { return s.Mem.Avail }) + out.Mem.Free = sumU(func(s Sample) uint64 { return s.Mem.Free }) + out.Mem.Cached = sumU(func(s Sample) uint64 { return s.Mem.Cached }) + out.Swap.Used = sumU(func(s Sample) uint64 { return s.Swap.Used }) + + // the rest are lists whose entries are matched by name across the bucket + temps := avgBy(b, func(s Sample) map[string][]float64 { + m := make(map[string][]float64, len(s.Temps)) + for _, x := range s.Temps { + m[x.Chip+"/"+x.Label] = []float64{x.C} + } + return m + }) + out.Temps = append([]Temp(nil), out.Temps...) + for i, x := range out.Temps { + if v, ok := temps[x.Chip+"/"+x.Label]; ok { + out.Temps[i].C = v[0] + } + } + + fans := avgBy(b, func(s Sample) map[string][]float64 { + m := make(map[string][]float64, len(s.Fans)) + for _, x := range s.Fans { + m[x.Chip+"/"+x.Label] = []float64{float64(x.RPM)} + } + return m + }) + out.Fans = append([]Fan(nil), out.Fans...) + for i, x := range out.Fans { + if v, ok := fans[x.Chip+"/"+x.Label]; ok { + out.Fans[i].RPM = int(v[0] + .5) + } + } + + nets := avgBy(b, func(s Sample) map[string][]float64 { + m := make(map[string][]float64, len(s.Net)) + for _, x := range s.Net { + m[x.Iface] = []float64{x.RxBps, x.TxBps} + } + return m + }) + out.Net = append([]Net(nil), out.Net...) + for i, x := range out.Net { + if v, ok := nets[x.Iface]; ok { + out.Net[i].RxBps, out.Net[i].TxBps = v[0], v[1] + } + } + + disks := avgBy(b, func(s Sample) map[string][]float64 { + m := make(map[string][]float64, len(s.Disks)) + for _, x := range s.Disks { + m[x.Name] = []float64{x.ReadBps, x.WriteBps, x.UtilPct} + } + return m + }) + out.Disks = append([]Disk(nil), out.Disks...) + for i, x := range out.Disks { + if v, ok := disks[x.Name]; ok { + out.Disks[i].ReadBps, out.Disks[i].WriteBps, out.Disks[i].UtilPct = v[0], v[1], v[2] + } + } + return out +} + +// avgBy averages each sample's named numbers across the bucket, per name. +func avgBy(b []Sample, of func(Sample) map[string][]float64) map[string][]float64 { + sums := map[string][]float64{} + count := map[string]float64{} + for _, s := range b { + for k, vs := range of(s) { + acc := sums[k] + if acc == nil { + acc = make([]float64, len(vs)) + } + for i, v := range vs { + if i < len(acc) { + acc[i] += v + } + } + sums[k], count[k] = acc, count[k]+1 + } + } + for k, acc := range sums { + for i := range acc { + acc[i] /= count[k] + } + } + return sums +} diff --git a/internal/metrics/fold_test.go b/internal/metrics/fold_test.go new file mode 100644 index 0000000..b099466 --- /dev/null +++ b/internal/metrics/fold_test.go @@ -0,0 +1,69 @@ +package metrics + +import ( + "testing" + "time" +) + +func TestFoldAverages(t *testing.T) { + b := []Sample{ + {T: 1, CPU: 10, Cores: []float64{10, 0}, Mem: Mem{Total: 100, Used: 10}, + Temps: []Temp{{Chip: "c", Label: "l", C: 40}}, Net: []Net{{Iface: "eth0", RxBps: 100}}, + Disks: []Disk{{Name: "sda", ReadBps: 10, UtilPct: 5}}}, + {T: 2, CPU: 30, Cores: []float64{30, 100}, Mem: Mem{Total: 100, Used: 30}, + Temps: []Temp{{Chip: "c", Label: "l", C: 60}}, Net: []Net{{Iface: "eth0", RxBps: 300}, {Iface: "new", RxBps: 7}}, + Disks: []Disk{{Name: "sda", ReadBps: 30, UtilPct: 15}}}, + } + f := fold(b) + if f.T != 2 { + t.Errorf("a folded point is stamped with the newest sample in it, got %d", f.T) + } + if f.CPU != 20 || f.Cores[0] != 20 || f.Cores[1] != 50 { + t.Errorf("cpu averaged: %v %v", f.CPU, f.Cores) + } + if f.Mem.Used != 20 || f.Mem.Total != 100 { + t.Errorf("memory: used averaged, total taken from the newest, got %+v", f.Mem) + } + if f.Temps[0].C != 50 || f.Net[0].RxBps != 200 || f.Disks[0].ReadBps != 20 || f.Disks[0].UtilPct != 10 { + t.Errorf("lists matched by name: %v %v %v", f.Temps, f.Net, f.Disks) + } + if len(f.Net) != 2 || f.Net[1].RxBps != 7 { + t.Errorf("an interface that appeared inside the bucket is averaged over the samples that have it: %v", f.Net) + } + // the template's own slices must not be touched: Latest() shares them + if b[1].Cores[0] != 30 || b[1].Temps[0].C != 60 { + t.Error("fold wrote through to the sample it templated from") + } +} + +func TestStrideAndTrim(t *testing.T) { + c := &Collector{interval: 2 * time.Second, window: DefaultWindow} + if got := c.stride(); got != 2*time.Second { + t.Errorf("a window of %v over %d points is finer than the interval, so the interval wins; got %v", DefaultWindow, History, got) + } + c.window = 24 * time.Hour + if got := c.stride(); got != 24*time.Hour/History { + t.Errorf("stride at a day: %v", got) + } + // a full day of samples at the day stride keeps exactly the budget + c.interval, c.window = time.Second, time.Hour + stride := c.stride().Milliseconds() + for i := 0; i < 5*History; i++ { + c.keep(Sample{T: int64(i) * stride}) + } + if len(c.ring) != History { + t.Errorf("the point count is the budget, got %d", len(c.ring)) + } + if span := c.ring[len(c.ring)-1].T - c.ring[0].T; span > c.window.Milliseconds() { + t.Errorf("the kept history reaches back further than the window: %d ms", span) + } + // narrowing the window drops what is now too old, at once + c.SetWindow(MinWindow) + if span := c.ring[len(c.ring)-1].T - c.ring[0].T; span > MinWindow.Milliseconds() { + t.Errorf("narrowing the window left %d ms of history", span) + } + c.SetWindow(365 * 24 * time.Hour) + if c.Window() != MaxWindow { + t.Errorf("a window past the maximum is the maximum, got %v", c.Window()) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 04e4f71..f8ea429 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -32,7 +32,12 @@ const ( DefaultInterval = 2 * time.Second MinInterval = 100 * time.Millisecond MaxInterval = 10 * time.Second - History = 300 // samples kept; the window is History × interval (30 s at 100 ms, 10 min at 2 s) + History = 300 // points kept per curve; the window below sets how far apart they are + // How far back the kept history reaches. The point count is fixed, so this sets the spacing: + // at a minute every sample is its own point, at a day each point is the average of ~5 minutes. + DefaultWindow = 10 * time.Minute + MinWindow = 1 * time.Minute + MaxWindow = 24 * time.Hour fsEvery = 30 * time.Second // filesystem usage changes slowly and statfs on network mounts isn't free upsEvery = 10 * time.Second // UPS daemons are an exec each; don't hammer them at fast intervals sectorSize = 512 // /proc/diskstats always counts 512-byte sectors @@ -138,8 +143,11 @@ type Sample struct { type Collector struct { mu sync.RWMutex - ring []Sample + ring []Sample // the kept history: one point per stride, each the average of its bucket + last Sample // the newest raw sample, whatever the stride + bucket []Sample // samples since the last point was kept interval time.Duration + window time.Duration wake chan struct{} // nudges Run when the interval changes prevCPU []cpuTimes @@ -162,7 +170,7 @@ type Collector struct { func NewCollector() *Collector { c := &Collector{prevNet: map[string][2]uint64{}, prevDisk: map[string][3]uint64{}, - interval: DefaultInterval, wake: make(chan struct{}, 1)} + interval: DefaultInterval, window: DefaultWindow, wake: make(chan struct{}, 1)} switch { case lookPath("pwrstat"): c.upsProbe = upsPwrstat @@ -223,13 +231,80 @@ func (c *Collector) Interval() time.Duration { return c.interval } +// SetWindow sets how far back the kept history reaches, clamped to [MinWindow, MaxWindow]. +func (c *Collector) SetWindow(d time.Duration) { + if d < MinWindow { + d = MinWindow + } + if d > MaxWindow { + d = MaxWindow + } + c.mu.Lock() + defer c.mu.Unlock() + c.window = d + c.trim() +} + +func (c *Collector) Window() time.Duration { + c.mu.RLock() + defer c.mu.RUnlock() + return c.window +} + +// Stride is how far apart the kept points are: the sampling interval until the window is +// wider than History samples of it, then the window split History ways. +func (c *Collector) Stride() time.Duration { + c.mu.RLock() + defer c.mu.RUnlock() + return c.stride() +} + +func (c *Collector) stride() time.Duration { + if s := c.window / History; s > c.interval { + return s + } + return c.interval +} + +// keep folds samples into the history, one point per stride. Averaged, not sampled: a CPU +// graph of the last six hours should show the load through each five minutes, not whichever +// two-second instant happened to land on the mark. +func (c *Collector) keep(s Sample) { + c.bucket = append(c.bucket, s) + if s.T-c.bucket[0].T < c.stride().Milliseconds() && len(c.bucket) > 1 { + return + } + c.ring = append(c.ring, fold(c.bucket)) + c.bucket = c.bucket[:0] + c.trim() +} + +// trim drops points past the count budget and past the window. Caller holds c.mu. +func (c *Collector) trim() { + if len(c.ring) > History { + c.ring = c.ring[len(c.ring)-History:] + } + n := len(c.ring) + if n == 0 { + return + } + cut := c.ring[n-1].T - c.window.Milliseconds() + i := 0 + for i < n && c.ring[i].T < cut { + i++ + } + if i > 0 { + c.ring = c.ring[i:] + } +} + func (c *Collector) Latest() (Sample, bool) { c.mu.RLock() defer c.mu.RUnlock() - if len(c.ring) == 0 { + if c.last.T == 0 { return Sample{}, false } - return c.ring[len(c.ring)-1], true + return c.last, true } // ---- sampling ---- @@ -265,10 +340,8 @@ func (c *Collector) sample() { return // deltas are meaningless on the priming pass } c.mu.Lock() - c.ring = append(c.ring, s) - if len(c.ring) > History { - c.ring = c.ring[len(c.ring)-History:] - } + c.last = s + c.keep(s) c.mu.Unlock() c.subMu.Lock() diff --git a/internal/store/store.go b/internal/store/store.go index 946efe5..442ee11 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -17,11 +17,12 @@ type Terminal struct { } type State struct { - Terminals []Terminal `json:"terminals"` - Sessions map[string]time.Time `json:"sessions"` // token -> expiry - Dashboard json.RawMessage `json:"dashboard,omitempty"` // UI layout prefs, opaque to the server - UI json.RawMessage `json:"ui,omitempty"` // theme etc., opaque to the server - Alerts json.RawMessage `json:"alerts,omitempty"` // rules, channels and recent events (internal/api/alerts.go) + Terminals []Terminal `json:"terminals"` + Sessions map[string]time.Time `json:"sessions"` // token -> expiry + Dashboard json.RawMessage `json:"dashboard,omitempty"` // UI layout prefs, opaque to the server + UI json.RawMessage `json:"ui,omitempty"` // theme etc., opaque to the server + Alerts json.RawMessage `json:"alerts,omitempty"` // rules, channels and recent events (internal/api/alerts.go) + HistorySec int `json:"history_sec,omitempty"` // how far back the graphs reach; the hub keeps every machine on the same one } type Store struct { diff --git a/web/src/App.tsx b/web/src/App.tsx index 4dded9e..ffefd94 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -4,7 +4,7 @@ import Login from './Login' import Shell from './Shell' import { Dialogs } from './ui' import { applyTheme, currentThemeId } from './themes' -import { getRefreshMs, setRefreshMs } from './prefs' +import { getHistorySec, getRefreshMs, setHistorySec, setRefreshMs } from './prefs' import { setAccents } from './accents' export default function App() { @@ -18,6 +18,9 @@ export default function App() { if (p.refreshMs && p.refreshMs !== getRefreshMs()) setRefreshMs(p.refreshMs) if (p.accents && typeof p.accents === 'object') setAccents(p.accents, false) }).catch(() => { /* keep local */ }) + // The graph window is the machines' own setting; the copy here only labels the curves + // and spaces the live points, so take the hub's word for it at every login. + api.historyWindow().then((h) => { if (h.seconds && h.seconds !== getHistorySec()) setHistorySec(h.seconds) }).catch(() => { /* keep local */ }) } catch (e) { if (e instanceof Unauthenticated) setMe(null) } }, []) diff --git a/web/src/Dashboard.tsx b/web/src/Dashboard.tsx index 964a76a..65f061f 100644 --- a/web/src/Dashboard.tsx +++ b/web/src/Dashboard.tsx @@ -2,7 +2,7 @@ import { ReactNode, useCallback, useEffect, useRef, useState } from 'react' import { api, DashPrefs, Sample, Series, Temp } from './api' import Sparkline from './Sparkline' import { Loading, Spinner, askYes, toast } from './ui' -import { fmtMs, useRefreshMs } from './prefs' +import { fmtMs, fmtWindowSec, strideMs, useHistorySec, useRefreshMs } from './prefs' import { fetchCatalog, fetchHistories, useSensorStream } from './sensors' import { Catalog, MONITOR_HISTORY, MONITOR_SHAPE, Values, mergeValues, sampleOf, seriesOf, stale } from './sampleAdapter' import { Widget, WidgetData, WidgetSpec, shapeOf } from './widgets' @@ -46,10 +46,17 @@ function normalize(raw: DashPrefs): Layout { widgets: (p.widgets ?? []).filter((w) => w && typeof w.id === 'string' && typeof w.sensor === 'string') } } const toPrefs = (l: Layout): DashPrefs => ({ order: l.order, hidden: [...l.hidden], hiddenItems: [...l.hiddenItems], intervalMs: l.intervalMs, widgets: l.widgets }) -const fmtWindow = (ms: number) => { const s = Math.round((MAX * ms) / 1000); return s >= 60 ? `${Math.round(s / 60)} min` : `${s} s` } // ---- formatting ---- -function push(arr: number[], v: number) { arr.push(v); if (arr.length > MAX) arr.shift() } +// A live value either starts a new point or is averaged into the one being built — the same +// rule the machine itself uses to keep its history, so the tail of a six-hour graph isn't a +// minute of 2-second samples crushed against the right edge. n is how many values this point +// has had, including this one. +function pushPoint(arr: number[], v: number, fresh: boolean, n: number) { + if (!fresh && arr.length) { arr[arr.length - 1] += (v - arr[arr.length - 1]) / n; return } + arr.push(v) + if (arr.length > MAX) arr.shift() +} function pad(arr: number[], n: number) { while (arr.length < n) arr.unshift(0) } export function fmtBytes(b: number): string { @@ -276,6 +283,7 @@ export default function Dashboard() { const hist = useRef(null) const saveTimer = useRef(undefined) const refreshMs = useRefreshMs() + const historySec = useHistorySec() // Persist layout changes server-side (debounced) so every device sees the same dashboard. const update = useCallback((fn: (l: Layout) => Layout) => { @@ -305,6 +313,7 @@ export default function Dashboard() { // thresholds; one batch of histories to seed the curves; one stream for the values. const catalog = useRef(new Map()) const refetching = useRef(false) + const binN = useRef(1) // values averaged into the point currently being built const stock = useSensorStream(MONITOR_SHAPE, { full: false, everyMs: refreshMs }) const seed = useCallback(async () => { try { @@ -324,8 +333,16 @@ export default function Dashboard() { const s = sampleOf(v, catalog.current) const h = hist.current if (h) { - push(h.t, s.t); push(h.cpu, s.cpu); push(h.mem, s.mem.used) + // the point being built keeps the time it started at, so the next one is a whole stride later + const last = h.t[h.t.length - 1] ?? 0 + const fresh = h.t.length === 0 || s.t - last >= strideMs(historySec, refreshMs) + binN.current = fresh ? 1 : binN.current + 1 + if (fresh) pushPoint(h.t, s.t, true, 1) const n = h.t.length + // a curve that has no point in this one yet starts one, whatever the others are doing: + // an interface or disk that appeared mid-point must still line up with the time axis + const push = (arr: number[], x: number) => pushPoint(arr, x, fresh || arr.length < n, binN.current) + push(h.cpu, s.cpu); push(h.mem, s.mem.used) for (const x of s.net) { const c = (h.net[x.iface] ??= { rx: [], tx: [] }) pad(c.rx, n - 1); pad(c.tx, n - 1); push(c.rx, x.rx_bps); push(c.tx, x.tx_bps) @@ -340,7 +357,7 @@ export default function Dashboard() { } } setLatest(s) - }, [stock.data]) + }, [stock.data, historySec, refreshMs]) useEffect(() => { if (stock.error && !latest) setErr(stock.error) }, [stock.error, latest]) // The sampler on this machine follows the global rate (clamped to what it supports). @@ -400,8 +417,8 @@ export default function Dashboard() {
{err && {err}} - {editing ? last {fmtWindow(latest.interval_ms || layout.intervalMs)} · ▲▼ reorder · ✕ hide · refresh rate is in Settings - : last {fmtWindow(latest.interval_ms || layout.intervalMs)} · {fmtMs(latest.interval_ms || layout.intervalMs)}} + {editing ? last {fmtWindowSec(historySec)} · ▲▼ reorder · ✕ hide · refresh rate is in Settings + : last {fmtWindowSec(historySec)} · {fmtMs(latest.interval_ms || layout.intervalMs)}} {saving === 'saving' && saving} {saving === 'saved' && saved ✓} diff --git a/web/src/Settings.tsx b/web/src/Settings.tsx index a1a8a22..74ba4df 100644 --- a/web/src/Settings.tsx +++ b/web/src/Settings.tsx @@ -4,7 +4,7 @@ import { api, Me } from './api' import { savePrefs } from './accents' import { applyTheme, currentThemeId, THEMES } from './themes' import { Btn, Loading, askYes, toast, useEscape } from './ui' -import { fmtMs, getRefreshMs, REFRESH_OPTIONS, setRefreshMs } from './prefs' +import { fmtMs, fmtWindowSec, getHistorySec, getRefreshMs, HISTORY_OPTIONS, HISTORY_POINTS, REFRESH_OPTIONS, setHistorySec, setRefreshMs, strideMs } from './prefs' const fmtUptime = (s: number) => { const d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60); return d > 0 ? `${d}d ${h}h` : h > 0 ? `${h}h ${m}m` : `${m}m` } @@ -33,6 +33,8 @@ function Preview({ vars }: { vars: Record }) { export default function Settings({ me }: { me: Me }) { const [theme, setTheme] = useState(currentThemeId()) const [refresh, setRefresh] = useState(getRefreshMs()) + const [history, setHistory] = useState(getHistorySec()) + const [histErr, setHistErr] = useState('') const [tokens, setTokens] = useState<{ name: string; created: string; scope?: string; sensors?: string[] }[] | null>(null) const [fresh, setFresh] = useState<{ name: string; token: string } | null>(null) const [making, setMaking] = useState(false) @@ -52,6 +54,14 @@ export default function Settings({ me }: { me: Me }) { setRefreshMs(ms); setRefresh(ms) await savePrefs() } + // The window is the machine's own setting, not a look: the hub takes it and passes it on. + useEffect(() => { api.historyWindow().then((h) => { setHistory(h.seconds); setHistorySec(h.seconds) }).catch(() => {}) }, []) + const pickHistory = async (sec: number) => { + const was = history + setHistory(sec); setHistorySec(sec); setHistErr('') + try { const h = await api.setHistoryWindow(sec); setHistory(h.seconds); setHistorySec(h.seconds) } + catch (e) { setHistory(was); setHistorySec(was); setHistErr(e instanceof Error ? e.message : 'failed') } + } async function changePassword(e: FormEvent) { e.preventDefault() @@ -85,11 +95,27 @@ export default function Settings({ me }: { me: Me }) { {REFRESH_OPTIONS.map((ms) => )}
- Every page polls at exactly this rate: dashboard, processes, disks, services, cron, updates, machines. The sampler on each machine follows it too, so charts keep 300 points and cover 300 × rate. + Every page polls at exactly this rate: dashboard, processes, disks, services, cron, updates, machines. The sampler on each machine follows it too, so it is also the finest a graph can be — how far back the graphs reach is the next setting down. Fast rates cost CPU on the host, especially on the list pages, which run commands to refresh. Nothing is throttled behind your back.
+

Graph history

every graph, on every machine
+
+
+ {HISTORY_OPTIONS.map((sec) => )} +
+ {histErr &&
{histErr}
} +
+ How far back every sparkline reaches — the dashboard, the processes page, the disks + page and any sparkline widget. Each machine keeps {HISTORY_POINTS} points per curve whatever you + pick, so a wider window is a coarser one: {fmtWindowSec(history)} puts a point every{' '} + {fmtWindowSec(strideMs(history, refresh) / 1000)}, each the average of what happened + in between rather than one instant out of it. Memory on the machine doesn't change. + The history is kept in memory, so a machine that restarts starts its graphs again. +
+
+

Account · {me.host}

diff --git a/web/src/api.ts b/web/src/api.ts index ecd02ba..33796be 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -82,6 +82,7 @@ export interface Series { // Dashboard layout. `order` lists module ids top-to-bottom; `hidden` are modules switched // off; `hiddenItems` are individual rows ("net:wlp1s0", "fs:/boot/efi", "disk:sda", "temp:chip/label"). +export interface HistoryWindow { seconds: number; points: number; stride_sec: number; min_seconds: number; max_seconds: number } export interface DashPrefs { order?: string[]; hidden?: string[]; hiddenItems?: string[]; intervalMs?: number; widgets?: { id: string; type: 'stat' | 'gauge' | 'bar' | 'spark' | 'list'; sensor: string; title?: string }[] } export interface ServiceUnit { name: string; description: string; load: string; active: string; sub: string; enabled: string; failed: boolean } @@ -179,6 +180,8 @@ export const api = { serviceFile: (unit: string) => req<{ content: string }>('GET', `/api/services/${encodeURIComponent(unit)}/file`), serviceLogs: (unit: string, n = 200) => req('GET', `/api/services/${encodeURIComponent(unit)}/logs?n=${n}`), serviceAction: (unit: string, action: string) => req('POST', `/api/services/${encodeURIComponent(unit)}/${action}`), + historyWindow: () => req('GET', '/api/prefs/history', undefined, true), + setHistoryWindow: (seconds: number) => req('PUT', '/api/prefs/history', { seconds }, true), dashPrefs: () => req('GET', '/api/prefs/dashboard'), saveDashPrefs: (p: DashPrefs) => req('PUT', '/api/prefs/dashboard', p), login: (password: string) => req('POST', '/api/login', { password }, true), diff --git a/web/src/prefs.ts b/web/src/prefs.ts index 079293b..796789d 100644 --- a/web/src/prefs.ts +++ b/web/src/prefs.ts @@ -21,6 +21,50 @@ export function setRefreshMs(ms: number) { export function fmtMs(ms: number): string { return ms >= 1000 ? `${ms / 1000} s` : `${ms} ms` } +// ---- how far back the graphs reach ---- +// Unlike the refresh rate this is the machine's own setting, not a look: the hub keeps the +// window and pushes it to every machine, so the graphs of any of them reach equally far back. +// The value is cached here only so a page can draw before the hub has answered. + +export const HISTORY_OPTIONS = [60, 300, 600, 1800, 3600, 3 * 3600, 6 * 3600, 12 * 3600, 24 * 3600] as const +export const DEFAULT_HISTORY_SEC = 600 +export const HISTORY_POINTS = 300 // what a machine keeps per curve, whatever the window +const HKEY = 'hamlab.historySec' + +export function getHistorySec(): number { + const v = Number(localStorage.getItem(HKEY)) + return v >= 60 && v <= 86400 ? v : DEFAULT_HISTORY_SEC +} + +export function setHistorySec(sec: number) { + if (!(sec >= 60 && sec <= 86400)) return + localStorage.setItem(HKEY, String(sec)) + window.dispatchEvent(new CustomEvent('hamlab:history', { detail: sec })) +} + +// useHistorySec re-renders the caller when the window changes. +export function useHistorySec(): number { + const [sec, setSec] = useState(getHistorySec()) + useEffect(() => { + const h = (e: Event) => setSec((e as CustomEvent).detail) + window.addEventListener('hamlab:history', h) + return () => window.removeEventListener('hamlab:history', h) + }, []) + return sec +} + +// How far apart the kept points are: the refresh rate until the window is wider than +// HISTORY_POINTS of it, then the window split that many ways. Mirrors the server. +export function strideMs(windowSec: number, refreshMs: number): number { + return Math.max(refreshMs, Math.round((windowSec * 1000) / HISTORY_POINTS)) +} + +export function fmtWindowSec(sec: number): string { + if (sec >= 3600) { const h = sec / 3600; return `${Number.isInteger(h) ? h : h.toFixed(1)} hour${h === 1 ? '' : 's'}` } + if (sec >= 60) { const m = sec / 60; return `${Number.isInteger(m) ? m : m.toFixed(1)} min` } + return `${Math.round(sec)} s` +} + // useRefreshMs re-renders the caller when the global rate changes. export function useRefreshMs(floor = 0): number { const [ms, setMs] = useState(getRefreshMs()) diff --git a/web/src/widgets.tsx b/web/src/widgets.tsx index 8466d83..60711b4 100644 --- a/web/src/widgets.tsx +++ b/web/src/widgets.tsx @@ -41,17 +41,31 @@ function Gauge({ s }: { s: Sensor }) { function Spark({ s, src }: { s: Sensor; src?: Source }) { const pts = useRef([]) const lastT = useRef(0) + const stride = useRef(0) // how far apart the machine's own points are, as they arrive const [, bump] = useState(0) useEffect(() => { let stop = false - pts.current = []; lastT.current = 0 - fetchHistory(s.id, src).then((h) => { if (!stop && h) { pts.current = h.v.slice(-MAX_POINTS); lastT.current = h.t[h.t.length - 1] ?? 0; bump((n) => n + 1) } }) + pts.current = []; lastT.current = 0; stride.current = 0 + fetchHistory(s.id, src).then((h) => { + if (stop || !h) return + pts.current = h.v.slice(-MAX_POINTS); lastT.current = h.t[h.t.length - 1] ?? 0 + const t = h.t.slice(-MAX_POINTS) + const gaps = t.slice(1).map((x, i) => x - t[i]).sort((a, b) => a - b) + stride.current = gaps.length ? gaps[gaps.length >> 1] : 0 // the median: one long gap is a restart, not the spacing + bump((n) => n + 1) + }) return () => { stop = true } }, [s.id, src?.machine, src?.token]) // eslint-disable-line react-hooks/exhaustive-deps if (typeof s.value === 'number' && s.t > lastT.current) { - lastT.current = s.t - pts.current.push(s.value) - if (pts.current.length > MAX_POINTS) pts.current.shift() + // a new point once a stride has passed; until then the newest point keeps being refined, + // so a wide window's curve isn't squeezed by a fast stream of live values + if (pts.current.length && s.t - lastT.current < stride.current) { + pts.current[pts.current.length - 1] = s.value + } else { + lastT.current = s.t + pts.current.push(s.value) + if (pts.current.length > MAX_POINTS) pts.current.shift() + } } return fmtValue(v, s.unit)} height={40} /> }