Settings: one window for how far back every graph reaches, 1 minute to 24 hours

Each machine keeps 300 points per curve whatever the window, so memory doesn't follow it;
a wider window means wider points, each the average of its bucket rather than one instant
out of it. The hub passes the setting to every machine it manages.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
2026-09-20 19:07:55 -07:00
co-authored by Claude Opus 5
parent 885b050d02
commit 04dd98e2cd
13 changed files with 533 additions and 33 deletions
+66 -1
View File
@@ -92,8 +92,16 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
s.alerts = newAlertEngine(s)
// The saved dashboard layout carries the sampling interval; apply it on boot.
var raw json.RawMessage
st.View(func(x *store.State) { raw = x.Dashboard })
var histSec int
st.View(func(x *store.State) { raw, histSec = x.Dashboard, x.HistorySec })
s.applyPrefs(raw)
if histSec > 0 {
col.SetWindow(time.Duration(histSec) * time.Second)
}
// A node that restarted, or was added since, has its own idea of the window: tell them all.
if histSec > 0 && nm != nil {
go s.pushHistoryWindow(histSec)
}
m := http.NewServeMux()
m.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(204) })
@@ -215,6 +223,8 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api")
pprof.Index(w, r)
}))
m.HandleFunc("GET /api/prefs/history", s.auth(s.getHistoryWindow))
m.HandleFunc("PUT /api/prefs/history", s.auth(s.putHistoryWindow))
m.HandleFunc("GET /api/prefs/dashboard", s.auth(s.getDashPrefs))
m.HandleFunc("PUT /api/prefs/dashboard", s.auth(s.putDashPrefs))
m.HandleFunc("/api/", func(w http.ResponseWriter, _ *http.Request) { jsonErr(w, 404, "not found") })
@@ -1888,6 +1898,61 @@ func (s *Server) putUIPrefs(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(204)
}
// ---- how far back the graphs reach ----
//
// One window for the whole fleet: the hub keeps it and pushes it to every machine, so a
// graph of `grid` opened from here reaches as far back as one of the hub. It is the
// machine's own setting either way — a node asked directly answers with what it was told.
func (s *Server) getHistoryWindow(w http.ResponseWriter, _ *http.Request) {
jsonOK(w, 200, map[string]any{
"seconds": int(s.metrics.Window().Seconds()),
"points": metrics.History,
"stride_sec": s.metrics.Stride().Seconds(),
"min_seconds": int(metrics.MinWindow.Seconds()),
"max_seconds": int(metrics.MaxWindow.Seconds()),
})
}
func (s *Server) putHistoryWindow(w http.ResponseWriter, r *http.Request) {
var body struct {
Seconds int `json:"seconds"`
}
if json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&body) != nil {
jsonErr(w, 400, "expected {\"seconds\": <number>}")
return
}
lo, hi := int(metrics.MinWindow.Seconds()), int(metrics.MaxWindow.Seconds())
if body.Seconds < lo || body.Seconds > hi {
jsonErr(w, 400, fmt.Sprintf("a window is between %d and %d seconds", lo, hi))
return
}
s.metrics.SetWindow(time.Duration(body.Seconds) * time.Second)
if err := s.store.Update(func(st *store.State) { st.HistorySec = body.Seconds }); err != nil {
jsonErr(w, 500, err.Error())
return
}
if s.nodes != nil && len(s.nodes.Names()) > 0 && r.Header.Get("X-Hamlab-From-Hub") == "" {
go s.pushHistoryWindow(body.Seconds)
}
s.getHistoryWindow(w, r)
}
// pushHistoryWindow tells every managed machine the window. Best effort: a machine that is
// off or older simply keeps its own, and says so on its own pages.
func (s *Server) pushHistoryWindow(seconds int) {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
body, _ := json.Marshal(map[string]int{"seconds": seconds})
hdr := http.Header{"X-Hamlab-From-Hub": []string{"1"}, "Content-Type": []string{"application/json"}}
for _, name := range s.nodes.Names() {
code, _, err := s.nodes.Do(ctx, name, "PUT", "/api/prefs/history", body, hdr)
if err != nil || code >= 400 {
log.Printf("history window: %s did not take %ds (%d %v)", name, seconds, code, err)
}
}
}
// ---- dashboard layout prefs (opaque JSON object, owned by the UI) ----
func (s *Server) getDashPrefs(w http.ResponseWriter, _ *http.Request) {
+31 -2
View File
@@ -27,12 +27,21 @@ type statusSensors struct {
cpuPrev map[string]cpuMark // container ID → CPU time at the previous read
hist map[string]*sensors.Series // numeric sensors only, statusHistory points each
acc map[string]*bucket // the point being built for each, when the window is wider than the read rate
}
// statusHistory is how many readings each status sensor keeps: at 15 s that is over an
// hour of container memory; at 10 min, two days of drive wear.
const statusHistory = 300
// A point being averaged, the same idea as the sampler's: these sources are read every 15 to
// 300 seconds, so a wide window keeps one point per stride rather than one per read.
type bucket struct {
start int64
sum float64
n float64
}
const (
servicesEvery = 30 * time.Second
containersEvery = 15 * time.Second
@@ -42,7 +51,7 @@ const (
)
func newStatusSensors(s *Server) *statusSensors {
p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}, hist: map[string]*sensors.Series{}}
p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}, hist: map[string]*sensors.Series{}, acc: map[string]*bucket{}}
go p.loop("services", servicesEvery, p.services)
go p.loop("containers", containersEvery, p.containers)
go p.loop("updates", updatesEvery, p.updates)
@@ -58,6 +67,8 @@ func (p *statusSensors) loop(name string, every time.Duration, read func(context
cancel()
p.mu.Lock()
p.by[name] = got
window := p.s.metrics.Window()
stride := window.Milliseconds() / statusHistory
live := map[string]bool{}
for _, x := range got {
v, ok := x.Value.(float64)
@@ -70,14 +81,32 @@ func (p *statusSensors) loop(name string, every time.Duration, read func(context
h = &sensors.Series{ID: x.ID, Unit: x.Unit}
p.hist[x.ID] = h
}
h.T, h.V = append(h.T, x.T), append(h.V, v)
b := p.acc[x.ID]
if b == nil {
b = &bucket{start: x.T}
p.acc[x.ID] = b
}
b.sum, b.n = b.sum+v, b.n+1
if x.T-b.start < stride && b.n > 1 {
continue // still filling this point
}
h.T, h.V = append(h.T, x.T), append(h.V, b.sum/b.n)
*b = bucket{start: x.T + 1}
if len(h.T) > statusHistory {
h.T, h.V = h.T[len(h.T)-statusHistory:], h.V[len(h.V)-statusHistory:]
}
if cut := x.T - window.Milliseconds(); h.T[0] < cut { // and nothing older than the window
i := 0
for i < len(h.T) && h.T[i] < cut {
i++
}
h.T, h.V = h.T[i:], h.V[i:]
}
}
for id := range p.hist { // a container that is gone takes its history with it
if strings.HasPrefix(id, prefixOf[name]) && !live[id] && prefixOf[name] != "" {
delete(p.hist, id)
delete(p.acc, id)
}
}
p.mu.Unlock()