Settings: one window for how far back every graph reaches, 1 minute to 24 hours
Each machine keeps 300 points per curve whatever the window, so memory doesn't follow it; a wider window means wider points, each the average of its bucket rather than one instant out of it. The hub passes the setting to every machine it manages. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
885b050d02
commit
04dd98e2cd
+66
-1
@@ -92,8 +92,16 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
|
||||
s.alerts = newAlertEngine(s)
|
||||
// The saved dashboard layout carries the sampling interval; apply it on boot.
|
||||
var raw json.RawMessage
|
||||
st.View(func(x *store.State) { raw = x.Dashboard })
|
||||
var histSec int
|
||||
st.View(func(x *store.State) { raw, histSec = x.Dashboard, x.HistorySec })
|
||||
s.applyPrefs(raw)
|
||||
if histSec > 0 {
|
||||
col.SetWindow(time.Duration(histSec) * time.Second)
|
||||
}
|
||||
// A node that restarted, or was added since, has its own idea of the window: tell them all.
|
||||
if histSec > 0 && nm != nil {
|
||||
go s.pushHistoryWindow(histSec)
|
||||
}
|
||||
|
||||
m := http.NewServeMux()
|
||||
m.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(204) })
|
||||
@@ -215,6 +223,8 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
|
||||
r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api")
|
||||
pprof.Index(w, r)
|
||||
}))
|
||||
m.HandleFunc("GET /api/prefs/history", s.auth(s.getHistoryWindow))
|
||||
m.HandleFunc("PUT /api/prefs/history", s.auth(s.putHistoryWindow))
|
||||
m.HandleFunc("GET /api/prefs/dashboard", s.auth(s.getDashPrefs))
|
||||
m.HandleFunc("PUT /api/prefs/dashboard", s.auth(s.putDashPrefs))
|
||||
m.HandleFunc("/api/", func(w http.ResponseWriter, _ *http.Request) { jsonErr(w, 404, "not found") })
|
||||
@@ -1888,6 +1898,61 @@ func (s *Server) putUIPrefs(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
// ---- how far back the graphs reach ----
|
||||
//
|
||||
// One window for the whole fleet: the hub keeps it and pushes it to every machine, so a
|
||||
// graph of `grid` opened from here reaches as far back as one of the hub. It is the
|
||||
// machine's own setting either way — a node asked directly answers with what it was told.
|
||||
|
||||
func (s *Server) getHistoryWindow(w http.ResponseWriter, _ *http.Request) {
|
||||
jsonOK(w, 200, map[string]any{
|
||||
"seconds": int(s.metrics.Window().Seconds()),
|
||||
"points": metrics.History,
|
||||
"stride_sec": s.metrics.Stride().Seconds(),
|
||||
"min_seconds": int(metrics.MinWindow.Seconds()),
|
||||
"max_seconds": int(metrics.MaxWindow.Seconds()),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) putHistoryWindow(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Seconds int `json:"seconds"`
|
||||
}
|
||||
if json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&body) != nil {
|
||||
jsonErr(w, 400, "expected {\"seconds\": <number>}")
|
||||
return
|
||||
}
|
||||
lo, hi := int(metrics.MinWindow.Seconds()), int(metrics.MaxWindow.Seconds())
|
||||
if body.Seconds < lo || body.Seconds > hi {
|
||||
jsonErr(w, 400, fmt.Sprintf("a window is between %d and %d seconds", lo, hi))
|
||||
return
|
||||
}
|
||||
s.metrics.SetWindow(time.Duration(body.Seconds) * time.Second)
|
||||
if err := s.store.Update(func(st *store.State) { st.HistorySec = body.Seconds }); err != nil {
|
||||
jsonErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
if s.nodes != nil && len(s.nodes.Names()) > 0 && r.Header.Get("X-Hamlab-From-Hub") == "" {
|
||||
go s.pushHistoryWindow(body.Seconds)
|
||||
}
|
||||
s.getHistoryWindow(w, r)
|
||||
}
|
||||
|
||||
// pushHistoryWindow tells every managed machine the window. Best effort: a machine that is
|
||||
// off or older simply keeps its own, and says so on its own pages.
|
||||
func (s *Server) pushHistoryWindow(seconds int) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
body, _ := json.Marshal(map[string]int{"seconds": seconds})
|
||||
hdr := http.Header{"X-Hamlab-From-Hub": []string{"1"}, "Content-Type": []string{"application/json"}}
|
||||
for _, name := range s.nodes.Names() {
|
||||
code, _, err := s.nodes.Do(ctx, name, "PUT", "/api/prefs/history", body, hdr)
|
||||
if err != nil || code >= 400 {
|
||||
log.Printf("history window: %s did not take %ds (%d %v)", name, seconds, code, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- dashboard layout prefs (opaque JSON object, owned by the UI) ----
|
||||
|
||||
func (s *Server) getDashPrefs(w http.ResponseWriter, _ *http.Request) {
|
||||
|
||||
@@ -27,12 +27,21 @@ type statusSensors struct {
|
||||
cpuPrev map[string]cpuMark // container ID → CPU time at the previous read
|
||||
|
||||
hist map[string]*sensors.Series // numeric sensors only, statusHistory points each
|
||||
acc map[string]*bucket // the point being built for each, when the window is wider than the read rate
|
||||
}
|
||||
|
||||
// statusHistory is how many readings each status sensor keeps: at 15 s that is over an
|
||||
// hour of container memory; at 10 min, two days of drive wear.
|
||||
const statusHistory = 300
|
||||
|
||||
// A point being averaged, the same idea as the sampler's: these sources are read every 15 to
|
||||
// 300 seconds, so a wide window keeps one point per stride rather than one per read.
|
||||
type bucket struct {
|
||||
start int64
|
||||
sum float64
|
||||
n float64
|
||||
}
|
||||
|
||||
const (
|
||||
servicesEvery = 30 * time.Second
|
||||
containersEvery = 15 * time.Second
|
||||
@@ -42,7 +51,7 @@ const (
|
||||
)
|
||||
|
||||
func newStatusSensors(s *Server) *statusSensors {
|
||||
p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}, hist: map[string]*sensors.Series{}}
|
||||
p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}, hist: map[string]*sensors.Series{}, acc: map[string]*bucket{}}
|
||||
go p.loop("services", servicesEvery, p.services)
|
||||
go p.loop("containers", containersEvery, p.containers)
|
||||
go p.loop("updates", updatesEvery, p.updates)
|
||||
@@ -58,6 +67,8 @@ func (p *statusSensors) loop(name string, every time.Duration, read func(context
|
||||
cancel()
|
||||
p.mu.Lock()
|
||||
p.by[name] = got
|
||||
window := p.s.metrics.Window()
|
||||
stride := window.Milliseconds() / statusHistory
|
||||
live := map[string]bool{}
|
||||
for _, x := range got {
|
||||
v, ok := x.Value.(float64)
|
||||
@@ -70,14 +81,32 @@ func (p *statusSensors) loop(name string, every time.Duration, read func(context
|
||||
h = &sensors.Series{ID: x.ID, Unit: x.Unit}
|
||||
p.hist[x.ID] = h
|
||||
}
|
||||
h.T, h.V = append(h.T, x.T), append(h.V, v)
|
||||
b := p.acc[x.ID]
|
||||
if b == nil {
|
||||
b = &bucket{start: x.T}
|
||||
p.acc[x.ID] = b
|
||||
}
|
||||
b.sum, b.n = b.sum+v, b.n+1
|
||||
if x.T-b.start < stride && b.n > 1 {
|
||||
continue // still filling this point
|
||||
}
|
||||
h.T, h.V = append(h.T, x.T), append(h.V, b.sum/b.n)
|
||||
*b = bucket{start: x.T + 1}
|
||||
if len(h.T) > statusHistory {
|
||||
h.T, h.V = h.T[len(h.T)-statusHistory:], h.V[len(h.V)-statusHistory:]
|
||||
}
|
||||
if cut := x.T - window.Milliseconds(); h.T[0] < cut { // and nothing older than the window
|
||||
i := 0
|
||||
for i < len(h.T) && h.T[i] < cut {
|
||||
i++
|
||||
}
|
||||
h.T, h.V = h.T[i:], h.V[i:]
|
||||
}
|
||||
}
|
||||
for id := range p.hist { // a container that is gone takes its history with it
|
||||
if strings.HasPrefix(id, prefixOf[name]) && !live[id] && prefixOf[name] != "" {
|
||||
delete(p.hist, id)
|
||||
delete(p.acc, id)
|
||||
}
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
Reference in New Issue
Block a user