Settings: one window for how far back every graph reaches, 1 minute to 24 hours

Each machine keeps 300 points per curve whatever the window, so memory doesn't follow it;
a wider window means wider points, each the average of its bucket rather than one instant
out of it. The hub passes the setting to every machine it manages.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
2026-09-20 19:07:55 -07:00
co-authored by Claude Opus 5
parent 885b050d02
commit 04dd98e2cd
13 changed files with 533 additions and 33 deletions
+31 -2
View File
@@ -27,12 +27,21 @@ type statusSensors struct {
cpuPrev map[string]cpuMark // container ID → CPU time at the previous read
hist map[string]*sensors.Series // numeric sensors only, statusHistory points each
acc map[string]*bucket // the point being built for each, when the window is wider than the read rate
}
// statusHistory is how many readings each status sensor keeps: at 15 s that is over an
// hour of container memory; at 10 min, two days of drive wear.
const statusHistory = 300
// A point being averaged, the same idea as the sampler's: these sources are read every 15 to
// 300 seconds, so a wide window keeps one point per stride rather than one per read.
type bucket struct {
start int64
sum float64
n float64
}
const (
servicesEvery = 30 * time.Second
containersEvery = 15 * time.Second
@@ -42,7 +51,7 @@ const (
)
func newStatusSensors(s *Server) *statusSensors {
p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}, hist: map[string]*sensors.Series{}}
p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}, hist: map[string]*sensors.Series{}, acc: map[string]*bucket{}}
go p.loop("services", servicesEvery, p.services)
go p.loop("containers", containersEvery, p.containers)
go p.loop("updates", updatesEvery, p.updates)
@@ -58,6 +67,8 @@ func (p *statusSensors) loop(name string, every time.Duration, read func(context
cancel()
p.mu.Lock()
p.by[name] = got
window := p.s.metrics.Window()
stride := window.Milliseconds() / statusHistory
live := map[string]bool{}
for _, x := range got {
v, ok := x.Value.(float64)
@@ -70,14 +81,32 @@ func (p *statusSensors) loop(name string, every time.Duration, read func(context
h = &sensors.Series{ID: x.ID, Unit: x.Unit}
p.hist[x.ID] = h
}
h.T, h.V = append(h.T, x.T), append(h.V, v)
b := p.acc[x.ID]
if b == nil {
b = &bucket{start: x.T}
p.acc[x.ID] = b
}
b.sum, b.n = b.sum+v, b.n+1
if x.T-b.start < stride && b.n > 1 {
continue // still filling this point
}
h.T, h.V = append(h.T, x.T), append(h.V, b.sum/b.n)
*b = bucket{start: x.T + 1}
if len(h.T) > statusHistory {
h.T, h.V = h.T[len(h.T)-statusHistory:], h.V[len(h.V)-statusHistory:]
}
if cut := x.T - window.Milliseconds(); h.T[0] < cut { // and nothing older than the window
i := 0
for i < len(h.T) && h.T[i] < cut {
i++
}
h.T, h.V = h.T[i:], h.V[i:]
}
}
for id := range p.hist { // a container that is gone takes its history with it
if strings.HasPrefix(id, prefixOf[name]) && !live[id] && prefixOf[name] != "" {
delete(p.hist, id)
delete(p.acc, id)
}
}
p.mu.Unlock()