diff --git a/BUGS.md b/BUGS.md index b61a82e..7e951b3 100644 --- a/BUGS.md +++ b/BUGS.md @@ -478,6 +478,14 @@ is 11px wide" at 1392); run after, nothing, in both browsers. 9. ✅ **Idle curves vanished**: a core at 10 % rounded to no dots at all in its mini graph. Anything above zero lights the bottom row, which is also btop's rule. +10. ✅ **Found by the deploy of fix 2 itself: telling the machines once is not enough.** The hub + restarted first and told three nodes that did not yet have the endpoint (404); they then + updated and carried on at 1 s. The same would happen to any node that is off or + restarting when a setting changes. The hub now *keeps* the fleet in step: fifteen seconds + after it starts and every two minutes after, it asks each machine for its sampling rate + and history window and corrects only what differs (logged as `fleet: grid had ms=1000, + now 250`). + ## Skins ✅ Top now wears the skin. The ground is the skin's terminal background and every curve and diff --git a/internal/api/server.go b/internal/api/server.go index 52bc5c2..183f9cd 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -113,12 +113,9 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me RefreshMs float64 `json:"refreshMs"` } if json.Unmarshal(uiRaw, &ui) == nil && ui.RefreshMs >= 100 { - go s.setSampler(int(ui.RefreshMs), true) + go s.setSampler(int(ui.RefreshMs), false) } - } - // A node that restarted, or was added since, has its own idea of the window: tell them all. - if histSec > 0 && nm != nil { - go s.pushHistoryWindow(histSec) + go s.keepFleetInStep() } m := http.NewServeMux() @@ -1955,6 +1952,55 @@ func (s *Server) setSampler(ms int, fanOut bool) { } } +// keepFleetInStep makes the two fleet-wide settings — the sampling rate and the history window — +// true of every machine, and keeps them true. Telling the machines once, when a setting +// changes, is not enough: a node that was off, restarting or mid-update at that moment never +// hears it (the first deploy of this very feature told three nodes that did not yet know the +// endpoint). So the hub asks each machine what it has, now and every two minutes, and corrects +// only what differs. +func (s *Server) keepFleetInStep() { + for first := true; ; first = false { + if first { + time.Sleep(15 * time.Second) // let a fleet that restarted together come up + } else { + time.Sleep(2 * time.Minute) + } + var ui struct { + RefreshMs float64 `json:"refreshMs"` + } + var histSec int + s.store.View(func(st *store.State) { _ = json.Unmarshal(st.UI, &ui); histSec = st.HistorySec }) + want := map[string][2]any{} // path → {field, value} + if ui.RefreshMs >= 100 { + ms := int(ui.RefreshMs) + if hi := int(metrics.MaxInterval.Milliseconds()); ms > hi { + ms = hi + } + want["/api/prefs/sampler"] = [2]any{"ms", ms} + } + if histSec > 0 { + want["/api/prefs/history"] = [2]any{"seconds", histSec} + } + hdr := http.Header{"X-Hamlab-From-Hub": []string{"1"}, "Content-Type": []string{"application/json"}} + for _, name := range s.nodes.Names() { + for path, fv := range want { + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + code, b, err := s.nodes.Do(ctx, name, "GET", path, nil, hdr) + var have map[string]any + if err == nil && code == 200 && json.Unmarshal(b, &have) == nil { + if got, _ := have[fv[0].(string)].(float64); int(got) != fv[1].(int) { + body, _ := json.Marshal(map[string]any{fv[0].(string): fv[1]}) + if c, _, e := s.nodes.Do(ctx, name, "PUT", path, body, hdr); e == nil && c < 400 { + log.Printf("fleet: %s had %s=%v, now %v", name, fv[0], got, fv[1]) + } + } + } + cancel() + } + } + } +} + func (s *Server) getSampler(w http.ResponseWriter, _ *http.Request) { jsonOK(w, 200, map[string]any{"ms": s.metrics.Interval().Milliseconds(), "min_ms": metrics.MinInterval.Milliseconds(), "max_ms": metrics.MaxInterval.Milliseconds()}) }