From 192504ead30bad602263591856d8b6f1fba761d5 Mon Sep 17 00:00:00 2001 From: Mitchell-Johnson Date: Thu, 24 Sep 2026 20:11:36 -0700 Subject: [PATCH] docker: the eight things Dockhand did that Ham Lab did not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dockhand is being retired, so the Docker tab has to cover what it actually earned its keep doing. Eight pieces, all per container or per machine: - Stats: CPU, memory and IO over time, from sensors already collected. - Health: the container's own healthcheck, with the last few probe outputs — which is where a failing check says why. - Events: `docker events` keeps a minute of backlog and nothing more, so Ham Lab follows the stream and keeps a ring of its own. A restart at 4am is still there at 9. - Shell: `docker exec` into a container, in the terminal already here. - Limits: memory, CPU and restart policy changed live, without hand-editing the compose file for a number you want to try. - Updates: what the registry has that this machine does not, for every image at once rather than one stack at a time. - The same across every machine, asked of the hub — which is the only thing that can reach them all. - Schedules: a real line in /etc/cron.d/hamlab-docker calling the binary, in the same shape as the ZFS snapshot schedules. Dockhand updated DNS and ingress nightly from a ticker inside a web app; a cron line keeps running when the console does not, and is visible to anyone reading the machine the normal way. internal/compose is new and shared, so a deploy from the console and a deploy from cron invoke compose identically — including the case where compose has to run inside the orchestrator's own container, where the paths are that container's and not the host's. Co-Authored-By: Claude Opus 5 --- cmd/hamlab/main.go | 49 +++++- internal/api/docker.go | 4 + internal/api/docker_events.go | 4 +- internal/api/dockersched.go | 322 ++++++++++++++++++++++++++++++++++ internal/api/server.go | 6 + internal/api/stacks.go | 161 +++++++++++------ internal/compose/compose.go | 50 ++++++ internal/storage/snapshots.go | 21 ++- internal/store/store.go | 3 +- web/src/Docker.tsx | 218 ++++++++++++++++++++++- web/src/api.ts | 11 ++ web/src/styles.css | 3 + 12 files changed, 783 insertions(+), 69 deletions(-) create mode 100644 internal/api/dockersched.go create mode 100644 internal/compose/compose.go diff --git a/cmd/hamlab/main.go b/cmd/hamlab/main.go index f642c36..1708d52 100644 --- a/cmd/hamlab/main.go +++ b/cmd/hamlab/main.go @@ -32,8 +32,8 @@ import ( "hamlab/internal/files" "hamlab/internal/metrics" "hamlab/internal/nodes" - "hamlab/internal/services" "hamlab/internal/priv" + "hamlab/internal/services" "hamlab/internal/setup" "hamlab/internal/space" "hamlab/internal/storage" @@ -62,6 +62,7 @@ func main() { passwd [user] set a user's password (makes the account if it is new; reads stdin when piped) users list the accounts zfs-snapshot run one ZFS snapshot schedule (what its cron line calls) + docker-update run one image-update schedule (what its cron line calls) user-rm remove an account and sign it out everywhere check verify the config and state load, then exit version @@ -132,6 +133,10 @@ func main() { if err := passwd(cfg, flag.Arg(1)); err != nil { log.Fatal(err) } + case "docker-update": + if err := dockerUpdateJob(cfg, flag.Arg(1)); err != nil { + log.Fatal(err) + } case "zfs-snapshot": if err := zfsSnapshot(cfg, flag.Arg(1)); err != nil { log.Fatal(err) @@ -185,6 +190,48 @@ func main() { // migrateUsers gives a pre-users config its first account, named after this OS user, so // the CLI sees the same accounts the daemon will. +// dockerUpdateJob pulls a stack's images, and deploys them when the schedule says to. This +// is what the cron line Ham Lab writes actually calls, so image updates keep happening +// whether or not the console is up. +func dockerUpdateJob(cfg *config.Config, id string) error { + if strings.TrimSpace(id) == "" { + return errors.New("usage: hamlab docker-update ") + } + st, err := store.Open(filepath.Join(cfg.DataDir(), "state.json")) + if err != nil { + return err + } + var raw json.RawMessage + st.View(func(s *store.State) { raw = s.DockerJobs }) + var d struct { + Jobs []api.UpdateJob `json:"jobs"` + } + if len(raw) > 0 { + if err := json.Unmarshal(raw, &d); err != nil { + return fmt.Errorf("reading the schedules: %w", err) + } + } + for _, j := range d.Jobs { + if j.ID != id { + continue + } + // A pull that never finishes must not leave a cron job resident until the next reboot. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) + defer cancel() + out, runErr := api.RunUpdate(ctx, j, api.DockerTool()) + rec := storage.RunRecord{At: time.Now(), Taken: j.Stack} + if runErr != nil { + rec.Error = runErr.Error() + } + if err := storage.WriteRun(cfg.DataDir(), "docker-"+id, rec); err != nil { + fmt.Fprintln(os.Stderr, "recording the run:", err) + } + fmt.Print(out) + return runErr + } + return fmt.Errorf("no image-update schedule %q — it may have been removed; this cron line is stale", id) +} + // zfsSnapshot runs one snapshot schedule: take it, then expire what has aged out. This is // what the cron line Ham Lab writes actually calls, so a schedule keeps running whether or // not the console is up — and keeps running if Ham Lab is never opened again. diff --git a/internal/api/docker.go b/internal/api/docker.go index cfc8e4a..47886b5 100644 --- a/internal/api/docker.go +++ b/internal/api/docker.go @@ -49,6 +49,10 @@ func dockerTool() string { } // dockerRun runs the tool as this user, and again as root if the socket said no. +// DockerTool is whichever of docker or podman this machine has, for callers outside the +// package — the scheduled updates the CLI runs. +func DockerTool() string { return dockerTool() } + func dockerRun(ctx context.Context, args ...string) ([]byte, error) { tool := dockerTool() if tool == "" { diff --git a/internal/api/docker_events.go b/internal/api/docker_events.go index f193f45..30c895e 100644 --- a/internal/api/docker_events.go +++ b/internal/api/docker_events.go @@ -31,7 +31,7 @@ import ( const ( eventsKept = 4000 - eventsSaveEeem = 30 * time.Second + eventsSaveEvery = 30 * time.Second ) // ContainerEvent is one thing that happened to one container. @@ -118,7 +118,7 @@ func (e *eventLog) save() { func (s *Server) watchDockerEvents(ctx context.Context) { s.events.load(filepath.Join(s.cfg.DataDir(), "docker-events.json")) go func() { // a periodic flush, so a crash loses at most half a minute of history - t := time.NewTicker(eventsSaveEeem) + t := time.NewTicker(eventsSaveEvery) defer t.Stop() for { select { diff --git a/internal/api/dockersched.go b/internal/api/dockersched.go new file mode 100644 index 0000000..7597139 --- /dev/null +++ b/internal/api/dockersched.go @@ -0,0 +1,322 @@ +package api + +// Scheduled image updates. +// +// Dockhand kept three containers up to date nightly — unbound, nginx and pihole, which is +// DNS and ingress — and that is the one thing it does that Ham Lab did not. This replaces +// it, and deliberately in the same shape as the ZFS snapshot schedules: a real line in a +// cron file Ham Lab owns, calling the binary, rather than a ticker inside the daemon. +// Updating an image is not something that should stop happening because a web console is +// down, and a cron line is visible to anyone looking at the machine the normal way. +// +// A schedule stores everything the run needs — project, compose files, how compose has to +// be invoked for that stack — so the cron job is self-contained and does not have to +// rediscover the world before it can act. + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "hamlab/internal/compose" + "hamlab/internal/cron" + "hamlab/internal/storage" + "hamlab/internal/store" +) + +// UpdateJob is a standing instruction to pull a stack's images, and optionally deploy them. +type UpdateJob struct { + ID string `json:"id"` + Name string `json:"name"` + Stack string `json:"stack"` + Machine string `json:"machine,omitempty"` // for display; the job runs where it was made + // Services narrows it to some of the stack's services; empty is all of them. + Services []string `json:"services,omitempty"` + Cron string `json:"cron"` + // Apply says whether a newer image is deployed or only fetched. Check-only still pulls, + // so the Docker tab shows what is behind without a registry round trip. + Apply bool `json:"apply"` + Enabled bool `json:"enabled"` + + // How to run compose for this stack, captured when the schedule was made. + Spec composeSpec `json:"spec"` +} + +type composeSpec struct { + Project string `json:"project"` + Files []string `json:"files"` + Via string `json:"via,omitempty"` + Remote string `json:"remote,omitempty"` +} + +func (c composeSpec) toSpec(tool string) compose.Spec { + return compose.Spec{Project: c.Project, Files: c.Files, Via: c.Via, Remote: c.Remote, Tool: tool} +} + +const dockerCronFile = "/etc/cron.d/hamlab-docker" + +type dockerSchedState struct { + Jobs []UpdateJob `json:"jobs"` +} + +func (s *Server) dsLoad() dockerSchedState { + var raw json.RawMessage + s.store.View(func(st *store.State) { raw = st.DockerJobs }) + var d dockerSchedState + if len(raw) > 0 { + _ = json.Unmarshal(raw, &d) + } + if d.Jobs == nil { + d.Jobs = []UpdateJob{} + } + return d +} + +func (s *Server) dsSave(d dockerSchedState) error { + b, err := json.Marshal(d) + if err != nil { + return err + } + return s.store.Update(func(st *store.State) { st.DockerJobs = b }) +} + +// dsCronText renders the schedules as cron lines. Same shape as the ZFS file: Ham Lab owns +// it whole and rewrites it on every change. +func dsCronText(jobs []UpdateJob, user, exe, config string) (string, error) { + var b strings.Builder + b.WriteString("# Ham Lab image-update schedules — written by the Docker tab, rewritten whole on every change.\n") + b.WriteString("# Edit them there, not here: anything added by hand is lost the next time a schedule changes.\n") + b.WriteString("SHELL=/bin/sh\nPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n") + for i := range jobs { + j := &jobs[i] + if !j.Enabled { + b.WriteString("# " + j.Name + " — paused\n") + continue + } + expr := strings.Join(strings.Fields(j.Cron), " ") + if err := cron.ValidSchedule(expr); err != nil { + return "", err + } + what := "pull only" + if j.Apply { + what = "pull and deploy" + } + b.WriteString("# " + j.Name + " — " + j.Stack + ", " + what + "\n") + b.WriteString(expr + " " + user + " " + exe + " -config " + config + " docker-update " + j.ID + "\n\n") + } + return b.String(), nil +} + +func (s *Server) dsSyncCron(ctx context.Context, d dockerSchedState) error { + live := false + for i := range d.Jobs { + if d.Jobs[i].Enabled { + live = true + } + } + if !live { + return s.storage.RemoveRootFile(ctx, dockerCronFile) + } + exe, err := os.Executable() + if err != nil { + return err + } + exe, _ = filepath.EvalSymlinks(exe) + text, err := dsCronText(d.Jobs, s.user, exe, s.cfg.Path()) + if err != nil { + return err + } + return s.storage.WriteRootFile(ctx, dockerCronFile, text) +} + +// RunUpdate pulls, and deploys when the schedule says to. Shared with the CLI so the button +// and the cron line cannot drift apart. +func RunUpdate(ctx context.Context, j UpdateJob, tool string) (string, error) { + spec := j.Spec.toSpec(tool) + var log strings.Builder + run := func(args ...string) error { + cmd := compose.Cmd(ctx, spec, args...) + out, err := cmd.CombinedOutput() + log.Write(out) + return err + } + args := append([]string{"pull"}, j.Services...) + if err := run(args...); err != nil { + return log.String(), err + } + if !j.Apply { + return log.String(), nil + } + up := append([]string{"up", "-d"}, j.Services...) + if err := run(up...); err != nil { + return log.String(), err + } + return log.String(), nil +} + +// ---- endpoints ---- + +func (s *Server) dsList(w http.ResponseWriter, _ *http.Request) { + d := s.dsLoad() + now := time.Now() + out := make([]map[string]any, 0, len(d.Jobs)) + for _, j := range d.Jobs { + m := map[string]any{"job": j, "last": storage.ReadRun(s.cfg.DataDir(), "docker-"+j.ID)} + if j.Enabled { + if n, ok := cron.NextRun(strings.Join(strings.Fields(j.Cron), " "), now); ok { + m["next"] = n + } + } + out = append(out, m) + } + jsonOK(w, 200, map[string]any{"jobs": out, "file": dockerCronFile}) +} + +func (s *Server) dsPut(w http.ResponseWriter, r *http.Request) { + var j UpdateJob + if err := json.NewDecoder(io.LimitReader(r.Body, 32<<10)).Decode(&j); err != nil { + jsonErr(w, 400, "bad request") + return + } + j.Cron = strings.Join(strings.Fields(j.Cron), " ") + if err := cron.ValidSchedule(j.Cron); err != nil { + jsonErr(w, 400, "schedule: "+err.Error()) + return + } + if strings.TrimSpace(j.Stack) == "" { + jsonErr(w, 400, "which stack?") + return + } + // resolve the stack now, so the cron job never has to + ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + defer cancel() + stacks, err := s.findStacks(ctx) + if err != nil { + jsonErr(w, 500, err.Error()) + return + } + found := false + for _, st := range stacks { + if st.Name == j.Stack { + // seenFiles is the list as compose itself sees it: host paths normally, the + // orchestrator container's own paths when compose runs inside one. Getting this + // backwards gives a cron line that works from the console and fails at 3am. + j.Spec = composeSpec{Project: st.Name, Files: seenFiles(st, -1, ""), Via: st.Via, Remote: st.Remote} + if len(j.Spec.Files) == 0 { + jsonErr(w, 409, "that stack's compose file isn't anywhere this console can reach") + return + } + found = true + } + } + if !found { + jsonErr(w, 404, "no stack called "+j.Stack+" on this machine") + return + } + if strings.TrimSpace(j.Name) == "" { + j.Name = j.Stack + } + d := s.dsLoad() + replaced := false + for i := range d.Jobs { + if d.Jobs[i].ID == j.ID && j.ID != "" { + d.Jobs[i] = j + replaced = true + } + } + if !replaced { + b := make([]byte, 6) + if _, err := rand.Read(b); err != nil { + jsonErr(w, 500, err.Error()) + return + } + j.ID = hex.EncodeToString(b) + d.Jobs = append(d.Jobs, j) + } + if err := s.dsSave(d); err != nil { + jsonErr(w, 500, err.Error()) + return + } + if err := s.dsSyncCron(r.Context(), d); err != nil { + jsonErr(w, 500, "saved, but writing "+dockerCronFile+" failed: "+err.Error()) + return + } + log.Printf("docker schedule %q saved by %s (%s, %q, %s)", j.Name, clientIP(r), j.Stack, j.Cron, map[bool]string{true: "pull and deploy", false: "pull only"}[j.Apply]) + jsonOK(w, 200, j) +} + +func (s *Server) dsDelete(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + d := s.dsLoad() + kept := d.Jobs[:0] + found := false + for _, j := range d.Jobs { + if j.ID == id { + found = true + continue + } + kept = append(kept, j) + } + if !found { + jsonErr(w, 404, "no such schedule") + return + } + d.Jobs = kept + if err := s.dsSave(d); err != nil { + jsonErr(w, 500, err.Error()) + return + } + if err := s.dsSyncCron(r.Context(), d); err != nil { + jsonErr(w, 500, "removed, but writing "+dockerCronFile+" failed: "+err.Error()) + return + } + storage.ForgetRun(s.cfg.DataDir(), "docker-"+id) + log.Printf("docker schedule %s removed by %s", id, clientIP(r)) + w.WriteHeader(204) +} + +func (s *Server) dsRun(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + var job UpdateJob + found := false + for _, j := range s.dsLoad().Jobs { + if j.ID == id { + job, found = j, true + } + } + if !found { + jsonErr(w, 404, "no such schedule") + return + } + // Outliving the request: a pull the browser gave up on still finishes, but not forever. + ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 30*time.Minute) + defer cancel() + out, err := RunUpdate(ctx, job, dockerTool()) + rec := storage.RunRecord{At: time.Now(), Taken: job.Stack} + if err != nil { + rec.Error = err.Error() + } + _ = storage.WriteRun(s.cfg.DataDir(), "docker-"+id, rec) + stackUpdatesForget("/" + job.Stack) + if err != nil { + jsonErr(w, 500, err.Error()+"\n"+tail(out, 2000)) + return + } + log.Printf("docker schedule %q run by %s", job.Name, clientIP(r)) + jsonOK(w, 200, map[string]string{"output": tail(out, 4000)}) +} + +func tail(s string, n int) string { + if len(s) <= n { + return s + } + return "…" + s[len(s)-n:] +} diff --git a/internal/api/server.go b/internal/api/server.go index 1dce958..d7db17c 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -192,6 +192,12 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me m.HandleFunc("POST /api/docker/stacks/{name}/{action}", s.auth(s.stackAction)) m.HandleFunc("GET /api/docker/stacks/{name}/diff", s.auth(s.stackDiff)) m.HandleFunc("GET /api/docker/stacks/{name}/updates", s.auth(s.stackUpdates)) + m.HandleFunc("GET /api/docker/updates", s.auth(s.machineUpdates)) + m.HandleFunc("GET /api/docker/schedules", s.auth(s.dsList)) + m.HandleFunc("PUT /api/docker/schedules", s.auth(s.dsPut)) + m.HandleFunc("DELETE /api/docker/schedules/{id}", s.auth(s.dsDelete)) + m.HandleFunc("POST /api/docker/schedules/{id}/run", s.auth(s.dsRun)) + m.HandleFunc("GET /api/nodes/{node}/docker/updates", s.auth(s.machineUpdates)) // The same, asked of a node through this hub: the hub answers for the stacks whose files // it holds (deployed to the node over tcp) and passes everything else to the node. m.HandleFunc("GET /api/nodes/{node}/docker/stacks", s.auth(s.stacksList)) diff --git a/internal/api/stacks.go b/internal/api/stacks.go index d009d73..ce141f0 100644 --- a/internal/api/stacks.go +++ b/internal/api/stacks.go @@ -39,6 +39,7 @@ import ( "sync" "time" + "hamlab/internal/compose" "hamlab/internal/priv" ) @@ -511,24 +512,13 @@ func (s *Server) stackFile(w http.ResponseWriter, r *http.Request) { // composeCmd is `docker compose …` for this stack, run where its files make sense and // pointed at the Docker that runs its containers. files are given as compose will see them. func composeCmd(ctx context.Context, st stack, files []string, args ...string) *exec.Cmd { - tool := dockerTool() - base := []string{"compose", "-p", st.Name} - for _, f := range files { - base = append(base, "-f", f) - } - if st.Via != "" { - pre := []string{"exec"} - if st.Remote != "" { - pre = append(pre, "-e", "DOCKER_HOST="+st.Remote) - } - return exec.CommandContext(ctx, tool, append(append(append(pre, st.Via, "docker"), base...), args...)...) - } - cmd := exec.CommandContext(ctx, tool, append(base, args...)...) - cmd.Dir = filepath.Dir(files[0]) - if st.Remote != "" { - cmd.Env = append(os.Environ(), "DOCKER_HOST="+st.Remote) - } - return cmd + return compose.Cmd(ctx, specOf(st, files), args...) +} + +// specOf is a stack as the compose runner needs it — and as a scheduled update stores it, +// so the cron job and the console invoke compose identically. +func specOf(st stack, files []string) compose.Spec { + return compose.Spec{Project: st.Name, Files: files, Via: st.Via, Remote: st.Remote, Tool: dockerTool()} } // seenFiles is the stack's file list as compose runs it, with file i replaced by a candidate. @@ -788,6 +778,103 @@ var ( func stackUpdatesForget(key string) { updMu.Lock(); delete(updCache, key); updMu.Unlock() } +// checkImages asks the registry whether each image is still the one we hold. remote is a +// DOCKER_HOST for a machine whose images live elsewhere, "" for this one. Each image is one +// registry request, which Docker Hub counts — so this is only ever called when asked for. +func checkImages(ctx context.Context, images []string, remote string) []imageCheck { + local := func(args ...string) ([]byte, error) { // the Docker that holds the images + if remote != "" { + args = append([]string{"-H", remote}, args...) + } + return dockerRun(ctx, args...) + } + res := []imageCheck{} + for _, img := range images { + ic := imageCheck{Image: img, State: "unknown"} + out, err := local("image", "inspect", "-f", "{{json .RepoDigests}}", img) + var have []string + if err != nil || json.Unmarshal(bytes.TrimSpace(out), &have) != nil || len(have) == 0 { + ic.Detail = "built or loaded here, not pulled from a registry" + res = append(res, ic) + continue + } + rem, err := dockerRun(ctx, "buildx", "imagetools", "inspect", img, "--format", "{{.Manifest.Digest}}") + digest := strings.TrimSpace(string(rem)) + if i := strings.LastIndex(digest, "\n"); i >= 0 { + digest = digest[i+1:] + } + if err != nil || !strings.HasPrefix(digest, "sha256:") { + ic.Detail = "the registry didn't answer" + if err != nil { + ic.Detail += ": " + err.Error() + } + res = append(res, ic) + continue + } + ic.State = "update" + for _, h := range have { + if strings.HasSuffix(h, "@"+digest) { + ic.State = "current" + } + } + res = append(res, ic) + } + return res +} + +// machineUpdates checks every image on this machine in one pass, so the container list can +// be coloured and the fleet can be asked "what is behind?" without walking stack by stack. +func (s *Server) machineUpdates(w http.ResponseWriter, r *http.Request) { + node := r.PathValue("node") + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute) + defer cancel() + stacks, err := s.findStacks(ctx) + if node != "" { + stacks, err = s.hubStacks(ctx, node) + } + if err != nil { + jsonErr(w, 500, err.Error()) + return + } + // one entry per image, however many stacks use it + type want struct{ remote string } + imgs := map[string]want{} + byImage := map[string][]string{} // image -> stacks + for _, st := range stacks { + for _, img := range st.Images { + imgs[img] = want{remote: st.Remote} + byImage[img] = append(byImage[img], st.Name) + } + } + refresh := r.URL.Query().Get("refresh") == "1" + key := "machine:" + node + updMu.Lock() + c, ok := updCache[key] + updMu.Unlock() + if ok && time.Since(c.at) < 6*time.Hour && !refresh { + jsonOK(w, 200, map[string]any{"images": c.res, "checked": c.at.UnixMilli(), "stacks": byImage}) + return + } + if !refresh { + jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0, "stacks": byImage}) + return + } + var list []string + remote := "" + for img, wnt := range imgs { + list = append(list, img) + if wnt.remote != "" { + remote = wnt.remote + } + } + sort.Strings(list) + res := checkImages(ctx, list, remote) + updMu.Lock() + updCache[key] = updAnswer{time.Now(), res} + updMu.Unlock() + jsonOK(w, 200, map[string]any{"images": res, "checked": time.Now().UnixMilli(), "stacks": byImage}) +} + // stackUpdates asks each image's registry for the digest its tag points at now and compares // it with the digest that was pulled. It is asked for, never polled: every answer is a // registry request, and Docker Hub counts those. Answers are kept for six hours. @@ -816,43 +903,7 @@ func (s *Server) stackUpdates(w http.ResponseWriter, r *http.Request) { jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0}) return } - local := func(args ...string) ([]byte, error) { // the Docker that holds the images - if st.Remote != "" { - args = append([]string{"-H", st.Remote}, args...) - } - return dockerRun(ctx, args...) - } - res := []imageCheck{} - for _, img := range st.Images { - ic := imageCheck{Image: img, State: "unknown"} - out, err := local("image", "inspect", "-f", "{{json .RepoDigests}}", img) - var have []string - if err != nil || json.Unmarshal(bytes.TrimSpace(out), &have) != nil || len(have) == 0 { - ic.Detail = "built or loaded here, not pulled from a registry" - res = append(res, ic) - continue - } - remote, err := dockerRun(ctx, "buildx", "imagetools", "inspect", img, "--format", "{{.Manifest.Digest}}") - digest := strings.TrimSpace(string(remote)) - if i := strings.LastIndex(digest, "\n"); i >= 0 { - digest = digest[i+1:] - } - if err != nil || !strings.HasPrefix(digest, "sha256:") { - ic.Detail = "the registry didn't answer" - if err != nil { - ic.Detail += ": " + err.Error() - } - res = append(res, ic) - continue - } - ic.State = "update" - for _, h := range have { - if strings.HasSuffix(h, "@"+digest) { - ic.State = "current" - } - } - res = append(res, ic) - } + res := checkImages(ctx, st.Images, st.Remote) updMu.Lock() updCache[key] = updAnswer{time.Now(), res} updMu.Unlock() diff --git a/internal/compose/compose.go b/internal/compose/compose.go new file mode 100644 index 0000000..7d5eed5 --- /dev/null +++ b/internal/compose/compose.go @@ -0,0 +1,50 @@ +// Package compose runs docker compose the way a stack was deployed — which is not always +// simply "on this host". A stack deployed by an orchestrator running inside its own +// container records paths in that container's filesystem, so compose has to run in there; +// a stack deployed to another machine over tcp needs DOCKER_HOST pointed at it. Both the +// console and the scheduled updates it writes must invoke it identically, or a redeploy +// from one would not match the other's labels. +package compose + +import ( + "context" + "os" + "os/exec" + "path/filepath" +) + +// Spec is everything needed to run compose for one stack. +type Spec struct { + Project string // -p + Files []string // -f, in order + Via string // a container to run compose inside; "" is this host + Remote string // a DOCKER_HOST; "" is this machine's Docker + Tool string // "docker" or "podman" +} + +// Cmd builds the command. args are compose's own, e.g. "pull" or "up", "-d". +func Cmd(ctx context.Context, s Spec, args ...string) *exec.Cmd { + tool := s.Tool + if tool == "" { + tool = "docker" + } + base := []string{"compose", "-p", s.Project} + for _, f := range s.Files { + base = append(base, "-f", f) + } + if s.Via != "" { + pre := []string{"exec"} + if s.Remote != "" { + pre = append(pre, "-e", "DOCKER_HOST="+s.Remote) + } + return exec.CommandContext(ctx, tool, append(append(append(pre, s.Via, "docker"), base...), args...)...) + } + cmd := exec.CommandContext(ctx, tool, append(base, args...)...) + if len(s.Files) > 0 { + cmd.Dir = filepath.Dir(s.Files[0]) + } + if s.Remote != "" { + cmd.Env = append(os.Environ(), "DOCKER_HOST="+s.Remote) + } + return cmd +} diff --git a/internal/storage/snapshots.go b/internal/storage/snapshots.go index 66cce28..9f5768e 100644 --- a/internal/storage/snapshots.go +++ b/internal/storage/snapshots.go @@ -452,8 +452,25 @@ func (m *Manager) WriteCronFile(ctx context.Context, policies []Policy, user, ex return m.writeRoot(ctx, CronFile, text) } -// writeRoot puts text at a root-owned path, through the same privilege escalation every -// other write here uses. +// WriteRootFile puts text at a root-owned path, through the same privilege escalation every +// other write here uses. Shared by the snapshot schedules and the image-update schedules, +// which both own a file in /etc/cron.d. +func (m *Manager) WriteRootFile(ctx context.Context, path, text string) error { + return m.writeRoot(ctx, path, text) +} + +// RemoveRootFile deletes one, for when the last schedule of a kind goes away. +func (m *Manager) RemoveRootFile(ctx context.Context, path string) error { + out, code, err := m.run(ctx, "rm", "-f", path) + if err != nil { + return err + } + if code != 0 { + return errors.New(strings.TrimSpace(string(out))) + } + return nil +} + func (m *Manager) writeRoot(ctx context.Context, path, text string) error { args := []string{"tee", path} if m.sudo { diff --git a/internal/store/store.go b/internal/store/store.go index 958e2ac..5e20ec6 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -52,7 +52,8 @@ type State struct { // account. Kept so an old state file still reads. UI json.RawMessage `json:"ui,omitempty"` Alerts json.RawMessage `json:"alerts,omitempty"` - ZFS json.RawMessage `json:"zfs,omitempty"` // snapshot schedules (internal/api/zfs.go) // rules, channels and recent events (internal/api/alerts.go) + ZFS json.RawMessage `json:"zfs,omitempty"` // snapshot schedules (internal/api/zfs.go) + DockerJobs json.RawMessage `json:"docker_jobs,omitempty"` // image-update schedules (internal/api/dockersched.go) // rules, channels and recent events (internal/api/alerts.go) SamplerMs int `json:"sampler_ms,omitempty"` // how often this machine samples; the hub keeps every machine on the console's refresh rate HistorySec int `json:"history_sec,omitempty"` // how far back the graphs reach; the hub keeps every machine on the same one } diff --git a/web/src/Docker.tsx b/web/src/Docker.tsx index cec19bf..2a2cc93 100644 --- a/web/src/Docker.tsx +++ b/web/src/Docker.tsx @@ -2,7 +2,7 @@ // logs that follow, and inspect — then images, volumes and networks. Live CPU and memory // come from the same container sensors everything else uses. import { FormEvent, useCallback, useEffect, useMemo, useRef, useState } from 'react' -import { api, DockerContainer, DockerEvent, DockerHealth, DockerInfo, ImageCheck, Stack, StackDiff } from './api' +import { api, apiPath, DockerContainer, DockerEvent, DockerHealth, DockerInfo, ImageCheck, MachineStatus, MachineUpdates, Stack, StackDiff, UpdateJob, UpdateJobRow } from './api' import TaskOutput from './TaskOutput' import StackNew, { DupFrom } from './StackNew' import { Sensor, fetchHistories, fmtValue, useSensorStream } from './sensors' @@ -12,7 +12,7 @@ import Sparkline from './Sparkline' import { fmtBytes } from './fmt' import { askText, askYes, Btn, CloseBtn, Loading, setUnsaved, toast, useEscape } from './ui' -type Tab = 'containers' | 'stacks' | 'images' | 'volumes' | 'networks' +type Tab = 'containers' | 'stacks' | 'updates' | 'images' | 'volumes' | 'networks' const dot = (c: DockerContainer) => (c.state !== 'running' ? (c.state === 'exited' || c.state === 'dead' ? 'bad' : 'warn') : c.health === 'unhealthy' ? 'bad' : c.health === 'starting' ? 'warn' : 'ok') const ports = (p: string) => [...new Set(p.split(', ').map((x) => x.replace(/^\[?::\]?:/, '').replace(/^0\.0\.0\.0:/, '').replace(/^[\d.]+:/, '')).filter((x) => x.includes('->')))].join(' ') @@ -368,6 +368,12 @@ export default function Docker() { // what a new stack could join, and what it could collide with const nets = useMemo(() => (d?.networks ?? []).map((n) => String(n.Name)).filter((n) => !['bridge', 'host', 'none'].includes(n)).sort(), [d]) const taken = useMemo(() => { const ports: Record = {}; for (const c of d?.containers ?? []) for (const m of (c.ports ?? '').matchAll(/:(\d+)->/g)) ports[m[1]] = c.name; return { names: (d?.containers ?? []).map((c) => c.name), ports } }, [d]) + // What the registry last said about each image, so a container can be marked without + // asking again. Only ever read from the cache here — checking is a deliberate act. + const [upd, setUpd] = useState(null) + useEffect(() => { api.machineUpdates().then(setUpd).catch(() => setUpd(null)) }, []) + const imgState = useMemo(() => Object.fromEntries((upd?.images ?? []).map((i) => [i.image, i.state])), [upd]) + const behind = (upd?.images ?? []).filter((i) => i.state === 'update').length if (!d) return
{err ?
{err === 'not found' ? "This machine's Ham Lab is older than the Docker tab. Update it from Machines." : err}
: }
if (!d.available) return
{d.message}
const selStack = sel.startsWith('stack:') ? sel.slice(6) : '' @@ -375,12 +381,12 @@ export default function Docker() { const shownTab: Tab = selStack ? 'stacks' : tab const up = d.containers.filter((c) => c.state === 'running').length const sid = (c: DockerContainer, k: string) => sensors[`container.${c.name.toLowerCase().replace(/[^a-z0-9-]+/g, '_').replace(/^_|_$/g, '')}.${k}`] - const cols: Record, [string, string][]> = { images: [['Repository', 'Repository'], ['Tag', 'Tag'], ['Size', 'Size'], ['CreatedSince', 'Created']], volumes: [['Name', 'Name'], ['Driver', 'Driver'], ['Mountpoint', 'Mountpoint']], networks: [['Name', 'Name'], ['Driver', 'Driver'], ['Scope', 'Scope']] } + const cols: Record, [string, string][]> = { images: [['Repository', 'Repository'], ['Tag', 'Tag'], ['Size', 'Size'], ['CreatedSince', 'Created']], volumes: [['Name', 'Name'], ['Driver', 'Driver'], ['Mountpoint', 'Mountpoint']], networks: [['Name', 'Name'], ['Driver', 'Driver'], ['Scope', 'Scope']] } return (
-
{(['containers', 'stacks', 'images', 'volumes', 'networks'] as Tab[]).map((t) => )}
+
{(['containers', 'stacks', 'updates', 'images', 'volumes', 'networks'] as Tab[]).map((t) => )}
{(shownTab === 'containers' || shownTab === 'stacks') &&
} {shownTab === 'images' &&
{ e.preventDefault(); try { setPullTask((await api.pullImage(pull.trim())).id) } catch (er) { toast(er instanceof Error ? er.message : 'failed', 'err') } }}> setPull(e.target.value)} placeholder="pull an image — nginx:alpine, ghcr.io/owner/name:tag" spellCheck={false} autoCapitalize="off" autoCorrect="off" /> @@ -407,7 +413,7 @@ export default function Docker() { {list.map((c) => (
setSel(c.name)}> -
{c.name}
{c.image}
+
{c.name}
{c.image}
{c.state === 'running' ? <>{fmtValue(sid(c, 'mem')?.value ?? c.mem, 'B')}
{fmtValue(sid(c, 'cpu')?.value, '%')} : c.state}
))} @@ -425,16 +431,212 @@ export default function Docker() { return {anon ? <>{name.slice(0, 12)}… anonymous : name}{users.length ? users.join(', ') : nothing}{users.length ? in use : } })} {d.volumes.length === 0 && No volumes.} )} - {shownTab !== 'containers' && shownTab !== 'stacks' && shownTab !== 'volumes' && ( - {cols[shownTab as Exclude].map(([, h]) => )}{shownTab === 'images' && - {d[shownTab as Exclude].map((row, i) => {cols[shownTab as Exclude].map(([k]) => )}{shownTab === 'images' && })}
{h}}
{String(row[k] ?? '')}{row.in_use ? in use : { const ref = row.Tag && row.Tag !== '' ? `${row.Repository}:${row.Tag}` : String(row.ID); try { await api.removeImage(ref); toast(`${ref} removed`); load() } catch (er) { toast(er instanceof Error ? er.message : 'failed', 'err') } }}>✕}
+ {shownTab !== 'containers' && shownTab !== 'stacks' && shownTab !== 'volumes' && shownTab !== 'updates' && ( + {cols[shownTab as Exclude].map(([, h]) => )}{shownTab === 'images' && + {d[shownTab as Exclude].map((row, i) => {cols[shownTab as Exclude].map(([k]) => )}{shownTab === 'images' && })}
{h}}
{String(row[k] ?? '')}{row.in_use ? in use : { const ref = row.Tag && row.Tag !== '' ? `${row.Repository}:${row.Tag}` : String(row.ID); try { await api.removeImage(ref); toast(`${ref} removed`); load() } catch (er) { toast(er instanceof Error ? er.message : 'failed', 'err') } }}>✕}
)}
{shownTab === 'containers' && !cur && sel && !selStack &&
setSel('')} />
There is no container called {sel} on this machine. It may have been removed, renamed or recreated under another name.
} {shownTab === 'containers' && (cur ? setSel('')} onChanged={load} /> : (sel && !selStack ? null :
Select a container.
))} {making && { setMaking(''); setDup(null) }} onCreated={async (name, deploy) => { setMaking(''); setDup(null); setTab('stacks'); if (deploy) { try { setStartTask({ name, id: (await api.stackAction(name, 'deploy')).id }) } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } } else setStartTask(null); await load(); setSel('stack:' + name) }} />} + {shownTab === 'updates' && } {shownTab === 'stacks' && (selStack ? setSel('')} onChanged={load} onDuplicate={(f) => { setDup(f); setMaking('stack') }} /> :
Select a stack to edit its compose file.
)}
) } + +// A container's image against the registry, as a single mark: nothing when nobody has +// asked, so an uncoloured list means "not checked" rather than "all current". +function UpdateDot({ state }: { state?: string }) { + if (!state || state === 'current') return null + const known = state === 'update' + return {known ? '↑' : '?'} +} + +// What is out of date here, and the schedules that keep it from happening again. +function Updates({ upd, onChecked }: { upd: MachineUpdates | null; onChecked: (u: MachineUpdates) => void }) { + const onHub = apiPath('/api/x') === '/api/x' + const [busy, setBusy] = useState(false) + const [jobs, setJobs] = useState(null) + const [file, setFile] = useState('') + const [editing, setEditing] = useState | null>(null) + const [stacks, setStacks] = useState([]) + const loadJobs = useCallback(async () => { + try { const r = await api.updateJobs(); setJobs(r.jobs); setFile(r.file) } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err'); setJobs([]) } + }, []) + useEffect(() => { loadJobs(); api.stacks().then((r) => setStacks(r.stacks.map((s) => s.name))).catch(() => {}) }, [loadJobs]) + const check = async () => { + setBusy(true) + try { onChecked(await api.machineUpdates(true)); toast('checked') } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } finally { setBusy(false) } + } + const run = async (j: UpdateJob) => { + if (!await askYes(`Run “${j.name}” now? It pulls ${j.stack}'s images${j.apply ? ' and deploys them, which restarts its containers' : ', without deploying them'}.`, 'Run now')) return + try { await api.runUpdateJob(j.id); toast(`${j.name}: done`); loadJobs() } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } + } + const remove = async (j: UpdateJob) => { + if (!await askYes(`Remove the schedule “${j.name}”? Nothing is updated or rolled back — the schedule simply stops.`, 'Remove')) return + try { await api.deleteUpdateJob(j.id); toast('removed'); loadJobs() } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } + } + const imgs = upd?.images ?? [] + const behind = imgs.filter((i) => i.state === 'update') + return ( +
+

What is out of date

+ {upd?.checked ? `checked ${new Date(upd.checked).toLocaleString()}` : 'not checked yet'} + Check now +
+
Each check is one request per image, which registries count — so it happens when asked, and the answer is kept for six hours.
+ {imgs.length === 0 &&
Nothing checked yet on this machine.
} + {imgs.length > 0 && ( +
+ {behind.length === 0 &&
Every image that could be checked is current.
} + {imgs.filter((i) => i.state !== 'current').map((i) => ( +
+ {i.image} + {i.state === 'update' ? 'a newer image is published' : i.detail || 'unknown'} + {(upd?.stacks?.[i.image] ?? []).join(' ')} +
+ ))} + {behind.length === 0 && imgs.some((i) => i.state === 'current') &&
{imgs.filter((i) => i.state === 'current').length} current.
} +
+ )} + + {onHub && } + +

Schedules

+ setEditing({ cron: '0 3 * * *', apply: true, enabled: true })}>+ New schedule +
+
Written to {file || '/etc/cron.d/hamlab-docker'}, which Ham Lab owns — so they keep running whether or not this console is up.
+ {jobs === null && } + {jobs?.length === 0 &&
No schedule yet. Nothing on this machine updates itself.
} +
+ {(jobs ?? []).map(({ job, next, last }) => ( +
+
+ + {job.name} + {!job.enabled && paused} + + {job.enabled && next ? new Date(next).toLocaleString() : 'not scheduled'} +
+
+
Stack{job.stack}{job.services?.length ? ` · ${job.services.join(' ')}` : ''}
+
Cron{job.cron}
+
Does{job.apply ? 'pulls and deploys' : 'pulls only — you deploy'}
+
Last run{last?.at ? new Date(last.at).toLocaleString() : 'never'}
+
+ {last?.error &&
{last.error}
} +
+ run(job)}>Run now + + { await api.saveUpdateJob({ ...job, enabled: !job.enabled }); loadJobs() }}>{job.enabled ? 'Pause' : 'Resume'} + remove(job)}>Remove +
+
+ ))} +
+ {editing && setEditing(null)} onSaved={() => { setEditing(null); loadJobs() }} />} +
+ ) +} + +// Every other machine at once. Only shown on the hub, because only the hub can reach them, +// and each answer is that machine's own six-hour cache — nothing is checked here without +// being asked for. +function Fleet() { + const [rows, setRows] = useState<{ name: string; online: boolean; upd?: MachineUpdates; err?: string }[] | null>(null) + const [busy, setBusy] = useState('') + const load = useCallback(async (refresh?: string) => { + let nodes: MachineStatus[] = [] + try { nodes = (await api.machines()).nodes } catch { setRows([]); return } + const out = await Promise.all(nodes.map(async (n) => { + if (!n.online) return { name: n.name, online: false } + try { return { name: n.name, online: true, upd: await api.nodeUpdates(n.name, refresh === n.name) } } + catch (e) { return { name: n.name, online: true, err: e instanceof Error ? e.message : 'failed' } } + })) + setRows(out) + }, []) + useEffect(() => { load() }, [load]) + if (rows === null) return
+ if (rows.length === 0) return null + return ( + <> +

The other machines

+
+ {rows.map((r) => { + const behind = (r.upd?.images ?? []).filter((i) => i.state === 'update') + return ( +
+ {r.name} + + {!r.online ? offline + : r.err ? {r.err} + : !r.upd?.checked ? not checked + : behind.length === 0 ? all current + : i.image).join('\n')}>{behind.length} behind} + + {r.online && !r.err && { setBusy(r.name); try { await load(r.name) } finally { setBusy('') } }}>check} +
+ ) + })} +
+
Open a machine from the switcher to schedule anything on it — a schedule lives on the machine that runs it.
+ + ) +} + +function JobDialog({ j, stacks, onClose, onSaved }: { j: Partial; stacks: string[]; onClose: () => void; onSaved: () => void }) { + const [f, setF] = useState>(j) + const [svc, setSvc] = useState((j.services ?? []).join(' ')) + const [err, setErr] = useState('') + const [busy, setBusy] = useState(false) + useEscape(onClose) + const set = (k: keyof UpdateJob, v: unknown) => setF((x) => ({ ...x, [k]: v })) + const save = async (e: FormEvent) => { + e.preventDefault(); setErr(''); setBusy(true) + try { await api.saveUpdateJob({ ...f, services: svc.split(/\s+/).filter(Boolean) }); onSaved() } + catch (e) { setErr(e instanceof Error ? e.message : 'failed'); setBusy(false) } + } + return ( +
+ e.stopPropagation()} onSubmit={save}> +

{f.id ? 'Edit schedule' : 'Keep a stack up to date'}

+ + + + + + +
+ {f.apply + ? 'Deploying restarts the stack’s containers. On something that carries DNS or ingress, pick an hour you are asleep.' + : 'Pulling only means the new image is on the machine and the list shows it as available — nothing restarts until you deploy.'} +
+ {err &&
{err}
} +
+ + + +
+ +
+ ) +} diff --git a/web/src/api.ts b/web/src/api.ts index 9d97676..1b750af 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -109,6 +109,10 @@ export interface StorageDevice { name: string; kind: string; model?: string; ven export interface SmartAttr { id: number; name: string; value: number; worst: number; thresh: number; raw: string; failed?: string; warn: boolean } export interface SmartInfo { device: string; available: boolean; supported: boolean; standby: boolean; passed: boolean | null; temp: number; power_on_hours: number; power_cycles: number; model?: string; serial?: string; firmware?: string; attrs: SmartAttr[]; nvme?: { critical_warning: number; percentage_used: number; available_spare: number; spare_threshold: number; media_errors: number; error_log_entries: number; data_read_bytes: number; data_written_bytes: number; unsafe_shutdowns: number }; mmc?: { life_a: number; life_b: number; pre_eol: number }; self_test?: string; self_test_remaining?: number; self_test_supported: boolean; self_test_note?: string; warnings: string[]; error?: string; fetched_at: number } export interface DockerContainer { id: string; name: string; image: string; state: string; status: string; health: string; ports: string; stack: string; service: string; created: string; mem: number } +export interface UpdateJob { id: string; name: string; stack: string; machine?: string; services?: string[]; cron: string; apply: boolean; enabled: boolean } +export interface UpdateJobRow { job: UpdateJob; next?: string; last?: { at?: string; taken?: string; error?: string } } +export interface MachineUpdates { images: ImageCheck[]; checked: number; stacks: Record } + export interface DockerHealthProbe { Start: string; End: string; ExitCode: number; Output: string } export interface DockerHealth { configured: boolean; state: string; status?: string; failing_streak?: number; log?: DockerHealthProbe[] } export interface DockerEvent { t: number; name: string; action: string; image?: string; exit_code?: string; signal?: string; health?: string } @@ -252,6 +256,13 @@ export const api = { dockerAction: (id: string, action: 'start' | 'stop' | 'restart' | 'remove') => req('POST', `/api/docker/containers/${encodeURIComponent(id)}/${action}`), dockerLogs: (id: string, since = '') => req<{ lines: { t: string; m: string }[] }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/logs${since ? `?since=${encodeURIComponent(since)}` : '?n=300'}`), dockerInspect: (id: string) => req>('GET', `/api/docker/containers/${encodeURIComponent(id)}/inspect`), + machineUpdates: (refresh = false) => req('GET', `/api/docker/updates${refresh ? '?refresh=1' : ''}`), + // One named machine, asked of the hub — the fleet view, which is on the hub by definition. + nodeUpdates: (node: string, refresh = false) => req('GET', `/api/nodes/${encodeURIComponent(node)}/docker/updates${refresh ? '?refresh=1' : ''}`, undefined, true), + updateJobs: () => req<{ jobs: UpdateJobRow[]; file: string }>('GET', '/api/docker/schedules'), + saveUpdateJob: (j: Partial) => req('PUT', '/api/docker/schedules', j), + deleteUpdateJob: (id: string) => req('DELETE', `/api/docker/schedules/${encodeURIComponent(id)}`), + runUpdateJob: (id: string) => req<{ output: string }>('POST', `/api/docker/schedules/${encodeURIComponent(id)}/run`), dockerHealth: (id: string) => req('GET', `/api/docker/containers/${encodeURIComponent(id)}/health`), dockerEvents: (id: string, since = '7d') => req<{ events: DockerEvent[]; since: string; kept: number; capacity: number }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/events?since=${since}`), dockerExec: (id: string) => req<{ id: string; name: string }>('POST', `/api/docker/containers/${encodeURIComponent(id)}/exec`, {}), diff --git a/web/src/styles.css b/web/src/styles.css index e897844..f046f2a 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -756,3 +756,6 @@ pre.diff .add { color: var(--ok); background: color-mix(in srgb, var(--ok) 12%, .kvm-frame.expanded iframe { height: 100%; } .kvm-shrink { position: absolute; right: max(10px, env(safe-area-inset-right)); bottom: max(10px, env(safe-area-inset-bottom)); z-index: 61; background: var(--bg2); border: 1px solid var(--border); opacity: .75; } .kvm-shrink:hover, .kvm-shrink:active { opacity: 1; } +.updot { margin-left: 6px; font-size: 11px; font-weight: 700; } +.updot.behind { color: var(--s2); } .updot.unknown { color: var(--muted); } +.card > .row > .row-end { text-align: right; white-space: nowrap; }