package api // Scheduled image updates. // // Dockhand kept three containers up to date nightly — unbound, nginx and pihole, which is // DNS and ingress — and that is the one thing it does that Ham Lab did not. This replaces // it, and deliberately in the same shape as the ZFS snapshot schedules: a real line in a // cron file Ham Lab owns, calling the binary, rather than a ticker inside the daemon. // Updating an image is not something that should stop happening because a web console is // down, and a cron line is visible to anyone looking at the machine the normal way. // // A schedule stores everything the run needs — project, compose files, how compose has to // be invoked for that stack — so the cron job is self-contained and does not have to // rediscover the world before it can act. import ( "context" "crypto/rand" "encoding/hex" "encoding/json" "errors" "io" "log" "net/http" "os" "path/filepath" "strings" "time" "hamlab/internal/compose" "hamlab/internal/cron" "hamlab/internal/storage" "hamlab/internal/store" ) // UpdateJob is a standing instruction to pull a stack's images, and optionally deploy them. type UpdateJob struct { ID string `json:"id"` Name string `json:"name"` Stack string `json:"stack"` Machine string `json:"machine,omitempty"` // for display; the job runs where it was made // Services narrows it to some of the stack's services; empty is all of them. Services []string `json:"services,omitempty"` Cron string `json:"cron"` // Apply says whether a newer image is deployed or only fetched. Check-only still pulls, // so the Docker tab shows what is behind without a registry round trip. Apply bool `json:"apply"` Enabled bool `json:"enabled"` // How to run compose for this stack, captured when the schedule was made. Spec composeSpec `json:"spec"` } type composeSpec struct { Project string `json:"project"` Files []string `json:"files"` Via string `json:"via,omitempty"` // Remote is read but never written: schedules made before v0.2 could point compose at // another machine's Docker over tcp. Such a job is refused rather than run against this // machine's Docker, which is where it would land without the variable. Remote string `json:"remote,omitempty"` } func (c composeSpec) toSpec(tool string) compose.Spec { return compose.Spec{Project: c.Project, Files: c.Files, Via: c.Via, Tool: tool} } const dockerCronFile = "/etc/cron.d/hamlab-docker" type dockerSchedState struct { Jobs []UpdateJob `json:"jobs"` } func (s *Server) dsLoad() dockerSchedState { var raw json.RawMessage s.store.View(func(st *store.State) { raw = st.DockerJobs }) var d dockerSchedState if len(raw) > 0 { _ = json.Unmarshal(raw, &d) } if d.Jobs == nil { d.Jobs = []UpdateJob{} } return d } func (s *Server) dsSave(d dockerSchedState) error { b, err := json.Marshal(d) if err != nil { return err } return s.store.Update(func(st *store.State) { st.DockerJobs = b }) } // dsCronText renders the schedules as cron lines. Same shape as the ZFS file: Ham Lab owns // it whole and rewrites it on every change. func dsCronText(jobs []UpdateJob, user, exe, config string) (string, error) { var b strings.Builder b.WriteString("# Ham Lab image-update schedules — written by the Docker tab, rewritten whole on every change.\n") b.WriteString("# Edit them there, not here: anything added by hand is lost the next time a schedule changes.\n") b.WriteString("SHELL=/bin/sh\nPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n") for i := range jobs { j := &jobs[i] if !j.Enabled { b.WriteString("# " + j.Name + " — paused\n") continue } expr := strings.Join(strings.Fields(j.Cron), " ") if err := cron.ValidSchedule(expr); err != nil { return "", err } what := "pull only" if j.Apply { what = "pull and deploy" } b.WriteString("# " + j.Name + " — " + j.Stack + ", " + what + "\n") b.WriteString(expr + " " + user + " " + exe + " -config " + config + " docker-update " + j.ID + "\n\n") } return b.String(), nil } func (s *Server) dsSyncCron(ctx context.Context, d dockerSchedState) error { live := false for i := range d.Jobs { if d.Jobs[i].Enabled { live = true } } if !live { return s.storage.RemoveRootFile(ctx, dockerCronFile) } exe, err := os.Executable() if err != nil { return err } exe, _ = filepath.EvalSymlinks(exe) text, err := dsCronText(d.Jobs, s.user, exe, s.cfg.Path()) if err != nil { return err } return s.storage.WriteRootFile(ctx, dockerCronFile, text) } // repoint keeps a schedule working when what deployed its stack has been removed. // // A schedule captures how to invoke compose at the moment it is made, so the cron job never // has to rediscover the world at 3am. But "run compose inside this container" stops being // true the day that container is deleted — which is exactly what happens when an // orchestrator is retired, and would otherwise break every schedule made before it went. // So: if the container named is not running, the files are looked up again, through live // mounts and remembered ones both, and compose runs on the host instead. func repoint(ctx context.Context, db *store.Store, spec compose.Spec) compose.Spec { if spec.Via == "" || db == nil { return spec } if out, err := dockerRun(ctx, "inspect", "-f", "{{.State.Running}}", spec.Via); err == nil && strings.TrimSpace(string(out)) == "true" { return spec } st := stack{Name: spec.Project, File: spec.Files[0], Files: spec.Files} st.resolve(mountsSeen(ctx, db)) // read-only: this also runs from cron, where the daemon owns state.json if !st.Editable || st.Via != "" { return spec // still only reachable through a container, or not reachable at all } log.Printf("docker schedule %s: %s is gone, running compose here against %s", spec.Project, spec.Via, st.HostFiles[0]) spec.Files, spec.Via = st.HostFiles, "" return spec } // RunUpdate pulls, and deploys when the schedule says to. Shared with the CLI so the button // and the cron line cannot drift apart. func RunUpdate(ctx context.Context, db *store.Store, j UpdateJob, tool string) (string, error) { if len(j.Spec.Files) == 0 { return "", errors.New("this schedule has no compose file recorded — open it in the Docker tab and save it again") } if j.Spec.Remote != "" { return "", errors.New("this schedule was made to deploy to another machine over tcp, which this console no longer does — make it again on that machine") } spec := repoint(ctx, db, j.Spec.toSpec(tool)) var log strings.Builder run := func(args ...string) error { cmd := compose.Cmd(ctx, spec, args...) out, err := cmd.CombinedOutput() log.Write(out) return err } args := append([]string{"pull"}, j.Services...) if err := run(args...); err != nil { return log.String(), err } if !j.Apply { return log.String(), nil } up := append([]string{"up", "-d"}, j.Services...) if err := run(up...); err != nil { return log.String(), err } return log.String(), nil } // ---- endpoints ---- func (s *Server) dsList(w http.ResponseWriter, _ *http.Request) { d := s.dsLoad() now := time.Now() out := make([]map[string]any, 0, len(d.Jobs)) for _, j := range d.Jobs { m := map[string]any{"job": j, "last": storage.ReadRun(s.cfg.DataDir(), "docker-"+j.ID)} if j.Enabled { if n, ok := cron.NextRun(strings.Join(strings.Fields(j.Cron), " "), now); ok { m["next"] = n } } out = append(out, m) } jsonOK(w, 200, map[string]any{"jobs": out, "file": dockerCronFile}) } func (s *Server) dsPut(w http.ResponseWriter, r *http.Request) { var j UpdateJob if err := json.NewDecoder(io.LimitReader(r.Body, 32<<10)).Decode(&j); err != nil { jsonErr(w, 400, "bad request") return } j.Cron = strings.Join(strings.Fields(j.Cron), " ") if err := cron.ValidSchedule(j.Cron); err != nil { jsonErr(w, 400, "schedule: "+err.Error()) return } if strings.TrimSpace(j.Stack) == "" { jsonErr(w, 400, "which stack?") return } // resolve the stack now, so the cron job never has to ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) defer cancel() stacks, err := s.findStacks(ctx) if err != nil { jsonErr(w, 500, err.Error()) return } found := false for _, st := range stacks { if st.Name == j.Stack { // seenFiles is the list as compose itself sees it: host paths normally, the // orchestrator container's own paths when compose runs inside one. Getting this // backwards gives a cron line that works from the console and fails at 3am. j.Spec = composeSpec{Project: st.Name, Files: seenFiles(st, -1, ""), Via: st.Via} if len(j.Spec.Files) == 0 { jsonErr(w, 409, "that stack's compose file isn't anywhere this console can reach") return } found = true } } if !found { jsonErr(w, 404, "no stack called "+j.Stack+" on this machine") return } if strings.TrimSpace(j.Name) == "" { j.Name = j.Stack } d := s.dsLoad() replaced := false for i := range d.Jobs { if d.Jobs[i].ID == j.ID && j.ID != "" { d.Jobs[i] = j replaced = true } } if !replaced { b := make([]byte, 6) if _, err := rand.Read(b); err != nil { jsonErr(w, 500, err.Error()) return } j.ID = hex.EncodeToString(b) d.Jobs = append(d.Jobs, j) } if err := s.dsSave(d); err != nil { jsonErr(w, 500, err.Error()) return } if err := s.dsSyncCron(r.Context(), d); err != nil { jsonErr(w, 500, "saved, but writing "+dockerCronFile+" failed: "+err.Error()) return } log.Printf("docker schedule %q saved by %s (%s, %q, %s)", j.Name, clientIP(r), j.Stack, j.Cron, map[bool]string{true: "pull and deploy", false: "pull only"}[j.Apply]) jsonOK(w, 200, j) } func (s *Server) dsDelete(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") d := s.dsLoad() kept := d.Jobs[:0] found := false for _, j := range d.Jobs { if j.ID == id { found = true continue } kept = append(kept, j) } if !found { jsonErr(w, 404, "no such schedule") return } d.Jobs = kept if err := s.dsSave(d); err != nil { jsonErr(w, 500, err.Error()) return } if err := s.dsSyncCron(r.Context(), d); err != nil { jsonErr(w, 500, "removed, but writing "+dockerCronFile+" failed: "+err.Error()) return } storage.ForgetRun(s.cfg.DataDir(), "docker-"+id) log.Printf("docker schedule %s removed by %s", id, clientIP(r)) w.WriteHeader(204) } func (s *Server) dsRun(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") var job UpdateJob found := false for _, j := range s.dsLoad().Jobs { if j.ID == id { job, found = j, true } } if !found { jsonErr(w, 404, "no such schedule") return } // Outliving the request: a pull the browser gave up on still finishes, but not forever. ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 30*time.Minute) defer cancel() out, err := RunUpdate(ctx, s.store, job, dockerTool()) rec := storage.RunRecord{At: time.Now(), Taken: job.Stack} if err != nil { rec.Error = err.Error() } _ = storage.WriteRun(s.cfg.DataDir(), "docker-"+id, rec) if err != nil { jsonErr(w, 500, err.Error()+"\n"+tail(out, 2000)) return } log.Printf("docker schedule %q run by %s", job.Name, clientIP(r)) jsonOK(w, 200, map[string]string{"output": tail(out, 4000)}) } func tail(s string, n int) string { if len(s) <= n { return s } return "…" + s[len(s)-n:] }