docker: the eight things Dockhand did that Ham Lab did not

Dockhand is being retired, so the Docker tab has to cover what it actually earned its keep doing. Eight pieces, all per container or per machine: - Stats: CPU, memory and IO over time, from sensors already collected. - Health: the container's own healthcheck, with the last few probe outputs — which is where a failing check says why. - Events: `docker events` keeps a minute of backlog and nothing more, so Ham Lab follows the stream and keeps a ring of its own. A restart at 4am is still there at 9. - Shell: `docker exec` into a container, in the terminal already here. - Limits: memory, CPU and restart policy changed live, without hand-editing the compose file for a number you want to try. - Updates: what the registry has that this machine does not, for every image at once rather than one stack at a time. - The same across every machine, asked of the hub — which is the only thing that can reach them all. - Schedules: a real line in /etc/cron.d/hamlab-docker calling the binary, in the same shape as the ZFS snapshot schedules. Dockhand updated DNS and ingress nightly from a ticker inside a web app; a cron line keeps running when the console does not, and is visible to anyone reading the machine the normal way. internal/compose is new and shared, so a deploy from the console and a deploy from cron invoke compose identically — including the case where compose has to run inside the orchestrator's own container, where the paths are that container's and not the host's. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-24 20:11:36 -07:00
parent a70f3e2ed2
commit 192504ead3
12 changed files with 783 additions and 69 deletions
@@ -49,6 +49,10 @@ func dockerTool() string {
 }

 // dockerRun runs the tool as this user, and again as root if the socket said no.
+// DockerTool is whichever of docker or podman this machine has, for callers outside the
+// package — the scheduled updates the CLI runs.
+func DockerTool() string { return dockerTool() }
+
 func dockerRun(ctx context.Context, args ...string) ([]byte, error) {
 	tool := dockerTool()
 	if tool == "" {
@@ -31,7 +31,7 @@ import (

 const (
 	eventsKept     = 4000
-	eventsSaveEeem = 30 * time.Second
+	eventsSaveEvery = 30 * time.Second
 )

 // ContainerEvent is one thing that happened to one container.
@@ -118,7 +118,7 @@ func (e *eventLog) save() {
 func (s *Server) watchDockerEvents(ctx context.Context) {
 	s.events.load(filepath.Join(s.cfg.DataDir(), "docker-events.json"))
 	go func() { // a periodic flush, so a crash loses at most half a minute of history
-		t := time.NewTicker(eventsSaveEeem)
+		t := time.NewTicker(eventsSaveEvery)
 		defer t.Stop()
 		for {
 			select {
@@ -0,0 +1,322 @@
+package api
+
+// Scheduled image updates.
+//
+// Dockhand kept three containers up to date nightly — unbound, nginx and pihole, which is
+// DNS and ingress — and that is the one thing it does that Ham Lab did not. This replaces
+// it, and deliberately in the same shape as the ZFS snapshot schedules: a real line in a
+// cron file Ham Lab owns, calling the binary, rather than a ticker inside the daemon.
+// Updating an image is not something that should stop happening because a web console is
+// down, and a cron line is visible to anyone looking at the machine the normal way.
+//
+// A schedule stores everything the run needs — project, compose files, how compose has to
+// be invoked for that stack — so the cron job is self-contained and does not have to
+// rediscover the world before it can act.
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"hamlab/internal/compose"
+	"hamlab/internal/cron"
+	"hamlab/internal/storage"
+	"hamlab/internal/store"
+)
+
+// UpdateJob is a standing instruction to pull a stack's images, and optionally deploy them.
+type UpdateJob struct {
+	ID      string `json:"id"`
+	Name    string `json:"name"`
+	Stack   string `json:"stack"`
+	Machine string `json:"machine,omitempty"` // for display; the job runs where it was made
+	// Services narrows it to some of the stack's services; empty is all of them.
+	Services []string `json:"services,omitempty"`
+	Cron     string   `json:"cron"`
+	// Apply says whether a newer image is deployed or only fetched. Check-only still pulls,
+	// so the Docker tab shows what is behind without a registry round trip.
+	Apply   bool `json:"apply"`
+	Enabled bool `json:"enabled"`
+
+	// How to run compose for this stack, captured when the schedule was made.
+	Spec composeSpec `json:"spec"`
+}
+
+type composeSpec struct {
+	Project string   `json:"project"`
+	Files   []string `json:"files"`
+	Via     string   `json:"via,omitempty"`
+	Remote  string   `json:"remote,omitempty"`
+}
+
+func (c composeSpec) toSpec(tool string) compose.Spec {
+	return compose.Spec{Project: c.Project, Files: c.Files, Via: c.Via, Remote: c.Remote, Tool: tool}
+}
+
+const dockerCronFile = "/etc/cron.d/hamlab-docker"
+
+type dockerSchedState struct {
+	Jobs []UpdateJob `json:"jobs"`
+}
+
+func (s *Server) dsLoad() dockerSchedState {
+	var raw json.RawMessage
+	s.store.View(func(st *store.State) { raw = st.DockerJobs })
+	var d dockerSchedState
+	if len(raw) > 0 {
+		_ = json.Unmarshal(raw, &d)
+	}
+	if d.Jobs == nil {
+		d.Jobs = []UpdateJob{}
+	}
+	return d
+}
+
+func (s *Server) dsSave(d dockerSchedState) error {
+	b, err := json.Marshal(d)
+	if err != nil {
+		return err
+	}
+	return s.store.Update(func(st *store.State) { st.DockerJobs = b })
+}
+
+// dsCronText renders the schedules as cron lines. Same shape as the ZFS file: Ham Lab owns
+// it whole and rewrites it on every change.
+func dsCronText(jobs []UpdateJob, user, exe, config string) (string, error) {
+	var b strings.Builder
+	b.WriteString("# Ham Lab image-update schedules — written by the Docker tab, rewritten whole on every change.\n")
+	b.WriteString("# Edit them there, not here: anything added by hand is lost the next time a schedule changes.\n")
+	b.WriteString("SHELL=/bin/sh\nPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n")
+	for i := range jobs {
+		j := &jobs[i]
+		if !j.Enabled {
+			b.WriteString("# " + j.Name + " — paused\n")
+			continue
+		}
+		expr := strings.Join(strings.Fields(j.Cron), " ")
+		if err := cron.ValidSchedule(expr); err != nil {
+			return "", err
+		}
+		what := "pull only"
+		if j.Apply {
+			what = "pull and deploy"
+		}
+		b.WriteString("# " + j.Name + " — " + j.Stack + ", " + what + "\n")
+		b.WriteString(expr + " " + user + " " + exe + " -config " + config + " docker-update " + j.ID + "\n\n")
+	}
+	return b.String(), nil
+}
+
+func (s *Server) dsSyncCron(ctx context.Context, d dockerSchedState) error {
+	live := false
+	for i := range d.Jobs {
+		if d.Jobs[i].Enabled {
+			live = true
+		}
+	}
+	if !live {
+		return s.storage.RemoveRootFile(ctx, dockerCronFile)
+	}
+	exe, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	exe, _ = filepath.EvalSymlinks(exe)
+	text, err := dsCronText(d.Jobs, s.user, exe, s.cfg.Path())
+	if err != nil {
+		return err
+	}
+	return s.storage.WriteRootFile(ctx, dockerCronFile, text)
+}
+
+// RunUpdate pulls, and deploys when the schedule says to. Shared with the CLI so the button
+// and the cron line cannot drift apart.
+func RunUpdate(ctx context.Context, j UpdateJob, tool string) (string, error) {
+	spec := j.Spec.toSpec(tool)
+	var log strings.Builder
+	run := func(args ...string) error {
+		cmd := compose.Cmd(ctx, spec, args...)
+		out, err := cmd.CombinedOutput()
+		log.Write(out)
+		return err
+	}
+	args := append([]string{"pull"}, j.Services...)
+	if err := run(args...); err != nil {
+		return log.String(), err
+	}
+	if !j.Apply {
+		return log.String(), nil
+	}
+	up := append([]string{"up", "-d"}, j.Services...)
+	if err := run(up...); err != nil {
+		return log.String(), err
+	}
+	return log.String(), nil
+}
+
+// ---- endpoints ----
+
+func (s *Server) dsList(w http.ResponseWriter, _ *http.Request) {
+	d := s.dsLoad()
+	now := time.Now()
+	out := make([]map[string]any, 0, len(d.Jobs))
+	for _, j := range d.Jobs {
+		m := map[string]any{"job": j, "last": storage.ReadRun(s.cfg.DataDir(), "docker-"+j.ID)}
+		if j.Enabled {
+			if n, ok := cron.NextRun(strings.Join(strings.Fields(j.Cron), " "), now); ok {
+				m["next"] = n
+			}
+		}
+		out = append(out, m)
+	}
+	jsonOK(w, 200, map[string]any{"jobs": out, "file": dockerCronFile})
+}
+
+func (s *Server) dsPut(w http.ResponseWriter, r *http.Request) {
+	var j UpdateJob
+	if err := json.NewDecoder(io.LimitReader(r.Body, 32<<10)).Decode(&j); err != nil {
+		jsonErr(w, 400, "bad request")
+		return
+	}
+	j.Cron = strings.Join(strings.Fields(j.Cron), " ")
+	if err := cron.ValidSchedule(j.Cron); err != nil {
+		jsonErr(w, 400, "schedule: "+err.Error())
+		return
+	}
+	if strings.TrimSpace(j.Stack) == "" {
+		jsonErr(w, 400, "which stack?")
+		return
+	}
+	// resolve the stack now, so the cron job never has to
+	ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
+	defer cancel()
+	stacks, err := s.findStacks(ctx)
+	if err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	found := false
+	for _, st := range stacks {
+		if st.Name == j.Stack {
+			// seenFiles is the list as compose itself sees it: host paths normally, the
+			// orchestrator container's own paths when compose runs inside one. Getting this
+			// backwards gives a cron line that works from the console and fails at 3am.
+			j.Spec = composeSpec{Project: st.Name, Files: seenFiles(st, -1, ""), Via: st.Via, Remote: st.Remote}
+			if len(j.Spec.Files) == 0 {
+				jsonErr(w, 409, "that stack's compose file isn't anywhere this console can reach")
+				return
+			}
+			found = true
+		}
+	}
+	if !found {
+		jsonErr(w, 404, "no stack called "+j.Stack+" on this machine")
+		return
+	}
+	if strings.TrimSpace(j.Name) == "" {
+		j.Name = j.Stack
+	}
+	d := s.dsLoad()
+	replaced := false
+	for i := range d.Jobs {
+		if d.Jobs[i].ID == j.ID && j.ID != "" {
+			d.Jobs[i] = j
+			replaced = true
+		}
+	}
+	if !replaced {
+		b := make([]byte, 6)
+		if _, err := rand.Read(b); err != nil {
+			jsonErr(w, 500, err.Error())
+			return
+		}
+		j.ID = hex.EncodeToString(b)
+		d.Jobs = append(d.Jobs, j)
+	}
+	if err := s.dsSave(d); err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	if err := s.dsSyncCron(r.Context(), d); err != nil {
+		jsonErr(w, 500, "saved, but writing "+dockerCronFile+" failed: "+err.Error())
+		return
+	}
+	log.Printf("docker schedule %q saved by %s (%s, %q, %s)", j.Name, clientIP(r), j.Stack, j.Cron, map[bool]string{true: "pull and deploy", false: "pull only"}[j.Apply])
+	jsonOK(w, 200, j)
+}
+
+func (s *Server) dsDelete(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	d := s.dsLoad()
+	kept := d.Jobs[:0]
+	found := false
+	for _, j := range d.Jobs {
+		if j.ID == id {
+			found = true
+			continue
+		}
+		kept = append(kept, j)
+	}
+	if !found {
+		jsonErr(w, 404, "no such schedule")
+		return
+	}
+	d.Jobs = kept
+	if err := s.dsSave(d); err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	if err := s.dsSyncCron(r.Context(), d); err != nil {
+		jsonErr(w, 500, "removed, but writing "+dockerCronFile+" failed: "+err.Error())
+		return
+	}
+	storage.ForgetRun(s.cfg.DataDir(), "docker-"+id)
+	log.Printf("docker schedule %s removed by %s", id, clientIP(r))
+	w.WriteHeader(204)
+}
+
+func (s *Server) dsRun(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	var job UpdateJob
+	found := false
+	for _, j := range s.dsLoad().Jobs {
+		if j.ID == id {
+			job, found = j, true
+		}
+	}
+	if !found {
+		jsonErr(w, 404, "no such schedule")
+		return
+	}
+	// Outliving the request: a pull the browser gave up on still finishes, but not forever.
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 30*time.Minute)
+	defer cancel()
+	out, err := RunUpdate(ctx, job, dockerTool())
+	rec := storage.RunRecord{At: time.Now(), Taken: job.Stack}
+	if err != nil {
+		rec.Error = err.Error()
+	}
+	_ = storage.WriteRun(s.cfg.DataDir(), "docker-"+id, rec)
+	stackUpdatesForget("/" + job.Stack)
+	if err != nil {
+		jsonErr(w, 500, err.Error()+"\n"+tail(out, 2000))
+		return
+	}
+	log.Printf("docker schedule %q run by %s", job.Name, clientIP(r))
+	jsonOK(w, 200, map[string]string{"output": tail(out, 4000)})
+}
+
+func tail(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return "…" + s[len(s)-n:]
+}
@@ -192,6 +192,12 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
 	m.HandleFunc("POST /api/docker/stacks/{name}/{action}", s.auth(s.stackAction))
 	m.HandleFunc("GET /api/docker/stacks/{name}/diff", s.auth(s.stackDiff))
 	m.HandleFunc("GET /api/docker/stacks/{name}/updates", s.auth(s.stackUpdates))
+	m.HandleFunc("GET /api/docker/updates", s.auth(s.machineUpdates))
+	m.HandleFunc("GET /api/docker/schedules", s.auth(s.dsList))
+	m.HandleFunc("PUT /api/docker/schedules", s.auth(s.dsPut))
+	m.HandleFunc("DELETE /api/docker/schedules/{id}", s.auth(s.dsDelete))
+	m.HandleFunc("POST /api/docker/schedules/{id}/run", s.auth(s.dsRun))
+	m.HandleFunc("GET /api/nodes/{node}/docker/updates", s.auth(s.machineUpdates))
 	// The same, asked of a node through this hub: the hub answers for the stacks whose files
 	// it holds (deployed to the node over tcp) and passes everything else to the node.
 	m.HandleFunc("GET /api/nodes/{node}/docker/stacks", s.auth(s.stacksList))
@@ -39,6 +39,7 @@ import (
 	"sync"
 	"time"

+	"hamlab/internal/compose"
 	"hamlab/internal/priv"
 )

@@ -511,24 +512,13 @@ func (s *Server) stackFile(w http.ResponseWriter, r *http.Request) {
 // composeCmd is `docker compose …` for this stack, run where its files make sense and
 // pointed at the Docker that runs its containers. files are given as compose will see them.
 func composeCmd(ctx context.Context, st stack, files []string, args ...string) *exec.Cmd {
-	tool := dockerTool()
-	base := []string{"compose", "-p", st.Name}
-	for _, f := range files {
-		base = append(base, "-f", f)
-	}
-	if st.Via != "" {
-		pre := []string{"exec"}
-		if st.Remote != "" {
-			pre = append(pre, "-e", "DOCKER_HOST="+st.Remote)
-		}
-		return exec.CommandContext(ctx, tool, append(append(append(pre, st.Via, "docker"), base...), args...)...)
-	}
-	cmd := exec.CommandContext(ctx, tool, append(base, args...)...)
-	cmd.Dir = filepath.Dir(files[0])
-	if st.Remote != "" {
-		cmd.Env = append(os.Environ(), "DOCKER_HOST="+st.Remote)
-	}
-	return cmd
+	return compose.Cmd(ctx, specOf(st, files), args...)
+}
+
+// specOf is a stack as the compose runner needs it — and as a scheduled update stores it,
+// so the cron job and the console invoke compose identically.
+func specOf(st stack, files []string) compose.Spec {
+	return compose.Spec{Project: st.Name, Files: files, Via: st.Via, Remote: st.Remote, Tool: dockerTool()}
 }

 // seenFiles is the stack's file list as compose runs it, with file i replaced by a candidate.
@@ -788,6 +778,103 @@ var (

 func stackUpdatesForget(key string) { updMu.Lock(); delete(updCache, key); updMu.Unlock() }

+// checkImages asks the registry whether each image is still the one we hold. remote is a
+// DOCKER_HOST for a machine whose images live elsewhere, "" for this one. Each image is one
+// registry request, which Docker Hub counts — so this is only ever called when asked for.
+func checkImages(ctx context.Context, images []string, remote string) []imageCheck {
+	local := func(args ...string) ([]byte, error) { // the Docker that holds the images
+		if remote != "" {
+			args = append([]string{"-H", remote}, args...)
+		}
+		return dockerRun(ctx, args...)
+	}
+	res := []imageCheck{}
+	for _, img := range images {
+		ic := imageCheck{Image: img, State: "unknown"}
+		out, err := local("image", "inspect", "-f", "{{json .RepoDigests}}", img)
+		var have []string
+		if err != nil || json.Unmarshal(bytes.TrimSpace(out), &have) != nil || len(have) == 0 {
+			ic.Detail = "built or loaded here, not pulled from a registry"
+			res = append(res, ic)
+			continue
+		}
+		rem, err := dockerRun(ctx, "buildx", "imagetools", "inspect", img, "--format", "{{.Manifest.Digest}}")
+		digest := strings.TrimSpace(string(rem))
+		if i := strings.LastIndex(digest, "\n"); i >= 0 {
+			digest = digest[i+1:]
+		}
+		if err != nil || !strings.HasPrefix(digest, "sha256:") {
+			ic.Detail = "the registry didn't answer"
+			if err != nil {
+				ic.Detail += ": " + err.Error()
+			}
+			res = append(res, ic)
+			continue
+		}
+		ic.State = "update"
+		for _, h := range have {
+			if strings.HasSuffix(h, "@"+digest) {
+				ic.State = "current"
+			}
+		}
+		res = append(res, ic)
+	}
+	return res
+}
+
+// machineUpdates checks every image on this machine in one pass, so the container list can
+// be coloured and the fleet can be asked "what is behind?" without walking stack by stack.
+func (s *Server) machineUpdates(w http.ResponseWriter, r *http.Request) {
+	node := r.PathValue("node")
+	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute)
+	defer cancel()
+	stacks, err := s.findStacks(ctx)
+	if node != "" {
+		stacks, err = s.hubStacks(ctx, node)
+	}
+	if err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	// one entry per image, however many stacks use it
+	type want struct{ remote string }
+	imgs := map[string]want{}
+	byImage := map[string][]string{} // image -> stacks
+	for _, st := range stacks {
+		for _, img := range st.Images {
+			imgs[img] = want{remote: st.Remote}
+			byImage[img] = append(byImage[img], st.Name)
+		}
+	}
+	refresh := r.URL.Query().Get("refresh") == "1"
+	key := "machine:" + node
+	updMu.Lock()
+	c, ok := updCache[key]
+	updMu.Unlock()
+	if ok && time.Since(c.at) < 6*time.Hour && !refresh {
+		jsonOK(w, 200, map[string]any{"images": c.res, "checked": c.at.UnixMilli(), "stacks": byImage})
+		return
+	}
+	if !refresh {
+		jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0, "stacks": byImage})
+		return
+	}
+	var list []string
+	remote := ""
+	for img, wnt := range imgs {
+		list = append(list, img)
+		if wnt.remote != "" {
+			remote = wnt.remote
+		}
+	}
+	sort.Strings(list)
+	res := checkImages(ctx, list, remote)
+	updMu.Lock()
+	updCache[key] = updAnswer{time.Now(), res}
+	updMu.Unlock()
+	jsonOK(w, 200, map[string]any{"images": res, "checked": time.Now().UnixMilli(), "stacks": byImage})
+}
+
 // stackUpdates asks each image's registry for the digest its tag points at now and compares
 // it with the digest that was pulled. It is asked for, never polled: every answer is a
 // registry request, and Docker Hub counts those. Answers are kept for six hours.
@@ -816,43 +903,7 @@ func (s *Server) stackUpdates(w http.ResponseWriter, r *http.Request) {
 		jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0})
 		return
 	}
-	local := func(args ...string) ([]byte, error) { // the Docker that holds the images
-		if st.Remote != "" {
-			args = append([]string{"-H", st.Remote}, args...)
-		}
-		return dockerRun(ctx, args...)
-	}
-	res := []imageCheck{}
-	for _, img := range st.Images {
-		ic := imageCheck{Image: img, State: "unknown"}
-		out, err := local("image", "inspect", "-f", "{{json .RepoDigests}}", img)
-		var have []string
-		if err != nil || json.Unmarshal(bytes.TrimSpace(out), &have) != nil || len(have) == 0 {
-			ic.Detail = "built or loaded here, not pulled from a registry"
-			res = append(res, ic)
-			continue
-		}
-		remote, err := dockerRun(ctx, "buildx", "imagetools", "inspect", img, "--format", "{{.Manifest.Digest}}")
-		digest := strings.TrimSpace(string(remote))
-		if i := strings.LastIndex(digest, "\n"); i >= 0 {
-			digest = digest[i+1:]
-		}
-		if err != nil || !strings.HasPrefix(digest, "sha256:") {
-			ic.Detail = "the registry didn't answer"
-			if err != nil {
-				ic.Detail += ": " + err.Error()
-			}
-			res = append(res, ic)
-			continue
-		}
-		ic.State = "update"
-		for _, h := range have {
-			if strings.HasSuffix(h, "@"+digest) {
-				ic.State = "current"
-			}
-		}
-		res = append(res, ic)
-	}
+	res := checkImages(ctx, st.Images, st.Remote)
 	updMu.Lock()
 	updCache[key] = updAnswer{time.Now(), res}
 	updMu.Unlock()
@@ -0,0 +1,50 @@
+// Package compose runs docker compose the way a stack was deployed — which is not always
+// simply "on this host". A stack deployed by an orchestrator running inside its own
+// container records paths in that container's filesystem, so compose has to run in there;
+// a stack deployed to another machine over tcp needs DOCKER_HOST pointed at it. Both the
+// console and the scheduled updates it writes must invoke it identically, or a redeploy
+// from one would not match the other's labels.
+package compose
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+// Spec is everything needed to run compose for one stack.
+type Spec struct {
+	Project string   // -p
+	Files   []string // -f, in order
+	Via     string   // a container to run compose inside; "" is this host
+	Remote  string   // a DOCKER_HOST; "" is this machine's Docker
+	Tool    string   // "docker" or "podman"
+}
+
+// Cmd builds the command. args are compose's own, e.g. "pull" or "up", "-d".
+func Cmd(ctx context.Context, s Spec, args ...string) *exec.Cmd {
+	tool := s.Tool
+	if tool == "" {
+		tool = "docker"
+	}
+	base := []string{"compose", "-p", s.Project}
+	for _, f := range s.Files {
+		base = append(base, "-f", f)
+	}
+	if s.Via != "" {
+		pre := []string{"exec"}
+		if s.Remote != "" {
+			pre = append(pre, "-e", "DOCKER_HOST="+s.Remote)
+		}
+		return exec.CommandContext(ctx, tool, append(append(append(pre, s.Via, "docker"), base...), args...)...)
+	}
+	cmd := exec.CommandContext(ctx, tool, append(base, args...)...)
+	if len(s.Files) > 0 {
+		cmd.Dir = filepath.Dir(s.Files[0])
+	}
+	if s.Remote != "" {
+		cmd.Env = append(os.Environ(), "DOCKER_HOST="+s.Remote)
+	}
+	return cmd
+}
@@ -452,8 +452,25 @@ func (m *Manager) WriteCronFile(ctx context.Context, policies []Policy, user, ex
 	return m.writeRoot(ctx, CronFile, text)
 }

-// writeRoot puts text at a root-owned path, through the same privilege escalation every
-// other write here uses.
+// WriteRootFile puts text at a root-owned path, through the same privilege escalation every
+// other write here uses. Shared by the snapshot schedules and the image-update schedules,
+// which both own a file in /etc/cron.d.
+func (m *Manager) WriteRootFile(ctx context.Context, path, text string) error {
+	return m.writeRoot(ctx, path, text)
+}
+
+// RemoveRootFile deletes one, for when the last schedule of a kind goes away.
+func (m *Manager) RemoveRootFile(ctx context.Context, path string) error {
+	out, code, err := m.run(ctx, "rm", "-f", path)
+	if err != nil {
+		return err
+	}
+	if code != 0 {
+		return errors.New(strings.TrimSpace(string(out)))
+	}
+	return nil
+}
+
 func (m *Manager) writeRoot(ctx context.Context, path, text string) error {
 	args := []string{"tee", path}
 	if m.sudo {
@@ -52,7 +52,8 @@ type State struct {
 	// account. Kept so an old state file still reads.
 	UI         json.RawMessage `json:"ui,omitempty"`
 	Alerts     json.RawMessage `json:"alerts,omitempty"`
-	ZFS        json.RawMessage `json:"zfs,omitempty"`  // snapshot schedules (internal/api/zfs.go)      // rules, channels and recent events (internal/api/alerts.go)
+	ZFS        json.RawMessage `json:"zfs,omitempty"`         // snapshot schedules (internal/api/zfs.go)
+	DockerJobs json.RawMessage `json:"docker_jobs,omitempty"` // image-update schedules (internal/api/dockersched.go)      // rules, channels and recent events (internal/api/alerts.go)
 	SamplerMs  int             `json:"sampler_ms,omitempty"`  // how often this machine samples; the hub keeps every machine on the console's refresh rate
 	HistorySec int             `json:"history_sec,omitempty"` // how far back the graphs reach; the hub keeps every machine on the same one
 }