docker: the eight things Dockhand did that Ham Lab did not

Dockhand is being retired, so the Docker tab has to cover what it actually
earned its keep doing. Eight pieces, all per container or per machine:

- Stats: CPU, memory and IO over time, from sensors already collected.
- Health: the container's own healthcheck, with the last few probe outputs —
  which is where a failing check says why.
- Events: `docker events` keeps a minute of backlog and nothing more, so Ham
  Lab follows the stream and keeps a ring of its own. A restart at 4am is
  still there at 9.
- Shell: `docker exec` into a container, in the terminal already here.
- Limits: memory, CPU and restart policy changed live, without hand-editing
  the compose file for a number you want to try.
- Updates: what the registry has that this machine does not, for every image
  at once rather than one stack at a time.
- The same across every machine, asked of the hub — which is the only thing
  that can reach them all.
- Schedules: a real line in /etc/cron.d/hamlab-docker calling the binary,
  in the same shape as the ZFS snapshot schedules. Dockhand updated DNS and
  ingress nightly from a ticker inside a web app; a cron line keeps running
  when the console does not, and is visible to anyone reading the machine the
  normal way.

internal/compose is new and shared, so a deploy from the console and a deploy
from cron invoke compose identically — including the case where compose has to
run inside the orchestrator's own container, where the paths are that
container's and not the host's.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

cmd/hamlab/main.go

@@ -32,8 +32,8 @@ import (
 	"hamlab/internal/files"
 	"hamlab/internal/metrics"
 	"hamlab/internal/nodes"
-	"hamlab/internal/services"
 	"hamlab/internal/priv"
+	"hamlab/internal/services"
 	"hamlab/internal/setup"
 	"hamlab/internal/space"
 	"hamlab/internal/storage"
@@ -62,6 +62,7 @@ func main() {
   passwd [user]        set a user's password (makes the account if it is new; reads stdin when piped)
   users                list the accounts
   zfs-snapshot <id>    run one ZFS snapshot schedule (what its cron line calls)
+  docker-update <id>   run one image-update schedule (what its cron line calls)
   user-rm <user>       remove an account and sign it out everywhere
   check                verify the config and state load, then exit
   version
@@ -132,6 +133,10 @@ func main() {
 		if err := passwd(cfg, flag.Arg(1)); err != nil {
 			log.Fatal(err)
 		}
+	case "docker-update":
+		if err := dockerUpdateJob(cfg, flag.Arg(1)); err != nil {
+			log.Fatal(err)
+		}
 	case "zfs-snapshot":
 		if err := zfsSnapshot(cfg, flag.Arg(1)); err != nil {
 			log.Fatal(err)
@@ -185,6 +190,48 @@ func main() {
 
 // migrateUsers gives a pre-users config its first account, named after this OS user, so
 // the CLI sees the same accounts the daemon will.
+// dockerUpdateJob pulls a stack's images, and deploys them when the schedule says to. This
+// is what the cron line Ham Lab writes actually calls, so image updates keep happening
+// whether or not the console is up.
+func dockerUpdateJob(cfg *config.Config, id string) error {
+	if strings.TrimSpace(id) == "" {
+		return errors.New("usage: hamlab docker-update <schedule-id>")
+	}
+	st, err := store.Open(filepath.Join(cfg.DataDir(), "state.json"))
+	if err != nil {
+		return err
+	}
+	var raw json.RawMessage
+	st.View(func(s *store.State) { raw = s.DockerJobs })
+	var d struct {
+		Jobs []api.UpdateJob `json:"jobs"`
+	}
+	if len(raw) > 0 {
+		if err := json.Unmarshal(raw, &d); err != nil {
+			return fmt.Errorf("reading the schedules: %w", err)
+		}
+	}
+	for _, j := range d.Jobs {
+		if j.ID != id {
+			continue
+		}
+		// A pull that never finishes must not leave a cron job resident until the next reboot.
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour)
+		defer cancel()
+		out, runErr := api.RunUpdate(ctx, j, api.DockerTool())
+		rec := storage.RunRecord{At: time.Now(), Taken: j.Stack}
+		if runErr != nil {
+			rec.Error = runErr.Error()
+		}
+		if err := storage.WriteRun(cfg.DataDir(), "docker-"+id, rec); err != nil {
+			fmt.Fprintln(os.Stderr, "recording the run:", err)
+		}
+		fmt.Print(out)
+		return runErr
+	}
+	return fmt.Errorf("no image-update schedule %q — it may have been removed; this cron line is stale", id)
+}
+
 // zfsSnapshot runs one snapshot schedule: take it, then expire what has aged out. This is
 // what the cron line Ham Lab writes actually calls, so a schedule keeps running whether or
 // not the console is up — and keeps running if Ham Lab is never opened again.

internal/api/docker.go

@@ -49,6 +49,10 @@ func dockerTool() string {
 }
 
 // dockerRun runs the tool as this user, and again as root if the socket said no.
+// DockerTool is whichever of docker or podman this machine has, for callers outside the
+// package — the scheduled updates the CLI runs.
+func DockerTool() string { return dockerTool() }
+
 func dockerRun(ctx context.Context, args ...string) ([]byte, error) {
 	tool := dockerTool()
 	if tool == "" {

internal/api/docker_events.go

@@ -31,7 +31,7 @@ import (
 
 const (
 	eventsKept     = 4000
-	eventsSaveEeem = 30 * time.Second
+	eventsSaveEvery = 30 * time.Second
 )
 
 // ContainerEvent is one thing that happened to one container.
@@ -118,7 +118,7 @@ func (e *eventLog) save() {
 func (s *Server) watchDockerEvents(ctx context.Context) {
 	s.events.load(filepath.Join(s.cfg.DataDir(), "docker-events.json"))
 	go func() { // a periodic flush, so a crash loses at most half a minute of history
-		t := time.NewTicker(eventsSaveEeem)
+		t := time.NewTicker(eventsSaveEvery)
 		defer t.Stop()
 		for {
 			select {

internal/api/dockersched.go

@@ -0,0 +1,322 @@
+package api
+
+// Scheduled image updates.
+//
+// Dockhand kept three containers up to date nightly — unbound, nginx and pihole, which is
+// DNS and ingress — and that is the one thing it does that Ham Lab did not. This replaces
+// it, and deliberately in the same shape as the ZFS snapshot schedules: a real line in a
+// cron file Ham Lab owns, calling the binary, rather than a ticker inside the daemon.
+// Updating an image is not something that should stop happening because a web console is
+// down, and a cron line is visible to anyone looking at the machine the normal way.
+//
+// A schedule stores everything the run needs — project, compose files, how compose has to
+// be invoked for that stack — so the cron job is self-contained and does not have to
+// rediscover the world before it can act.
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"hamlab/internal/compose"
+	"hamlab/internal/cron"
+	"hamlab/internal/storage"
+	"hamlab/internal/store"
+)
+
+// UpdateJob is a standing instruction to pull a stack's images, and optionally deploy them.
+type UpdateJob struct {
+	ID      string `json:"id"`
+	Name    string `json:"name"`
+	Stack   string `json:"stack"`
+	Machine string `json:"machine,omitempty"` // for display; the job runs where it was made
+	// Services narrows it to some of the stack's services; empty is all of them.
+	Services []string `json:"services,omitempty"`
+	Cron     string   `json:"cron"`
+	// Apply says whether a newer image is deployed or only fetched. Check-only still pulls,
+	// so the Docker tab shows what is behind without a registry round trip.
+	Apply   bool `json:"apply"`
+	Enabled bool `json:"enabled"`
+
+	// How to run compose for this stack, captured when the schedule was made.
+	Spec composeSpec `json:"spec"`
+}
+
+type composeSpec struct {
+	Project string   `json:"project"`
+	Files   []string `json:"files"`
+	Via     string   `json:"via,omitempty"`
+	Remote  string   `json:"remote,omitempty"`
+}
+
+func (c composeSpec) toSpec(tool string) compose.Spec {
+	return compose.Spec{Project: c.Project, Files: c.Files, Via: c.Via, Remote: c.Remote, Tool: tool}
+}
+
+const dockerCronFile = "/etc/cron.d/hamlab-docker"
+
+type dockerSchedState struct {
+	Jobs []UpdateJob `json:"jobs"`
+}
+
+func (s *Server) dsLoad() dockerSchedState {
+	var raw json.RawMessage
+	s.store.View(func(st *store.State) { raw = st.DockerJobs })
+	var d dockerSchedState
+	if len(raw) > 0 {
+		_ = json.Unmarshal(raw, &d)
+	}
+	if d.Jobs == nil {
+		d.Jobs = []UpdateJob{}
+	}
+	return d
+}
+
+func (s *Server) dsSave(d dockerSchedState) error {
+	b, err := json.Marshal(d)
+	if err != nil {
+		return err
+	}
+	return s.store.Update(func(st *store.State) { st.DockerJobs = b })
+}
+
+// dsCronText renders the schedules as cron lines. Same shape as the ZFS file: Ham Lab owns
+// it whole and rewrites it on every change.
+func dsCronText(jobs []UpdateJob, user, exe, config string) (string, error) {
+	var b strings.Builder
+	b.WriteString("# Ham Lab image-update schedules — written by the Docker tab, rewritten whole on every change.\n")
+	b.WriteString("# Edit them there, not here: anything added by hand is lost the next time a schedule changes.\n")
+	b.WriteString("SHELL=/bin/sh\nPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n")
+	for i := range jobs {
+		j := &jobs[i]
+		if !j.Enabled {
+			b.WriteString("# " + j.Name + " — paused\n")
+			continue
+		}
+		expr := strings.Join(strings.Fields(j.Cron), " ")
+		if err := cron.ValidSchedule(expr); err != nil {
+			return "", err
+		}
+		what := "pull only"
+		if j.Apply {
+			what = "pull and deploy"
+		}
+		b.WriteString("# " + j.Name + " — " + j.Stack + ", " + what + "\n")
+		b.WriteString(expr + " " + user + " " + exe + " -config " + config + " docker-update " + j.ID + "\n\n")
+	}
+	return b.String(), nil
+}
+
+func (s *Server) dsSyncCron(ctx context.Context, d dockerSchedState) error {
+	live := false
+	for i := range d.Jobs {
+		if d.Jobs[i].Enabled {
+			live = true
+		}
+	}
+	if !live {
+		return s.storage.RemoveRootFile(ctx, dockerCronFile)
+	}
+	exe, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	exe, _ = filepath.EvalSymlinks(exe)
+	text, err := dsCronText(d.Jobs, s.user, exe, s.cfg.Path())
+	if err != nil {
+		return err
+	}
+	return s.storage.WriteRootFile(ctx, dockerCronFile, text)
+}
+
+// RunUpdate pulls, and deploys when the schedule says to. Shared with the CLI so the button
+// and the cron line cannot drift apart.
+func RunUpdate(ctx context.Context, j UpdateJob, tool string) (string, error) {
+	spec := j.Spec.toSpec(tool)
+	var log strings.Builder
+	run := func(args ...string) error {
+		cmd := compose.Cmd(ctx, spec, args...)
+		out, err := cmd.CombinedOutput()
+		log.Write(out)
+		return err
+	}
+	args := append([]string{"pull"}, j.Services...)
+	if err := run(args...); err != nil {
+		return log.String(), err
+	}
+	if !j.Apply {
+		return log.String(), nil
+	}
+	up := append([]string{"up", "-d"}, j.Services...)
+	if err := run(up...); err != nil {
+		return log.String(), err
+	}
+	return log.String(), nil
+}
+
+// ---- endpoints ----
+
+func (s *Server) dsList(w http.ResponseWriter, _ *http.Request) {
+	d := s.dsLoad()
+	now := time.Now()
+	out := make([]map[string]any, 0, len(d.Jobs))
+	for _, j := range d.Jobs {
+		m := map[string]any{"job": j, "last": storage.ReadRun(s.cfg.DataDir(), "docker-"+j.ID)}
+		if j.Enabled {
+			if n, ok := cron.NextRun(strings.Join(strings.Fields(j.Cron), " "), now); ok {
+				m["next"] = n
+			}
+		}
+		out = append(out, m)
+	}
+	jsonOK(w, 200, map[string]any{"jobs": out, "file": dockerCronFile})
+}
+
+func (s *Server) dsPut(w http.ResponseWriter, r *http.Request) {
+	var j UpdateJob
+	if err := json.NewDecoder(io.LimitReader(r.Body, 32<<10)).Decode(&j); err != nil {
+		jsonErr(w, 400, "bad request")
+		return
+	}
+	j.Cron = strings.Join(strings.Fields(j.Cron), " ")
+	if err := cron.ValidSchedule(j.Cron); err != nil {
+		jsonErr(w, 400, "schedule: "+err.Error())
+		return
+	}
+	if strings.TrimSpace(j.Stack) == "" {
+		jsonErr(w, 400, "which stack?")
+		return
+	}
+	// resolve the stack now, so the cron job never has to
+	ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
+	defer cancel()
+	stacks, err := s.findStacks(ctx)
+	if err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	found := false
+	for _, st := range stacks {
+		if st.Name == j.Stack {
+			// seenFiles is the list as compose itself sees it: host paths normally, the
+			// orchestrator container's own paths when compose runs inside one. Getting this
+			// backwards gives a cron line that works from the console and fails at 3am.
+			j.Spec = composeSpec{Project: st.Name, Files: seenFiles(st, -1, ""), Via: st.Via, Remote: st.Remote}
+			if len(j.Spec.Files) == 0 {
+				jsonErr(w, 409, "that stack's compose file isn't anywhere this console can reach")
+				return
+			}
+			found = true
+		}
+	}
+	if !found {
+		jsonErr(w, 404, "no stack called "+j.Stack+" on this machine")
+		return
+	}
+	if strings.TrimSpace(j.Name) == "" {
+		j.Name = j.Stack
+	}
+	d := s.dsLoad()
+	replaced := false
+	for i := range d.Jobs {
+		if d.Jobs[i].ID == j.ID && j.ID != "" {
+			d.Jobs[i] = j
+			replaced = true
+		}
+	}
+	if !replaced {
+		b := make([]byte, 6)
+		if _, err := rand.Read(b); err != nil {
+			jsonErr(w, 500, err.Error())
+			return
+		}
+		j.ID = hex.EncodeToString(b)
+		d.Jobs = append(d.Jobs, j)
+	}
+	if err := s.dsSave(d); err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	if err := s.dsSyncCron(r.Context(), d); err != nil {
+		jsonErr(w, 500, "saved, but writing "+dockerCronFile+" failed: "+err.Error())
+		return
+	}
+	log.Printf("docker schedule %q saved by %s (%s, %q, %s)", j.Name, clientIP(r), j.Stack, j.Cron, map[bool]string{true: "pull and deploy", false: "pull only"}[j.Apply])
+	jsonOK(w, 200, j)
+}
+
+func (s *Server) dsDelete(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	d := s.dsLoad()
+	kept := d.Jobs[:0]
+	found := false
+	for _, j := range d.Jobs {
+		if j.ID == id {
+			found = true
+			continue
+		}
+		kept = append(kept, j)
+	}
+	if !found {
+		jsonErr(w, 404, "no such schedule")
+		return
+	}
+	d.Jobs = kept
+	if err := s.dsSave(d); err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	if err := s.dsSyncCron(r.Context(), d); err != nil {
+		jsonErr(w, 500, "removed, but writing "+dockerCronFile+" failed: "+err.Error())
+		return
+	}
+	storage.ForgetRun(s.cfg.DataDir(), "docker-"+id)
+	log.Printf("docker schedule %s removed by %s", id, clientIP(r))
+	w.WriteHeader(204)
+}
+
+func (s *Server) dsRun(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	var job UpdateJob
+	found := false
+	for _, j := range s.dsLoad().Jobs {
+		if j.ID == id {
+			job, found = j, true
+		}
+	}
+	if !found {
+		jsonErr(w, 404, "no such schedule")
+		return
+	}
+	// Outliving the request: a pull the browser gave up on still finishes, but not forever.
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 30*time.Minute)
+	defer cancel()
+	out, err := RunUpdate(ctx, job, dockerTool())
+	rec := storage.RunRecord{At: time.Now(), Taken: job.Stack}
+	if err != nil {
+		rec.Error = err.Error()
+	}
+	_ = storage.WriteRun(s.cfg.DataDir(), "docker-"+id, rec)
+	stackUpdatesForget("/" + job.Stack)
+	if err != nil {
+		jsonErr(w, 500, err.Error()+"\n"+tail(out, 2000))
+		return
+	}
+	log.Printf("docker schedule %q run by %s", job.Name, clientIP(r))
+	jsonOK(w, 200, map[string]string{"output": tail(out, 4000)})
+}
+
+func tail(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return "…" + s[len(s)-n:]
+}

internal/api/server.go

@@ -192,6 +192,12 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
 	m.HandleFunc("POST /api/docker/stacks/{name}/{action}", s.auth(s.stackAction))
 	m.HandleFunc("GET /api/docker/stacks/{name}/diff", s.auth(s.stackDiff))
 	m.HandleFunc("GET /api/docker/stacks/{name}/updates", s.auth(s.stackUpdates))
+	m.HandleFunc("GET /api/docker/updates", s.auth(s.machineUpdates))
+	m.HandleFunc("GET /api/docker/schedules", s.auth(s.dsList))
+	m.HandleFunc("PUT /api/docker/schedules", s.auth(s.dsPut))
+	m.HandleFunc("DELETE /api/docker/schedules/{id}", s.auth(s.dsDelete))
+	m.HandleFunc("POST /api/docker/schedules/{id}/run", s.auth(s.dsRun))
+	m.HandleFunc("GET /api/nodes/{node}/docker/updates", s.auth(s.machineUpdates))
 	// The same, asked of a node through this hub: the hub answers for the stacks whose files
 	// it holds (deployed to the node over tcp) and passes everything else to the node.
 	m.HandleFunc("GET /api/nodes/{node}/docker/stacks", s.auth(s.stacksList))

internal/api/stacks.go

@@ -39,6 +39,7 @@ import (
 	"sync"
 	"time"
 
+	"hamlab/internal/compose"
 	"hamlab/internal/priv"
 )
 
@@ -511,24 +512,13 @@ func (s *Server) stackFile(w http.ResponseWriter, r *http.Request) {
 // composeCmd is `docker compose …` for this stack, run where its files make sense and
 // pointed at the Docker that runs its containers. files are given as compose will see them.
 func composeCmd(ctx context.Context, st stack, files []string, args ...string) *exec.Cmd {
-	tool := dockerTool()
+	return compose.Cmd(ctx, specOf(st, files), args...)
-	base := []string{"compose", "-p", st.Name}
-	for _, f := range files {
-		base = append(base, "-f", f)
 }
-	if st.Via != "" {
+
-		pre := []string{"exec"}
+// specOf is a stack as the compose runner needs it — and as a scheduled update stores it,
-		if st.Remote != "" {
+// so the cron job and the console invoke compose identically.
-			pre = append(pre, "-e", "DOCKER_HOST="+st.Remote)
+func specOf(st stack, files []string) compose.Spec {
-		}
+	return compose.Spec{Project: st.Name, Files: files, Via: st.Via, Remote: st.Remote, Tool: dockerTool()}
-		return exec.CommandContext(ctx, tool, append(append(append(pre, st.Via, "docker"), base...), args...)...)
-	}
-	cmd := exec.CommandContext(ctx, tool, append(base, args...)...)
-	cmd.Dir = filepath.Dir(files[0])
-	if st.Remote != "" {
-		cmd.Env = append(os.Environ(), "DOCKER_HOST="+st.Remote)
-	}
-	return cmd
 }
 
 // seenFiles is the stack's file list as compose runs it, with file i replaced by a candidate.
@@ -788,6 +778,103 @@ var (
 
 func stackUpdatesForget(key string) { updMu.Lock(); delete(updCache, key); updMu.Unlock() }
 
+// checkImages asks the registry whether each image is still the one we hold. remote is a
+// DOCKER_HOST for a machine whose images live elsewhere, "" for this one. Each image is one
+// registry request, which Docker Hub counts — so this is only ever called when asked for.
+func checkImages(ctx context.Context, images []string, remote string) []imageCheck {
+	local := func(args ...string) ([]byte, error) { // the Docker that holds the images
+		if remote != "" {
+			args = append([]string{"-H", remote}, args...)
+		}
+		return dockerRun(ctx, args...)
+	}
+	res := []imageCheck{}
+	for _, img := range images {
+		ic := imageCheck{Image: img, State: "unknown"}
+		out, err := local("image", "inspect", "-f", "{{json .RepoDigests}}", img)
+		var have []string
+		if err != nil || json.Unmarshal(bytes.TrimSpace(out), &have) != nil || len(have) == 0 {
+			ic.Detail = "built or loaded here, not pulled from a registry"
+			res = append(res, ic)
+			continue
+		}
+		rem, err := dockerRun(ctx, "buildx", "imagetools", "inspect", img, "--format", "{{.Manifest.Digest}}")
+		digest := strings.TrimSpace(string(rem))
+		if i := strings.LastIndex(digest, "\n"); i >= 0 {
+			digest = digest[i+1:]
+		}
+		if err != nil || !strings.HasPrefix(digest, "sha256:") {
+			ic.Detail = "the registry didn't answer"
+			if err != nil {
+				ic.Detail += ": " + err.Error()
+			}
+			res = append(res, ic)
+			continue
+		}
+		ic.State = "update"
+		for _, h := range have {
+			if strings.HasSuffix(h, "@"+digest) {
+				ic.State = "current"
+			}
+		}
+		res = append(res, ic)
+	}
+	return res
+}
+
+// machineUpdates checks every image on this machine in one pass, so the container list can
+// be coloured and the fleet can be asked "what is behind?" without walking stack by stack.
+func (s *Server) machineUpdates(w http.ResponseWriter, r *http.Request) {
+	node := r.PathValue("node")
+	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute)
+	defer cancel()
+	stacks, err := s.findStacks(ctx)
+	if node != "" {
+		stacks, err = s.hubStacks(ctx, node)
+	}
+	if err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	// one entry per image, however many stacks use it
+	type want struct{ remote string }
+	imgs := map[string]want{}
+	byImage := map[string][]string{} // image -> stacks
+	for _, st := range stacks {
+		for _, img := range st.Images {
+			imgs[img] = want{remote: st.Remote}
+			byImage[img] = append(byImage[img], st.Name)
+		}
+	}
+	refresh := r.URL.Query().Get("refresh") == "1"
+	key := "machine:" + node
+	updMu.Lock()
+	c, ok := updCache[key]
+	updMu.Unlock()
+	if ok && time.Since(c.at) < 6*time.Hour && !refresh {
+		jsonOK(w, 200, map[string]any{"images": c.res, "checked": c.at.UnixMilli(), "stacks": byImage})
+		return
+	}
+	if !refresh {
+		jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0, "stacks": byImage})
+		return
+	}
+	var list []string
+	remote := ""
+	for img, wnt := range imgs {
+		list = append(list, img)
+		if wnt.remote != "" {
+			remote = wnt.remote
+		}
+	}
+	sort.Strings(list)
+	res := checkImages(ctx, list, remote)
+	updMu.Lock()
+	updCache[key] = updAnswer{time.Now(), res}
+	updMu.Unlock()
+	jsonOK(w, 200, map[string]any{"images": res, "checked": time.Now().UnixMilli(), "stacks": byImage})
+}
+
 // stackUpdates asks each image's registry for the digest its tag points at now and compares
 // it with the digest that was pulled. It is asked for, never polled: every answer is a
 // registry request, and Docker Hub counts those. Answers are kept for six hours.
@@ -816,43 +903,7 @@ func (s *Server) stackUpdates(w http.ResponseWriter, r *http.Request) {
 		jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0})
 		return
 	}
-	local := func(args ...string) ([]byte, error) { // the Docker that holds the images
+	res := checkImages(ctx, st.Images, st.Remote)
-		if st.Remote != "" {
-			args = append([]string{"-H", st.Remote}, args...)
-		}
-		return dockerRun(ctx, args...)
-	}
-	res := []imageCheck{}
-	for _, img := range st.Images {
-		ic := imageCheck{Image: img, State: "unknown"}
-		out, err := local("image", "inspect", "-f", "{{json .RepoDigests}}", img)
-		var have []string
-		if err != nil || json.Unmarshal(bytes.TrimSpace(out), &have) != nil || len(have) == 0 {
-			ic.Detail = "built or loaded here, not pulled from a registry"
-			res = append(res, ic)
-			continue
-		}
-		remote, err := dockerRun(ctx, "buildx", "imagetools", "inspect", img, "--format", "{{.Manifest.Digest}}")
-		digest := strings.TrimSpace(string(remote))
-		if i := strings.LastIndex(digest, "\n"); i >= 0 {
-			digest = digest[i+1:]
-		}
-		if err != nil || !strings.HasPrefix(digest, "sha256:") {
-			ic.Detail = "the registry didn't answer"
-			if err != nil {
-				ic.Detail += ": " + err.Error()
-			}
-			res = append(res, ic)
-			continue
-		}
-		ic.State = "update"
-		for _, h := range have {
-			if strings.HasSuffix(h, "@"+digest) {
-				ic.State = "current"
-			}
-		}
-		res = append(res, ic)
-	}
 	updMu.Lock()
 	updCache[key] = updAnswer{time.Now(), res}
 	updMu.Unlock()

internal/compose/compose.go

@@ -0,0 +1,50 @@
+// Package compose runs docker compose the way a stack was deployed — which is not always
+// simply "on this host". A stack deployed by an orchestrator running inside its own
+// container records paths in that container's filesystem, so compose has to run in there;
+// a stack deployed to another machine over tcp needs DOCKER_HOST pointed at it. Both the
+// console and the scheduled updates it writes must invoke it identically, or a redeploy
+// from one would not match the other's labels.
+package compose
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+// Spec is everything needed to run compose for one stack.
+type Spec struct {
+	Project string   // -p
+	Files   []string // -f, in order
+	Via     string   // a container to run compose inside; "" is this host
+	Remote  string   // a DOCKER_HOST; "" is this machine's Docker
+	Tool    string   // "docker" or "podman"
+}
+
+// Cmd builds the command. args are compose's own, e.g. "pull" or "up", "-d".
+func Cmd(ctx context.Context, s Spec, args ...string) *exec.Cmd {
+	tool := s.Tool
+	if tool == "" {
+		tool = "docker"
+	}
+	base := []string{"compose", "-p", s.Project}
+	for _, f := range s.Files {
+		base = append(base, "-f", f)
+	}
+	if s.Via != "" {
+		pre := []string{"exec"}
+		if s.Remote != "" {
+			pre = append(pre, "-e", "DOCKER_HOST="+s.Remote)
+		}
+		return exec.CommandContext(ctx, tool, append(append(append(pre, s.Via, "docker"), base...), args...)...)
+	}
+	cmd := exec.CommandContext(ctx, tool, append(base, args...)...)
+	if len(s.Files) > 0 {
+		cmd.Dir = filepath.Dir(s.Files[0])
+	}
+	if s.Remote != "" {
+		cmd.Env = append(os.Environ(), "DOCKER_HOST="+s.Remote)
+	}
+	return cmd
+}

internal/storage/snapshots.go

@@ -452,8 +452,25 @@ func (m *Manager) WriteCronFile(ctx context.Context, policies []Policy, user, ex
 	return m.writeRoot(ctx, CronFile, text)
 }
 
-// writeRoot puts text at a root-owned path, through the same privilege escalation every
+// WriteRootFile puts text at a root-owned path, through the same privilege escalation every
-// other write here uses.
+// other write here uses. Shared by the snapshot schedules and the image-update schedules,
+// which both own a file in /etc/cron.d.
+func (m *Manager) WriteRootFile(ctx context.Context, path, text string) error {
+	return m.writeRoot(ctx, path, text)
+}
+
+// RemoveRootFile deletes one, for when the last schedule of a kind goes away.
+func (m *Manager) RemoveRootFile(ctx context.Context, path string) error {
+	out, code, err := m.run(ctx, "rm", "-f", path)
+	if err != nil {
+		return err
+	}
+	if code != 0 {
+		return errors.New(strings.TrimSpace(string(out)))
+	}
+	return nil
+}
+
 func (m *Manager) writeRoot(ctx context.Context, path, text string) error {
 	args := []string{"tee", path}
 	if m.sudo {

internal/store/store.go

@@ -52,7 +52,8 @@ type State struct {
 	// account. Kept so an old state file still reads.
 	UI         json.RawMessage `json:"ui,omitempty"`
 	Alerts     json.RawMessage `json:"alerts,omitempty"`
-	ZFS        json.RawMessage `json:"zfs,omitempty"`  // snapshot schedules (internal/api/zfs.go)      // rules, channels and recent events (internal/api/alerts.go)
+	ZFS        json.RawMessage `json:"zfs,omitempty"`         // snapshot schedules (internal/api/zfs.go)
+	DockerJobs json.RawMessage `json:"docker_jobs,omitempty"` // image-update schedules (internal/api/dockersched.go)      // rules, channels and recent events (internal/api/alerts.go)
 	SamplerMs  int             `json:"sampler_ms,omitempty"`  // how often this machine samples; the hub keeps every machine on the console's refresh rate
 	HistorySec int             `json:"history_sec,omitempty"` // how far back the graphs reach; the hub keeps every machine on the same one
 }

web/src/Docker.tsx

@@ -2,7 +2,7 @@
 // logs that follow, and inspect — then images, volumes and networks. Live CPU and memory
 // come from the same container sensors everything else uses.
 import { FormEvent, useCallback, useEffect, useMemo, useRef, useState } from 'react'
-import { api, DockerContainer, DockerEvent, DockerHealth, DockerInfo, ImageCheck, Stack, StackDiff } from './api'
+import { api, apiPath, DockerContainer, DockerEvent, DockerHealth, DockerInfo, ImageCheck, MachineStatus, MachineUpdates, Stack, StackDiff, UpdateJob, UpdateJobRow } from './api'
 import TaskOutput from './TaskOutput'
 import StackNew, { DupFrom } from './StackNew'
 import { Sensor, fetchHistories, fmtValue, useSensorStream } from './sensors'
@@ -12,7 +12,7 @@ import Sparkline from './Sparkline'
 import { fmtBytes } from './fmt'
 import { askText, askYes, Btn, CloseBtn, Loading, setUnsaved, toast, useEscape } from './ui'
 
-type Tab = 'containers' | 'stacks' | 'images' | 'volumes' | 'networks'
+type Tab = 'containers' | 'stacks' | 'updates' | 'images' | 'volumes' | 'networks'
 const dot = (c: DockerContainer) => (c.state !== 'running' ? (c.state === 'exited' || c.state === 'dead' ? 'bad' : 'warn') : c.health === 'unhealthy' ? 'bad' : c.health === 'starting' ? 'warn' : 'ok')
 const ports = (p: string) => [...new Set(p.split(', ').map((x) => x.replace(/^\[?::\]?:/, '').replace(/^0\.0\.0\.0:/, '').replace(/^[\d.]+:/, '')).filter((x) => x.includes('->')))].join('  ')
 
@@ -368,6 +368,12 @@ export default function Docker() {
   // what a new stack could join, and what it could collide with
   const nets = useMemo(() => (d?.networks ?? []).map((n) => String(n.Name)).filter((n) => !['bridge', 'host', 'none'].includes(n)).sort(), [d])
   const taken = useMemo(() => { const ports: Record<string, string> = {}; for (const c of d?.containers ?? []) for (const m of (c.ports ?? '').matchAll(/:(\d+)->/g)) ports[m[1]] = c.name; return { names: (d?.containers ?? []).map((c) => c.name), ports } }, [d])
+  // What the registry last said about each image, so a container can be marked without
+  // asking again. Only ever read from the cache here — checking is a deliberate act.
+  const [upd, setUpd] = useState<MachineUpdates | null>(null)
+  useEffect(() => { api.machineUpdates().then(setUpd).catch(() => setUpd(null)) }, [])
+  const imgState = useMemo(() => Object.fromEntries((upd?.images ?? []).map((i) => [i.image, i.state])), [upd])
+  const behind = (upd?.images ?? []).filter((i) => i.state === 'update').length
   if (!d) return <div className="dash">{err ? <div className="notice">{err === 'not found' ? "This machine's Ham Lab is older than the Docker tab. Update it from Machines." : err}</div> : <Loading label="Asking Docker…" />}</div>
   if (!d.available) return <div className="dash"><div className="notice">{d.message}</div></div>
   const selStack = sel.startsWith('stack:') ? sel.slice(6) : ''
@@ -375,12 +381,12 @@ export default function Docker() {
   const shownTab: Tab = selStack ? 'stacks' : tab
   const up = d.containers.filter((c) => c.state === 'running').length
   const sid = (c: DockerContainer, k: string) => sensors[`container.${c.name.toLowerCase().replace(/[^a-z0-9-]+/g, '_').replace(/^_|_$/g, '')}.${k}`]
-  const cols: Record<Exclude<Tab, 'containers' | 'stacks'>, [string, string][]> = { images: [['Repository', 'Repository'], ['Tag', 'Tag'], ['Size', 'Size'], ['CreatedSince', 'Created']], volumes: [['Name', 'Name'], ['Driver', 'Driver'], ['Mountpoint', 'Mountpoint']], networks: [['Name', 'Name'], ['Driver', 'Driver'], ['Scope', 'Scope']] }
+  const cols: Record<Exclude<Tab, 'containers' | 'stacks' | 'updates'>, [string, string][]> = { images: [['Repository', 'Repository'], ['Tag', 'Tag'], ['Size', 'Size'], ['CreatedSince', 'Created']], volumes: [['Name', 'Name'], ['Driver', 'Driver'], ['Mountpoint', 'Mountpoint']], networks: [['Name', 'Name'], ['Driver', 'Driver'], ['Scope', 'Scope']] }
   return (
     <div className={'svc docker' + (((cur || sel) && shownTab === 'containers') || selStack ? ' has-sel' : '')}>
       <div className="svc-list">
         <div className="svc-tools">
-          <div className="chips">{(['containers', 'stacks', 'images', 'volumes', 'networks'] as Tab[]).map((t) => <button key={t} className={'chip' + (shownTab === t ? ' on' : '')} onClick={() => { setTab(t); if (sel) setSel('') }}>{t} · {t === 'containers' ? `${up}/${d.containers.length}` : t === 'stacks' ? (stacks?.length ?? '…') : d[t].length}</button>)}</div>
+          <div className="chips">{(['containers', 'stacks', 'updates', 'images', 'volumes', 'networks'] as Tab[]).map((t) => <button key={t} className={'chip' + (shownTab === t ? ' on' : '')} onClick={() => { setTab(t); if (sel) setSel('') }}>{t}{t === 'containers' ? ` · ${up}/${d.containers.length}` : t === 'stacks' ? ` · ${stacks?.length ?? '…'}` : t === 'updates' ? (behind > 0 ? ` · ${behind}` : '') : ` · ${d[t].length}`}</button>)}</div>
           {(shownTab === 'containers' || shownTab === 'stacks') && <div className="svc-actions"><button className="btn" onClick={() => setMaking(shownTab === 'stacks' ? 'stack' : 'container')}>{shownTab === 'stacks' ? '+ New stack' : '+ New container'}</button></div>}
           {shownTab === 'images' && <form className="chan-add" onSubmit={async (e) => { e.preventDefault(); try { setPullTask((await api.pullImage(pull.trim())).id) } catch (er) { toast(er instanceof Error ? er.message : 'failed', 'err') } }}>
             <input className="mono" value={pull} onChange={(e) => setPull(e.target.value)} placeholder="pull an image — nginx:alpine, ghcr.io/owner/name:tag" spellCheck={false} autoCapitalize="off" autoCorrect="off" />
@@ -407,7 +413,7 @@ export default function Docker() {
               {list.map((c) => (
                 <div key={c.id} className={'svc-row' + (c.name === sel ? ' active' : '')} onClick={() => setSel(c.name)}>
                   <span className={'sdot ' + dot(c)} />
-                  <div className="svc-name"><div className="n">{c.name}</div><div className="d">{c.image}</div></div>
+                  <div className="svc-name"><div className="n">{c.name}<UpdateDot state={imgState[c.image]} /></div><div className="d">{c.image}</div></div>
                   <div className="svc-state">{c.state === 'running' ? <>{fmtValue(sid(c, 'mem')?.value ?? c.mem, 'B')}<br />{fmtValue(sid(c, 'cpu')?.value, '%')}</> : c.state}</div>
                 </div>
               ))}
@@ -425,16 +431,212 @@ export default function Docker() {
                 return <tr key={name}><td className="mono small" title={String(row.Mountpoint ?? '')}>{anon ? <>{name.slice(0, 12)}… <span className="muted">anonymous</span></> : name}</td><td className="small">{users.length ? users.join(', ') : <span className="muted">nothing</span>}</td><td className="row-end">{users.length ? <span className="badge">in use</span> : <button className="icon-btn" title="delete this volume and its data" onClick={remove}>✕</button>}</td></tr> })}
                 {d.volumes.length === 0 && <tr><td colSpan={3} className="muted small">No volumes.</td></tr>}</tbody></table>
           )}
-          {shownTab !== 'containers' && shownTab !== 'stacks' && shownTab !== 'volumes' && (
+          {shownTab !== 'containers' && shownTab !== 'stacks' && shownTab !== 'volumes' && shownTab !== 'updates' && (
-            <table className="dtable"><thead><tr>{cols[shownTab as Exclude<Tab, 'containers' | 'stacks'>].map(([, h]) => <th key={h}>{h}</th>)}{shownTab === 'images' && <th />}</tr></thead>
+            <table className="dtable"><thead><tr>{cols[shownTab as Exclude<Tab, 'containers' | 'stacks' | 'volumes' | 'updates'>].map(([, h]) => <th key={h}>{h}</th>)}{shownTab === 'images' && <th />}</tr></thead>
-              <tbody>{d[shownTab as Exclude<Tab, 'containers' | 'stacks'>].map((row, i) => <tr key={i}>{cols[shownTab as Exclude<Tab, 'containers' | 'stacks'>].map(([k]) => <td key={k} className="mono small">{String(row[k] ?? '')}</td>)}{shownTab === 'images' && <td className="row-end">{row.in_use ? <span className="badge" title="a container uses this image">in use</span> : <Btn className="icon-btn" title="remove this image" confirm={`Remove the image ${row.Repository}:${row.Tag}? Docker refuses if a container still uses it.`} onClick={async () => { const ref = row.Tag && row.Tag !== '<none>' ? `${row.Repository}:${row.Tag}` : String(row.ID); try { await api.removeImage(ref); toast(`${ref} removed`); load() } catch (er) { toast(er instanceof Error ? er.message : 'failed', 'err') } }}>✕</Btn>}</td>}</tr>)}</tbody></table>
+              <tbody>{d[shownTab as Exclude<Tab, 'containers' | 'stacks' | 'volumes' | 'updates'>].map((row, i) => <tr key={i}>{cols[shownTab as Exclude<Tab, 'containers' | 'stacks' | 'volumes' | 'updates'>].map(([k]) => <td key={k} className="mono small">{String(row[k] ?? '')}</td>)}{shownTab === 'images' && <td className="row-end">{row.in_use ? <span className="badge" title="a container uses this image">in use</span> : <Btn className="icon-btn" title="remove this image" confirm={`Remove the image ${row.Repository}:${row.Tag}? Docker refuses if a container still uses it.`} onClick={async () => { const ref = row.Tag && row.Tag !== '<none>' ? `${row.Repository}:${row.Tag}` : String(row.ID); try { await api.removeImage(ref); toast(`${ref} removed`); load() } catch (er) { toast(er instanceof Error ? er.message : 'failed', 'err') } }}>✕</Btn>}</td>}</tr>)}</tbody></table>
           )}
         </div>
       </div>
       {shownTab === 'containers' && !cur && sel && !selStack && <div className="svc-detail"><CloseBtn onClick={() => setSel('')} /><button className="btn back" onClick={() => setSel('')}>‹ Back</button><div className="notice">There is no container called <b className="mono">{sel}</b> on this machine. It may have been removed, renamed or recreated under another name. <button className="link" onClick={() => setSel('')}>Show them all</button></div></div>}
       {shownTab === 'containers' && (cur ? <Detail c={cur} live={sensors} onBack={() => setSel('')} onChanged={load} /> : (sel && !selStack ? null : <div className="svc-empty muted">Select a container.</div>))}
       {making && <StackNew key={'new:' + (dup?.name ?? making)} mode={making} from={dup ?? undefined} networks={nets} taken={taken} onClose={() => { setMaking(''); setDup(null) }} onCreated={async (name, deploy) => { setMaking(''); setDup(null); setTab('stacks'); if (deploy) { try { setStartTask({ name, id: (await api.stackAction(name, 'deploy')).id }) } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } } else setStartTask(null); await load(); setSel('stack:' + name) }} />}
+      {shownTab === 'updates' && <Updates upd={upd} onChecked={setUpd} />}
       {shownTab === 'stacks' && (selStack ? <StackDetail key={selStack + (startTask?.name === selStack ? startTask.id : '')} name={selStack} startTask={startTask?.name === selStack ? startTask.id : undefined} onBack={() => setSel('')} onChanged={load} onDuplicate={(f) => { setDup(f); setMaking('stack') }} /> : <div className="svc-empty muted">Select a stack to edit its compose file.</div>)}
     </div>
   )
 }
+
+// A container's image against the registry, as a single mark: nothing when nobody has
+// asked, so an uncoloured list means "not checked" rather than "all current".
+function UpdateDot({ state }: { state?: string }) {
+  if (!state || state === 'current') return null
+  const known = state === 'update'
+  return <span className={'updot' + (known ? ' behind' : ' unknown')} title={known ? 'a newer image has been published' : 'the registry could not say'}>{known ? '↑' : '?'}</span>
+}
+
+// What is out of date here, and the schedules that keep it from happening again.
+function Updates({ upd, onChecked }: { upd: MachineUpdates | null; onChecked: (u: MachineUpdates) => void }) {
+  const onHub = apiPath('/api/x') === '/api/x'
+  const [busy, setBusy] = useState(false)
+  const [jobs, setJobs] = useState<UpdateJobRow[] | null>(null)
+  const [file, setFile] = useState('')
+  const [editing, setEditing] = useState<Partial<UpdateJob> | null>(null)
+  const [stacks, setStacks] = useState<string[]>([])
+  const loadJobs = useCallback(async () => {
+    try { const r = await api.updateJobs(); setJobs(r.jobs); setFile(r.file) } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err'); setJobs([]) }
+  }, [])
+  useEffect(() => { loadJobs(); api.stacks().then((r) => setStacks(r.stacks.map((s) => s.name))).catch(() => {}) }, [loadJobs])
+  const check = async () => {
+    setBusy(true)
+    try { onChecked(await api.machineUpdates(true)); toast('checked') } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } finally { setBusy(false) }
+  }
+  const run = async (j: UpdateJob) => {
+    if (!await askYes(`Run “${j.name}” now? It pulls ${j.stack}'s images${j.apply ? ' and deploys them, which restarts its containers' : ', without deploying them'}.`, 'Run now')) return
+    try { await api.runUpdateJob(j.id); toast(`${j.name}: done`); loadJobs() } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') }
+  }
+  const remove = async (j: UpdateJob) => {
+    if (!await askYes(`Remove the schedule “${j.name}”? Nothing is updated or rolled back — the schedule simply stops.`, 'Remove')) return
+    try { await api.deleteUpdateJob(j.id); toast('removed'); loadJobs() } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') }
+  }
+  const imgs = upd?.images ?? []
+  const behind = imgs.filter((i) => i.state === 'update')
+  return (
+    <div className="svc-detail">
+      <div className="dash-bar"><h2 className="section">What is out of date</h2><span className="spacer" />
+        <span className="muted small">{upd?.checked ? `checked ${new Date(upd.checked).toLocaleString()}` : 'not checked yet'}</span>
+        <Btn className="btn" busy={busy} onClick={check}>Check now</Btn>
+      </div>
+      <div className="muted small">Each check is one request per image, which registries count — so it happens when asked, and the answer is kept for six hours.</div>
+      {imgs.length === 0 && <div className="notice" style={{ marginTop: 8 }}>Nothing checked yet on this machine.</div>}
+      {imgs.length > 0 && (
+        <section className="card" style={{ marginTop: 8 }}>
+          {behind.length === 0 && <div className="muted">Every image that could be checked is current.</div>}
+          {imgs.filter((i) => i.state !== 'current').map((i) => (
+            <div className="row" key={i.image}>
+              <span className="k mono small">{i.image}</span>
+              <span className={i.state === 'update' ? 'warn-text' : 'muted small'}>{i.state === 'update' ? 'a newer image is published' : i.detail || 'unknown'}</span>
+              <span className="muted small">{(upd?.stacks?.[i.image] ?? []).join(' ')}</span>
+            </div>
+          ))}
+          {behind.length === 0 && imgs.some((i) => i.state === 'current') && <div className="muted small" style={{ marginTop: 6 }}>{imgs.filter((i) => i.state === 'current').length} current.</div>}
+        </section>
+      )}
+
+      {onHub && <Fleet />}
+
+      <div className="dash-bar" style={{ marginTop: 14 }}><h2 className="section">Schedules</h2><span className="spacer" />
+        <Btn className="btn on" onClick={() => setEditing({ cron: '0 3 * * *', apply: true, enabled: true })}>+ New schedule</Btn>
+      </div>
+      <div className="muted small">Written to <span className="mono">{file || '/etc/cron.d/hamlab-docker'}</span>, which Ham Lab owns — so they keep running whether or not this console is up.</div>
+      {jobs === null && <Loading label="Reading…" />}
+      {jobs?.length === 0 && <div className="notice" style={{ marginTop: 8 }}>No schedule yet. Nothing on this machine updates itself.</div>}
+      <div className="cards" style={{ marginTop: 8 }}>
+        {(jobs ?? []).map(({ job, next, last }) => (
+          <section className="card" key={job.id}>
+            <div className="sub-head">
+              <span className={'dot ' + (last?.error ? 'bad' : job.enabled ? 'ok' : '')} />
+              <span className="dname">{job.name}</span>
+              {!job.enabled && <span className="badge">paused</span>}
+              <span className="spacer" />
+              <span className="muted small">{job.enabled && next ? new Date(next).toLocaleString() : 'not scheduled'}</span>
+            </div>
+            <div className="props" style={{ marginTop: 6 }}>
+              <div><span className="k">Stack</span><span>{job.stack}{job.services?.length ? ` · ${job.services.join(' ')}` : ''}</span></div>
+              <div><span className="k">Cron</span><span className="mono small">{job.cron}</span></div>
+              <div><span className="k">Does</span><span>{job.apply ? 'pulls and deploys' : 'pulls only — you deploy'}</span></div>
+              <div><span className="k">Last run</span><span>{last?.at ? new Date(last.at).toLocaleString() : 'never'}</span></div>
+            </div>
+            {last?.error && <div className="error" style={{ marginTop: 6 }}>{last.error}</div>}
+            <div className="svc-actions">
+              <Btn className="btn on" onClick={() => run(job)}>Run now</Btn>
+              <button className="btn" onClick={() => setEditing(job)}>Edit</button>
+              <Btn className="btn" onClick={async () => { await api.saveUpdateJob({ ...job, enabled: !job.enabled }); loadJobs() }}>{job.enabled ? 'Pause' : 'Resume'}</Btn>
+              <Btn className="btn danger" onClick={() => remove(job)}>Remove</Btn>
+            </div>
+          </section>
+        ))}
+      </div>
+      {editing && <JobDialog j={editing} stacks={stacks} onClose={() => setEditing(null)} onSaved={() => { setEditing(null); loadJobs() }} />}
+    </div>
+  )
+}
+
+// Every other machine at once. Only shown on the hub, because only the hub can reach them,
+// and each answer is that machine's own six-hour cache — nothing is checked here without
+// being asked for.
+function Fleet() {
+  const [rows, setRows] = useState<{ name: string; online: boolean; upd?: MachineUpdates; err?: string }[] | null>(null)
+  const [busy, setBusy] = useState('')
+  const load = useCallback(async (refresh?: string) => {
+    let nodes: MachineStatus[] = []
+    try { nodes = (await api.machines()).nodes } catch { setRows([]); return }
+    const out = await Promise.all(nodes.map(async (n) => {
+      if (!n.online) return { name: n.name, online: false }
+      try { return { name: n.name, online: true, upd: await api.nodeUpdates(n.name, refresh === n.name) } }
+      catch (e) { return { name: n.name, online: true, err: e instanceof Error ? e.message : 'failed' } }
+    }))
+    setRows(out)
+  }, [])
+  useEffect(() => { load() }, [load])
+  if (rows === null) return <div style={{ marginTop: 14 }}><Loading label="Asking the other machines…" /></div>
+  if (rows.length === 0) return null
+  return (
+    <>
+      <h2 className="section" style={{ marginTop: 14 }}>The other machines</h2>
+      <section className="card">
+        {rows.map((r) => {
+          const behind = (r.upd?.images ?? []).filter((i) => i.state === 'update')
+          return (
+            <div className="row" key={r.name}>
+              <span className="k">{r.name}</span>
+              <span>
+                {!r.online ? <span className="muted">offline</span>
+                  : r.err ? <span className="muted small">{r.err}</span>
+                  : !r.upd?.checked ? <span className="muted">not checked</span>
+                  : behind.length === 0 ? <span className="muted">all current</span>
+                  : <span className="warn-text" title={behind.map((i) => i.image).join('\n')}>{behind.length} behind</span>}
+              </span>
+              <span className="row-end">{r.online && !r.err && <Btn className="link" busy={busy === r.name} onClick={async () => { setBusy(r.name); try { await load(r.name) } finally { setBusy('') } }}>check</Btn>}</span>
+            </div>
+          )
+        })}
+      </section>
+      <div className="muted small">Open a machine from the switcher to schedule anything on it — a schedule lives on the machine that runs it.</div>
+    </>
+  )
+}
+
+function JobDialog({ j, stacks, onClose, onSaved }: { j: Partial<UpdateJob>; stacks: string[]; onClose: () => void; onSaved: () => void }) {
+  const [f, setF] = useState<Partial<UpdateJob>>(j)
+  const [svc, setSvc] = useState((j.services ?? []).join(' '))
+  const [err, setErr] = useState('')
+  const [busy, setBusy] = useState(false)
+  useEscape(onClose)
+  const set = (k: keyof UpdateJob, v: unknown) => setF((x) => ({ ...x, [k]: v }))
+  const save = async (e: FormEvent) => {
+    e.preventDefault(); setErr(''); setBusy(true)
+    try { await api.saveUpdateJob({ ...f, services: svc.split(/\s+/).filter(Boolean) }); onSaved() }
+    catch (e) { setErr(e instanceof Error ? e.message : 'failed'); setBusy(false) }
+  }
+  return (
+    <div className="modal-back" onClick={onClose}>
+      <form className="modal" onClick={(e) => e.stopPropagation()} onSubmit={save}>
+        <h3>{f.id ? 'Edit schedule' : 'Keep a stack up to date'}</h3>
+        <label className="muted">Stack
+          <select value={f.stack ?? ''} onChange={(e) => set('stack', e.target.value)} required>
+            <option value="">choose…</option>
+            {stacks.map((s) => <option key={s} value={s}>{s}</option>)}
+          </select>
+        </label>
+        <label className="muted">Name — optional
+          <input value={f.name ?? ''} onChange={(e) => set('name', e.target.value)} placeholder={f.stack ?? 'nightly'} maxLength={64} />
+        </label>
+        <label className="muted">Only these services — optional, space separated; blank means the whole stack
+          <input className="mono" value={svc} onChange={(e) => setSvc(e.target.value)} placeholder="all of them" spellCheck={false} />
+        </label>
+        <label className="muted">When — a cron expression
+          <input className="mono" value={f.cron ?? ''} onChange={(e) => set('cron', e.target.value)} placeholder="0 3 * * *" spellCheck={false} required />
+        </label>
+        <label className="muted" style={{ flexDirection: 'row', alignItems: 'center', gap: 8 }}>
+          <input type="checkbox" checked={!!f.apply} onChange={(e) => set('apply', e.target.checked)} style={{ width: 'auto' }} />
+          Deploy the new image, not just fetch it
+        </label>
+        <label className="muted" style={{ flexDirection: 'row', alignItems: 'center', gap: 8 }}>
+          <input type="checkbox" checked={!!f.enabled} onChange={(e) => set('enabled', e.target.checked)} style={{ width: 'auto' }} />
+          Enabled
+        </label>
+        <div className="notice">
+          {f.apply
+            ? 'Deploying restarts the stack’s containers. On something that carries DNS or ingress, pick an hour you are asleep.'
+            : 'Pulling only means the new image is on the machine and the list shows it as available — nothing restarts until you deploy.'}
+        </div>
+        {err && <div className="error">{err}</div>}
+        <div className="svc-actions">
+          <button type="submit" className="btn on" disabled={busy || !f.stack || !f.cron}>Save</button>
+          <span className="spacer" />
+          <button type="button" className="btn" onClick={onClose}>Cancel</button>
+        </div>
+      </form>
+    </div>
+  )
+}

web/src/api.ts

@@ -109,6 +109,10 @@ export interface StorageDevice { name: string; kind: string; model?: string; ven
 export interface SmartAttr { id: number; name: string; value: number; worst: number; thresh: number; raw: string; failed?: string; warn: boolean }
 export interface SmartInfo { device: string; available: boolean; supported: boolean; standby: boolean; passed: boolean | null; temp: number; power_on_hours: number; power_cycles: number; model?: string; serial?: string; firmware?: string; attrs: SmartAttr[]; nvme?: { critical_warning: number; percentage_used: number; available_spare: number; spare_threshold: number; media_errors: number; error_log_entries: number; data_read_bytes: number; data_written_bytes: number; unsafe_shutdowns: number }; mmc?: { life_a: number; life_b: number; pre_eol: number }; self_test?: string; self_test_remaining?: number; self_test_supported: boolean; self_test_note?: string; warnings: string[]; error?: string; fetched_at: number }
 export interface DockerContainer { id: string; name: string; image: string; state: string; status: string; health: string; ports: string; stack: string; service: string; created: string; mem: number }
+export interface UpdateJob { id: string; name: string; stack: string; machine?: string; services?: string[]; cron: string; apply: boolean; enabled: boolean }
+export interface UpdateJobRow { job: UpdateJob; next?: string; last?: { at?: string; taken?: string; error?: string } }
+export interface MachineUpdates { images: ImageCheck[]; checked: number; stacks: Record<string, string[]> }
+
 export interface DockerHealthProbe { Start: string; End: string; ExitCode: number; Output: string }
 export interface DockerHealth { configured: boolean; state: string; status?: string; failing_streak?: number; log?: DockerHealthProbe[] }
 export interface DockerEvent { t: number; name: string; action: string; image?: string; exit_code?: string; signal?: string; health?: string }
@@ -252,6 +256,13 @@ export const api = {
   dockerAction: (id: string, action: 'start' | 'stop' | 'restart' | 'remove') => req<void>('POST', `/api/docker/containers/${encodeURIComponent(id)}/${action}`),
   dockerLogs: (id: string, since = '') => req<{ lines: { t: string; m: string }[] }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/logs${since ? `?since=${encodeURIComponent(since)}` : '?n=300'}`),
   dockerInspect: (id: string) => req<Record<string, unknown>>('GET', `/api/docker/containers/${encodeURIComponent(id)}/inspect`),
+  machineUpdates: (refresh = false) => req<MachineUpdates>('GET', `/api/docker/updates${refresh ? '?refresh=1' : ''}`),
+  // One named machine, asked of the hub — the fleet view, which is on the hub by definition.
+  nodeUpdates: (node: string, refresh = false) => req<MachineUpdates>('GET', `/api/nodes/${encodeURIComponent(node)}/docker/updates${refresh ? '?refresh=1' : ''}`, undefined, true),
+  updateJobs: () => req<{ jobs: UpdateJobRow[]; file: string }>('GET', '/api/docker/schedules'),
+  saveUpdateJob: (j: Partial<UpdateJob>) => req<UpdateJob>('PUT', '/api/docker/schedules', j),
+  deleteUpdateJob: (id: string) => req<void>('DELETE', `/api/docker/schedules/${encodeURIComponent(id)}`),
+  runUpdateJob: (id: string) => req<{ output: string }>('POST', `/api/docker/schedules/${encodeURIComponent(id)}/run`),
   dockerHealth: (id: string) => req<DockerHealth>('GET', `/api/docker/containers/${encodeURIComponent(id)}/health`),
   dockerEvents: (id: string, since = '7d') => req<{ events: DockerEvent[]; since: string; kept: number; capacity: number }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/events?since=${since}`),
   dockerExec: (id: string) => req<{ id: string; name: string }>('POST', `/api/docker/containers/${encodeURIComponent(id)}/exec`, {}),

web/src/styles.css

@@ -756,3 +756,6 @@ pre.diff .add { color: var(--ok); background: color-mix(in srgb, var(--ok) 12%,
 .kvm-frame.expanded iframe { height: 100%; }
 .kvm-shrink { position: absolute; right: max(10px, env(safe-area-inset-right)); bottom: max(10px, env(safe-area-inset-bottom)); z-index: 61; background: var(--bg2); border: 1px solid var(--border); opacity: .75; }
 .kvm-shrink:hover, .kvm-shrink:active { opacity: 1; }
+.updot { margin-left: 6px; font-size: 11px; font-weight: 700; }
+.updot.behind { color: var(--s2); } .updot.unknown { color: var(--muted); }
+.card > .row > .row-end { text-align: right; white-space: nowrap; }