Stacks: remote stacks editable from the hub, .env editing, deploy diff, image update checks; a gone container says so

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-19 16:42:54 -07:00
parent eddba4250b
commit 1d48267d90
7 changed files with 553 additions and 132 deletions
@@ -133,6 +133,18 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
 	m.HandleFunc("GET /api/docker/stacks/{name}", s.auth(s.stackFile))
 	m.HandleFunc("PUT /api/docker/stacks/{name}", s.auth(s.stackSave))
 	m.HandleFunc("POST /api/docker/stacks/{name}/{action}", s.auth(s.stackAction))
+	m.HandleFunc("GET /api/docker/stacks/{name}/diff", s.auth(s.stackDiff))
+	m.HandleFunc("GET /api/docker/stacks/{name}/updates", s.auth(s.stackUpdates))
+	// The same, asked of a node through this hub: the hub answers for the stacks whose files
+	// it holds (deployed to the node over tcp) and passes everything else to the node.
+	m.HandleFunc("GET /api/nodes/{node}/docker/stacks", s.auth(s.stacksList))
+	m.HandleFunc("GET /api/nodes/{node}/docker/stacks/{name}", s.auth(s.stackFile))
+	m.HandleFunc("PUT /api/nodes/{node}/docker/stacks/{name}", s.auth(s.stackSave))
+	m.HandleFunc("GET /api/nodes/{node}/docker/stacks/{name}/diff", s.auth(s.stackDiff))
+	m.HandleFunc("GET /api/nodes/{node}/docker/stacks/{name}/updates", s.auth(s.stackUpdates))
+	m.HandleFunc("POST /api/nodes/{node}/docker/stacks/{name}/{action}", s.auth(s.stackAction))
+	m.HandleFunc("GET /api/nodes/{node}/tasks/{id}", s.auth(s.taskOnHub(s.getTask)))
+	m.HandleFunc("GET /api/nodes/{node}/tasks/{id}/stream", s.auth(s.taskOnHub(s.streamTask)))
 	m.HandleFunc("POST /api/docker/containers/{id}/{action}", s.auth(s.dockerAction))
 	m.HandleFunc("GET /api/docker/containers/{id}/logs", s.auth(s.dockerLogs))
 	m.HandleFunc("GET /api/docker/containers/{id}/inspect", s.auth(s.dockerInspect))
@@ -674,7 +686,11 @@ func (s *Server) updateAllMachines(w http.ResponseWriter, r *http.Request) {
 }

 func (s *Server) proxyNode(w http.ResponseWriter, r *http.Request) {
-	p, ok := s.nodes.Proxy(r.PathValue("name"))
+	node := r.PathValue("node") // routes that also carry a {name} of their own call the machine {node}
+	if node == "" {
+		node = r.PathValue("name")
+	}
+	p, ok := s.nodes.Proxy(node)
 	if !ok {
 		jsonErr(w, 404, "no such machine")
 		return
@@ -3,15 +3,20 @@ package api
 // Stacks: the compose projects behind the containers, with their files to edit and a way to
 // redeploy them.
 //
-// A stack is found from its containers' labels (project, config file). The file may not be
+// A stack is found from its containers' labels (project, config files). The files may not be
 // where the label says: an orchestrator that runs compose from inside its own container —
-// Dockhand, Dockge, Portainer — records a path in *its* filesystem. So the path is looked
-// up twice: as it stands on this host, and through every container's mounts
+// Dockhand, Dockge, Portainer — records paths in *its* filesystem. So each path is looked up
+// twice: as it stands on this host, and through every container's mounts
 // (/app/data/stacks/… in `dockhand` is /home/core/apps/dockhand/stacks/… here). Compose is
-// then run where the file makes sense — inside that container when it came from one, so
-// relative paths, the .env beside it and the labels all stay exactly as they were deployed —
-// else on the host. A stack whose file is on no path this machine can see (deployed from
-// another host over tcp) is listed read-only, and says so.
+// then run where the files make sense — inside that container when they came from one, so
+// relative paths, the .env beside them and the labels all stay exactly as they were
+// deployed — else on the host.
+//
+// A stack can also be deployed *from another machine*: Dockhand on the hub drives a node's
+// Docker over tcp, so the node runs the containers and the hub holds the files. The node
+// alone can only list such a stack read-only. The hub, asked about one of its nodes, fills
+// in what the node couldn't (hubStacks): it resolves the files on itself and runs compose
+// here with DOCKER_HOST pointed at the node.

 import (
 	"bytes"
@@ -24,12 +29,13 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"sort"
-	"strconv"
 	"strings"
+	"sync"
 	"time"

 	"hamlab/internal/priv"
@@ -38,12 +44,15 @@ import (
 type stack struct {
 	Name       string   `json:"name"`
 	Containers []string `json:"containers"`
+	Images     []string `json:"images"`
 	Running    int      `json:"running"`
 	File       string   `json:"file"`       // as the label has it (the first of Files)
-	HostFile   string   `json:"host_file"`  // where it is on this machine; "" when it isn't
+	HostFile   string   `json:"host_file"`  // where it is on the machine that holds it; "" when unknown
 	Files      []string `json:"files"`      // every file it was deployed from, in order: -f a -f b
-	HostFiles  []string `json:"host_files"` // the same, on this machine
+	HostFiles  []string `json:"host_files"` // the same, on the machine that holds them
 	Via        string   `json:"via"`        // container compose runs inside; "" = on the host
+	Remote     string   `json:"remote"`     // DOCKER_HOST compose is pointed at; "" = this machine's Docker
+	HeldBy     string   `json:"held_by"`    // the machine holding the files, when it isn't the one running them
 	Editable   bool     `json:"editable"`
 	Note       string   `json:"note,omitempty"`
 	HasEnv     bool     `json:"has_env"`
@@ -90,6 +99,41 @@ func readable(p string) bool {
 	return false
 }

+// resolve finds every file of a stack on this machine. Every one must be found — a stack
+// deployed from `-f compose.yaml -f override.yaml` and redeployed from the first alone would
+// quietly lose everything the override set.
+func (st *stack) resolve(maps []mountMap) {
+	st.HostFiles, st.HostFile, st.Via, st.Editable, st.HasEnv = nil, "", "", false, false
+	if st.File == "" {
+		st.Note = "its containers don't say which file deployed them"
+		return
+	}
+	one := func(f string) (host, via string) {
+		if readable(f) {
+			return f, ""
+		}
+		for _, m := range maps {
+			if rest, ok := strings.CutPrefix(f, m.dest+"/"); ok && readable(filepath.Join(m.src, rest)) {
+				return filepath.Join(m.src, rest), m.container
+			}
+		}
+		return "", ""
+	}
+	for i, f := range st.Files {
+		host, via := one(f)
+		if host == "" || (i > 0 && via != st.Via) {
+			st.HostFiles, st.Via = nil, ""
+			st.Note = "deployed from somewhere this machine can't see (" + f + "); edit it there"
+			return
+		}
+		st.HostFiles, st.Via = append(st.HostFiles, host), via
+	}
+	st.HostFile, st.Editable, st.Note = st.HostFiles[0], true, ""
+	st.HasEnv = readable(st.envPath())
+}
+
+func (st stack) envPath() string { return filepath.Join(filepath.Dir(st.HostFiles[0]), ".env") }
+
 func (s *Server) findStacks(ctx context.Context) ([]stack, error) {
 	// Labels come from inspect, not `ps`: ps joins every label with commas, and the label that
 	// matters here — the list of compose files — is itself a comma-separated list.
@@ -99,26 +143,29 @@ func (s *Server) findStacks(ctx context.Context) ([]stack, error) {
 	}
 	by := map[string]*stack{}
 	if len(bytes.TrimSpace(ids)) > 0 {
-		const f = `{{.Name}}|{{.State.Status}}|{{index .Config.Labels "com.docker.compose.project"}}|{{index .Config.Labels "com.docker.compose.project.config_files"}}`
+		const f = `{{.Name}}|{{.State.Status}}|{{.Config.Image}}|{{index .Config.Labels "com.docker.compose.project"}}|{{index .Config.Labels "com.docker.compose.project.config_files"}}`
 		out, err := dockerRun(ctx, append([]string{"inspect", "-f", f}, strings.Fields(string(ids))...)...)
 		if err != nil {
 			return nil, err
 		}
 		for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
-			p := strings.SplitN(line, "|", 4)
-			if len(p) < 4 || p[2] == "" {
+			p := strings.SplitN(line, "|", 5)
+			if len(p) < 5 || p[3] == "" {
 				continue
 			}
-			st := by[p[2]]
+			st := by[p[3]]
 			if st == nil {
-				st = &stack{Name: p[2]}
-				if p[3] != "" {
-					st.Files = strings.Split(p[3], ",")
+				st = &stack{Name: p[3]}
+				if p[4] != "" {
+					st.Files = strings.Split(p[4], ",")
 					st.File = st.Files[0]
 				}
-				by[p[2]] = st
+				by[p[3]] = st
 			}
 			st.Containers = append(st.Containers, strings.TrimPrefix(p[0], "/"))
+			if !contains(st.Images, p[2]) {
+				st.Images = append(st.Images, p[2])
+			}
 			if p[1] == "running" {
 				st.Running++
 			}
@@ -128,61 +175,123 @@ func (s *Server) findStacks(ctx context.Context) ([]stack, error) {
 	stacks := []stack{}
 	for _, st := range by {
 		sort.Strings(st.Containers)
-		// Every file must be found — a stack deployed from `-f compose.yaml -f override.yaml` and
-		// redeployed from the first alone would quietly lose everything the override set.
-		resolve := func(f string) (host, via string) {
-			if readable(f) {
-				return f, ""
-			}
-			for _, m := range maps {
-				if rest, ok := strings.CutPrefix(f, m.dest+"/"); ok && readable(filepath.Join(m.src, rest)) {
-					return filepath.Join(m.src, rest), m.container
-				}
-			}
-			return "", ""
-		}
-		if st.File == "" {
-			st.Note = "its containers don't say which file deployed them"
-		} else {
-			for i, f := range st.Files {
-				host, via := resolve(f)
-				if host == "" || (i > 0 && via != st.Via) {
-					st.HostFiles, st.Via = nil, ""
-					st.Note = "deployed from somewhere this machine can't see (" + f + "); edit it there"
-					break
-				}
-				st.HostFiles, st.Via = append(st.HostFiles, host), via
-			}
-			if len(st.HostFiles) > 0 {
-				st.HostFile = st.HostFiles[0]
-			}
-		}
-		if st.HostFile != "" {
-			st.Editable = true
-			st.HasEnv = readable(filepath.Join(filepath.Dir(st.HostFile), ".env"))
-		}
+		sort.Strings(st.Images)
+		st.resolve(maps)
 		stacks = append(stacks, *st)
 	}
 	sort.Slice(stacks, func(i, j int) bool { return stacks[i].Name < stacks[j].Name })
 	return stacks, nil
 }

-func (s *Server) stackByName(ctx context.Context, name string) (stack, error) {
-	all, err := s.findStacks(ctx)
-	if err != nil {
-		return stack{}, err
-	}
-	for _, st := range all {
-		if st.Name == name {
-			return st, nil
+func contains(list []string, s string) bool {
+	for _, x := range list {
+		if x == s {
+			return true
 		}
 	}
-	return stack{}, errors.New("no such stack")
+	return false
+}
+
+// hubStacks is a node's stack list with the hub's help: what the node could not find is
+// looked for here, and if it is here — and the node's Docker answers over tcp, which is how
+// it was deployed in the first place — the stack becomes editable from this hub.
+func (s *Server) hubStacks(ctx context.Context, node string) ([]stack, error) {
+	code, b, err := s.nodes.Do(ctx, node, "GET", "/api/docker/stacks", nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	var res struct{ Stacks []stack }
+	if code != 200 || json.Unmarshal(b, &res) != nil {
+		return nil, fmt.Errorf("%s answered %d", node, code)
+	}
+	remote, maps := "", []mountMap(nil)
+	for i := range res.Stacks {
+		st := &res.Stacks[i]
+		if st.Editable || len(st.Files) == 0 {
+			continue
+		}
+		if remote == "" {
+			remote, maps = s.nodeDocker(ctx, node), stackMounts(ctx)
+		}
+		if remote == "-" {
+			continue
+		}
+		note := st.Note
+		if st.resolve(maps); !st.Editable {
+			st.Note = note
+			continue
+		}
+		st.Remote, st.HeldBy = remote, s.host
+	}
+	return res.Stacks, nil
+}
+
+var (
+	nodeDockerMu sync.Mutex
+	nodeDockerOK = map[string]string{} // node → "tcp://host:2375", or "-" when it doesn't answer
+)
+
+// nodeDocker is the node's Docker endpoint as Dockhand-style deployments use it, if it answers.
+func (s *Server) nodeDocker(ctx context.Context, node string) string {
+	nodeDockerMu.Lock()
+	defer nodeDockerMu.Unlock()
+	if v, ok := nodeDockerOK[node]; ok && v != "-" {
+		return v
+	}
+	ep := "-"
+	for _, n := range s.nodes.List() {
+		if n.Name != node {
+			continue
+		}
+		if u, err := url.Parse(n.URL); err == nil && u.Hostname() != "" {
+			cand := "tcp://" + u.Hostname() + ":2375"
+			c, cancel := context.WithTimeout(ctx, 5*time.Second)
+			if exec.CommandContext(c, dockerTool(), "-H", cand, "version", "--format", "{{.Server.Version}}").Run() == nil {
+				ep = cand
+			}
+			cancel()
+		}
+	}
+	nodeDockerOK[node] = ep
+	return ep
+}
+
+// stackFor finds the stack a request means: on this machine, or (on a hub route) one of a
+// node's stacks that this hub holds the files for. handled is false when the request should
+// simply be passed on to the node.
+func (s *Server) stackFor(ctx context.Context, r *http.Request) (st stack, handled bool, err error) {
+	var all []stack
+	if node := r.PathValue("node"); node != "" {
+		all, err = s.hubStacks(ctx, node)
+	} else {
+		all, err = s.findStacks(ctx)
+	}
+	if err != nil {
+		return stack{}, true, err
+	}
+	for _, x := range all {
+		if x.Name == r.PathValue("name") {
+			if r.PathValue("node") != "" && x.Remote == "" {
+				return x, false, nil // the node can serve this one itself
+			}
+			return x, true, nil
+		}
+	}
+	return stack{}, true, errors.New("no such stack")
 }

 func (s *Server) stacksList(w http.ResponseWriter, r *http.Request) {
-	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
+	ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second)
 	defer cancel()
+	if node := r.PathValue("node"); node != "" {
+		stacks, err := s.hubStacks(ctx, node)
+		if err != nil {
+			s.proxyNode(w, r) // let the node's own answer (or failure) through
+			return
+		}
+		jsonOK(w, 200, map[string]any{"stacks": stacks})
+		return
+	}
 	if dockerTool() == "" { // no Docker is an empty answer, not a failure
 		jsonOK(w, 200, map[string]any{"stacks": []stack{}})
 		return
@@ -206,29 +315,41 @@ func readAsRoot(ctx context.Context, path string) ([]byte, error) {

 func sumOf(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:8]) }

-// fileIndex is which of the stack's files a request means: ?file=1 is the first override.
-func fileIndex(r *http.Request, st stack) (int, bool) {
-	i, err := strconv.Atoi(r.URL.Query().Get("file"))
-	if r.URL.Query().Get("file") == "" {
-		return 0, true
+// target is which of the stack's files a request means: ?file=1 is the first override,
+// ?file=env the .env beside the first file.
+func target(r *http.Request, st stack) (host string, index int, ok bool) {
+	q := r.URL.Query().Get("file")
+	switch {
+	case q == "env":
+		return st.envPath(), -1, st.HasEnv
+	case q == "":
+		return st.HostFiles[0], 0, true
 	}
-	return i, err == nil && i >= 0 && i < len(st.HostFiles)
+	var i int
+	if _, err := fmt.Sscanf(q, "%d", &i); err != nil || i < 0 || i >= len(st.HostFiles) {
+		return "", 0, false
+	}
+	return st.HostFiles[i], i, true
 }

 func (s *Server) stackFile(w http.ResponseWriter, r *http.Request) {
-	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
+	ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second)
 	defer cancel()
-	st, err := s.stackByName(ctx, r.PathValue("name"))
-	if err != nil || !st.Editable {
-		jsonErr(w, 404, "that stack's compose file isn't on this machine")
+	st, handled, err := s.stackFor(ctx, r)
+	if !handled {
+		s.proxyNode(w, r)
 		return
 	}
-	i, ok := fileIndex(r, st)
+	if err != nil || !st.Editable {
+		jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach")
+		return
+	}
+	path, i, ok := target(r, st)
 	if !ok {
 		jsonErr(w, 400, "no such file in this stack")
 		return
 	}
-	b, err := readAsRoot(ctx, st.HostFiles[i])
+	b, err := readAsRoot(ctx, path)
 	if err != nil {
 		jsonErr(w, 500, err.Error())
 		return
@@ -237,8 +358,8 @@ func (s *Server) stackFile(w http.ResponseWriter, r *http.Request) {
 	jsonOK(w, 200, map[string]any{"stack": st, "text": string(b), "sum": sumOf(b), "index": i})
 }

-// composeCmd is `docker compose …` for this stack, run where its file makes sense.
-// files are given as compose will see them: the label's paths inside a container, host paths otherwise.
+// composeCmd is `docker compose …` for this stack, run where its files make sense and
+// pointed at the Docker that runs its containers. files are given as compose will see them.
 func composeCmd(ctx context.Context, st stack, files []string, args ...string) *exec.Cmd {
 	tool := dockerTool()
 	base := []string{"compose", "-p", st.Name}
@@ -246,10 +367,17 @@ func composeCmd(ctx context.Context, st stack, files []string, args ...string) *
 		base = append(base, "-f", f)
 	}
 	if st.Via != "" {
-		return exec.CommandContext(ctx, tool, append(append([]string{"exec", st.Via, "docker"}, base...), args...)...)
+		pre := []string{"exec"}
+		if st.Remote != "" {
+			pre = append(pre, "-e", "DOCKER_HOST="+st.Remote)
+		}
+		return exec.CommandContext(ctx, tool, append(append(append(pre, st.Via, "docker"), base...), args...)...)
 	}
 	cmd := exec.CommandContext(ctx, tool, append(base, args...)...)
 	cmd.Dir = filepath.Dir(files[0])
+	if st.Remote != "" {
+		cmd.Env = append(os.Environ(), "DOCKER_HOST="+st.Remote)
+	}
 	return cmd
 }

@@ -265,72 +393,156 @@ func seenFiles(st stack, i int, candidate string) []string {
 	return out
 }

+func (s *Server) writeAsOwner(ctx context.Context, path, text string) error {
+	err := s.fsLocal.Write(ctx, path, strings.NewReader(text)) // ours to write, or root's
+	if err != nil {
+		err = s.fsSudo.Write(ctx, path, strings.NewReader(text))
+	}
+	return err
+}
+
 // stackSave validates the new text with compose itself, keeps the old file beside it as
 // <file>.hamlab-bak, and writes. It does not deploy: that is its own button.
 func (s *Server) stackSave(w http.ResponseWriter, r *http.Request) {
 	ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
 	defer cancel()
-	st, err := s.stackByName(ctx, r.PathValue("name"))
+	st, handled, err := s.stackFor(ctx, r)
+	if !handled {
+		s.proxyNode(w, r)
+		return
+	}
 	if err != nil || !st.Editable {
-		jsonErr(w, 404, "that stack's compose file isn't on this machine")
+		jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach")
 		return
 	}
 	var body struct {
 		Text string `json:"text"`
 		Sum  string `json:"sum"` // the sum the editor was given when it opened the file
 	}
-	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&body); err != nil || strings.TrimSpace(body.Text) == "" {
-		jsonErr(w, 400, "the file can't be empty")
+	path, i, ok := target(r, st)
+	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&body); err != nil || !ok || (i >= 0 && strings.TrimSpace(body.Text) == "") {
+		jsonErr(w, 400, "no such file in this stack, or nothing to write")
 		return
 	}
-	i, ok := fileIndex(r, st)
-	if !ok {
-		jsonErr(w, 400, "no such file in this stack")
-		return
-	}
-	target := st.HostFiles[i]
-	if now, err := readAsRoot(ctx, target); err == nil && body.Sum != "" && sumOf(now) != body.Sum {
+	if now, err := readAsRoot(ctx, path); err == nil && body.Sum != "" && sumOf(now) != body.Sum {
 		jsonErr(w, 409, "the file has changed on disk since you opened it (Dockhand, or another editor). Reload it to see the change; your text is still in the box.")
 		return
 	}
-	// Validate a candidate beside the real file, so `.env` and relative paths resolve as they will.
-	cand := target + ".hamlab-new"
-	err = s.fsLocal.Write(ctx, cand, strings.NewReader(body.Text)) // ours to write, or root's
-	if err != nil {
-		err = s.fsSudo.Write(ctx, cand, strings.NewReader(body.Text))
-	}
-	if err != nil {
-		jsonErr(w, 500, "couldn't write beside the compose file: "+err.Error())
+	// A candidate goes beside the real file, so `.env` and relative paths resolve as they will.
+	// A compose file is validated as that candidate; a .env has to be in place to be read at all,
+	// so it is swapped in, validated, and swapped back out if compose objects.
+	cand := path + ".hamlab-new"
+	if err := s.writeAsOwner(ctx, cand, body.Text); err != nil {
+		jsonErr(w, 500, "couldn't write beside the file: "+err.Error())
 		return
 	}
-	inner := cand
-	if st.Via != "" {
-		inner = st.Files[i] + ".hamlab-new"
+	sudo := func(args ...string) error {
+		full := priv.Wrap(args...)
+		if msg, err := exec.CommandContext(ctx, full[0], full[1:]...).CombinedOutput(); err != nil {
+			return errors.New(strings.TrimSpace(string(msg)))
+		}
+		return nil
 	}
-	if msg, err := composeCmd(ctx, st, seenFiles(st, i, inner), "config", "-q").CombinedOutput(); err != nil {
-		_ = s.fsSudo.Delete(ctx, cand, false)
+	if err := sudo("cp", "-p", path, path+".hamlab-bak"); err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	files := seenFiles(st, -1, "")
+	if i >= 0 {
+		inner := cand
+		if st.Via != "" {
+			inner = st.Files[i] + ".hamlab-new"
+		}
+		files = seenFiles(st, i, inner)
+	} else if err := sudo("mv", cand, path); err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	if msg, err := composeCmd(ctx, st, files, "config", "-q").CombinedOutput(); err != nil {
+		if i >= 0 {
+			_ = s.fsSudo.Delete(ctx, cand, false)
+		} else {
+			_ = sudo("cp", "-p", path+".hamlab-bak", path) // put the .env back
+		}
 		jsonErr(w, 400, "compose rejects it: "+strings.TrimSpace(string(msg)))
 		return
 	}
-	for _, mv := range [][]string{{"cp", "-p", target, target + ".hamlab-bak"}, {"mv", cand, target}} {
-		full := priv.Wrap(mv...)
-		if msg, err := exec.CommandContext(ctx, full[0], full[1:]...).CombinedOutput(); err != nil {
-			jsonErr(w, 500, strings.TrimSpace(string(msg)))
+	if i >= 0 {
+		if err := sudo("mv", cand, path); err != nil {
+			jsonErr(w, 500, err.Error())
 			return
 		}
 	}
-	log.Printf("stack %s: compose file saved (from %s)", st.Name, clientIP(r))
+	log.Printf("stack %s: %s saved (from %s)", st.Name, filepath.Base(path), clientIP(r))
 	w.WriteHeader(204)
 }

+// stackDiff is what a Deploy would change: each file now, against the copy kept the last
+// time this console deployed it (<file>.hamlab-deployed), or else the last saved-over copy.
+func (s *Server) stackDiff(w http.ResponseWriter, r *http.Request) {
+	ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second)
+	defer cancel()
+	st, handled, err := s.stackFor(ctx, r)
+	if !handled {
+		s.proxyNode(w, r)
+		return
+	}
+	if err != nil || !st.Editable {
+		jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach")
+		return
+	}
+	type fileDiff struct {
+		Name string `json:"name"`
+		Base string `json:"base"` // deployed | saved | ""
+		Old  string `json:"old"`
+		New  string `json:"new"`
+	}
+	paths := append([]string{}, st.HostFiles...)
+	if st.HasEnv {
+		paths = append(paths, st.envPath())
+	}
+	out := []fileDiff{}
+	for _, p := range paths {
+		now, _ := readAsRoot(ctx, p)
+		d := fileDiff{Name: filepath.Base(p), New: string(now)}
+		for _, b := range [][2]string{{".hamlab-deployed", "deployed"}, {".hamlab-bak", "saved"}} {
+			if old, err := readAsRoot(ctx, p+b[0]); err == nil {
+				d.Base, d.Old = b[1], string(old)
+				break
+			}
+		}
+		if filepath.Base(p) == ".env" { // values are secrets: say which keys changed, not what to
+			d.Old, d.New = maskEnv(d.Old), maskEnv(d.New)
+		}
+		out = append(out, d)
+	}
+	jsonOK(w, 200, map[string]any{"files": out})
+}
+
+// maskEnv keeps a .env's keys and hides its values, tagging each distinct value so a change
+// still shows as a change.
+func maskEnv(text string) string {
+	lines := strings.Split(text, "\n")
+	for i, l := range lines {
+		if k, v, ok := strings.Cut(l, "="); ok && !strings.HasPrefix(strings.TrimSpace(l), "#") && v != "" {
+			lines[i] = k + "=•••• (" + sumOf([]byte(v))[:6] + ")"
+		}
+	}
+	return strings.Join(lines, "\n")
+}
+
 // stackAction runs deploy (up -d), update (pull, then up -d), restart or stop as a task
 // whose output the page follows.
 func (s *Server) stackAction(w http.ResponseWriter, r *http.Request) {
-	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
+	ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second)
 	defer cancel()
-	st, err := s.stackByName(ctx, r.PathValue("name"))
+	st, handled, err := s.stackFor(ctx, r)
+	if !handled {
+		s.proxyNode(w, r)
+		return
+	}
 	if err != nil || !st.Editable {
-		jsonErr(w, 404, "that stack's compose file isn't on this machine")
+		jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach")
 		return
 	}
 	files := seenFiles(st, -1, "")
@@ -339,18 +551,35 @@ func (s *Server) stackAction(w http.ResponseWriter, r *http.Request) {
 		fl = append(fl, "-f", f)
 	}
 	flags := priv.Quote(append([]string{"-p", st.Name}, fl...))
-	var script string
 	base := func(args string) string {
 		if st.Via != "" {
-			return fmt.Sprintf("%s exec %s docker compose %s %s", dockerTool(), priv.Quote([]string{st.Via}), flags, args)
+			env := ""
+			if st.Remote != "" {
+				env = "-e " + priv.Quote([]string{"DOCKER_HOST=" + st.Remote}) + " "
+			}
+			return fmt.Sprintf("%s exec %s%s docker compose %s %s", dockerTool(), env, priv.Quote([]string{st.Via}), flags, args)
 		}
-		return fmt.Sprintf("cd %s && %s compose %s %s", priv.Quote([]string{filepath.Dir(files[0])}), dockerTool(), flags, args)
+		env := ""
+		if st.Remote != "" {
+			env = priv.Quote([]string{"DOCKER_HOST=" + st.Remote}) + " "
+		}
+		return fmt.Sprintf("cd %s && %s%s compose %s %s", priv.Quote([]string{filepath.Dir(files[0])}), env, dockerTool(), flags, args)
 	}
+	// After a deploy that worked, keep a copy of exactly what was deployed: the next diff is against it.
+	remember := ""
+	paths := append([]string{}, st.HostFiles...)
+	if st.HasEnv {
+		paths = append(paths, st.envPath())
+	}
+	for _, p := range paths {
+		remember += " && " + priv.Quote(priv.Wrap("cp", "-p", p, p+".hamlab-deployed"))
+	}
+	var script string
 	switch r.PathValue("action") {
 	case "deploy":
-		script = base("up -d --remove-orphans")
+		script = base("up -d --remove-orphans") + remember
 	case "update":
-		script = base("pull") + " && " + base("up -d --remove-orphans")
+		script = base("pull") + " && " + base("up -d --remove-orphans") + remember
 	case "restart":
 		script = base("restart")
 	case "stop":
@@ -368,6 +597,110 @@ func (s *Server) stackAction(w http.ResponseWriter, r *http.Request) {
 		jsonErr(w, 500, err.Error())
 		return
 	}
+	stackUpdatesForget(r.PathValue("node") + "/" + st.Name)
 	log.Printf("stack %s: %s started as task %s (from %s)", st.Name, r.PathValue("action"), t.ID, clientIP(r))
 	jsonOK(w, 202, t.Snapshot(false))
 }
+
+// ---- is there a newer image? ----
+
+type imageCheck struct {
+	Image  string `json:"image"`
+	State  string `json:"state"` // current | update | unknown
+	Detail string `json:"detail,omitempty"`
+}
+
+type updAnswer struct {
+	at  time.Time
+	res []imageCheck
+}
+
+var (
+	updMu    sync.Mutex
+	updCache = map[string]updAnswer{}
+)
+
+func stackUpdatesForget(key string) { updMu.Lock(); delete(updCache, key); updMu.Unlock() }
+
+// stackUpdates asks each image's registry for the digest its tag points at now and compares
+// it with the digest that was pulled. It is asked for, never polled: every answer is a
+// registry request, and Docker Hub counts those. Answers are kept for six hours.
+func (s *Server) stackUpdates(w http.ResponseWriter, r *http.Request) {
+	ctx, cancel := context.WithTimeout(r.Context(), 90*time.Second)
+	defer cancel()
+	st, handled, err := s.stackFor(ctx, r)
+	if !handled {
+		s.proxyNode(w, r)
+		return
+	}
+	if err != nil {
+		jsonErr(w, 404, err.Error())
+		return
+	}
+	key := r.PathValue("node") + "/" + st.Name
+	updMu.Lock()
+	c, ok := updCache[key]
+	updMu.Unlock()
+	refresh := r.URL.Query().Get("refresh") == "1"
+	if ok && time.Since(c.at) < 6*time.Hour && !refresh {
+		jsonOK(w, 200, map[string]any{"images": c.res, "checked": c.at.UnixMilli()})
+		return
+	}
+	if !refresh {
+		jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0})
+		return
+	}
+	local := func(args ...string) ([]byte, error) { // the Docker that holds the images
+		if st.Remote != "" {
+			args = append([]string{"-H", st.Remote}, args...)
+		}
+		return dockerRun(ctx, args...)
+	}
+	res := []imageCheck{}
+	for _, img := range st.Images {
+		ic := imageCheck{Image: img, State: "unknown"}
+		out, err := local("image", "inspect", "-f", "{{json .RepoDigests}}", img)
+		var have []string
+		if err != nil || json.Unmarshal(bytes.TrimSpace(out), &have) != nil || len(have) == 0 {
+			ic.Detail = "built or loaded here, not pulled from a registry"
+			res = append(res, ic)
+			continue
+		}
+		remote, err := dockerRun(ctx, "buildx", "imagetools", "inspect", img, "--format", "{{.Manifest.Digest}}")
+		digest := strings.TrimSpace(string(remote))
+		if i := strings.LastIndex(digest, "\n"); i >= 0 {
+			digest = digest[i+1:]
+		}
+		if err != nil || !strings.HasPrefix(digest, "sha256:") {
+			ic.Detail = "the registry didn't answer"
+			if err != nil {
+				ic.Detail += ": " + err.Error()
+			}
+			res = append(res, ic)
+			continue
+		}
+		ic.State = "update"
+		for _, h := range have {
+			if strings.HasSuffix(h, "@"+digest) {
+				ic.State = "current"
+			}
+		}
+		res = append(res, ic)
+	}
+	updMu.Lock()
+	updCache[key] = updAnswer{time.Now(), res}
+	updMu.Unlock()
+	jsonOK(w, 200, map[string]any{"images": res, "checked": time.Now().UnixMilli()})
+}
+
+// taskOnHub serves a task this hub is running for one of its nodes' stacks; anything else
+// under /api/nodes/<node>/tasks belongs to the node.
+func (s *Server) taskOnHub(local http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if _, ok := s.tasks.Get(r.PathValue("id")); ok {
+			local(w, r)
+			return
+		}
+		s.proxyNode(w, r)
+	}
+}