stacks: remember where a container's bind mounts pointed

Every stack on core and grid is labelled with a path inside Dockhand's filesystem — /app/data/stacks/… — because Dockhand ran compose from inside its own container. Ham Lab resolves those by walking the bind mounts of running containers, which works right up until the container doing the mounting is removed. Retiring Dockhand would, in one step, make seventeen stacks on core and five on grid unfindable. So the mounts are written down as they are seen, and a path no live container explains is looked for in what was seen before. A remembered mount resolves with no container to run in, which is the right answer: compose runs on the host, against the host's own copy of the file. The same problem bites a scheduled update, which captures how to invoke compose when it is made so the cron job never has to rediscover the world at 3am. repoint() checks the named container is still running and falls back the same way if it is not — and pointedly leaves the spec alone if nothing explains the path, so it fails loudly rather than deploying a guess. Out in cron the record is read and never added to: the daemon owns state.json and would clobber anything written from there. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-24 20:19:54 -07:00
parent 192504ead3
commit f17a365ea1
5 changed files with 273 additions and 8 deletions
@@ -18,6 +18,7 @@ import (
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"io"
 	"log"
 	"net/http"
@@ -137,10 +138,38 @@ func (s *Server) dsSyncCron(ctx context.Context, d dockerSchedState) error {
 	return s.storage.WriteRootFile(ctx, dockerCronFile, text)
 }

+// repoint keeps a schedule working when what deployed its stack has been removed.
+//
+// A schedule captures how to invoke compose at the moment it is made, so the cron job never
+// has to rediscover the world at 3am. But "run compose inside this container" stops being
+// true the day that container is deleted — which is exactly what happens when an
+// orchestrator is retired, and would otherwise break every schedule made before it went.
+// So: if the container named is not running, the files are looked up again, through live
+// mounts and remembered ones both, and compose runs on the host instead.
+func repoint(ctx context.Context, db *store.Store, spec compose.Spec) compose.Spec {
+	if spec.Via == "" || db == nil {
+		return spec
+	}
+	if out, err := dockerRun(ctx, "inspect", "-f", "{{.State.Running}}", spec.Via); err == nil && strings.TrimSpace(string(out)) == "true" {
+		return spec
+	}
+	st := stack{Name: spec.Project, File: spec.Files[0], Files: spec.Files}
+	st.resolve(mountsSeen(ctx, db)) // read-only: this also runs from cron, where the daemon owns state.json
+	if !st.Editable || st.Via != "" {
+		return spec // still only reachable through a container, or not reachable at all
+	}
+	log.Printf("docker schedule %s: %s is gone, running compose here against %s", spec.Project, spec.Via, st.HostFiles[0])
+	spec.Files, spec.Via = st.HostFiles, ""
+	return spec
+}
+
 // RunUpdate pulls, and deploys when the schedule says to. Shared with the CLI so the button
 // and the cron line cannot drift apart.
-func RunUpdate(ctx context.Context, j UpdateJob, tool string) (string, error) {
-	spec := j.Spec.toSpec(tool)
+func RunUpdate(ctx context.Context, db *store.Store, j UpdateJob, tool string) (string, error) {
+	if len(j.Spec.Files) == 0 {
+		return "", errors.New("this schedule has no compose file recorded — open it in the Docker tab and save it again")
+	}
+	spec := repoint(ctx, db, j.Spec.toSpec(tool))
 	var log strings.Builder
 	run := func(args ...string) error {
 		cmd := compose.Cmd(ctx, spec, args...)
@@ -299,7 +328,7 @@ func (s *Server) dsRun(w http.ResponseWriter, r *http.Request) {
 	// Outliving the request: a pull the browser gave up on still finishes, but not forever.
 	ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 30*time.Minute)
 	defer cancel()
-	out, err := RunUpdate(ctx, job, dockerTool())
+	out, err := RunUpdate(ctx, s.store, job, dockerTool())
 	rec := storage.RunRecord{At: time.Now(), Taken: job.Stack}
 	if err != nil {
 		rec.Error = err.Error()
@@ -12,6 +12,12 @@ package api
 // relative paths, the .env beside them and the labels all stay exactly as they were
 // deployed — else on the host.
 //
+// Those mounts are also written down. An orchestrator can be removed, and when it is, the
+// containers it left behind still carry labels naming paths inside a filesystem that no
+// longer exists; without a record of where that filesystem was mounted, every stack on the
+// machine would become unfindable at once. A remembered mount resolves with no container to
+// run in, which is right: compose runs on the host against the host's copy of the file.
+//
 // A stack can also be deployed *from another machine*: Dockhand on the hub drives a node's
 // Docker over tcp, so the node runs the containers and the hub holds the files. The node
 // alone can only list such a stack read-only. The hub, asked about one of its nodes, fills
@@ -41,6 +47,7 @@ import (

 	"hamlab/internal/compose"
 	"hamlab/internal/priv"
+	"hamlab/internal/store"
 )

 type stack struct {
@@ -90,6 +97,100 @@ func stackMounts(ctx context.Context) []mountMap {
 	return maps
 }

+// mountsFor is stackMounts plus memory. An orchestrator that deployed everything here and
+// has since been removed takes the only record of where its files really are with it: the
+// containers it left behind are labelled with paths inside *its* filesystem. So every map
+// seen is written down, and a path that no live container explains is looked for in what
+// was seen before. A remembered map resolves with no `via`, which is correct — the
+// container is gone, so compose runs on the host, against the host's copy of the file.
+func mountsFor(ctx context.Context, db *store.Store) []mountMap {
+	live := stackMounts(ctx)
+	rememberMounts(db, live)
+	return append(live, recallMounts(db, live)...)
+}
+
+// mountsSeen is the same without writing anything down. The daemon owns state.json; a cron
+// job that wrote to it would be clobbered by the next thing the daemon saved, so out there
+// the record is read and never added to.
+func mountsSeen(ctx context.Context, db *store.Store) []mountMap {
+	live := stackMounts(ctx)
+	return append(live, recallMounts(db, live)...)
+}
+
+// rememberMounts folds what is mounted now into what was mounted before, and forgets a map
+// whose source has gone from disk — that one is not coming back, and keeping it would make
+// a stack look editable when its file no longer exists.
+//
+// This is on the path of every stack listing, which the Docker tab polls, so it reads first
+// and only writes when the set actually changed.
+func rememberMounts(db *store.Store, live []mountMap) {
+	if len(live) == 0 {
+		return // docker is down or answered nothing; that is not evidence of anything
+	}
+	key := func(c, d string) string { return c + "\x00" + d }
+	var have []store.RememberedMount
+	db.View(func(st *store.State) { have = append([]store.RememberedMount(nil), st.Mounts...) })
+	at := map[string]int{}
+	for i, m := range have {
+		at[key(m.Container, m.Dest)] = i
+	}
+	now, add, moved := time.Now(), 0, false
+	for _, m := range live {
+		i, ok := at[key(m.container, m.dest)]
+		if !ok {
+			add++
+			continue
+		}
+		if have[i].Src != m.src {
+			moved = true
+		}
+	}
+	// Pruning is the expensive half — a stat, or a sudo test(1), per remembered map — so it
+	// only happens when something else has already made a write necessary.
+	if add == 0 && !moved {
+		return
+	}
+	_ = db.Update(func(st *store.State) {
+		at := map[string]int{}
+		for i, m := range st.Mounts {
+			at[key(m.Container, m.Dest)] = i
+		}
+		for _, m := range live {
+			if i, ok := at[key(m.container, m.dest)]; ok {
+				st.Mounts[i].Src, st.Mounts[i].Seen = m.src, now
+				continue
+			}
+			st.Mounts = append(st.Mounts, store.RememberedMount{Container: m.container, Dest: m.dest, Src: m.src, Seen: now})
+		}
+		kept := st.Mounts[:0]
+		for _, m := range st.Mounts {
+			if readable(m.Src) {
+				kept = append(kept, m)
+			}
+		}
+		st.Mounts = kept
+	})
+}
+
+// recallMounts is what was seen before and is not running now.
+func recallMounts(db *store.Store, live []mountMap) []mountMap {
+	var remembered []store.RememberedMount
+	db.View(func(st *store.State) { remembered = append([]store.RememberedMount(nil), st.Mounts...) })
+	now := map[string]bool{}
+	for _, m := range live {
+		now[m.container+"\x00"+m.dest] = true
+	}
+	var out []mountMap
+	for _, m := range remembered {
+		if now[m.Container+"\x00"+m.Dest] {
+			continue
+		}
+		out = append(out, mountMap{"", filepath.Clean(m.Dest), filepath.Clean(m.Src)}) // no container: run compose here
+	}
+	sort.Slice(out, func(i, j int) bool { return len(out[i].dest) > len(out[j].dest) })
+	return out
+}
+
 func readable(p string) bool {
 	if _, err := os.Stat(p); err == nil {
 		return true
@@ -173,7 +274,7 @@ func (s *Server) findStacks(ctx context.Context) ([]stack, error) {
 			}
 		}
 	}
-	maps := stackMounts(ctx)
+	maps := mountsFor(ctx, s.store)
 	stacks := []stack{}
 	for _, st := range by {
 		sort.Strings(st.Containers)
@@ -354,7 +455,7 @@ func (s *Server) hubStacks(ctx context.Context, node string) ([]stack, error) {
 			continue
 		}
 		if remote == "" {
-			remote, maps = s.nodeDocker(ctx, node), stackMounts(ctx)
+			remote, maps = s.nodeDocker(ctx, node), mountsFor(ctx, s.store)
 		}
 		if remote == "-" {
 			continue
@@ -0,0 +1,121 @@
+package api
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"hamlab/internal/compose"
+	"hamlab/internal/store"
+)
+
+// A stack deployed by an orchestrator running in a container records the path its compose
+// file had inside that container. Resolving it has to work both while the orchestrator is
+// running — compose is then run inside it — and after it has been removed, where the same
+// mount is only remembered and compose has to run on the host instead.
+func TestResolveThroughMounts(t *testing.T) {
+	root := t.TempDir()
+	dir := filepath.Join(root, "orch", "stacks", "app")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	file := filepath.Join(dir, "compose.yaml")
+	if err := os.WriteFile(file, []byte("services: {}\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	inner := "/app/data/stacks/app/compose.yaml"
+
+	for _, tc := range []struct {
+		name, container, wantVia string
+	}{
+		{"orchestrator running", "orch", "orch"},
+		{"orchestrator removed", "", ""},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			st := stack{Name: "app", File: inner, Files: []string{inner}}
+			st.resolve([]mountMap{{tc.container, "/app/data", filepath.Join(root, "orch")}})
+			if !st.Editable {
+				t.Fatalf("not editable: %s", st.Note)
+			}
+			if st.HostFile != file {
+				t.Errorf("host file = %q, want %q", st.HostFile, file)
+			}
+			if st.Via != tc.wantVia {
+				t.Errorf("via = %q, want %q", st.Via, tc.wantVia)
+			}
+		})
+	}
+
+	// Nothing explains the path: the stack is listed, but read-only and says why.
+	st := stack{Name: "app", File: inner, Files: []string{inner}}
+	st.resolve(nil)
+	if st.Editable || st.Note == "" {
+		t.Errorf("unresolvable stack should be read-only with a note, got editable=%v note=%q", st.Editable, st.Note)
+	}
+}
+
+// Every file of a multi-file stack must resolve through the same place. Redeploying from
+// the first file alone would silently drop whatever the override set.
+func TestResolveRefusesSplitStack(t *testing.T) {
+	root := t.TempDir()
+	dir := filepath.Join(root, "orch", "stacks", "app")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "compose.yaml"), []byte("services: {}\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	st := stack{Name: "app", File: "/app/data/stacks/app/compose.yaml", Files: []string{
+		"/app/data/stacks/app/compose.yaml",
+		"/app/data/stacks/app/override.yaml", // never written: not on disk
+	}}
+	st.resolve([]mountMap{{"orch", "/app/data", filepath.Join(root, "orch")}})
+	if st.Editable {
+		t.Error("a stack with a file that cannot be found should not be editable")
+	}
+}
+
+// A schedule made while an orchestrator was running says "run compose inside that
+// container". When the container is gone the schedule must still work, against the same
+// files on the host — otherwise retiring the orchestrator silently breaks every scheduled
+// update made before it went.
+func TestRepointSurvivesAMissingContainer(t *testing.T) {
+	root := t.TempDir()
+	dir := filepath.Join(root, "orch", "stacks", "app")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	host := filepath.Join(dir, "compose.yaml")
+	if err := os.WriteFile(host, []byte("services: {}\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	db, err := store.Open(filepath.Join(root, "state.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Update(func(st *store.State) {
+		st.Mounts = []store.RememberedMount{{Container: "gone-orchestrator", Dest: "/app/data", Src: filepath.Join(root, "orch")}}
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	in := compose.Spec{Project: "app", Files: []string{"/app/data/stacks/app/compose.yaml"}, Via: "gone-orchestrator", Tool: "docker"}
+	got := repoint(context.Background(), db, in)
+	if got.Via != "" {
+		t.Errorf("via = %q, want it dropped", got.Via)
+	}
+	if len(got.Files) != 1 || got.Files[0] != host {
+		t.Errorf("files = %v, want [%s]", got.Files, host)
+	}
+
+	// Nothing remembered: the schedule is left exactly as it was rather than pointed at a
+	// guess. It will fail loudly, which is better than deploying the wrong file.
+	empty, err := store.Open(filepath.Join(root, "empty.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := repoint(context.Background(), empty, in); got.Via != in.Via || len(got.Files) != 1 || got.Files[0] != in.Files[0] {
+		t.Errorf("unresolvable spec was changed: %+v", got)
+	}
+}