diff --git a/cmd/hamlab/main.go b/cmd/hamlab/main.go index 1708d52..5fba657 100644 --- a/cmd/hamlab/main.go +++ b/cmd/hamlab/main.go @@ -218,7 +218,7 @@ func dockerUpdateJob(cfg *config.Config, id string) error { // A pull that never finishes must not leave a cron job resident until the next reboot. ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) defer cancel() - out, runErr := api.RunUpdate(ctx, j, api.DockerTool()) + out, runErr := api.RunUpdate(ctx, st, j, api.DockerTool()) rec := storage.RunRecord{At: time.Now(), Taken: j.Stack} if runErr != nil { rec.Error = runErr.Error() diff --git a/internal/api/dockersched.go b/internal/api/dockersched.go index 7597139..398a206 100644 --- a/internal/api/dockersched.go +++ b/internal/api/dockersched.go @@ -18,6 +18,7 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "errors" "io" "log" "net/http" @@ -137,10 +138,38 @@ func (s *Server) dsSyncCron(ctx context.Context, d dockerSchedState) error { return s.storage.WriteRootFile(ctx, dockerCronFile, text) } +// repoint keeps a schedule working when what deployed its stack has been removed. +// +// A schedule captures how to invoke compose at the moment it is made, so the cron job never +// has to rediscover the world at 3am. But "run compose inside this container" stops being +// true the day that container is deleted — which is exactly what happens when an +// orchestrator is retired, and would otherwise break every schedule made before it went. +// So: if the container named is not running, the files are looked up again, through live +// mounts and remembered ones both, and compose runs on the host instead. +func repoint(ctx context.Context, db *store.Store, spec compose.Spec) compose.Spec { + if spec.Via == "" || db == nil { + return spec + } + if out, err := dockerRun(ctx, "inspect", "-f", "{{.State.Running}}", spec.Via); err == nil && strings.TrimSpace(string(out)) == "true" { + return spec + } + st := stack{Name: spec.Project, File: spec.Files[0], Files: spec.Files} + st.resolve(mountsSeen(ctx, db)) // read-only: this also runs from cron, where the daemon owns state.json + if !st.Editable || st.Via != "" { + return spec // still only reachable through a container, or not reachable at all + } + log.Printf("docker schedule %s: %s is gone, running compose here against %s", spec.Project, spec.Via, st.HostFiles[0]) + spec.Files, spec.Via = st.HostFiles, "" + return spec +} + // RunUpdate pulls, and deploys when the schedule says to. Shared with the CLI so the button // and the cron line cannot drift apart. -func RunUpdate(ctx context.Context, j UpdateJob, tool string) (string, error) { - spec := j.Spec.toSpec(tool) +func RunUpdate(ctx context.Context, db *store.Store, j UpdateJob, tool string) (string, error) { + if len(j.Spec.Files) == 0 { + return "", errors.New("this schedule has no compose file recorded — open it in the Docker tab and save it again") + } + spec := repoint(ctx, db, j.Spec.toSpec(tool)) var log strings.Builder run := func(args ...string) error { cmd := compose.Cmd(ctx, spec, args...) @@ -299,7 +328,7 @@ func (s *Server) dsRun(w http.ResponseWriter, r *http.Request) { // Outliving the request: a pull the browser gave up on still finishes, but not forever. ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 30*time.Minute) defer cancel() - out, err := RunUpdate(ctx, job, dockerTool()) + out, err := RunUpdate(ctx, s.store, job, dockerTool()) rec := storage.RunRecord{At: time.Now(), Taken: job.Stack} if err != nil { rec.Error = err.Error() diff --git a/internal/api/stacks.go b/internal/api/stacks.go index ce141f0..d2ee5ea 100644 --- a/internal/api/stacks.go +++ b/internal/api/stacks.go @@ -12,6 +12,12 @@ package api // relative paths, the .env beside them and the labels all stay exactly as they were // deployed — else on the host. // +// Those mounts are also written down. An orchestrator can be removed, and when it is, the +// containers it left behind still carry labels naming paths inside a filesystem that no +// longer exists; without a record of where that filesystem was mounted, every stack on the +// machine would become unfindable at once. A remembered mount resolves with no container to +// run in, which is right: compose runs on the host against the host's copy of the file. +// // A stack can also be deployed *from another machine*: Dockhand on the hub drives a node's // Docker over tcp, so the node runs the containers and the hub holds the files. The node // alone can only list such a stack read-only. The hub, asked about one of its nodes, fills @@ -41,6 +47,7 @@ import ( "hamlab/internal/compose" "hamlab/internal/priv" + "hamlab/internal/store" ) type stack struct { @@ -90,6 +97,100 @@ func stackMounts(ctx context.Context) []mountMap { return maps } +// mountsFor is stackMounts plus memory. An orchestrator that deployed everything here and +// has since been removed takes the only record of where its files really are with it: the +// containers it left behind are labelled with paths inside *its* filesystem. So every map +// seen is written down, and a path that no live container explains is looked for in what +// was seen before. A remembered map resolves with no `via`, which is correct — the +// container is gone, so compose runs on the host, against the host's copy of the file. +func mountsFor(ctx context.Context, db *store.Store) []mountMap { + live := stackMounts(ctx) + rememberMounts(db, live) + return append(live, recallMounts(db, live)...) +} + +// mountsSeen is the same without writing anything down. The daemon owns state.json; a cron +// job that wrote to it would be clobbered by the next thing the daemon saved, so out there +// the record is read and never added to. +func mountsSeen(ctx context.Context, db *store.Store) []mountMap { + live := stackMounts(ctx) + return append(live, recallMounts(db, live)...) +} + +// rememberMounts folds what is mounted now into what was mounted before, and forgets a map +// whose source has gone from disk — that one is not coming back, and keeping it would make +// a stack look editable when its file no longer exists. +// +// This is on the path of every stack listing, which the Docker tab polls, so it reads first +// and only writes when the set actually changed. +func rememberMounts(db *store.Store, live []mountMap) { + if len(live) == 0 { + return // docker is down or answered nothing; that is not evidence of anything + } + key := func(c, d string) string { return c + "\x00" + d } + var have []store.RememberedMount + db.View(func(st *store.State) { have = append([]store.RememberedMount(nil), st.Mounts...) }) + at := map[string]int{} + for i, m := range have { + at[key(m.Container, m.Dest)] = i + } + now, add, moved := time.Now(), 0, false + for _, m := range live { + i, ok := at[key(m.container, m.dest)] + if !ok { + add++ + continue + } + if have[i].Src != m.src { + moved = true + } + } + // Pruning is the expensive half — a stat, or a sudo test(1), per remembered map — so it + // only happens when something else has already made a write necessary. + if add == 0 && !moved { + return + } + _ = db.Update(func(st *store.State) { + at := map[string]int{} + for i, m := range st.Mounts { + at[key(m.Container, m.Dest)] = i + } + for _, m := range live { + if i, ok := at[key(m.container, m.dest)]; ok { + st.Mounts[i].Src, st.Mounts[i].Seen = m.src, now + continue + } + st.Mounts = append(st.Mounts, store.RememberedMount{Container: m.container, Dest: m.dest, Src: m.src, Seen: now}) + } + kept := st.Mounts[:0] + for _, m := range st.Mounts { + if readable(m.Src) { + kept = append(kept, m) + } + } + st.Mounts = kept + }) +} + +// recallMounts is what was seen before and is not running now. +func recallMounts(db *store.Store, live []mountMap) []mountMap { + var remembered []store.RememberedMount + db.View(func(st *store.State) { remembered = append([]store.RememberedMount(nil), st.Mounts...) }) + now := map[string]bool{} + for _, m := range live { + now[m.container+"\x00"+m.dest] = true + } + var out []mountMap + for _, m := range remembered { + if now[m.Container+"\x00"+m.Dest] { + continue + } + out = append(out, mountMap{"", filepath.Clean(m.Dest), filepath.Clean(m.Src)}) // no container: run compose here + } + sort.Slice(out, func(i, j int) bool { return len(out[i].dest) > len(out[j].dest) }) + return out +} + func readable(p string) bool { if _, err := os.Stat(p); err == nil { return true @@ -173,7 +274,7 @@ func (s *Server) findStacks(ctx context.Context) ([]stack, error) { } } } - maps := stackMounts(ctx) + maps := mountsFor(ctx, s.store) stacks := []stack{} for _, st := range by { sort.Strings(st.Containers) @@ -354,7 +455,7 @@ func (s *Server) hubStacks(ctx context.Context, node string) ([]stack, error) { continue } if remote == "" { - remote, maps = s.nodeDocker(ctx, node), stackMounts(ctx) + remote, maps = s.nodeDocker(ctx, node), mountsFor(ctx, s.store) } if remote == "-" { continue diff --git a/internal/api/stacks_test.go b/internal/api/stacks_test.go new file mode 100644 index 0000000..cf2052b --- /dev/null +++ b/internal/api/stacks_test.go @@ -0,0 +1,121 @@ +package api + +import ( + "context" + "os" + "path/filepath" + "testing" + + "hamlab/internal/compose" + "hamlab/internal/store" +) + +// A stack deployed by an orchestrator running in a container records the path its compose +// file had inside that container. Resolving it has to work both while the orchestrator is +// running — compose is then run inside it — and after it has been removed, where the same +// mount is only remembered and compose has to run on the host instead. +func TestResolveThroughMounts(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "orch", "stacks", "app") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + file := filepath.Join(dir, "compose.yaml") + if err := os.WriteFile(file, []byte("services: {}\n"), 0o644); err != nil { + t.Fatal(err) + } + inner := "/app/data/stacks/app/compose.yaml" + + for _, tc := range []struct { + name, container, wantVia string + }{ + {"orchestrator running", "orch", "orch"}, + {"orchestrator removed", "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + st := stack{Name: "app", File: inner, Files: []string{inner}} + st.resolve([]mountMap{{tc.container, "/app/data", filepath.Join(root, "orch")}}) + if !st.Editable { + t.Fatalf("not editable: %s", st.Note) + } + if st.HostFile != file { + t.Errorf("host file = %q, want %q", st.HostFile, file) + } + if st.Via != tc.wantVia { + t.Errorf("via = %q, want %q", st.Via, tc.wantVia) + } + }) + } + + // Nothing explains the path: the stack is listed, but read-only and says why. + st := stack{Name: "app", File: inner, Files: []string{inner}} + st.resolve(nil) + if st.Editable || st.Note == "" { + t.Errorf("unresolvable stack should be read-only with a note, got editable=%v note=%q", st.Editable, st.Note) + } +} + +// Every file of a multi-file stack must resolve through the same place. Redeploying from +// the first file alone would silently drop whatever the override set. +func TestResolveRefusesSplitStack(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "orch", "stacks", "app") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "compose.yaml"), []byte("services: {}\n"), 0o644); err != nil { + t.Fatal(err) + } + st := stack{Name: "app", File: "/app/data/stacks/app/compose.yaml", Files: []string{ + "/app/data/stacks/app/compose.yaml", + "/app/data/stacks/app/override.yaml", // never written: not on disk + }} + st.resolve([]mountMap{{"orch", "/app/data", filepath.Join(root, "orch")}}) + if st.Editable { + t.Error("a stack with a file that cannot be found should not be editable") + } +} + +// A schedule made while an orchestrator was running says "run compose inside that +// container". When the container is gone the schedule must still work, against the same +// files on the host — otherwise retiring the orchestrator silently breaks every scheduled +// update made before it went. +func TestRepointSurvivesAMissingContainer(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "orch", "stacks", "app") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + host := filepath.Join(dir, "compose.yaml") + if err := os.WriteFile(host, []byte("services: {}\n"), 0o644); err != nil { + t.Fatal(err) + } + db, err := store.Open(filepath.Join(root, "state.json")) + if err != nil { + t.Fatal(err) + } + if err := db.Update(func(st *store.State) { + st.Mounts = []store.RememberedMount{{Container: "gone-orchestrator", Dest: "/app/data", Src: filepath.Join(root, "orch")}} + }); err != nil { + t.Fatal(err) + } + + in := compose.Spec{Project: "app", Files: []string{"/app/data/stacks/app/compose.yaml"}, Via: "gone-orchestrator", Tool: "docker"} + got := repoint(context.Background(), db, in) + if got.Via != "" { + t.Errorf("via = %q, want it dropped", got.Via) + } + if len(got.Files) != 1 || got.Files[0] != host { + t.Errorf("files = %v, want [%s]", got.Files, host) + } + + // Nothing remembered: the schedule is left exactly as it was rather than pointed at a + // guess. It will fail loudly, which is better than deploying the wrong file. + empty, err := store.Open(filepath.Join(root, "empty.json")) + if err != nil { + t.Fatal(err) + } + if got := repoint(context.Background(), empty, in); got.Via != in.Via || len(got.Files) != 1 || got.Files[0] != in.Files[0] { + t.Errorf("unresolvable spec was changed: %+v", got) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 5e20ec6..2746d02 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -44,6 +44,16 @@ type UserState struct { UI json.RawMessage `json:"ui,omitempty"` // theme, accents; opaque to the server } +// RememberedMount is one bind mount of a container that was running when Ham Lab last +// looked. A compose stack records the path its file had *inside* whatever deployed it, so +// when that container is gone the path is the only way back to the file on disk. +type RememberedMount struct { + Container string `json:"container"` + Dest string `json:"dest"` // the path inside the container + Src string `json:"src"` // the path on the machine + Seen time.Time `json:"seen"` +} + type State struct { Terminals []Terminal `json:"terminals"` Sessions map[string]Session `json:"sessions"` // token -> session @@ -51,9 +61,13 @@ type State struct { // UI is the pre-users machine-wide look; on first boot with users it moves to the first // account. Kept so an old state file still reads. UI json.RawMessage `json:"ui,omitempty"` - Alerts json.RawMessage `json:"alerts,omitempty"` + Alerts json.RawMessage `json:"alerts,omitempty"` // rules, channels and recent events (internal/api/alerts.go) ZFS json.RawMessage `json:"zfs,omitempty"` // snapshot schedules (internal/api/zfs.go) - DockerJobs json.RawMessage `json:"docker_jobs,omitempty"` // image-update schedules (internal/api/dockersched.go) // rules, channels and recent events (internal/api/alerts.go) + DockerJobs json.RawMessage `json:"docker_jobs,omitempty"` // image-update schedules (internal/api/dockersched.go) + // Mounts remembers where a container's bind mounts pointed, so a stack deployed by an + // orchestrator that has since been removed can still be found on disk. See + // internal/api/stacks.go. + Mounts []RememberedMount `json:"mounts,omitempty"` SamplerMs int `json:"sampler_ms,omitempty"` // how often this machine samples; the hub keeps every machine on the console's refresh rate HistorySec int `json:"history_sec,omitempty"` // how far back the graphs reach; the hub keeps every machine on the same one }