package api // Stacks: the compose projects behind the containers, with their files to edit and a way to // redeploy them. // // A stack is found from its containers' labels (project, config files). The files may not be // where the label says: an orchestrator that runs compose from inside its own container — // Dockhand, Dockge, Portainer — records paths in *its* filesystem. So each path is looked up // twice: as it stands on this host, and through every container's mounts // (/app/data/stacks/… in `dockhand` is /home/core/apps/dockhand/stacks/… here). Compose is // then run where the files make sense — inside that container when they came from one, so // relative paths, the .env beside them and the labels all stay exactly as they were // deployed — else on the host. // // Those mounts are also written down. An orchestrator can be removed, and when it is, the // containers it left behind still carry labels naming paths inside a filesystem that no // longer exists; without a record of where that filesystem was mounted, every stack on the // machine would become unfindable at once. A remembered mount resolves with no container to // run in, which is right: compose runs on the host against the host's copy of the file. // // A stack can also be deployed *from another machine*: Dockhand on the hub drives a node's // Docker over tcp, so the node runs the containers and the hub holds the files. The node // alone can only list such a stack read-only. The hub, asked about one of its nodes, fills // in what the node couldn't (hubStacks): it resolves the files on itself and runs compose // here with DOCKER_HOST pointed at the node. import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "log" "net/http" "net/url" "os" "os/exec" "path/filepath" "regexp" "sort" "strings" "sync" "time" "hamlab/internal/compose" "hamlab/internal/priv" "hamlab/internal/store" ) type stack struct { Name string `json:"name"` Containers []string `json:"containers"` Images []string `json:"images"` Running int `json:"running"` File string `json:"file"` // as the label has it (the first of Files) HostFile string `json:"host_file"` // where it is on the machine that holds it; "" when unknown Files []string `json:"files"` // every file it was deployed from, in order: -f a -f b HostFiles []string `json:"host_files"` // the same, on the machine that holds them Via string `json:"via"` // container compose runs inside; "" = on the host Remote string `json:"remote"` // DOCKER_HOST compose is pointed at; "" = this machine's Docker HeldBy string `json:"held_by"` // the machine holding the files, when it isn't the one running them Editable bool `json:"editable"` Note string `json:"note,omitempty"` HasEnv bool `json:"has_env"` } type mountMap struct{ container, dest, src string } func stackMounts(ctx context.Context) []mountMap { out, err := dockerRun(ctx, "ps", "-q") if err != nil || len(bytes.TrimSpace(out)) == 0 { return nil } insp, err := dockerRun(ctx, append([]string{"inspect"}, strings.Fields(string(out))...)...) if err != nil { return nil } var list []struct { Name string Mounts []struct{ Type, Source, Destination string } } if json.Unmarshal(insp, &list) != nil { return nil } var maps []mountMap for _, c := range list { for _, m := range c.Mounts { if m.Type == "bind" && m.Destination != "/" && m.Source != "" { maps = append(maps, mountMap{strings.TrimPrefix(c.Name, "/"), filepath.Clean(m.Destination), filepath.Clean(m.Source)}) } } } sort.Slice(maps, func(i, j int) bool { return len(maps[i].dest) > len(maps[j].dest) }) // the most specific mount wins return maps } // mountsFor is stackMounts plus memory. An orchestrator that deployed everything here and // has since been removed takes the only record of where its files really are with it: the // containers it left behind are labelled with paths inside *its* filesystem. So every map // seen is written down, and a path that no live container explains is looked for in what // was seen before. A remembered map resolves with no `via`, which is correct — the // container is gone, so compose runs on the host, against the host's copy of the file. func mountsFor(ctx context.Context, db *store.Store) []mountMap { live := stackMounts(ctx) rememberMounts(db, live) return append(live, recallMounts(db, live)...) } // mountsSeen is the same without writing anything down. The daemon owns state.json; a cron // job that wrote to it would be clobbered by the next thing the daemon saved, so out there // the record is read and never added to. func mountsSeen(ctx context.Context, db *store.Store) []mountMap { live := stackMounts(ctx) return append(live, recallMounts(db, live)...) } // rememberMounts folds what is mounted now into what was mounted before, and forgets a map // whose source has gone from disk — that one is not coming back, and keeping it would make // a stack look editable when its file no longer exists. // // This is on the path of every stack listing, which the Docker tab polls, so it reads first // and only writes when the set actually changed. func rememberMounts(db *store.Store, live []mountMap) { if len(live) == 0 { return // docker is down or answered nothing; that is not evidence of anything } key := func(c, d string) string { return c + "\x00" + d } var have []store.RememberedMount db.View(func(st *store.State) { have = append([]store.RememberedMount(nil), st.Mounts...) }) at := map[string]int{} for i, m := range have { at[key(m.Container, m.Dest)] = i } now, add, moved := time.Now(), 0, false for _, m := range live { i, ok := at[key(m.container, m.dest)] if !ok { add++ continue } if have[i].Src != m.src { moved = true } } // Pruning is the expensive half — a stat, or a sudo test(1), per remembered map — so it // only happens when something else has already made a write necessary. if add == 0 && !moved { return } _ = db.Update(func(st *store.State) { at := map[string]int{} for i, m := range st.Mounts { at[key(m.Container, m.Dest)] = i } for _, m := range live { if i, ok := at[key(m.container, m.dest)]; ok { st.Mounts[i].Src, st.Mounts[i].Seen = m.src, now continue } st.Mounts = append(st.Mounts, store.RememberedMount{Container: m.container, Dest: m.dest, Src: m.src, Seen: now}) } kept := st.Mounts[:0] for _, m := range st.Mounts { if readable(m.Src) { kept = append(kept, m) } } st.Mounts = kept }) } // recallMounts is what was seen before and is not running now. func recallMounts(db *store.Store, live []mountMap) []mountMap { var remembered []store.RememberedMount db.View(func(st *store.State) { remembered = append([]store.RememberedMount(nil), st.Mounts...) }) now := map[string]bool{} for _, m := range live { now[m.container+"\x00"+m.dest] = true } var out []mountMap for _, m := range remembered { if now[m.Container+"\x00"+m.Dest] { continue } out = append(out, mountMap{"", filepath.Clean(m.Dest), filepath.Clean(m.Src)}) // no container: run compose here } sort.Slice(out, func(i, j int) bool { return len(out[i].dest) > len(out[j].dest) }) return out } func readable(p string) bool { if _, err := os.Stat(p); err == nil { return true } if priv.Available() { // stacks kept by a root-owned orchestrator are often unreadable to us full := priv.Wrap("test", "-f", p) return exec.Command(full[0], full[1:]...).Run() == nil } return false } // resolve finds every file of a stack on this machine. Every one must be found — a stack // deployed from `-f compose.yaml -f override.yaml` and redeployed from the first alone would // quietly lose everything the override set. func (st *stack) resolve(maps []mountMap) { st.HostFiles, st.HostFile, st.Via, st.Editable, st.HasEnv = nil, "", "", false, false if st.File == "" { st.Note = "its containers don't say which file deployed them" return } one := func(f string) (host, via string) { if readable(f) { return f, "" } for _, m := range maps { if rest, ok := strings.CutPrefix(f, m.dest+"/"); ok && readable(filepath.Join(m.src, rest)) { return filepath.Join(m.src, rest), m.container } } return "", "" } for i, f := range st.Files { host, via := one(f) if host == "" || (i > 0 && via != st.Via) { st.HostFiles, st.Via = nil, "" st.Note = "deployed from somewhere this machine can't see (" + f + "); edit it there" return } st.HostFiles, st.Via = append(st.HostFiles, host), via } st.HostFile, st.Editable, st.Note = st.HostFiles[0], true, "" st.HasEnv = readable(st.envPath()) } func (st stack) envPath() string { return filepath.Join(filepath.Dir(st.HostFiles[0]), ".env") } func (s *Server) findStacks(ctx context.Context) ([]stack, error) { // Labels come from inspect, not `ps`: ps joins every label with commas, and the label that // matters here — the list of compose files — is itself a comma-separated list. ids, err := dockerRun(ctx, "ps", "-aq") if err != nil { return nil, err } by := map[string]*stack{} if len(bytes.TrimSpace(ids)) > 0 { const f = `{{.Name}}|{{.State.Status}}|{{.Config.Image}}|{{index .Config.Labels "com.docker.compose.project"}}|{{index .Config.Labels "com.docker.compose.project.config_files"}}` out, err := dockerRun(ctx, append([]string{"inspect", "-f", f}, strings.Fields(string(ids))...)...) if err != nil { return nil, err } for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { p := strings.SplitN(line, "|", 5) if len(p) < 5 || p[3] == "" { continue } st := by[p[3]] if st == nil { st = &stack{Name: p[3]} if p[4] != "" { st.Files = strings.Split(p[4], ",") st.File = st.Files[0] } by[p[3]] = st } st.Containers = append(st.Containers, strings.TrimPrefix(p[0], "/")) if !contains(st.Images, p[2]) { st.Images = append(st.Images, p[2]) } if p[1] == "running" { st.Running++ } } } maps := mountsFor(ctx, s.store) stacks := []stack{} for _, st := range by { sort.Strings(st.Containers) sort.Strings(st.Images) st.resolve(maps) stacks = append(stacks, *st) } stacks = append(stacks, scanHomes(ctx, s.homesOf(stacks, true), stacks)...) sort.Slice(stacks, func(i, j int) bool { return stacks[i].Name < stacks[j].Name }) return stacks, nil } // ---- where stacks live ---- // A home is a directory of stacks, one folder each: where new ones are made, and where ones // that aren't deployed (never were, or were taken down) are still found. Homes aren't // configured; they are wherever this machine's stacks already are. With none at all, ~/stacks. type stackHome struct { Dir string `json:"dir"` // on the machine that holds the files Inner string `json:"inner"` // the same directory as compose sees it (inside Via); Dir otherwise Via string `json:"via"` // container compose runs inside Remote string `json:"remote"` // DOCKER_HOST, when the containers run on another machine HeldBy string `json:"held_by"` // the machine holding the files, when it isn't the one running them Stacks int `json:"stacks"` Default bool `json:"default"` } func (s *Server) homesOf(stacks []stack, withFallback bool) []stackHome { type tally struct { home stackHome deployed int byVia map[string]int inner map[string]string } by := map[string]*tally{} for _, st := range stacks { if !st.Editable || len(st.HostFiles) == 0 || len(st.Files) == 0 { continue } dir := filepath.Dir(filepath.Dir(st.HostFiles[0])) if strings.Count(dir, "/") < 2 { // "/" or "/tmp": not somewhere to make folders in continue } key := dir + "|" + st.Remote t := by[key] if t == nil { t = &tally{home: stackHome{Dir: dir, Remote: st.Remote, HeldBy: st.HeldBy}, byVia: map[string]int{}, inner: map[string]string{}} by[key] = t } t.home.Stacks++ if len(st.Containers) > 0 { t.deployed++ t.byVia[st.Via]++ } t.inner[st.Via] = dir if st.Via != "" { t.inner[st.Via] = filepath.Dir(filepath.Dir(st.Files[0])) } } // A folder is a home when stacks are kept there as a habit: two or more deployed from it. // One stack on its own (Dockhand's own compose file in ~/apps/dockhand) says nothing about // where the next should go, and would have every compose file near it listed as a stack. // New stacks are deployed the way most of their neighbours were. var homes, lone []stackHome for _, t := range by { best, n := "", -1 for via, c := range t.byVia { if c > n || (c == n && via > best) { best, n = via, c } } if n < 0 { for via := range t.inner { best = via } } t.home.Via, t.home.Inner = best, t.inner[best] if t.home.Inner == "" { t.home.Inner = t.home.Dir } if t.deployed >= 2 { homes = append(homes, t.home) } else { lone = append(lone, t.home) } } if len(homes) == 0 { homes = lone } sort.Slice(homes, func(i, j int) bool { if homes[i].Stacks != homes[j].Stacks { return homes[i].Stacks > homes[j].Stacks } return homes[i].Dir < homes[j].Dir }) if len(homes) == 0 && withFallback { if home, err := os.UserHomeDir(); err == nil { homes = append(homes, stackHome{Dir: filepath.Join(home, "stacks"), Inner: filepath.Join(home, "stacks")}) } } if len(homes) > 0 { homes[0].Default = true } return homes } var composeNames = []string{"compose.yaml", "compose.yml", "docker-compose.yml", "docker-compose.yaml"} // scanHomes finds the stacks in these homes that have no containers: a folder with a compose file. func scanHomes(ctx context.Context, homes []stackHome, known []stack) []stack { have := map[string]bool{} for _, st := range known { have[st.Name] = true for _, f := range st.HostFiles { have[filepath.Dir(f)] = true } } var out []stack for _, h := range homes { args := []string{h.Dir, "-mindepth", "2", "-maxdepth", "2", "-type", "f", "("} for i, n := range composeNames { if i > 0 { args = append(args, "-o") } args = append(args, "-name", n) } args = append(args, ")") found, err := exec.CommandContext(ctx, "find", args...).Output() if err != nil && priv.Available() { full := priv.Wrap(append([]string{"find"}, args...)...) found, _ = exec.CommandContext(ctx, full[0], full[1:]...).Output() } for _, f := range strings.Split(strings.TrimSpace(string(found)), "\n") { if f == "" { continue } dir := filepath.Dir(f) name := strings.ToLower(filepath.Base(dir)) if have[name] || have[dir] || strings.HasPrefix(filepath.Base(dir), ".") { continue } have[name], have[dir] = true, true st := stack{Name: name, Containers: []string{}, Images: []string{}, HostFiles: []string{f}, HostFile: f, Via: h.Via, Remote: h.Remote, HeldBy: h.HeldBy, Editable: true, Note: "not deployed"} st.Files = []string{filepath.Join(h.Inner, filepath.Base(dir), filepath.Base(f))} st.File = st.Files[0] st.HasEnv = readable(st.envPath()) out = append(out, st) } } return out } func contains(list []string, s string) bool { for _, x := range list { if x == s { return true } } return false } // hubStacks is a node's stack list with the hub's help: what the node could not find is // looked for here, and if it is here — and the node's Docker answers over tcp, which is how // it was deployed in the first place — the stack becomes editable from this hub. func (s *Server) hubStacks(ctx context.Context, node string) ([]stack, error) { code, b, err := s.nodes.Do(ctx, node, "GET", "/api/docker/stacks", nil, nil) if err != nil { return nil, err } var res struct{ Stacks []stack } if code != 200 || json.Unmarshal(b, &res) != nil { return nil, fmt.Errorf("%s answered %d", node, code) } remote, maps := "", []mountMap(nil) for i := range res.Stacks { st := &res.Stacks[i] if st.Editable || len(st.Files) == 0 { continue } if remote == "" { remote, maps = s.nodeDocker(ctx, node), mountsFor(ctx, s.store) } if remote == "-" { continue } note := st.Note if st.resolve(maps); !st.Editable { st.Note = note continue } st.Remote, st.HeldBy = remote, s.hostName() } var held []stack for _, st := range res.Stacks { if st.HeldBy != "" { held = append(held, st) } } res.Stacks = append(res.Stacks, scanHomes(ctx, s.homesOf(held, false), res.Stacks)...) sort.Slice(res.Stacks, func(i, j int) bool { return res.Stacks[i].Name < res.Stacks[j].Name }) return res.Stacks, nil } var ( nodeDockerMu sync.Mutex nodeDockerOK = map[string]string{} // node → "tcp://host:2375", or "-" when it doesn't answer ) // nodeDocker is the node's Docker endpoint as Dockhand-style deployments use it, if it answers. func (s *Server) nodeDocker(ctx context.Context, node string) string { nodeDockerMu.Lock() defer nodeDockerMu.Unlock() if v, ok := nodeDockerOK[node]; ok && v != "-" { return v } ep := "-" for _, n := range s.nodes.List() { if n.Name != node { continue } if u, err := url.Parse(n.URL); err == nil && u.Hostname() != "" { cand := "tcp://" + u.Hostname() + ":2375" c, cancel := context.WithTimeout(ctx, 5*time.Second) if exec.CommandContext(c, dockerTool(), "-H", cand, "version", "--format", "{{.Server.Version}}").Run() == nil { ep = cand } cancel() } } nodeDockerOK[node] = ep return ep } // stackFor finds the stack a request means: on this machine, or (on a hub route) one of a // node's stacks that this hub holds the files for. handled is false when the request should // simply be passed on to the node. func (s *Server) stackFor(ctx context.Context, r *http.Request) (st stack, handled bool, err error) { var all []stack if node := r.PathValue("node"); node != "" { all, err = s.hubStacks(ctx, node) } else { all, err = s.findStacks(ctx) } if err != nil { return stack{}, true, err } for _, x := range all { if x.Name == r.PathValue("name") { if r.PathValue("node") != "" && x.Remote == "" { return x, false, nil // the node can serve this one itself } return x, true, nil } } return stack{}, true, errors.New("no such stack") } func (s *Server) stacksList(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second) defer cancel() if node := r.PathValue("node"); node != "" { stacks, err := s.hubStacks(ctx, node) if err != nil { s.proxyNode(w, r) // let the node's own answer (or failure) through return } jsonOK(w, 200, map[string]any{"stacks": stacks}) return } if dockerTool() == "" { // no Docker is an empty answer, not a failure jsonOK(w, 200, map[string]any{"stacks": []stack{}}) return } stacks, err := s.findStacks(ctx) if err != nil { jsonErr(w, 500, err.Error()) return } jsonOK(w, 200, map[string]any{"stacks": stacks}) } func readAsRoot(ctx context.Context, path string) ([]byte, error) { b, err := os.ReadFile(path) if err != nil && priv.Available() { full := priv.Wrap("cat", path) b, err = exec.CommandContext(ctx, full[0], full[1:]...).Output() } return b, err } func sumOf(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:8]) } // target is which of the stack's files a request means: ?file=1 is the first override, // ?file=env the .env beside the first file. func target(r *http.Request, st stack) (host string, index int, ok bool) { q := r.URL.Query().Get("file") switch { case q == "env": return st.envPath(), -1, st.HasEnv case q == "": return st.HostFiles[0], 0, true } var i int if _, err := fmt.Sscanf(q, "%d", &i); err != nil || i < 0 || i >= len(st.HostFiles) { return "", 0, false } return st.HostFiles[i], i, true } func (s *Server) stackFile(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second) defer cancel() st, handled, err := s.stackFor(ctx, r) if !handled { s.proxyNode(w, r) return } if err != nil || !st.Editable { jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach") return } path, i, ok := target(r, st) if !ok { jsonErr(w, 400, "no such file in this stack") return } b, err := readAsRoot(ctx, path) if err != nil { jsonErr(w, 500, err.Error()) return } // sum is what the editor hands back when it saves, so a file changed meanwhile is noticed jsonOK(w, 200, map[string]any{"stack": st, "text": string(b), "sum": sumOf(b), "index": i}) } // composeCmd is `docker compose …` for this stack, run where its files make sense and // pointed at the Docker that runs its containers. files are given as compose will see them. func composeCmd(ctx context.Context, st stack, files []string, args ...string) *exec.Cmd { return compose.Cmd(ctx, specOf(st, files), args...) } // specOf is a stack as the compose runner needs it — and as a scheduled update stores it, // so the cron job and the console invoke compose identically. func specOf(st stack, files []string) compose.Spec { return compose.Spec{Project: st.Name, Files: files, Via: st.Via, Remote: st.Remote, Tool: dockerTool()} } // seenFiles is the stack's file list as compose runs it, with file i replaced by a candidate. func seenFiles(st stack, i int, candidate string) []string { out := append([]string{}, st.HostFiles...) if st.Via != "" { out = append([]string{}, st.Files...) } if i >= 0 { out[i] = candidate } return out } func (s *Server) writeAsOwner(ctx context.Context, path, text string) error { err := s.fsLocal.Write(ctx, path, strings.NewReader(text)) // ours to write, or root's if err != nil { err = s.fsSudo.Write(ctx, path, strings.NewReader(text)) } return err } // stackSave validates the new text with compose itself, keeps the old file beside it as // .hamlab-bak, and writes. It does not deploy: that is its own button. func (s *Server) stackSave(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) defer cancel() st, handled, err := s.stackFor(ctx, r) if !handled { s.proxyNode(w, r) return } if err != nil || !st.Editable { jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach") return } var body struct { Text string `json:"text"` Sum string `json:"sum"` // the sum the editor was given when it opened the file } path, i, ok := target(r, st) if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&body); err != nil || !ok || (i >= 0 && strings.TrimSpace(body.Text) == "") { jsonErr(w, 400, "no such file in this stack, or nothing to write") return } if now, err := readAsRoot(ctx, path); err == nil && body.Sum != "" && sumOf(now) != body.Sum { jsonErr(w, 409, "the file has changed on disk since you opened it (Dockhand, or another editor). Reload it to see the change; your text is still in the box.") return } // A candidate goes beside the real file, so `.env` and relative paths resolve as they will. // A compose file is validated as that candidate; a .env has to be in place to be read at all, // so it is swapped in, validated, and swapped back out if compose objects. cand := path + ".hamlab-new" if err := s.writeAsOwner(ctx, cand, body.Text); err != nil { jsonErr(w, 500, "couldn't write beside the file: "+err.Error()) return } sudo := func(args ...string) error { full := priv.Wrap(args...) if msg, err := exec.CommandContext(ctx, full[0], full[1:]...).CombinedOutput(); err != nil { return errors.New(strings.TrimSpace(string(msg))) } return nil } if err := sudo("cp", "-p", path, path+".hamlab-bak"); err != nil { jsonErr(w, 500, err.Error()) return } files := seenFiles(st, -1, "") if i >= 0 { inner := cand if st.Via != "" { inner = st.Files[i] + ".hamlab-new" } files = seenFiles(st, i, inner) } else if err := sudo("mv", cand, path); err != nil { jsonErr(w, 500, err.Error()) return } if msg, err := composeCmd(ctx, st, files, "config", "-q").CombinedOutput(); err != nil { if i >= 0 { _ = s.fsSudo.Delete(ctx, cand, false) } else { _ = sudo("cp", "-p", path+".hamlab-bak", path) // put the .env back } jsonErr(w, 400, "compose rejects it: "+strings.TrimSpace(string(msg))) return } if i >= 0 { if err := sudo("mv", cand, path); err != nil { jsonErr(w, 500, err.Error()) return } } log.Printf("stack %s: %s saved (from %s)", st.Name, filepath.Base(path), clientIP(r)) w.WriteHeader(204) } // stackDiff is what a Deploy would change: each file now, against the copy kept the last // time this console deployed it (.hamlab-deployed), or else the last saved-over copy. func (s *Server) stackDiff(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second) defer cancel() st, handled, err := s.stackFor(ctx, r) if !handled { s.proxyNode(w, r) return } if err != nil || !st.Editable { jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach") return } type fileDiff struct { Name string `json:"name"` Base string `json:"base"` // deployed | saved | "" Old string `json:"old"` New string `json:"new"` } paths := append([]string{}, st.HostFiles...) if st.HasEnv { paths = append(paths, st.envPath()) } out := []fileDiff{} for _, p := range paths { now, _ := readAsRoot(ctx, p) d := fileDiff{Name: filepath.Base(p), New: string(now)} for _, b := range [][2]string{{".hamlab-deployed", "deployed"}, {".hamlab-bak", "saved"}} { if old, err := readAsRoot(ctx, p+b[0]); err == nil { d.Base, d.Old = b[1], string(old) break } } if filepath.Base(p) == ".env" { // values are secrets: say which keys changed, not what to d.Old, d.New = maskEnv(d.Old), maskEnv(d.New) } out = append(out, d) } jsonOK(w, 200, map[string]any{"files": out}) } // maskEnv keeps a .env's keys and hides its values, tagging each distinct value so a change // still shows as a change. func maskEnv(text string) string { lines := strings.Split(text, "\n") for i, l := range lines { if k, v, ok := strings.Cut(l, "="); ok && !strings.HasPrefix(strings.TrimSpace(l), "#") && v != "" { lines[i] = k + "=•••• (" + sumOf([]byte(v))[:6] + ")" } } return strings.Join(lines, "\n") } // stackAction runs deploy (up -d), update (pull, then up -d), restart or stop as a task // whose output the page follows. func (s *Server) stackAction(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second) defer cancel() st, handled, err := s.stackFor(ctx, r) if !handled { s.proxyNode(w, r) return } if err != nil || !st.Editable { jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach") return } files := seenFiles(st, -1, "") var fl []string for _, f := range files { fl = append(fl, "-f", f) } flags := priv.Quote(append([]string{"-p", st.Name}, fl...)) base := func(args string) string { if st.Via != "" { env := "" if st.Remote != "" { env = "-e " + priv.Quote([]string{"DOCKER_HOST=" + st.Remote}) + " " } return fmt.Sprintf("%s exec %s%s docker compose %s %s", dockerTool(), env, priv.Quote([]string{st.Via}), flags, args) } env := "" if st.Remote != "" { // through env(1): a quoted NAME=value in front of a command is a command name to sh, not an assignment env = "env " + priv.Quote([]string{"DOCKER_HOST=" + st.Remote}) + " " } return fmt.Sprintf("cd %s && %s%s compose %s %s", priv.Quote([]string{filepath.Dir(files[0])}), env, dockerTool(), flags, args) } // After a deploy that worked, keep a copy of exactly what was deployed: the next diff is against it. remember := "" paths := append([]string{}, st.HostFiles...) if st.HasEnv { paths = append(paths, st.envPath()) } for _, p := range paths { remember += " && " + priv.Quote(priv.Wrap("cp", "-p", p, p+".hamlab-deployed")) } var script string if a := r.PathValue("action"); len(st.Containers) == 0 && (a == "restart" || a == "stop" || a == "down") { jsonErr(w, 409, st.Name+" isn't deployed: there is nothing to "+a) return } switch r.PathValue("action") { case "deploy": script = base("up -d --remove-orphans") + remember case "update": script = base("pull") + " && " + base("up -d --remove-orphans") + remember case "restart": script = base("restart") case "stop": script = base("stop") case "down": // remove its containers and networks; its files, volumes and bind-mounted data stay script = base("down --remove-orphans") default: jsonErr(w, 400, "deploy, update, restart, stop or down") return } if t, busy := s.tasks.RunningOf("stack:" + st.Name); busy { jsonErr(w, 409, "this stack is already busy (task "+t.ID+")") return } t, err := s.tasks.Start("stack:"+st.Name, r.PathValue("action")+" "+st.Name, exec.Command("sh", "-c", script+" 2>&1")) if err != nil { jsonErr(w, 500, err.Error()) return } if st.Via != "" { // compose runs inside st.Via, started by `docker exec`: stopping the exec client does not // stop what it started in there, so a cancel has to go in after it. via, pattern := st.Via, "compose -p "+regexp.QuoteMeta(st.Name)+" -f " t.OnCancel(func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = exec.CommandContext(ctx, dockerTool(), "exec", via, "pkill", "-TERM", "-f", pattern).Run() }) } stackUpdatesForget(r.PathValue("node") + "/" + st.Name) log.Printf("stack %s: %s started as task %s (from %s)", st.Name, r.PathValue("action"), t.ID, clientIP(r)) jsonOK(w, 202, t.Snapshot(false)) } // ---- is there a newer image? ---- type imageCheck struct { Image string `json:"image"` State string `json:"state"` // current | update | unknown Detail string `json:"detail,omitempty"` } type updAnswer struct { at time.Time res []imageCheck } var ( updMu sync.Mutex updCache = map[string]updAnswer{} ) func stackUpdatesForget(key string) { updMu.Lock(); delete(updCache, key); updMu.Unlock() } // checkImages asks the registry whether each image is still the one we hold. remote is a // DOCKER_HOST for a machine whose images live elsewhere, "" for this one. Each image is one // registry request, which Docker Hub counts — so this is only ever called when asked for. func checkImages(ctx context.Context, images []string, remote string) []imageCheck { local := func(args ...string) ([]byte, error) { // the Docker that holds the images if remote != "" { args = append([]string{"-H", remote}, args...) } return dockerRun(ctx, args...) } res := []imageCheck{} for _, img := range images { ic := imageCheck{Image: img, State: "unknown"} out, err := local("image", "inspect", "-f", "{{json .RepoDigests}}", img) var have []string if err != nil || json.Unmarshal(bytes.TrimSpace(out), &have) != nil || len(have) == 0 { ic.Detail = "built or loaded here, not pulled from a registry" res = append(res, ic) continue } rem, err := dockerRun(ctx, "buildx", "imagetools", "inspect", img, "--format", "{{.Manifest.Digest}}") digest := strings.TrimSpace(string(rem)) if i := strings.LastIndex(digest, "\n"); i >= 0 { digest = digest[i+1:] } if err != nil || !strings.HasPrefix(digest, "sha256:") { ic.Detail = "the registry didn't answer" if err != nil { ic.Detail += ": " + err.Error() } res = append(res, ic) continue } ic.State = "update" for _, h := range have { if strings.HasSuffix(h, "@"+digest) { ic.State = "current" } } res = append(res, ic) } return res } // machineUpdates checks every image on this machine in one pass, so the container list can // be coloured and the fleet can be asked "what is behind?" without walking stack by stack. func (s *Server) machineUpdates(w http.ResponseWriter, r *http.Request) { node := r.PathValue("node") ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute) defer cancel() stacks, err := s.findStacks(ctx) if node != "" { stacks, err = s.hubStacks(ctx, node) } if err != nil { jsonErr(w, 500, err.Error()) return } // one entry per image, however many stacks use it type want struct{ remote string } imgs := map[string]want{} byImage := map[string][]string{} // image -> stacks for _, st := range stacks { for _, img := range st.Images { imgs[img] = want{remote: st.Remote} byImage[img] = append(byImage[img], st.Name) } } refresh := r.URL.Query().Get("refresh") == "1" key := "machine:" + node updMu.Lock() c, ok := updCache[key] updMu.Unlock() if ok && time.Since(c.at) < 6*time.Hour && !refresh { jsonOK(w, 200, map[string]any{"images": c.res, "checked": c.at.UnixMilli(), "stacks": byImage}) return } if !refresh { jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0, "stacks": byImage}) return } var list []string remote := "" for img, wnt := range imgs { list = append(list, img) if wnt.remote != "" { remote = wnt.remote } } sort.Strings(list) res := checkImages(ctx, list, remote) updMu.Lock() updCache[key] = updAnswer{time.Now(), res} updMu.Unlock() jsonOK(w, 200, map[string]any{"images": res, "checked": time.Now().UnixMilli(), "stacks": byImage}) } // stackUpdates asks each image's registry for the digest its tag points at now and compares // it with the digest that was pulled. It is asked for, never polled: every answer is a // registry request, and Docker Hub counts those. Answers are kept for six hours. func (s *Server) stackUpdates(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 90*time.Second) defer cancel() st, handled, err := s.stackFor(ctx, r) if !handled { s.proxyNode(w, r) return } if err != nil { jsonErr(w, 404, err.Error()) return } key := r.PathValue("node") + "/" + st.Name updMu.Lock() c, ok := updCache[key] updMu.Unlock() refresh := r.URL.Query().Get("refresh") == "1" if ok && time.Since(c.at) < 6*time.Hour && !refresh { jsonOK(w, 200, map[string]any{"images": c.res, "checked": c.at.UnixMilli()}) return } if !refresh { jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0}) return } res := checkImages(ctx, st.Images, st.Remote) updMu.Lock() updCache[key] = updAnswer{time.Now(), res} updMu.Unlock() jsonOK(w, 200, map[string]any{"images": res, "checked": time.Now().UnixMilli()}) } // taskOnHub serves a task this hub is running for one of its nodes' stacks; anything else // under /api/nodes//tasks belongs to the node. func (s *Server) taskOnHub(local http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if _, ok := s.tasks.Get(r.PathValue("id")); ok { local(w, r) return } s.proxyNode(w, r) } } // ---- making and unmaking stacks ---- var stackNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,62}$`) // homesFor is where a new stack could go, seen from this request: this machine's homes, or — // on a hub route — the node's own homes plus the ones this hub holds for it. func (s *Server) homesFor(ctx context.Context, r *http.Request) ([]stackHome, error) { node := r.PathValue("node") if node == "" { stacks, err := s.findStacks(ctx) if err != nil { return nil, err } return s.homesOf(stacks, true), nil } stacks, err := s.hubStacks(ctx, node) if err != nil { return nil, err } var held []stack for _, st := range stacks { if st.HeldBy != "" { held = append(held, st) } } homes := s.homesOf(held, false) if code, b, err := s.nodes.Do(ctx, node, "GET", "/api/docker/stacks/homes", nil, nil); err == nil && code == 200 { var res struct{ Homes []stackHome } if json.Unmarshal(b, &res) == nil { for _, h := range res.Homes { if len(homes) > 0 && h.Stacks == 0 { continue // the node's empty ~/stacks is only worth offering when there is nothing else } h.Default = false homes = append(homes, h) } } } if len(homes) > 0 { homes[0].Default = true } return homes, nil } func (s *Server) stackHomes(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second) defer cancel() if dockerTool() == "" && r.PathValue("node") == "" { jsonOK(w, 200, map[string]any{"homes": []stackHome{}}) return } homes, err := s.homesFor(ctx, r) if err != nil { jsonErr(w, 500, err.Error()) return } if homes == nil { // a machine with no Docker and nothing held for it: an empty list, never null homes = []stackHome{} } jsonOK(w, 200, map[string]any{"homes": homes}) } // stackCreate makes a stack: a folder in a home with compose.yaml (and .env) in it, checked by // compose before it is kept. It deploys nothing; Deploy is the next, separate act. func (s *Server) stackCreate(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) defer cancel() raw, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) var body struct { Name, Text, Env, Home string EnvFrom string `json:"env_from"` // a stack being duplicated: its .env is copied here, on the machine, unseen } if err != nil || json.Unmarshal(raw, &body) != nil { jsonErr(w, 400, "bad request") return } name := strings.TrimSpace(body.Name) if !stackNameRe.MatchString(name) || name == "homes" { jsonErr(w, 400, "a stack name is lower-case letters, digits, - and _, starting with a letter or digit") return } if strings.TrimSpace(body.Text) == "" { jsonErr(w, 400, "the compose file can't be empty") return } homes, err := s.homesFor(ctx, r) if err != nil { jsonErr(w, 500, err.Error()) return } var home *stackHome for i := range homes { if homes[i].Dir == body.Home || (body.Home == "" && homes[i].Default) { home = &homes[i] } } if home == nil { jsonErr(w, 400, "that isn't one of the places this machine keeps stacks") return } if r.PathValue("node") != "" && home.HeldBy == "" { // the node's own folder: the node makes it r.Body, r.ContentLength = io.NopCloser(bytes.NewReader(raw)), int64(len(raw)) s.proxyNode(w, r) return } var all []stack if node := r.PathValue("node"); node != "" { all, _ = s.hubStacks(ctx, node) } else { all, _ = s.findStacks(ctx) } for _, st := range all { if st.Name == name { jsonErr(w, 409, "there is already a stack called "+name) return } } if body.EnvFrom != "" && strings.TrimSpace(body.Env) == "" { var from *stack for i := range all { if all[i].Name == body.EnvFrom && all[i].Editable && all[i].HasEnv { from = &all[i] } } if from == nil { jsonErr(w, 400, body.EnvFrom+" has no .env this console can read") return } b, err := readAsRoot(ctx, from.envPath()) if err != nil { jsonErr(w, 500, "reading "+body.EnvFrom+"'s .env: "+err.Error()) return } body.Env = string(b) } dir := filepath.Join(home.Dir, name) if readable(dir) || exec.Command("test", "-e", dir).Run() == nil { jsonErr(w, 409, dir+" already exists") return } sudo := func(args ...string) error { if msg, err := exec.CommandContext(ctx, args[0], args[1:]...).CombinedOutput(); err != nil { full := priv.Wrap(args...) if msg2, err2 := exec.CommandContext(ctx, full[0], full[1:]...).CombinedOutput(); err2 != nil { return errors.New(strings.TrimSpace(string(msg) + " " + string(msg2))) } } return nil } if err := sudo("mkdir", "-p", dir); err != nil { jsonErr(w, 500, err.Error()) return } undo := func() { _ = sudo("rm", "-f", filepath.Join(dir, "compose.yaml"), filepath.Join(dir, ".env")) _ = sudo("rmdir", dir) } host := filepath.Join(dir, "compose.yaml") if err := s.writeAsOwner(ctx, host, body.Text); err != nil { undo() jsonErr(w, 500, err.Error()) return } if strings.TrimSpace(body.Env) != "" { if err := s.writeAsOwner(ctx, filepath.Join(dir, ".env"), body.Env); err != nil { undo() jsonErr(w, 500, err.Error()) return } _ = sudo("chmod", "600", filepath.Join(dir, ".env")) // it is where the secrets go } st := stack{Name: name, Containers: []string{}, Images: []string{}, HostFiles: []string{host}, HostFile: host, Files: []string{filepath.Join(home.Inner, name, "compose.yaml")}, Via: home.Via, Remote: home.Remote, HeldBy: home.HeldBy, Editable: true, Note: "not deployed", HasEnv: strings.TrimSpace(body.Env) != ""} st.File = st.Files[0] if msg, err := composeCmd(ctx, st, seenFiles(st, -1, ""), "config", "-q").CombinedOutput(); err != nil { undo() jsonErr(w, 400, "compose rejects it: "+strings.TrimSpace(string(msg))) return } log.Printf("stack %s: created in %s (from %s)", name, home.Dir, clientIP(r)) jsonOK(w, 201, map[string]any{"stack": st}) } // stackDelete takes a stack down and puts its files in /.hamlab-trash/-