hamlab/internal/api/stacks.go

package api

// Stacks: the compose projects behind the containers, with their files to edit and a way to
// redeploy them.
//
// A stack is found from its containers' labels (project, config files). The files may not be
// where the label says: an orchestrator that runs compose from inside its own container —
// Dockhand, Dockge, Portainer — records paths in *its* filesystem. So each path is looked up
// twice: as it stands on this host, and through every container's mounts
// (/app/data/stacks/… in `dockhand` is /home/core/apps/dockhand/stacks/… here). Compose is
// then run where the files make sense — inside that container when they came from one, so
// relative paths, the .env beside them and the labels all stay exactly as they were
// deployed — else on the host.
//
// A stack can also be deployed *from another machine*: Dockhand on the hub drives a node's
// Docker over tcp, so the node runs the containers and the hub holds the files. The node
// alone can only list such a stack read-only. The hub, asked about one of its nodes, fills
// in what the node couldn't (hubStacks): it resolves the files on itself and runs compose
// here with DOCKER_HOST pointed at the node.

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"hamlab/internal/compose"
	"hamlab/internal/priv"
)

type stack struct {
	Name       string   `json:"name"`
	Containers []string `json:"containers"`
	Images     []string `json:"images"`
	Running    int      `json:"running"`
	File       string   `json:"file"`       // as the label has it (the first of Files)
	HostFile   string   `json:"host_file"`  // where it is on the machine that holds it; "" when unknown
	Files      []string `json:"files"`      // every file it was deployed from, in order: -f a -f b
	HostFiles  []string `json:"host_files"` // the same, on the machine that holds them
	Via        string   `json:"via"`        // container compose runs inside; "" = on the host
	Remote     string   `json:"remote"`     // DOCKER_HOST compose is pointed at; "" = this machine's Docker
	HeldBy     string   `json:"held_by"`    // the machine holding the files, when it isn't the one running them
	Editable   bool     `json:"editable"`
	Note       string   `json:"note,omitempty"`
	HasEnv     bool     `json:"has_env"`
}

type mountMap struct{ container, dest, src string }

func stackMounts(ctx context.Context) []mountMap {
	out, err := dockerRun(ctx, "ps", "-q")
	if err != nil || len(bytes.TrimSpace(out)) == 0 {
		return nil
	}
	insp, err := dockerRun(ctx, append([]string{"inspect"}, strings.Fields(string(out))...)...)
	if err != nil {
		return nil
	}
	var list []struct {
		Name   string
		Mounts []struct{ Type, Source, Destination string }
	}
	if json.Unmarshal(insp, &list) != nil {
		return nil
	}
	var maps []mountMap
	for _, c := range list {
		for _, m := range c.Mounts {
			if m.Type == "bind" && m.Destination != "/" && m.Source != "" {
				maps = append(maps, mountMap{strings.TrimPrefix(c.Name, "/"), filepath.Clean(m.Destination), filepath.Clean(m.Source)})
			}
		}
	}
	sort.Slice(maps, func(i, j int) bool { return len(maps[i].dest) > len(maps[j].dest) }) // the most specific mount wins
	return maps
}

func readable(p string) bool {
	if _, err := os.Stat(p); err == nil {
		return true
	}
	if priv.Available() { // stacks kept by a root-owned orchestrator are often unreadable to us
		full := priv.Wrap("test", "-f", p)
		return exec.Command(full[0], full[1:]...).Run() == nil
	}
	return false
}

// resolve finds every file of a stack on this machine. Every one must be found — a stack
// deployed from `-f compose.yaml -f override.yaml` and redeployed from the first alone would
// quietly lose everything the override set.
func (st *stack) resolve(maps []mountMap) {
	st.HostFiles, st.HostFile, st.Via, st.Editable, st.HasEnv = nil, "", "", false, false
	if st.File == "" {
		st.Note = "its containers don't say which file deployed them"
		return
	}
	one := func(f string) (host, via string) {
		if readable(f) {
			return f, ""
		}
		for _, m := range maps {
			if rest, ok := strings.CutPrefix(f, m.dest+"/"); ok && readable(filepath.Join(m.src, rest)) {
				return filepath.Join(m.src, rest), m.container
			}
		}
		return "", ""
	}
	for i, f := range st.Files {
		host, via := one(f)
		if host == "" || (i > 0 && via != st.Via) {
			st.HostFiles, st.Via = nil, ""
			st.Note = "deployed from somewhere this machine can't see (" + f + "); edit it there"
			return
		}
		st.HostFiles, st.Via = append(st.HostFiles, host), via
	}
	st.HostFile, st.Editable, st.Note = st.HostFiles[0], true, ""
	st.HasEnv = readable(st.envPath())
}

func (st stack) envPath() string { return filepath.Join(filepath.Dir(st.HostFiles[0]), ".env") }

func (s *Server) findStacks(ctx context.Context) ([]stack, error) {
	// Labels come from inspect, not `ps`: ps joins every label with commas, and the label that
	// matters here — the list of compose files — is itself a comma-separated list.
	ids, err := dockerRun(ctx, "ps", "-aq")
	if err != nil {
		return nil, err
	}
	by := map[string]*stack{}
	if len(bytes.TrimSpace(ids)) > 0 {
		const f = `{{.Name}}|{{.State.Status}}|{{.Config.Image}}|{{index .Config.Labels "com.docker.compose.project"}}|{{index .Config.Labels "com.docker.compose.project.config_files"}}`
		out, err := dockerRun(ctx, append([]string{"inspect", "-f", f}, strings.Fields(string(ids))...)...)
		if err != nil {
			return nil, err
		}
		for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
			p := strings.SplitN(line, "|", 5)
			if len(p) < 5 || p[3] == "" {
				continue
			}
			st := by[p[3]]
			if st == nil {
				st = &stack{Name: p[3]}
				if p[4] != "" {
					st.Files = strings.Split(p[4], ",")
					st.File = st.Files[0]
				}
				by[p[3]] = st
			}
			st.Containers = append(st.Containers, strings.TrimPrefix(p[0], "/"))
			if !contains(st.Images, p[2]) {
				st.Images = append(st.Images, p[2])
			}
			if p[1] == "running" {
				st.Running++
			}
		}
	}
	maps := stackMounts(ctx)
	stacks := []stack{}
	for _, st := range by {
		sort.Strings(st.Containers)
		sort.Strings(st.Images)
		st.resolve(maps)
		stacks = append(stacks, *st)
	}
	stacks = append(stacks, scanHomes(ctx, s.homesOf(stacks, true), stacks)...)
	sort.Slice(stacks, func(i, j int) bool { return stacks[i].Name < stacks[j].Name })
	return stacks, nil
}

// ---- where stacks live ----

// A home is a directory of stacks, one folder each: where new ones are made, and where ones
// that aren't deployed (never were, or were taken down) are still found. Homes aren't
// configured; they are wherever this machine's stacks already are. With none at all, ~/stacks.
type stackHome struct {
	Dir     string `json:"dir"`     // on the machine that holds the files
	Inner   string `json:"inner"`   // the same directory as compose sees it (inside Via); Dir otherwise
	Via     string `json:"via"`     // container compose runs inside
	Remote  string `json:"remote"`  // DOCKER_HOST, when the containers run on another machine
	HeldBy  string `json:"held_by"` // the machine holding the files, when it isn't the one running them
	Stacks  int    `json:"stacks"`
	Default bool   `json:"default"`
}

func (s *Server) homesOf(stacks []stack, withFallback bool) []stackHome {
	type tally struct {
		home     stackHome
		deployed int
		byVia    map[string]int
		inner    map[string]string
	}
	by := map[string]*tally{}
	for _, st := range stacks {
		if !st.Editable || len(st.HostFiles) == 0 || len(st.Files) == 0 {
			continue
		}
		dir := filepath.Dir(filepath.Dir(st.HostFiles[0]))
		if strings.Count(dir, "/") < 2 { // "/" or "/tmp": not somewhere to make folders in
			continue
		}
		key := dir + "|" + st.Remote
		t := by[key]
		if t == nil {
			t = &tally{home: stackHome{Dir: dir, Remote: st.Remote, HeldBy: st.HeldBy}, byVia: map[string]int{}, inner: map[string]string{}}
			by[key] = t
		}
		t.home.Stacks++
		if len(st.Containers) > 0 {
			t.deployed++
			t.byVia[st.Via]++
		}
		t.inner[st.Via] = dir
		if st.Via != "" {
			t.inner[st.Via] = filepath.Dir(filepath.Dir(st.Files[0]))
		}
	}
	// A folder is a home when stacks are kept there as a habit: two or more deployed from it.
	// One stack on its own (Dockhand's own compose file in ~/apps/dockhand) says nothing about
	// where the next should go, and would have every compose file near it listed as a stack.
	// New stacks are deployed the way most of their neighbours were.
	var homes, lone []stackHome
	for _, t := range by {
		best, n := "", -1
		for via, c := range t.byVia {
			if c > n || (c == n && via > best) {
				best, n = via, c
			}
		}
		if n < 0 {
			for via := range t.inner {
				best = via
			}
		}
		t.home.Via, t.home.Inner = best, t.inner[best]
		if t.home.Inner == "" {
			t.home.Inner = t.home.Dir
		}
		if t.deployed >= 2 {
			homes = append(homes, t.home)
		} else {
			lone = append(lone, t.home)
		}
	}
	if len(homes) == 0 {
		homes = lone
	}
	sort.Slice(homes, func(i, j int) bool {
		if homes[i].Stacks != homes[j].Stacks {
			return homes[i].Stacks > homes[j].Stacks
		}
		return homes[i].Dir < homes[j].Dir
	})
	if len(homes) == 0 && withFallback {
		if home, err := os.UserHomeDir(); err == nil {
			homes = append(homes, stackHome{Dir: filepath.Join(home, "stacks"), Inner: filepath.Join(home, "stacks")})
		}
	}
	if len(homes) > 0 {
		homes[0].Default = true
	}
	return homes
}

var composeNames = []string{"compose.yaml", "compose.yml", "docker-compose.yml", "docker-compose.yaml"}

// scanHomes finds the stacks in these homes that have no containers: a folder with a compose file.
func scanHomes(ctx context.Context, homes []stackHome, known []stack) []stack {
	have := map[string]bool{}
	for _, st := range known {
		have[st.Name] = true
		for _, f := range st.HostFiles {
			have[filepath.Dir(f)] = true
		}
	}
	var out []stack
	for _, h := range homes {
		args := []string{h.Dir, "-mindepth", "2", "-maxdepth", "2", "-type", "f", "("}
		for i, n := range composeNames {
			if i > 0 {
				args = append(args, "-o")
			}
			args = append(args, "-name", n)
		}
		args = append(args, ")")
		found, err := exec.CommandContext(ctx, "find", args...).Output()
		if err != nil && priv.Available() {
			full := priv.Wrap(append([]string{"find"}, args...)...)
			found, _ = exec.CommandContext(ctx, full[0], full[1:]...).Output()
		}
		for _, f := range strings.Split(strings.TrimSpace(string(found)), "\n") {
			if f == "" {
				continue
			}
			dir := filepath.Dir(f)
			name := strings.ToLower(filepath.Base(dir))
			if have[name] || have[dir] || strings.HasPrefix(filepath.Base(dir), ".") {
				continue
			}
			have[name], have[dir] = true, true
			st := stack{Name: name, Containers: []string{}, Images: []string{}, HostFiles: []string{f}, HostFile: f, Via: h.Via, Remote: h.Remote, HeldBy: h.HeldBy, Editable: true, Note: "not deployed"}
			st.Files = []string{filepath.Join(h.Inner, filepath.Base(dir), filepath.Base(f))}
			st.File = st.Files[0]
			st.HasEnv = readable(st.envPath())
			out = append(out, st)
		}
	}
	return out
}

func contains(list []string, s string) bool {
	for _, x := range list {
		if x == s {
			return true
		}
	}
	return false
}

// hubStacks is a node's stack list with the hub's help: what the node could not find is
// looked for here, and if it is here — and the node's Docker answers over tcp, which is how
// it was deployed in the first place — the stack becomes editable from this hub.
func (s *Server) hubStacks(ctx context.Context, node string) ([]stack, error) {
	code, b, err := s.nodes.Do(ctx, node, "GET", "/api/docker/stacks", nil, nil)
	if err != nil {
		return nil, err
	}
	var res struct{ Stacks []stack }
	if code != 200 || json.Unmarshal(b, &res) != nil {
		return nil, fmt.Errorf("%s answered %d", node, code)
	}
	remote, maps := "", []mountMap(nil)
	for i := range res.Stacks {
		st := &res.Stacks[i]
		if st.Editable || len(st.Files) == 0 {
			continue
		}
		if remote == "" {
			remote, maps = s.nodeDocker(ctx, node), stackMounts(ctx)
		}
		if remote == "-" {
			continue
		}
		note := st.Note
		if st.resolve(maps); !st.Editable {
			st.Note = note
			continue
		}
		st.Remote, st.HeldBy = remote, s.hostName()
	}
	var held []stack
	for _, st := range res.Stacks {
		if st.HeldBy != "" {
			held = append(held, st)
		}
	}
	res.Stacks = append(res.Stacks, scanHomes(ctx, s.homesOf(held, false), res.Stacks)...)
	sort.Slice(res.Stacks, func(i, j int) bool { return res.Stacks[i].Name < res.Stacks[j].Name })
	return res.Stacks, nil
}

var (
	nodeDockerMu sync.Mutex
	nodeDockerOK = map[string]string{} // node → "tcp://host:2375", or "-" when it doesn't answer
)

// nodeDocker is the node's Docker endpoint as Dockhand-style deployments use it, if it answers.
func (s *Server) nodeDocker(ctx context.Context, node string) string {
	nodeDockerMu.Lock()
	defer nodeDockerMu.Unlock()
	if v, ok := nodeDockerOK[node]; ok && v != "-" {
		return v
	}
	ep := "-"
	for _, n := range s.nodes.List() {
		if n.Name != node {
			continue
		}
		if u, err := url.Parse(n.URL); err == nil && u.Hostname() != "" {
			cand := "tcp://" + u.Hostname() + ":2375"
			c, cancel := context.WithTimeout(ctx, 5*time.Second)
			if exec.CommandContext(c, dockerTool(), "-H", cand, "version", "--format", "{{.Server.Version}}").Run() == nil {
				ep = cand
			}
			cancel()
		}
	}
	nodeDockerOK[node] = ep
	return ep
}

// stackFor finds the stack a request means: on this machine, or (on a hub route) one of a
// node's stacks that this hub holds the files for. handled is false when the request should
// simply be passed on to the node.
func (s *Server) stackFor(ctx context.Context, r *http.Request) (st stack, handled bool, err error) {
	var all []stack
	if node := r.PathValue("node"); node != "" {
		all, err = s.hubStacks(ctx, node)
	} else {
		all, err = s.findStacks(ctx)
	}
	if err != nil {
		return stack{}, true, err
	}
	for _, x := range all {
		if x.Name == r.PathValue("name") {
			if r.PathValue("node") != "" && x.Remote == "" {
				return x, false, nil // the node can serve this one itself
			}
			return x, true, nil
		}
	}
	return stack{}, true, errors.New("no such stack")
}

func (s *Server) stacksList(w http.ResponseWriter, r *http.Request) {
	ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second)
	defer cancel()
	if node := r.PathValue("node"); node != "" {
		stacks, err := s.hubStacks(ctx, node)
		if err != nil {
			s.proxyNode(w, r) // let the node's own answer (or failure) through
			return
		}
		jsonOK(w, 200, map[string]any{"stacks": stacks})
		return
	}
	if dockerTool() == "" { // no Docker is an empty answer, not a failure
		jsonOK(w, 200, map[string]any{"stacks": []stack{}})
		return
	}
	stacks, err := s.findStacks(ctx)
	if err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	jsonOK(w, 200, map[string]any{"stacks": stacks})
}

func readAsRoot(ctx context.Context, path string) ([]byte, error) {
	b, err := os.ReadFile(path)
	if err != nil && priv.Available() {
		full := priv.Wrap("cat", path)
		b, err = exec.CommandContext(ctx, full[0], full[1:]...).Output()
	}
	return b, err
}

func sumOf(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:8]) }

// target is which of the stack's files a request means: ?file=1 is the first override,
// ?file=env the .env beside the first file.
func target(r *http.Request, st stack) (host string, index int, ok bool) {
	q := r.URL.Query().Get("file")
	switch {
	case q == "env":
		return st.envPath(), -1, st.HasEnv
	case q == "":
		return st.HostFiles[0], 0, true
	}
	var i int
	if _, err := fmt.Sscanf(q, "%d", &i); err != nil || i < 0 || i >= len(st.HostFiles) {
		return "", 0, false
	}
	return st.HostFiles[i], i, true
}

func (s *Server) stackFile(w http.ResponseWriter, r *http.Request) {
	ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second)
	defer cancel()
	st, handled, err := s.stackFor(ctx, r)
	if !handled {
		s.proxyNode(w, r)
		return
	}
	if err != nil || !st.Editable {
		jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach")
		return
	}
	path, i, ok := target(r, st)
	if !ok {
		jsonErr(w, 400, "no such file in this stack")
		return
	}
	b, err := readAsRoot(ctx, path)
	if err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	// sum is what the editor hands back when it saves, so a file changed meanwhile is noticed
	jsonOK(w, 200, map[string]any{"stack": st, "text": string(b), "sum": sumOf(b), "index": i})
}

// composeCmd is `docker compose …` for this stack, run where its files make sense and
// pointed at the Docker that runs its containers. files are given as compose will see them.
func composeCmd(ctx context.Context, st stack, files []string, args ...string) *exec.Cmd {
	return compose.Cmd(ctx, specOf(st, files), args...)
}

// specOf is a stack as the compose runner needs it — and as a scheduled update stores it,
// so the cron job and the console invoke compose identically.
func specOf(st stack, files []string) compose.Spec {
	return compose.Spec{Project: st.Name, Files: files, Via: st.Via, Remote: st.Remote, Tool: dockerTool()}
}

// seenFiles is the stack's file list as compose runs it, with file i replaced by a candidate.
func seenFiles(st stack, i int, candidate string) []string {
	out := append([]string{}, st.HostFiles...)
	if st.Via != "" {
		out = append([]string{}, st.Files...)
	}
	if i >= 0 {
		out[i] = candidate
	}
	return out
}

func (s *Server) writeAsOwner(ctx context.Context, path, text string) error {
	err := s.fsLocal.Write(ctx, path, strings.NewReader(text)) // ours to write, or root's
	if err != nil {
		err = s.fsSudo.Write(ctx, path, strings.NewReader(text))
	}
	return err
}

// stackSave validates the new text with compose itself, keeps the old file beside it as
// <file>.hamlab-bak, and writes. It does not deploy: that is its own button.
func (s *Server) stackSave(w http.ResponseWriter, r *http.Request) {
	ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
	defer cancel()
	st, handled, err := s.stackFor(ctx, r)
	if !handled {
		s.proxyNode(w, r)
		return
	}
	if err != nil || !st.Editable {
		jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach")
		return
	}
	var body struct {
		Text string `json:"text"`
		Sum  string `json:"sum"` // the sum the editor was given when it opened the file
	}
	path, i, ok := target(r, st)
	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&body); err != nil || !ok || (i >= 0 && strings.TrimSpace(body.Text) == "") {
		jsonErr(w, 400, "no such file in this stack, or nothing to write")
		return
	}
	if now, err := readAsRoot(ctx, path); err == nil && body.Sum != "" && sumOf(now) != body.Sum {
		jsonErr(w, 409, "the file has changed on disk since you opened it (Dockhand, or another editor). Reload it to see the change; your text is still in the box.")
		return
	}
	// A candidate goes beside the real file, so `.env` and relative paths resolve as they will.
	// A compose file is validated as that candidate; a .env has to be in place to be read at all,
	// so it is swapped in, validated, and swapped back out if compose objects.
	cand := path + ".hamlab-new"
	if err := s.writeAsOwner(ctx, cand, body.Text); err != nil {
		jsonErr(w, 500, "couldn't write beside the file: "+err.Error())
		return
	}
	sudo := func(args ...string) error {
		full := priv.Wrap(args...)
		if msg, err := exec.CommandContext(ctx, full[0], full[1:]...).CombinedOutput(); err != nil {
			return errors.New(strings.TrimSpace(string(msg)))
		}
		return nil
	}
	if err := sudo("cp", "-p", path, path+".hamlab-bak"); err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	files := seenFiles(st, -1, "")
	if i >= 0 {
		inner := cand
		if st.Via != "" {
			inner = st.Files[i] + ".hamlab-new"
		}
		files = seenFiles(st, i, inner)
	} else if err := sudo("mv", cand, path); err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	if msg, err := composeCmd(ctx, st, files, "config", "-q").CombinedOutput(); err != nil {
		if i >= 0 {
			_ = s.fsSudo.Delete(ctx, cand, false)
		} else {
			_ = sudo("cp", "-p", path+".hamlab-bak", path) // put the .env back
		}
		jsonErr(w, 400, "compose rejects it: "+strings.TrimSpace(string(msg)))
		return
	}
	if i >= 0 {
		if err := sudo("mv", cand, path); err != nil {
			jsonErr(w, 500, err.Error())
			return
		}
	}
	log.Printf("stack %s: %s saved (from %s)", st.Name, filepath.Base(path), clientIP(r))
	w.WriteHeader(204)
}

// stackDiff is what a Deploy would change: each file now, against the copy kept the last
// time this console deployed it (<file>.hamlab-deployed), or else the last saved-over copy.
func (s *Server) stackDiff(w http.ResponseWriter, r *http.Request) {
	ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second)
	defer cancel()
	st, handled, err := s.stackFor(ctx, r)
	if !handled {
		s.proxyNode(w, r)
		return
	}
	if err != nil || !st.Editable {
		jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach")
		return
	}
	type fileDiff struct {
		Name string `json:"name"`
		Base string `json:"base"` // deployed | saved | ""
		Old  string `json:"old"`
		New  string `json:"new"`
	}
	paths := append([]string{}, st.HostFiles...)
	if st.HasEnv {
		paths = append(paths, st.envPath())
	}
	out := []fileDiff{}
	for _, p := range paths {
		now, _ := readAsRoot(ctx, p)
		d := fileDiff{Name: filepath.Base(p), New: string(now)}
		for _, b := range [][2]string{{".hamlab-deployed", "deployed"}, {".hamlab-bak", "saved"}} {
			if old, err := readAsRoot(ctx, p+b[0]); err == nil {
				d.Base, d.Old = b[1], string(old)
				break
			}
		}
		if filepath.Base(p) == ".env" { // values are secrets: say which keys changed, not what to
			d.Old, d.New = maskEnv(d.Old), maskEnv(d.New)
		}
		out = append(out, d)
	}
	jsonOK(w, 200, map[string]any{"files": out})
}

// maskEnv keeps a .env's keys and hides its values, tagging each distinct value so a change
// still shows as a change.
func maskEnv(text string) string {
	lines := strings.Split(text, "\n")
	for i, l := range lines {
		if k, v, ok := strings.Cut(l, "="); ok && !strings.HasPrefix(strings.TrimSpace(l), "#") && v != "" {
			lines[i] = k + "=•••• (" + sumOf([]byte(v))[:6] + ")"
		}
	}
	return strings.Join(lines, "\n")
}

// stackAction runs deploy (up -d), update (pull, then up -d), restart or stop as a task
// whose output the page follows.
func (s *Server) stackAction(w http.ResponseWriter, r *http.Request) {
	ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second)
	defer cancel()
	st, handled, err := s.stackFor(ctx, r)
	if !handled {
		s.proxyNode(w, r)
		return
	}
	if err != nil || !st.Editable {
		jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach")
		return
	}
	files := seenFiles(st, -1, "")
	var fl []string
	for _, f := range files {
		fl = append(fl, "-f", f)
	}
	flags := priv.Quote(append([]string{"-p", st.Name}, fl...))
	base := func(args string) string {
		if st.Via != "" {
			env := ""
			if st.Remote != "" {
				env = "-e " + priv.Quote([]string{"DOCKER_HOST=" + st.Remote}) + " "
			}
			return fmt.Sprintf("%s exec %s%s docker compose %s %s", dockerTool(), env, priv.Quote([]string{st.Via}), flags, args)
		}
		env := ""
		if st.Remote != "" { // through env(1): a quoted NAME=value in front of a command is a command name to sh, not an assignment
			env = "env " + priv.Quote([]string{"DOCKER_HOST=" + st.Remote}) + " "
		}
		return fmt.Sprintf("cd %s && %s%s compose %s %s", priv.Quote([]string{filepath.Dir(files[0])}), env, dockerTool(), flags, args)
	}
	// After a deploy that worked, keep a copy of exactly what was deployed: the next diff is against it.
	remember := ""
	paths := append([]string{}, st.HostFiles...)
	if st.HasEnv {
		paths = append(paths, st.envPath())
	}
	for _, p := range paths {
		remember += " && " + priv.Quote(priv.Wrap("cp", "-p", p, p+".hamlab-deployed"))
	}
	var script string
	if a := r.PathValue("action"); len(st.Containers) == 0 && (a == "restart" || a == "stop" || a == "down") {
		jsonErr(w, 409, st.Name+" isn't deployed: there is nothing to "+a)
		return
	}
	switch r.PathValue("action") {
	case "deploy":
		script = base("up -d --remove-orphans") + remember
	case "update":
		script = base("pull") + " && " + base("up -d --remove-orphans") + remember
	case "restart":
		script = base("restart")
	case "stop":
		script = base("stop")
	case "down": // remove its containers and networks; its files, volumes and bind-mounted data stay
		script = base("down --remove-orphans")
	default:
		jsonErr(w, 400, "deploy, update, restart, stop or down")
		return
	}
	if t, busy := s.tasks.RunningOf("stack:" + st.Name); busy {
		jsonErr(w, 409, "this stack is already busy (task "+t.ID+")")
		return
	}
	t, err := s.tasks.Start("stack:"+st.Name, r.PathValue("action")+" "+st.Name, exec.Command("sh", "-c", script+" 2>&1"))
	if err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	if st.Via != "" {
		// compose runs inside st.Via, started by `docker exec`: stopping the exec client does not
		// stop what it started in there, so a cancel has to go in after it.
		via, pattern := st.Via, "compose -p "+regexp.QuoteMeta(st.Name)+" -f "
		t.OnCancel(func() {
			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
			defer cancel()
			_ = exec.CommandContext(ctx, dockerTool(), "exec", via, "pkill", "-TERM", "-f", pattern).Run()
		})
	}
	stackUpdatesForget(r.PathValue("node") + "/" + st.Name)
	log.Printf("stack %s: %s started as task %s (from %s)", st.Name, r.PathValue("action"), t.ID, clientIP(r))
	jsonOK(w, 202, t.Snapshot(false))
}

// ---- is there a newer image? ----

type imageCheck struct {
	Image  string `json:"image"`
	State  string `json:"state"` // current | update | unknown
	Detail string `json:"detail,omitempty"`
}

type updAnswer struct {
	at  time.Time
	res []imageCheck
}

var (
	updMu    sync.Mutex
	updCache = map[string]updAnswer{}
)

func stackUpdatesForget(key string) { updMu.Lock(); delete(updCache, key); updMu.Unlock() }

// checkImages asks the registry whether each image is still the one we hold. remote is a
// DOCKER_HOST for a machine whose images live elsewhere, "" for this one. Each image is one
// registry request, which Docker Hub counts — so this is only ever called when asked for.
func checkImages(ctx context.Context, images []string, remote string) []imageCheck {
	local := func(args ...string) ([]byte, error) { // the Docker that holds the images
		if remote != "" {
			args = append([]string{"-H", remote}, args...)
		}
		return dockerRun(ctx, args...)
	}
	res := []imageCheck{}
	for _, img := range images {
		ic := imageCheck{Image: img, State: "unknown"}
		out, err := local("image", "inspect", "-f", "{{json .RepoDigests}}", img)
		var have []string
		if err != nil || json.Unmarshal(bytes.TrimSpace(out), &have) != nil || len(have) == 0 {
			ic.Detail = "built or loaded here, not pulled from a registry"
			res = append(res, ic)
			continue
		}
		rem, err := dockerRun(ctx, "buildx", "imagetools", "inspect", img, "--format", "{{.Manifest.Digest}}")
		digest := strings.TrimSpace(string(rem))
		if i := strings.LastIndex(digest, "\n"); i >= 0 {
			digest = digest[i+1:]
		}
		if err != nil || !strings.HasPrefix(digest, "sha256:") {
			ic.Detail = "the registry didn't answer"
			if err != nil {
				ic.Detail += ": " + err.Error()
			}
			res = append(res, ic)
			continue
		}
		ic.State = "update"
		for _, h := range have {
			if strings.HasSuffix(h, "@"+digest) {
				ic.State = "current"
			}
		}
		res = append(res, ic)
	}
	return res
}

// machineUpdates checks every image on this machine in one pass, so the container list can
// be coloured and the fleet can be asked "what is behind?" without walking stack by stack.
func (s *Server) machineUpdates(w http.ResponseWriter, r *http.Request) {
	node := r.PathValue("node")
	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute)
	defer cancel()
	stacks, err := s.findStacks(ctx)
	if node != "" {
		stacks, err = s.hubStacks(ctx, node)
	}
	if err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	// one entry per image, however many stacks use it
	type want struct{ remote string }
	imgs := map[string]want{}
	byImage := map[string][]string{} // image -> stacks
	for _, st := range stacks {
		for _, img := range st.Images {
			imgs[img] = want{remote: st.Remote}
			byImage[img] = append(byImage[img], st.Name)
		}
	}
	refresh := r.URL.Query().Get("refresh") == "1"
	key := "machine:" + node
	updMu.Lock()
	c, ok := updCache[key]
	updMu.Unlock()
	if ok && time.Since(c.at) < 6*time.Hour && !refresh {
		jsonOK(w, 200, map[string]any{"images": c.res, "checked": c.at.UnixMilli(), "stacks": byImage})
		return
	}
	if !refresh {
		jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0, "stacks": byImage})
		return
	}
	var list []string
	remote := ""
	for img, wnt := range imgs {
		list = append(list, img)
		if wnt.remote != "" {
			remote = wnt.remote
		}
	}
	sort.Strings(list)
	res := checkImages(ctx, list, remote)
	updMu.Lock()
	updCache[key] = updAnswer{time.Now(), res}
	updMu.Unlock()
	jsonOK(w, 200, map[string]any{"images": res, "checked": time.Now().UnixMilli(), "stacks": byImage})
}

// stackUpdates asks each image's registry for the digest its tag points at now and compares
// it with the digest that was pulled. It is asked for, never polled: every answer is a
// registry request, and Docker Hub counts those. Answers are kept for six hours.
func (s *Server) stackUpdates(w http.ResponseWriter, r *http.Request) {
	ctx, cancel := context.WithTimeout(r.Context(), 90*time.Second)
	defer cancel()
	st, handled, err := s.stackFor(ctx, r)
	if !handled {
		s.proxyNode(w, r)
		return
	}
	if err != nil {
		jsonErr(w, 404, err.Error())
		return
	}
	key := r.PathValue("node") + "/" + st.Name
	updMu.Lock()
	c, ok := updCache[key]
	updMu.Unlock()
	refresh := r.URL.Query().Get("refresh") == "1"
	if ok && time.Since(c.at) < 6*time.Hour && !refresh {
		jsonOK(w, 200, map[string]any{"images": c.res, "checked": c.at.UnixMilli()})
		return
	}
	if !refresh {
		jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0})
		return
	}
	res := checkImages(ctx, st.Images, st.Remote)
	updMu.Lock()
	updCache[key] = updAnswer{time.Now(), res}
	updMu.Unlock()
	jsonOK(w, 200, map[string]any{"images": res, "checked": time.Now().UnixMilli()})
}

// taskOnHub serves a task this hub is running for one of its nodes' stacks; anything else
// under /api/nodes/<node>/tasks belongs to the node.
func (s *Server) taskOnHub(local http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if _, ok := s.tasks.Get(r.PathValue("id")); ok {
			local(w, r)
			return
		}
		s.proxyNode(w, r)
	}
}

// ---- making and unmaking stacks ----

var stackNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,62}$`)

// homesFor is where a new stack could go, seen from this request: this machine's homes, or —
// on a hub route — the node's own homes plus the ones this hub holds for it.
func (s *Server) homesFor(ctx context.Context, r *http.Request) ([]stackHome, error) {
	node := r.PathValue("node")
	if node == "" {
		stacks, err := s.findStacks(ctx)
		if err != nil {
			return nil, err
		}
		return s.homesOf(stacks, true), nil
	}
	stacks, err := s.hubStacks(ctx, node)
	if err != nil {
		return nil, err
	}
	var held []stack
	for _, st := range stacks {
		if st.HeldBy != "" {
			held = append(held, st)
		}
	}
	homes := s.homesOf(held, false)
	if code, b, err := s.nodes.Do(ctx, node, "GET", "/api/docker/stacks/homes", nil, nil); err == nil && code == 200 {
		var res struct{ Homes []stackHome }
		if json.Unmarshal(b, &res) == nil {
			for _, h := range res.Homes {
				if len(homes) > 0 && h.Stacks == 0 {
					continue // the node's empty ~/stacks is only worth offering when there is nothing else
				}
				h.Default = false
				homes = append(homes, h)
			}
		}
	}
	if len(homes) > 0 {
		homes[0].Default = true
	}
	return homes, nil
}

func (s *Server) stackHomes(w http.ResponseWriter, r *http.Request) {
	ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second)
	defer cancel()
	if dockerTool() == "" && r.PathValue("node") == "" {
		jsonOK(w, 200, map[string]any{"homes": []stackHome{}})
		return
	}
	homes, err := s.homesFor(ctx, r)
	if err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	if homes == nil { // a machine with no Docker and nothing held for it: an empty list, never null
		homes = []stackHome{}
	}
	jsonOK(w, 200, map[string]any{"homes": homes})
}

// stackCreate makes a stack: a folder in a home with compose.yaml (and .env) in it, checked by
// compose before it is kept. It deploys nothing; Deploy is the next, separate act.
func (s *Server) stackCreate(w http.ResponseWriter, r *http.Request) {
	ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
	defer cancel()
	raw, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
	var body struct {
		Name, Text, Env, Home string
		EnvFrom               string `json:"env_from"` // a stack being duplicated: its .env is copied here, on the machine, unseen
	}
	if err != nil || json.Unmarshal(raw, &body) != nil {
		jsonErr(w, 400, "bad request")
		return
	}
	name := strings.TrimSpace(body.Name)
	if !stackNameRe.MatchString(name) || name == "homes" {
		jsonErr(w, 400, "a stack name is lower-case letters, digits, - and _, starting with a letter or digit")
		return
	}
	if strings.TrimSpace(body.Text) == "" {
		jsonErr(w, 400, "the compose file can't be empty")
		return
	}
	homes, err := s.homesFor(ctx, r)
	if err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	var home *stackHome
	for i := range homes {
		if homes[i].Dir == body.Home || (body.Home == "" && homes[i].Default) {
			home = &homes[i]
		}
	}
	if home == nil {
		jsonErr(w, 400, "that isn't one of the places this machine keeps stacks")
		return
	}
	if r.PathValue("node") != "" && home.HeldBy == "" { // the node's own folder: the node makes it
		r.Body, r.ContentLength = io.NopCloser(bytes.NewReader(raw)), int64(len(raw))
		s.proxyNode(w, r)
		return
	}
	var all []stack
	if node := r.PathValue("node"); node != "" {
		all, _ = s.hubStacks(ctx, node)
	} else {
		all, _ = s.findStacks(ctx)
	}
	for _, st := range all {
		if st.Name == name {
			jsonErr(w, 409, "there is already a stack called "+name)
			return
		}
	}
	if body.EnvFrom != "" && strings.TrimSpace(body.Env) == "" {
		var from *stack
		for i := range all {
			if all[i].Name == body.EnvFrom && all[i].Editable && all[i].HasEnv {
				from = &all[i]
			}
		}
		if from == nil {
			jsonErr(w, 400, body.EnvFrom+" has no .env this console can read")
			return
		}
		b, err := readAsRoot(ctx, from.envPath())
		if err != nil {
			jsonErr(w, 500, "reading "+body.EnvFrom+"'s .env: "+err.Error())
			return
		}
		body.Env = string(b)
	}
	dir := filepath.Join(home.Dir, name)
	if readable(dir) || exec.Command("test", "-e", dir).Run() == nil {
		jsonErr(w, 409, dir+" already exists")
		return
	}
	sudo := func(args ...string) error {
		if msg, err := exec.CommandContext(ctx, args[0], args[1:]...).CombinedOutput(); err != nil {
			full := priv.Wrap(args...)
			if msg2, err2 := exec.CommandContext(ctx, full[0], full[1:]...).CombinedOutput(); err2 != nil {
				return errors.New(strings.TrimSpace(string(msg) + " " + string(msg2)))
			}
		}
		return nil
	}
	if err := sudo("mkdir", "-p", dir); err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	undo := func() {
		_ = sudo("rm", "-f", filepath.Join(dir, "compose.yaml"), filepath.Join(dir, ".env"))
		_ = sudo("rmdir", dir)
	}
	host := filepath.Join(dir, "compose.yaml")
	if err := s.writeAsOwner(ctx, host, body.Text); err != nil {
		undo()
		jsonErr(w, 500, err.Error())
		return
	}
	if strings.TrimSpace(body.Env) != "" {
		if err := s.writeAsOwner(ctx, filepath.Join(dir, ".env"), body.Env); err != nil {
			undo()
			jsonErr(w, 500, err.Error())
			return
		}
		_ = sudo("chmod", "600", filepath.Join(dir, ".env")) // it is where the secrets go
	}
	st := stack{Name: name, Containers: []string{}, Images: []string{}, HostFiles: []string{host}, HostFile: host, Files: []string{filepath.Join(home.Inner, name, "compose.yaml")},
		Via: home.Via, Remote: home.Remote, HeldBy: home.HeldBy, Editable: true, Note: "not deployed", HasEnv: strings.TrimSpace(body.Env) != ""}
	st.File = st.Files[0]
	if msg, err := composeCmd(ctx, st, seenFiles(st, -1, ""), "config", "-q").CombinedOutput(); err != nil {
		undo()
		jsonErr(w, 400, "compose rejects it: "+strings.TrimSpace(string(msg)))
		return
	}
	log.Printf("stack %s: created in %s (from %s)", name, home.Dir, clientIP(r))
	jsonOK(w, 201, map[string]any{"stack": st})
}

// stackDelete takes a stack down and puts its files in <home>/.hamlab-trash/<name>-<time>/ —
// moved, not erased, because a compose file is often the only record of how a service was set
// up. Only the stack's own files move (compose files, .env, this console's copies); anything
// else in the folder, its volumes and its bind-mounted data are left exactly where they are.
func (s *Server) stackDelete(w http.ResponseWriter, r *http.Request) {
	ctx, cancel := context.WithTimeout(r.Context(), 4*time.Minute)
	defer cancel()
	raw, _ := io.ReadAll(io.LimitReader(r.Body, 4096))
	st, handled, err := s.stackFor(ctx, r)
	if !handled {
		r.Body, r.ContentLength = io.NopCloser(bytes.NewReader(raw)), int64(len(raw))
		s.proxyNode(w, r)
		return
	}
	if err != nil || !st.Editable {
		jsonErr(w, 404, "that stack's compose file isn't anywhere this console can reach")
		return
	}
	var body struct{ Confirm string }
	if json.Unmarshal(raw, &body) != nil || body.Confirm != st.Name {
		jsonErr(w, 400, "type the stack's name to confirm")
		return
	}
	if len(st.Containers) > 0 {
		if msg, err := composeCmd(ctx, st, seenFiles(st, -1, ""), "down", "--remove-orphans").CombinedOutput(); err != nil {
			jsonErr(w, 500, "couldn't take it down, so nothing was deleted: "+strings.TrimSpace(string(msg)))
			return
		}
	}
	dir := filepath.Dir(st.HostFiles[0])
	trash := filepath.Join(filepath.Dir(dir), ".hamlab-trash", st.Name+"-"+time.Now().Format("20060102-150405"))
	run := func(args ...string) error {
		full := priv.Wrap(args...)
		if msg, err := exec.CommandContext(ctx, full[0], full[1:]...).CombinedOutput(); err != nil {
			return errors.New(strings.TrimSpace(string(msg)))
		}
		return nil
	}
	if err := run("mkdir", "-p", trash); err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	moved := 0
	for _, f := range append(append([]string{}, st.HostFiles...), st.envPath()) {
		for _, suffix := range []string{"", ".hamlab-bak", ".hamlab-deployed", ".hamlab-new"} {
			if filepath.Dir(f) == dir && readable(f+suffix) && run("mv", f+suffix, trash+"/") == nil {
				moved++
			}
		}
	}
	_ = run("rmdir", dir) // only if nothing else was in it
	stackUpdatesForget(r.PathValue("node") + "/" + st.Name)
	log.Printf("stack %s: taken down and %d file(s) moved to %s (from %s)", st.Name, moved, trash, clientIP(r))
	jsonOK(w, 200, map[string]any{"trash": trash, "moved": moved})
}