hamlab/internal/api/dockersched.go

package api

// Scheduled image updates.
//
// Dockhand kept three containers up to date nightly — unbound, nginx and pihole, which is
// DNS and ingress — and that is the one thing it does that Ham Lab did not. This replaces
// it, and deliberately in the same shape as the ZFS snapshot schedules: a real line in a
// cron file Ham Lab owns, calling the binary, rather than a ticker inside the daemon.
// Updating an image is not something that should stop happening because a web console is
// down, and a cron line is visible to anyone looking at the machine the normal way.
//
// A schedule stores everything the run needs — project, compose files, how compose has to
// be invoked for that stack — so the cron job is self-contained and does not have to
// rediscover the world before it can act.

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"hamlab/internal/compose"
	"hamlab/internal/cron"
	"hamlab/internal/storage"
	"hamlab/internal/store"
)

// UpdateJob is a standing instruction to pull a stack's images, and optionally deploy them.
type UpdateJob struct {
	ID      string `json:"id"`
	Name    string `json:"name"`
	Stack   string `json:"stack"`
	Machine string `json:"machine,omitempty"` // for display; the job runs where it was made
	// Services narrows it to some of the stack's services; empty is all of them.
	Services []string `json:"services,omitempty"`
	Cron     string   `json:"cron"`
	// Apply says whether a newer image is deployed or only fetched. Check-only still pulls,
	// so the Docker tab shows what is behind without a registry round trip.
	Apply   bool `json:"apply"`
	Enabled bool `json:"enabled"`

	// How to run compose for this stack, captured when the schedule was made.
	Spec composeSpec `json:"spec"`
}

type composeSpec struct {
	Project string   `json:"project"`
	Files   []string `json:"files"`
	Via     string   `json:"via,omitempty"`
	// Remote is read but never written: schedules made before v0.2 could point compose at
	// another machine's Docker over tcp. Such a job is refused rather than run against this
	// machine's Docker, which is where it would land without the variable.
	Remote string `json:"remote,omitempty"`
}

func (c composeSpec) toSpec(tool string) compose.Spec {
	return compose.Spec{Project: c.Project, Files: c.Files, Via: c.Via, Tool: tool}
}

const dockerCronFile = "/etc/cron.d/hamlab-docker"

type dockerSchedState struct {
	Jobs []UpdateJob `json:"jobs"`
}

func (s *Server) dsLoad() dockerSchedState {
	var raw json.RawMessage
	s.store.View(func(st *store.State) { raw = st.DockerJobs })
	var d dockerSchedState
	if len(raw) > 0 {
		_ = json.Unmarshal(raw, &d)
	}
	if d.Jobs == nil {
		d.Jobs = []UpdateJob{}
	}
	return d
}

func (s *Server) dsSave(d dockerSchedState) error {
	b, err := json.Marshal(d)
	if err != nil {
		return err
	}
	return s.store.Update(func(st *store.State) { st.DockerJobs = b })
}

// dsCronText renders the schedules as cron lines. Same shape as the ZFS file: Ham Lab owns
// it whole and rewrites it on every change.
func dsCronText(jobs []UpdateJob, user, exe, config string) (string, error) {
	var b strings.Builder
	b.WriteString("# Ham Lab image-update schedules — written by the Docker tab, rewritten whole on every change.\n")
	b.WriteString("# Edit them there, not here: anything added by hand is lost the next time a schedule changes.\n")
	b.WriteString("SHELL=/bin/sh\nPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n")
	for i := range jobs {
		j := &jobs[i]
		if !j.Enabled {
			b.WriteString("# " + j.Name + " — paused\n")
			continue
		}
		expr := strings.Join(strings.Fields(j.Cron), " ")
		if err := cron.ValidSchedule(expr); err != nil {
			return "", err
		}
		what := "pull only"
		if j.Apply {
			what = "pull and deploy"
		}
		b.WriteString("# " + j.Name + " — " + j.Stack + ", " + what + "\n")
		b.WriteString(expr + " " + user + " " + exe + " -config " + config + " docker-update " + j.ID + "\n\n")
	}
	return b.String(), nil
}

func (s *Server) dsSyncCron(ctx context.Context, d dockerSchedState) error {
	live := false
	for i := range d.Jobs {
		if d.Jobs[i].Enabled {
			live = true
		}
	}
	if !live {
		return s.storage.RemoveRootFile(ctx, dockerCronFile)
	}
	exe, err := os.Executable()
	if err != nil {
		return err
	}
	exe, _ = filepath.EvalSymlinks(exe)
	text, err := dsCronText(d.Jobs, s.user, exe, s.cfg.Path())
	if err != nil {
		return err
	}
	return s.storage.WriteRootFile(ctx, dockerCronFile, text)
}

// repoint keeps a schedule working when what deployed its stack has been removed.
//
// A schedule captures how to invoke compose at the moment it is made, so the cron job never
// has to rediscover the world at 3am. But "run compose inside this container" stops being
// true the day that container is deleted — which is exactly what happens when an
// orchestrator is retired, and would otherwise break every schedule made before it went.
// So: if the container named is not running, the files are looked up again, through live
// mounts and remembered ones both, and compose runs on the host instead.
func repoint(ctx context.Context, db *store.Store, spec compose.Spec) compose.Spec {
	if spec.Via == "" || db == nil {
		return spec
	}
	if out, err := dockerRun(ctx, "inspect", "-f", "{{.State.Running}}", spec.Via); err == nil && strings.TrimSpace(string(out)) == "true" {
		return spec
	}
	st := stack{Name: spec.Project, File: spec.Files[0], Files: spec.Files}
	st.resolve(mountsSeen(ctx, db)) // read-only: this also runs from cron, where the daemon owns state.json
	if !st.Editable || st.Via != "" {
		return spec // still only reachable through a container, or not reachable at all
	}
	log.Printf("docker schedule %s: %s is gone, running compose here against %s", spec.Project, spec.Via, st.HostFiles[0])
	spec.Files, spec.Via = st.HostFiles, ""
	return spec
}

// RunUpdate pulls, and deploys when the schedule says to. Shared with the CLI so the button
// and the cron line cannot drift apart.
func RunUpdate(ctx context.Context, db *store.Store, j UpdateJob, tool string) (string, error) {
	if len(j.Spec.Files) == 0 {
		return "", errors.New("this schedule has no compose file recorded — open it in the Docker tab and save it again")
	}
	if j.Spec.Remote != "" {
		return "", errors.New("this schedule was made to deploy to another machine over tcp, which this console no longer does — make it again on that machine")
	}
	spec := repoint(ctx, db, j.Spec.toSpec(tool))
	var log strings.Builder
	run := func(args ...string) error {
		cmd := compose.Cmd(ctx, spec, args...)
		out, err := cmd.CombinedOutput()
		log.Write(out)
		return err
	}
	args := append([]string{"pull"}, j.Services...)
	if err := run(args...); err != nil {
		return log.String(), err
	}
	if !j.Apply {
		return log.String(), nil
	}
	up := append([]string{"up", "-d"}, j.Services...)
	if err := run(up...); err != nil {
		return log.String(), err
	}
	return log.String(), nil
}

// ---- endpoints ----

func (s *Server) dsList(w http.ResponseWriter, _ *http.Request) {
	d := s.dsLoad()
	now := time.Now()
	out := make([]map[string]any, 0, len(d.Jobs))
	for _, j := range d.Jobs {
		m := map[string]any{"job": j, "last": storage.ReadRun(s.cfg.DataDir(), "docker-"+j.ID)}
		if j.Enabled {
			if n, ok := cron.NextRun(strings.Join(strings.Fields(j.Cron), " "), now); ok {
				m["next"] = n
			}
		}
		out = append(out, m)
	}
	jsonOK(w, 200, map[string]any{"jobs": out, "file": dockerCronFile})
}

func (s *Server) dsPut(w http.ResponseWriter, r *http.Request) {
	var j UpdateJob
	if err := json.NewDecoder(io.LimitReader(r.Body, 32<<10)).Decode(&j); err != nil {
		jsonErr(w, 400, "bad request")
		return
	}
	j.Cron = strings.Join(strings.Fields(j.Cron), " ")
	if err := cron.ValidSchedule(j.Cron); err != nil {
		jsonErr(w, 400, "schedule: "+err.Error())
		return
	}
	if strings.TrimSpace(j.Stack) == "" {
		jsonErr(w, 400, "which stack?")
		return
	}
	// resolve the stack now, so the cron job never has to
	ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
	defer cancel()
	stacks, err := s.findStacks(ctx)
	if err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	found := false
	for _, st := range stacks {
		if st.Name == j.Stack {
			// seenFiles is the list as compose itself sees it: host paths normally, the
			// orchestrator container's own paths when compose runs inside one. Getting this
			// backwards gives a cron line that works from the console and fails at 3am.
			j.Spec = composeSpec{Project: st.Name, Files: seenFiles(st, -1, ""), Via: st.Via}
			if len(j.Spec.Files) == 0 {
				jsonErr(w, 409, "that stack's compose file isn't anywhere this console can reach")
				return
			}
			found = true
		}
	}
	if !found {
		jsonErr(w, 404, "no stack called "+j.Stack+" on this machine")
		return
	}
	if strings.TrimSpace(j.Name) == "" {
		j.Name = j.Stack
	}
	d := s.dsLoad()
	replaced := false
	for i := range d.Jobs {
		if d.Jobs[i].ID == j.ID && j.ID != "" {
			d.Jobs[i] = j
			replaced = true
		}
	}
	if !replaced {
		b := make([]byte, 6)
		if _, err := rand.Read(b); err != nil {
			jsonErr(w, 500, err.Error())
			return
		}
		j.ID = hex.EncodeToString(b)
		d.Jobs = append(d.Jobs, j)
	}
	if err := s.dsSave(d); err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	if err := s.dsSyncCron(r.Context(), d); err != nil {
		jsonErr(w, 500, "saved, but writing "+dockerCronFile+" failed: "+err.Error())
		return
	}
	log.Printf("docker schedule %q saved by %s (%s, %q, %s)", j.Name, clientIP(r), j.Stack, j.Cron, map[bool]string{true: "pull and deploy", false: "pull only"}[j.Apply])
	jsonOK(w, 200, j)
}

func (s *Server) dsDelete(w http.ResponseWriter, r *http.Request) {
	id := r.PathValue("id")
	d := s.dsLoad()
	kept := d.Jobs[:0]
	found := false
	for _, j := range d.Jobs {
		if j.ID == id {
			found = true
			continue
		}
		kept = append(kept, j)
	}
	if !found {
		jsonErr(w, 404, "no such schedule")
		return
	}
	d.Jobs = kept
	if err := s.dsSave(d); err != nil {
		jsonErr(w, 500, err.Error())
		return
	}
	if err := s.dsSyncCron(r.Context(), d); err != nil {
		jsonErr(w, 500, "removed, but writing "+dockerCronFile+" failed: "+err.Error())
		return
	}
	storage.ForgetRun(s.cfg.DataDir(), "docker-"+id)
	log.Printf("docker schedule %s removed by %s", id, clientIP(r))
	w.WriteHeader(204)
}

func (s *Server) dsRun(w http.ResponseWriter, r *http.Request) {
	id := r.PathValue("id")
	var job UpdateJob
	found := false
	for _, j := range s.dsLoad().Jobs {
		if j.ID == id {
			job, found = j, true
		}
	}
	if !found {
		jsonErr(w, 404, "no such schedule")
		return
	}
	// Outliving the request: a pull the browser gave up on still finishes, but not forever.
	ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 30*time.Minute)
	defer cancel()
	out, err := RunUpdate(ctx, s.store, job, dockerTool())
	rec := storage.RunRecord{At: time.Now(), Taken: job.Stack}
	if err != nil {
		rec.Error = err.Error()
	}
	_ = storage.WriteRun(s.cfg.DataDir(), "docker-"+id, rec)
	stackUpdatesForget("/" + job.Stack)
	if err != nil {
		jsonErr(w, 500, err.Error()+"\n"+tail(out, 2000))
		return
	}
	log.Printf("docker schedule %q run by %s", job.Name, clientIP(r))
	jsonOK(w, 200, map[string]string{"output": tail(out, 4000)})
}

func tail(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return "…" + s[len(s)-n:]
}