docker: the eight things Dockhand did that Ham Lab did not
Dockhand is being retired, so the Docker tab has to cover what it actually earned its keep doing. Eight pieces, all per container or per machine: - Stats: CPU, memory and IO over time, from sensors already collected. - Health: the container's own healthcheck, with the last few probe outputs — which is where a failing check says why. - Events: `docker events` keeps a minute of backlog and nothing more, so Ham Lab follows the stream and keeps a ring of its own. A restart at 4am is still there at 9. - Shell: `docker exec` into a container, in the terminal already here. - Limits: memory, CPU and restart policy changed live, without hand-editing the compose file for a number you want to try. - Updates: what the registry has that this machine does not, for every image at once rather than one stack at a time. - The same across every machine, asked of the hub — which is the only thing that can reach them all. - Schedules: a real line in /etc/cron.d/hamlab-docker calling the binary, in the same shape as the ZFS snapshot schedules. Dockhand updated DNS and ingress nightly from a ticker inside a web app; a cron line keeps running when the console does not, and is visible to anyone reading the machine the normal way. internal/compose is new and shared, so a deploy from the console and a deploy from cron invoke compose identically — including the case where compose has to run inside the orchestrator's own container, where the paths are that container's and not the host's. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
co-authored by
Claude Opus 5
Claude Opus 5
parent
a70f3e2ed2
commit
192504ead3
+106
-55
@@ -39,6 +39,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"hamlab/internal/compose"
|
||||
"hamlab/internal/priv"
|
||||
)
|
||||
|
||||
@@ -511,24 +512,13 @@ func (s *Server) stackFile(w http.ResponseWriter, r *http.Request) {
|
||||
// composeCmd is `docker compose …` for this stack, run where its files make sense and
|
||||
// pointed at the Docker that runs its containers. files are given as compose will see them.
|
||||
func composeCmd(ctx context.Context, st stack, files []string, args ...string) *exec.Cmd {
|
||||
tool := dockerTool()
|
||||
base := []string{"compose", "-p", st.Name}
|
||||
for _, f := range files {
|
||||
base = append(base, "-f", f)
|
||||
}
|
||||
if st.Via != "" {
|
||||
pre := []string{"exec"}
|
||||
if st.Remote != "" {
|
||||
pre = append(pre, "-e", "DOCKER_HOST="+st.Remote)
|
||||
}
|
||||
return exec.CommandContext(ctx, tool, append(append(append(pre, st.Via, "docker"), base...), args...)...)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, tool, append(base, args...)...)
|
||||
cmd.Dir = filepath.Dir(files[0])
|
||||
if st.Remote != "" {
|
||||
cmd.Env = append(os.Environ(), "DOCKER_HOST="+st.Remote)
|
||||
}
|
||||
return cmd
|
||||
return compose.Cmd(ctx, specOf(st, files), args...)
|
||||
}
|
||||
|
||||
// specOf is a stack as the compose runner needs it — and as a scheduled update stores it,
|
||||
// so the cron job and the console invoke compose identically.
|
||||
func specOf(st stack, files []string) compose.Spec {
|
||||
return compose.Spec{Project: st.Name, Files: files, Via: st.Via, Remote: st.Remote, Tool: dockerTool()}
|
||||
}
|
||||
|
||||
// seenFiles is the stack's file list as compose runs it, with file i replaced by a candidate.
|
||||
@@ -788,6 +778,103 @@ var (
|
||||
|
||||
func stackUpdatesForget(key string) { updMu.Lock(); delete(updCache, key); updMu.Unlock() }
|
||||
|
||||
// checkImages asks the registry whether each image is still the one we hold. remote is a
|
||||
// DOCKER_HOST for a machine whose images live elsewhere, "" for this one. Each image is one
|
||||
// registry request, which Docker Hub counts — so this is only ever called when asked for.
|
||||
func checkImages(ctx context.Context, images []string, remote string) []imageCheck {
|
||||
local := func(args ...string) ([]byte, error) { // the Docker that holds the images
|
||||
if remote != "" {
|
||||
args = append([]string{"-H", remote}, args...)
|
||||
}
|
||||
return dockerRun(ctx, args...)
|
||||
}
|
||||
res := []imageCheck{}
|
||||
for _, img := range images {
|
||||
ic := imageCheck{Image: img, State: "unknown"}
|
||||
out, err := local("image", "inspect", "-f", "{{json .RepoDigests}}", img)
|
||||
var have []string
|
||||
if err != nil || json.Unmarshal(bytes.TrimSpace(out), &have) != nil || len(have) == 0 {
|
||||
ic.Detail = "built or loaded here, not pulled from a registry"
|
||||
res = append(res, ic)
|
||||
continue
|
||||
}
|
||||
rem, err := dockerRun(ctx, "buildx", "imagetools", "inspect", img, "--format", "{{.Manifest.Digest}}")
|
||||
digest := strings.TrimSpace(string(rem))
|
||||
if i := strings.LastIndex(digest, "\n"); i >= 0 {
|
||||
digest = digest[i+1:]
|
||||
}
|
||||
if err != nil || !strings.HasPrefix(digest, "sha256:") {
|
||||
ic.Detail = "the registry didn't answer"
|
||||
if err != nil {
|
||||
ic.Detail += ": " + err.Error()
|
||||
}
|
||||
res = append(res, ic)
|
||||
continue
|
||||
}
|
||||
ic.State = "update"
|
||||
for _, h := range have {
|
||||
if strings.HasSuffix(h, "@"+digest) {
|
||||
ic.State = "current"
|
||||
}
|
||||
}
|
||||
res = append(res, ic)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// machineUpdates checks every image on this machine in one pass, so the container list can
|
||||
// be coloured and the fleet can be asked "what is behind?" without walking stack by stack.
|
||||
func (s *Server) machineUpdates(w http.ResponseWriter, r *http.Request) {
|
||||
node := r.PathValue("node")
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute)
|
||||
defer cancel()
|
||||
stacks, err := s.findStacks(ctx)
|
||||
if node != "" {
|
||||
stacks, err = s.hubStacks(ctx, node)
|
||||
}
|
||||
if err != nil {
|
||||
jsonErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
// one entry per image, however many stacks use it
|
||||
type want struct{ remote string }
|
||||
imgs := map[string]want{}
|
||||
byImage := map[string][]string{} // image -> stacks
|
||||
for _, st := range stacks {
|
||||
for _, img := range st.Images {
|
||||
imgs[img] = want{remote: st.Remote}
|
||||
byImage[img] = append(byImage[img], st.Name)
|
||||
}
|
||||
}
|
||||
refresh := r.URL.Query().Get("refresh") == "1"
|
||||
key := "machine:" + node
|
||||
updMu.Lock()
|
||||
c, ok := updCache[key]
|
||||
updMu.Unlock()
|
||||
if ok && time.Since(c.at) < 6*time.Hour && !refresh {
|
||||
jsonOK(w, 200, map[string]any{"images": c.res, "checked": c.at.UnixMilli(), "stacks": byImage})
|
||||
return
|
||||
}
|
||||
if !refresh {
|
||||
jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0, "stacks": byImage})
|
||||
return
|
||||
}
|
||||
var list []string
|
||||
remote := ""
|
||||
for img, wnt := range imgs {
|
||||
list = append(list, img)
|
||||
if wnt.remote != "" {
|
||||
remote = wnt.remote
|
||||
}
|
||||
}
|
||||
sort.Strings(list)
|
||||
res := checkImages(ctx, list, remote)
|
||||
updMu.Lock()
|
||||
updCache[key] = updAnswer{time.Now(), res}
|
||||
updMu.Unlock()
|
||||
jsonOK(w, 200, map[string]any{"images": res, "checked": time.Now().UnixMilli(), "stacks": byImage})
|
||||
}
|
||||
|
||||
// stackUpdates asks each image's registry for the digest its tag points at now and compares
|
||||
// it with the digest that was pulled. It is asked for, never polled: every answer is a
|
||||
// registry request, and Docker Hub counts those. Answers are kept for six hours.
|
||||
@@ -816,43 +903,7 @@ func (s *Server) stackUpdates(w http.ResponseWriter, r *http.Request) {
|
||||
jsonOK(w, 200, map[string]any{"images": []imageCheck{}, "checked": 0})
|
||||
return
|
||||
}
|
||||
local := func(args ...string) ([]byte, error) { // the Docker that holds the images
|
||||
if st.Remote != "" {
|
||||
args = append([]string{"-H", st.Remote}, args...)
|
||||
}
|
||||
return dockerRun(ctx, args...)
|
||||
}
|
||||
res := []imageCheck{}
|
||||
for _, img := range st.Images {
|
||||
ic := imageCheck{Image: img, State: "unknown"}
|
||||
out, err := local("image", "inspect", "-f", "{{json .RepoDigests}}", img)
|
||||
var have []string
|
||||
if err != nil || json.Unmarshal(bytes.TrimSpace(out), &have) != nil || len(have) == 0 {
|
||||
ic.Detail = "built or loaded here, not pulled from a registry"
|
||||
res = append(res, ic)
|
||||
continue
|
||||
}
|
||||
remote, err := dockerRun(ctx, "buildx", "imagetools", "inspect", img, "--format", "{{.Manifest.Digest}}")
|
||||
digest := strings.TrimSpace(string(remote))
|
||||
if i := strings.LastIndex(digest, "\n"); i >= 0 {
|
||||
digest = digest[i+1:]
|
||||
}
|
||||
if err != nil || !strings.HasPrefix(digest, "sha256:") {
|
||||
ic.Detail = "the registry didn't answer"
|
||||
if err != nil {
|
||||
ic.Detail += ": " + err.Error()
|
||||
}
|
||||
res = append(res, ic)
|
||||
continue
|
||||
}
|
||||
ic.State = "update"
|
||||
for _, h := range have {
|
||||
if strings.HasSuffix(h, "@"+digest) {
|
||||
ic.State = "current"
|
||||
}
|
||||
}
|
||||
res = append(res, ic)
|
||||
}
|
||||
res := checkImages(ctx, st.Images, st.Remote)
|
||||
updMu.Lock()
|
||||
updCache[key] = updAnswer{time.Now(), res}
|
||||
updMu.Unlock()
|
||||
|
||||
Reference in New Issue
Block a user