diff --git a/internal/api/docker.go b/internal/api/docker.go index 0ca5a66..cfc8e4a 100644 --- a/internal/api/docker.go +++ b/internal/api/docker.go @@ -439,65 +439,6 @@ func (s *Server) dockerHealth(w http.ResponseWriter, r *http.Request) { }) } -// dockerEvents is what has happened to one container lately — started, died, killed, the -// health checks going bad. Asked of Docker on demand rather than kept: Docker already has -// the history, and a copy of it would only go stale. -func (s *Server) dockerEvents(w http.ResponseWriter, r *http.Request) { - id := r.PathValue("id") - if !dockerRef.MatchString(id) { - jsonErr(w, 400, "bad container") - return - } - since := r.URL.Query().Get("since") - if !regexp.MustCompile(`^[0-9]+[mhd]$`).MatchString(since) { - since = "7d" - } - ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) - defer cancel() - out, err := dockerRun(ctx, "events", "--since", since, "--until", "0s", - "--filter", "container="+id, "--format", "{{json .}}") - if err != nil { - jsonErr(w, 500, err.Error()) - return - } - events := []map[string]any{} - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - if !strings.HasPrefix(line, "{") { - continue - } - var e struct { - Action string `json:"Action"` - Time int64 `json:"time"` - TimeNano int64 `json:"timeNano"` - Actor map[string]any `json:"Actor"` - } - if json.Unmarshal([]byte(line), &e) != nil { - continue - } - m := map[string]any{"action": e.Action, "t": e.Time * 1000} - if e.TimeNano > 0 { - m["t"] = e.TimeNano / 1e6 - } - if at, ok := e.Actor["Attributes"].(map[string]any); ok { - // the interesting ones: why it stopped, and what the health check said - for _, k := range []string{"exitCode", "signal", "name", "image"} { - if v, ok := at[k]; ok && v != "" { - m[k] = v - } - } - } - events = append(events, m) - } - // newest first, and bounded: a flapping container can produce thousands - for i, j := 0, len(events)-1; i < j; i, j = i+1, j-1 { - events[i], events[j] = events[j], events[i] - } - if len(events) > 400 { - events = events[:400] - } - jsonOK(w, 200, map[string]any{"events": events, "since": since}) -} - // dockerExec opens a shell inside a container, as a terminal session like any other — so // it reattaches, resizes and scrolls back the same way, and works on a managed machine // through the same proxy. diff --git a/internal/api/docker_events.go b/internal/api/docker_events.go new file mode 100644 index 0000000..f193f45 --- /dev/null +++ b/internal/api/docker_events.go @@ -0,0 +1,229 @@ +package api + +// A container's history. +// +// `docker events` is a live stream with only a short in-memory backlog — ask it for the +// last thirty days and you get the last few minutes, because the daemon keeps events in a +// small ring and loses them all when it restarts. That is why Dockhand kept its own copy, +// and why this does too: a watcher follows the stream and appends the lifecycle events to a +// ring kept beside the config, so "what happened to this container last Tuesday" has an +// answer. +// +// Only lifecycle events are kept. A health check produces three exec events every probe, +// which is thousands a day and tells you nothing that the health panel does not. + +import ( + "bufio" + "context" + "encoding/json" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "hamlab/internal/priv" +) + +const ( + eventsKept = 4000 + eventsSaveEeem = 30 * time.Second +) + +// ContainerEvent is one thing that happened to one container. +type ContainerEvent struct { + T int64 `json:"t"` // unix ms + Name string `json:"name"` + Action string `json:"action"` + Image string `json:"image,omitempty"` + ExitCode string `json:"exit_code,omitempty"` + Signal string `json:"signal,omitempty"` + Health string `json:"health,omitempty"` +} + +type eventLog struct { + mu sync.Mutex + ring []ContainerEvent + dirty bool + path string +} + +func (e *eventLog) load(path string) { + e.mu.Lock() + defer e.mu.Unlock() + e.path = path + b, err := os.ReadFile(path) + if err != nil { + return + } + var list []ContainerEvent + if json.Unmarshal(b, &list) == nil { + e.ring = list + } +} + +func (e *eventLog) add(ev ContainerEvent) { + e.mu.Lock() + defer e.mu.Unlock() + e.ring = append(e.ring, ev) + if len(e.ring) > eventsKept { + e.ring = append([]ContainerEvent(nil), e.ring[len(e.ring)-eventsKept:]...) + } + e.dirty = true +} + +// forName is one container's events, newest first. +func (e *eventLog) forName(name string, since time.Time, limit int) []ContainerEvent { + e.mu.Lock() + defer e.mu.Unlock() + out := []ContainerEvent{} + for _, ev := range e.ring { + if ev.Name == name && ev.T >= since.UnixMilli() { + out = append(out, ev) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].T > out[j].T }) + if len(out) > limit { + out = out[:limit] + } + return out +} + +func (e *eventLog) save() { + e.mu.Lock() + if !e.dirty || e.path == "" { + e.mu.Unlock() + return + } + b, err := json.Marshal(e.ring) + e.dirty = false + path := e.path + e.mu.Unlock() + if err != nil { + return + } + _ = os.MkdirAll(filepath.Dir(path), 0o700) + tmp := path + ".tmp" + if os.WriteFile(tmp, b, 0o600) == nil { + _ = os.Rename(tmp, path) + } +} + +// watchDockerEvents follows the stream, restarting if Docker goes away. It is the only +// thing that writes the log, so nothing contends with it. +func (s *Server) watchDockerEvents(ctx context.Context) { + s.events.load(filepath.Join(s.cfg.DataDir(), "docker-events.json")) + go func() { // a periodic flush, so a crash loses at most half a minute of history + t := time.NewTicker(eventsSaveEeem) + defer t.Stop() + for { + select { + case <-ctx.Done(): + s.events.save() + return + case <-t.C: + s.events.save() + } + } + }() + for { + if ctx.Err() != nil { + return + } + if dockerTool() == "" { + time.Sleep(time.Minute) // no Docker here; look again later in case it is installed + continue + } + s.followEvents(ctx) + select { + case <-ctx.Done(): + return + case <-time.After(10 * time.Second): // the daemon restarted, or the pipe broke + } + } +} + +func (s *Server) followEvents(ctx context.Context) { + args := []string{"events", "--format", "{{json .}}", "--filter", "type=container"} + for _, e := range []string{"create", "start", "restart", "stop", "kill", "die", "destroy", "oom", "pause", "unpause", "rename", "update", "health_status"} { + args = append(args, "--filter", "event="+e) + } + tool := dockerTool() + full := append([]string{tool}, args...) + if !dockerReachable() && priv.Available() { + full = priv.Wrap(full...) + } + cmd := exec.CommandContext(ctx, full[0], full[1:]...) + out, err := cmd.StdoutPipe() + if err != nil { + return + } + if err := cmd.Start(); err != nil { + return + } + sc := bufio.NewScanner(out) + sc.Buffer(make([]byte, 64<<10), 1<<20) + n := 0 + for sc.Scan() { + var e struct { + Action string `json:"Action"` + Time int64 `json:"time"` + TimeNano int64 `json:"timeNano"` + Actor struct { + Attributes map[string]string `json:"Attributes"` + } `json:"Actor"` + } + if json.Unmarshal(sc.Bytes(), &e) != nil || e.Action == "" { + continue + } + at := e.Actor.Attributes + ev := ContainerEvent{ + T: e.Time * 1000, Name: at["name"], Image: at["image"], + Action: e.Action, ExitCode: at["exitCode"], Signal: at["signal"], + } + if e.TimeNano > 0 { + ev.T = e.TimeNano / 1e6 + } + // health_status comes through as "health_status: healthy" + if a, h, ok := strings.Cut(e.Action, ": "); ok { + ev.Action, ev.Health = a, h + } + if ev.Name == "" { + continue + } + s.events.add(ev) + n++ + } + _ = cmd.Wait() + if n > 0 { + log.Printf("docker events: stream ended after %d events; reconnecting", n) + } +} + +// dockerEvents answers from Ham Lab's own history, not Docker's short memory. +func (s *Server) dockerEvents(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + if !dockerRef.MatchString(id) { + jsonErr(w, 400, "bad container") + return + } + since := r.URL.Query().Get("since") + d := 7 * 24 * time.Hour + switch since { + case "24h": + d = 24 * time.Hour + case "30d": + d = 30 * 24 * time.Hour + default: + since = "7d" + } + ev := s.events.forName(id, time.Now().Add(-d), 500) + s.events.mu.Lock() + total := len(s.events.ring) + s.events.mu.Unlock() + jsonOK(w, 200, map[string]any{"events": ev, "since": since, "kept": total, "capacity": eventsKept}) +} diff --git a/internal/api/server.go b/internal/api/server.go index 79ef643..1dce958 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -69,6 +69,7 @@ type Server struct { storage *storage.Manager space *space.Manager update updateState + events eventLog kvms kvmProxies fsLocal files.FS fsSudo files.FS // nil without passwordless sudo @@ -138,6 +139,7 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me go s.keepFleetInStep() } go s.watchForUpdates(context.Background()) + go s.watchDockerEvents(context.Background()) m := http.NewServeMux() m.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(204) }) diff --git a/web/src/Docker.tsx b/web/src/Docker.tsx index bbf6dcb..cec19bf 100644 --- a/web/src/Docker.tsx +++ b/web/src/Docker.tsx @@ -180,10 +180,11 @@ function Health({ id }: { id: string }) { // What has happened to this container lately, asked of Docker rather than kept here. function Events({ id }: { id: string }) { const [ev, setEv] = useState(null) + const [kept, setKept] = useState(0) const [since, setSince] = useState('7d') const [err, setErr] = useState('') - useEffect(() => { setEv(null); setErr(''); api.dockerEvents(id, since).then((r) => setEv(r.events)).catch((e) => setErr(e instanceof Error ? e.message : 'failed')) }, [id, since]) - const bad = (a: string) => /die|kill|oom|destroy|health_status: unhealthy/.test(a) + useEffect(() => { setEv(null); setErr(''); api.dockerEvents(id, since).then((r) => { setEv(r.events); setKept(r.kept) }).catch((e) => setErr(e instanceof Error ? e.message : 'failed')) }, [id, since]) + const bad = (a: string, h?: string) => /die|kill|oom|destroy/.test(a) || (h !== undefined && h !== 'healthy') return ( <>
@@ -191,14 +192,14 @@ function Events({ id }: { id: string }) {
{err &&
{err}
} {!ev && !err && } - {ev && ev.length === 0 &&
Nothing in the last {since}. Docker keeps events only while the daemon has been up.
} + {ev && ev.length === 0 &&
Nothing for this container in the last {since}. Ham Lab keeps its own record — Docker's own only reaches back a few minutes — so history starts from when this version was installed. {kept} events kept across every container.
} {ev && ev.length > 0 && (
{ev.map((e, i) => (
- {e.action} + {e.action}{e.health ? `: ${e.health}` : ''} {new Date(e.t).toLocaleString()} - {e.exitCode !== undefined ? `exit ${e.exitCode}` : ''}{e.signal ? ` signal ${e.signal}` : ''} + {e.exit_code ? `exit ${e.exit_code}` : ''}{e.signal ? ` signal ${e.signal}` : ''}
diff --git a/web/src/api.ts b/web/src/api.ts index 65726ef..9d97676 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -111,7 +111,7 @@ export interface SmartInfo { device: string; available: boolean; supported: bool export interface DockerContainer { id: string; name: string; image: string; state: string; status: string; health: string; ports: string; stack: string; service: string; created: string; mem: number } export interface DockerHealthProbe { Start: string; End: string; ExitCode: number; Output: string } export interface DockerHealth { configured: boolean; state: string; status?: string; failing_streak?: number; log?: DockerHealthProbe[] } -export interface DockerEvent { action: string; t: number; exitCode?: string; signal?: string; name?: string; image?: string } +export interface DockerEvent { t: number; name: string; action: string; image?: string; exit_code?: string; signal?: string; health?: string } export interface DockerInfo { available: boolean; message?: string; tool?: string; containers: DockerContainer[]; images: Record[]; volumes: Record[]; networks: Record[] } export interface Stack { name: string; containers: string[]; running: number; file: string; host_file: string; files: string[]; host_files: string[] | null; images: string[] | null; via: string; remote: string; held_by: string; editable: boolean; note?: string; has_env: boolean } @@ -253,7 +253,7 @@ export const api = { dockerLogs: (id: string, since = '') => req<{ lines: { t: string; m: string }[] }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/logs${since ? `?since=${encodeURIComponent(since)}` : '?n=300'}`), dockerInspect: (id: string) => req>('GET', `/api/docker/containers/${encodeURIComponent(id)}/inspect`), dockerHealth: (id: string) => req('GET', `/api/docker/containers/${encodeURIComponent(id)}/health`), - dockerEvents: (id: string, since = '7d') => req<{ events: DockerEvent[]; since: string }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/events?since=${since}`), + dockerEvents: (id: string, since = '7d') => req<{ events: DockerEvent[]; since: string; kept: number; capacity: number }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/events?since=${since}`), dockerExec: (id: string) => req<{ id: string; name: string }>('POST', `/api/docker/containers/${encodeURIComponent(id)}/exec`, {}), dockerLimits: (id: string, p: { memory?: string; cpus?: string; restart?: string }) => req('POST', `/api/docker/containers/${encodeURIComponent(id)}/limits`, p), stacks: () => req<{ stacks: Stack[] }>('GET', '/api/docker/stacks'),