Container events are Ham Lab's own record, not Docker's short memory

docker events is a live stream with a small in-memory backlog: ask for thirty days and you
get a few minutes, and a daemon restart loses the lot. That is why Dockhand kept its own
copy. A watcher now follows the stream and appends lifecycle events to a ring beside the
config, flushed every thirty seconds, so the question of what happened to a container last
week has an answer. Health-check exec events are excluded — three per probe, thousands a
day, and the health panel says it better.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

internal/api/docker.go

@@ -439,65 +439,6 @@ func (s *Server) dockerHealth(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
-// dockerEvents is what has happened to one container lately — started, died, killed, the
-// health checks going bad. Asked of Docker on demand rather than kept: Docker already has
-// the history, and a copy of it would only go stale.
-func (s *Server) dockerEvents(w http.ResponseWriter, r *http.Request) {
-	id := r.PathValue("id")
-	if !dockerRef.MatchString(id) {
-		jsonErr(w, 400, "bad container")
-		return
-	}
-	since := r.URL.Query().Get("since")
-	if !regexp.MustCompile(`^[0-9]+[mhd]$`).MatchString(since) {
-		since = "7d"
-	}
-	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
-	defer cancel()
-	out, err := dockerRun(ctx, "events", "--since", since, "--until", "0s",
-		"--filter", "container="+id, "--format", "{{json .}}")
-	if err != nil {
-		jsonErr(w, 500, err.Error())
-		return
-	}
-	events := []map[string]any{}
-	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
-		if !strings.HasPrefix(line, "{") {
-			continue
-		}
-		var e struct {
-			Action   string         `json:"Action"`
-			Time     int64          `json:"time"`
-			TimeNano int64          `json:"timeNano"`
-			Actor    map[string]any `json:"Actor"`
-		}
-		if json.Unmarshal([]byte(line), &e) != nil {
-			continue
-		}
-		m := map[string]any{"action": e.Action, "t": e.Time * 1000}
-		if e.TimeNano > 0 {
-			m["t"] = e.TimeNano / 1e6
-		}
-		if at, ok := e.Actor["Attributes"].(map[string]any); ok {
-			// the interesting ones: why it stopped, and what the health check said
-			for _, k := range []string{"exitCode", "signal", "name", "image"} {
-				if v, ok := at[k]; ok && v != "" {
-					m[k] = v
-				}
-			}
-		}
-		events = append(events, m)
-	}
-	// newest first, and bounded: a flapping container can produce thousands
-	for i, j := 0, len(events)-1; i < j; i, j = i+1, j-1 {
-		events[i], events[j] = events[j], events[i]
-	}
-	if len(events) > 400 {
-		events = events[:400]
-	}
-	jsonOK(w, 200, map[string]any{"events": events, "since": since})
-}
-
 // dockerExec opens a shell inside a container, as a terminal session like any other — so
 // it reattaches, resizes and scrolls back the same way, and works on a managed machine
 // through the same proxy.

internal/api/docker_events.go

@@ -0,0 +1,229 @@
+package api
+
+// A container's history.
+//
+// `docker events` is a live stream with only a short in-memory backlog — ask it for the
+// last thirty days and you get the last few minutes, because the daemon keeps events in a
+// small ring and loses them all when it restarts. That is why Dockhand kept its own copy,
+// and why this does too: a watcher follows the stream and appends the lifecycle events to a
+// ring kept beside the config, so "what happened to this container last Tuesday" has an
+// answer.
+//
+// Only lifecycle events are kept. A health check produces three exec events every probe,
+// which is thousands a day and tells you nothing that the health panel does not.
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"hamlab/internal/priv"
+)
+
+const (
+	eventsKept     = 4000
+	eventsSaveEeem = 30 * time.Second
+)
+
+// ContainerEvent is one thing that happened to one container.
+type ContainerEvent struct {
+	T        int64  `json:"t"` // unix ms
+	Name     string `json:"name"`
+	Action   string `json:"action"`
+	Image    string `json:"image,omitempty"`
+	ExitCode string `json:"exit_code,omitempty"`
+	Signal   string `json:"signal,omitempty"`
+	Health   string `json:"health,omitempty"`
+}
+
+type eventLog struct {
+	mu    sync.Mutex
+	ring  []ContainerEvent
+	dirty bool
+	path  string
+}
+
+func (e *eventLog) load(path string) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.path = path
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return
+	}
+	var list []ContainerEvent
+	if json.Unmarshal(b, &list) == nil {
+		e.ring = list
+	}
+}
+
+func (e *eventLog) add(ev ContainerEvent) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.ring = append(e.ring, ev)
+	if len(e.ring) > eventsKept {
+		e.ring = append([]ContainerEvent(nil), e.ring[len(e.ring)-eventsKept:]...)
+	}
+	e.dirty = true
+}
+
+// forName is one container's events, newest first.
+func (e *eventLog) forName(name string, since time.Time, limit int) []ContainerEvent {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	out := []ContainerEvent{}
+	for _, ev := range e.ring {
+		if ev.Name == name && ev.T >= since.UnixMilli() {
+			out = append(out, ev)
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].T > out[j].T })
+	if len(out) > limit {
+		out = out[:limit]
+	}
+	return out
+}
+
+func (e *eventLog) save() {
+	e.mu.Lock()
+	if !e.dirty || e.path == "" {
+		e.mu.Unlock()
+		return
+	}
+	b, err := json.Marshal(e.ring)
+	e.dirty = false
+	path := e.path
+	e.mu.Unlock()
+	if err != nil {
+		return
+	}
+	_ = os.MkdirAll(filepath.Dir(path), 0o700)
+	tmp := path + ".tmp"
+	if os.WriteFile(tmp, b, 0o600) == nil {
+		_ = os.Rename(tmp, path)
+	}
+}
+
+// watchDockerEvents follows the stream, restarting if Docker goes away. It is the only
+// thing that writes the log, so nothing contends with it.
+func (s *Server) watchDockerEvents(ctx context.Context) {
+	s.events.load(filepath.Join(s.cfg.DataDir(), "docker-events.json"))
+	go func() { // a periodic flush, so a crash loses at most half a minute of history
+		t := time.NewTicker(eventsSaveEeem)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				s.events.save()
+				return
+			case <-t.C:
+				s.events.save()
+			}
+		}
+	}()
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+		if dockerTool() == "" {
+			time.Sleep(time.Minute) // no Docker here; look again later in case it is installed
+			continue
+		}
+		s.followEvents(ctx)
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(10 * time.Second): // the daemon restarted, or the pipe broke
+		}
+	}
+}
+
+func (s *Server) followEvents(ctx context.Context) {
+	args := []string{"events", "--format", "{{json .}}", "--filter", "type=container"}
+	for _, e := range []string{"create", "start", "restart", "stop", "kill", "die", "destroy", "oom", "pause", "unpause", "rename", "update", "health_status"} {
+		args = append(args, "--filter", "event="+e)
+	}
+	tool := dockerTool()
+	full := append([]string{tool}, args...)
+	if !dockerReachable() && priv.Available() {
+		full = priv.Wrap(full...)
+	}
+	cmd := exec.CommandContext(ctx, full[0], full[1:]...)
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		return
+	}
+	sc := bufio.NewScanner(out)
+	sc.Buffer(make([]byte, 64<<10), 1<<20)
+	n := 0
+	for sc.Scan() {
+		var e struct {
+			Action   string `json:"Action"`
+			Time     int64  `json:"time"`
+			TimeNano int64  `json:"timeNano"`
+			Actor    struct {
+				Attributes map[string]string `json:"Attributes"`
+			} `json:"Actor"`
+		}
+		if json.Unmarshal(sc.Bytes(), &e) != nil || e.Action == "" {
+			continue
+		}
+		at := e.Actor.Attributes
+		ev := ContainerEvent{
+			T: e.Time * 1000, Name: at["name"], Image: at["image"],
+			Action: e.Action, ExitCode: at["exitCode"], Signal: at["signal"],
+		}
+		if e.TimeNano > 0 {
+			ev.T = e.TimeNano / 1e6
+		}
+		// health_status comes through as "health_status: healthy"
+		if a, h, ok := strings.Cut(e.Action, ": "); ok {
+			ev.Action, ev.Health = a, h
+		}
+		if ev.Name == "" {
+			continue
+		}
+		s.events.add(ev)
+		n++
+	}
+	_ = cmd.Wait()
+	if n > 0 {
+		log.Printf("docker events: stream ended after %d events; reconnecting", n)
+	}
+}
+
+// dockerEvents answers from Ham Lab's own history, not Docker's short memory.
+func (s *Server) dockerEvents(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if !dockerRef.MatchString(id) {
+		jsonErr(w, 400, "bad container")
+		return
+	}
+	since := r.URL.Query().Get("since")
+	d := 7 * 24 * time.Hour
+	switch since {
+	case "24h":
+		d = 24 * time.Hour
+	case "30d":
+		d = 30 * 24 * time.Hour
+	default:
+		since = "7d"
+	}
+	ev := s.events.forName(id, time.Now().Add(-d), 500)
+	s.events.mu.Lock()
+	total := len(s.events.ring)
+	s.events.mu.Unlock()
+	jsonOK(w, 200, map[string]any{"events": ev, "since": since, "kept": total, "capacity": eventsKept})
+}

internal/api/server.go

@@ -69,6 +69,7 @@ type Server struct {
 	storage     *storage.Manager
 	space       *space.Manager
 	update      updateState
+	events      eventLog
 	kvms        kvmProxies
 	fsLocal     files.FS
 	fsSudo      files.FS // nil without passwordless sudo
@@ -138,6 +139,7 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
 		go s.keepFleetInStep()
 	}
 	go s.watchForUpdates(context.Background())
+	go s.watchDockerEvents(context.Background())
 
 	m := http.NewServeMux()
 	m.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(204) })

web/src/Docker.tsx

@@ -180,10 +180,11 @@ function Health({ id }: { id: string }) {
 // What has happened to this container lately, asked of Docker rather than kept here.
 function Events({ id }: { id: string }) {
   const [ev, setEv] = useState<DockerEvent[] | null>(null)
+  const [kept, setKept] = useState(0)
   const [since, setSince] = useState('7d')
   const [err, setErr] = useState('')
-  useEffect(() => { setEv(null); setErr(''); api.dockerEvents(id, since).then((r) => setEv(r.events)).catch((e) => setErr(e instanceof Error ? e.message : 'failed')) }, [id, since])
+  useEffect(() => { setEv(null); setErr(''); api.dockerEvents(id, since).then((r) => { setEv(r.events); setKept(r.kept) }).catch((e) => setErr(e instanceof Error ? e.message : 'failed')) }, [id, since])
-  const bad = (a: string) => /die|kill|oom|destroy|health_status: unhealthy/.test(a)
+  const bad = (a: string, h?: string) => /die|kill|oom|destroy/.test(a) || (h !== undefined && h !== 'healthy')
   return (
     <>
       <div className="chips" style={{ marginBottom: 8 }}>
@@ -191,14 +192,14 @@ function Events({ id }: { id: string }) {
       </div>
       {err && <div className="error">{err}</div>}
       {!ev && !err && <Loading label="Asking Docker…" />}
-      {ev && ev.length === 0 && <div className="notice">Nothing in the last {since}. Docker keeps events only while the daemon has been up.</div>}
+      {ev && ev.length === 0 && <div className="notice">Nothing for this container in the last {since}. Ham Lab keeps its own record — Docker's own only reaches back a few minutes — so history starts from when this version was installed. {kept} events kept across every container.</div>}
       {ev && ev.length > 0 && (
         <div className="snap-list">
           {ev.map((e, i) => (
             <div className="snap" key={i}>
-              <span className={bad(e.action) ? 'warn-text' : ''}>{e.action}</span>
+              <span className={bad(e.action, e.health) ? 'warn-text' : ''}>{e.action}{e.health ? `: ${e.health}` : ''}</span>
               <span className="muted">{new Date(e.t).toLocaleString()}</span>
-              <span className="muted small">{e.exitCode !== undefined ? `exit ${e.exitCode}` : ''}{e.signal ? ` signal ${e.signal}` : ''}</span>
+              <span className="muted small">{e.exit_code ? `exit ${e.exit_code}` : ''}{e.signal ? ` signal ${e.signal}` : ''}</span>
               <span />
               <span />
             </div>

web/src/api.ts

@@ -111,7 +111,7 @@ export interface SmartInfo { device: string; available: boolean; supported: bool
 export interface DockerContainer { id: string; name: string; image: string; state: string; status: string; health: string; ports: string; stack: string; service: string; created: string; mem: number }
 export interface DockerHealthProbe { Start: string; End: string; ExitCode: number; Output: string }
 export interface DockerHealth { configured: boolean; state: string; status?: string; failing_streak?: number; log?: DockerHealthProbe[] }
-export interface DockerEvent { action: string; t: number; exitCode?: string; signal?: string; name?: string; image?: string }
+export interface DockerEvent { t: number; name: string; action: string; image?: string; exit_code?: string; signal?: string; health?: string }
 
 export interface DockerInfo { available: boolean; message?: string; tool?: string; containers: DockerContainer[]; images: Record<string, unknown>[]; volumes: Record<string, unknown>[]; networks: Record<string, unknown>[] }
 export interface Stack { name: string; containers: string[]; running: number; file: string; host_file: string; files: string[]; host_files: string[] | null; images: string[] | null; via: string; remote: string; held_by: string; editable: boolean; note?: string; has_env: boolean }
@@ -253,7 +253,7 @@ export const api = {
   dockerLogs: (id: string, since = '') => req<{ lines: { t: string; m: string }[] }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/logs${since ? `?since=${encodeURIComponent(since)}` : '?n=300'}`),
   dockerInspect: (id: string) => req<Record<string, unknown>>('GET', `/api/docker/containers/${encodeURIComponent(id)}/inspect`),
   dockerHealth: (id: string) => req<DockerHealth>('GET', `/api/docker/containers/${encodeURIComponent(id)}/health`),
-  dockerEvents: (id: string, since = '7d') => req<{ events: DockerEvent[]; since: string }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/events?since=${since}`),
+  dockerEvents: (id: string, since = '7d') => req<{ events: DockerEvent[]; since: string; kept: number; capacity: number }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/events?since=${since}`),
   dockerExec: (id: string) => req<{ id: string; name: string }>('POST', `/api/docker/containers/${encodeURIComponent(id)}/exec`, {}),
   dockerLimits: (id: string, p: { memory?: string; cpus?: string; restart?: string }) => req<void>('POST', `/api/docker/containers/${encodeURIComponent(id)}/limits`, p),
   stacks: () => req<{ stacks: Stack[] }>('GET', '/api/docker/stacks'),