Container events are Ham Lab's own record, not Docker's short memory

docker events is a live stream with a small in-memory backlog: ask for thirty days and you
get a few minutes, and a daemon restart loses the lot. That is why Dockhand kept its own
copy. A watcher now follows the stream and appends lifecycle events to a ring beside the
config, flushed every thirty seconds, so the question of what happened to a container last
week has an answer. Health-check exec events are excluded — three per probe, thousands a
day, and the health panel says it better.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

internal/api/docker.go

@@ -394,3 +394,161 @@ func (s *Server) volumePrune(w http.ResponseWriter, r *http.Request) {
 	log.Printf("docker: volume prune (from %s)", clientIP(r))
 	jsonOK(w, 200, map[string]string{"result": lines[len(lines)-1]})
 }
+
+// ---- a container's health, its recent life, its console, and its limits ----
+
+// dockerHealth is the health check's verdict and the last few probe results. Docker keeps
+// the last five itself, which is all anyone wants: is it failing, and what did it say.
+func (s *Server) dockerHealth(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if !dockerRef.MatchString(id) {
+		jsonErr(w, 400, "bad container")
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+	defer cancel()
+	out, err := dockerRun(ctx, "inspect", "--format", "{{json .State}}", id)
+	if err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	var st struct {
+		Status string `json:"Status"`
+		Health *struct {
+			Status        string `json:"Status"`
+			FailingStreak int    `json:"FailingStreak"`
+			Log           []struct {
+				Start    time.Time `json:"Start"`
+				End      time.Time `json:"End"`
+				ExitCode int       `json:"ExitCode"`
+				Output   string    `json:"Output"`
+			} `json:"Log"`
+		} `json:"Health"`
+	}
+	if json.Unmarshal(out, &st) != nil {
+		jsonErr(w, 500, "could not read the container's state")
+		return
+	}
+	if st.Health == nil {
+		jsonOK(w, 200, map[string]any{"configured": false, "state": st.Status})
+		return
+	}
+	jsonOK(w, 200, map[string]any{
+		"configured": true, "state": st.Status, "status": st.Health.Status,
+		"failing_streak": st.Health.FailingStreak, "log": st.Health.Log,
+	})
+}
+
+// dockerExec opens a shell inside a container, as a terminal session like any other — so
+// it reattaches, resizes and scrolls back the same way, and works on a managed machine
+// through the same proxy.
+func (s *Server) dockerExec(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if !dockerRef.MatchString(id) {
+		jsonErr(w, 400, "bad container")
+		return
+	}
+	tool := dockerTool()
+	if tool == "" {
+		jsonErr(w, 500, "neither docker nor podman is installed here")
+		return
+	}
+	var body struct {
+		Shell string `json:"shell"`
+	}
+	_ = json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&body)
+	shell := strings.TrimSpace(body.Shell)
+	if shell == "" {
+		// most images have sh; bash is nicer where it exists, so try it first and fall back
+		shell = "if command -v bash >/dev/null 2>&1; then exec bash; else exec sh; fi"
+	} else if strings.ContainsAny(shell, "\n\r") {
+		jsonErr(w, 400, "bad shell")
+		return
+	}
+	argv := []string{tool, "exec", "-it", id, "sh", "-c", shell}
+	if priv.Available() && !dockerReachable() {
+		argv = priv.Wrap(argv...)
+	}
+	sess, err := s.mgr.CreateCmd("", "sh "+shortName(id), argv)
+	if err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	log.Printf("docker exec into %s as terminal %s by %s", id, sess.ID, clientIP(r))
+	jsonOK(w, 200, map[string]string{"id": sess.ID, "name": "sh " + shortName(id)})
+}
+
+func shortName(id string) string {
+	if len(id) > 20 {
+		return id[:20]
+	}
+	return id
+}
+
+// dockerReachable reports whether docker answers without help.
+func dockerReachable() bool {
+	tool := dockerTool()
+	if tool == "" {
+		return false
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	return exec.CommandContext(ctx, tool, "version", "--format", "{{.Server.Version}}").Run() == nil
+}
+
+// dockerUpdate changes a running container's limits and restart policy. `docker update`
+// does this live, without recreating it and without touching the compose file — which is
+// also the catch, and the caller is told: a deploy from compose puts back whatever compose
+// says.
+func (s *Server) dockerUpdate(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if !dockerRef.MatchString(id) {
+		jsonErr(w, 400, "bad container")
+		return
+	}
+	var body struct {
+		Memory  string `json:"memory"`  // "512m", "2g", or "" to leave alone; "0" removes the limit
+		CPUs    string `json:"cpus"`    // "1.5", or "0" for no limit
+		Restart string `json:"restart"` // no | on-failure | unless-stopped | always
+	}
+	if err := json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&body); err != nil {
+		jsonErr(w, 400, "bad request")
+		return
+	}
+	args := []string{"update"}
+	if m := strings.TrimSpace(body.Memory); m != "" {
+		if !regexp.MustCompile(`^[0-9]+([bkmgBKMG])?$`).MatchString(m) {
+			jsonErr(w, 400, "memory: a number, optionally with b, k, m or g — 0 removes the limit")
+			return
+		}
+		args = append(args, "--memory", m, "--memory-swap", m)
+	}
+	if c := strings.TrimSpace(body.CPUs); c != "" {
+		if !regexp.MustCompile(`^[0-9]+(\.[0-9]+)?$`).MatchString(c) {
+			jsonErr(w, 400, "cpus: a number like 1.5 — 0 removes the limit")
+			return
+		}
+		args = append(args, "--cpus", c)
+	}
+	if p := strings.TrimSpace(body.Restart); p != "" {
+		switch p {
+		case "no", "on-failure", "unless-stopped", "always":
+		default:
+			jsonErr(w, 400, "restart: no, on-failure, unless-stopped or always")
+			return
+		}
+		args = append(args, "--restart", p)
+	}
+	if len(args) == 1 {
+		jsonErr(w, 400, "nothing to change")
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
+	defer cancel()
+	if _, err := dockerRun(ctx, append(args, id)...); err != nil {
+		jsonErr(w, 500, err.Error())
+		return
+	}
+	log.Printf("docker update %s %v by %s", id, args[1:], clientIP(r))
+	w.WriteHeader(204)
+}

internal/api/docker_events.go

@@ -0,0 +1,229 @@
+package api
+
+// A container's history.
+//
+// `docker events` is a live stream with only a short in-memory backlog — ask it for the
+// last thirty days and you get the last few minutes, because the daemon keeps events in a
+// small ring and loses them all when it restarts. That is why Dockhand kept its own copy,
+// and why this does too: a watcher follows the stream and appends the lifecycle events to a
+// ring kept beside the config, so "what happened to this container last Tuesday" has an
+// answer.
+//
+// Only lifecycle events are kept. A health check produces three exec events every probe,
+// which is thousands a day and tells you nothing that the health panel does not.
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"hamlab/internal/priv"
+)
+
+const (
+	eventsKept     = 4000
+	eventsSaveEeem = 30 * time.Second
+)
+
+// ContainerEvent is one thing that happened to one container.
+type ContainerEvent struct {
+	T        int64  `json:"t"` // unix ms
+	Name     string `json:"name"`
+	Action   string `json:"action"`
+	Image    string `json:"image,omitempty"`
+	ExitCode string `json:"exit_code,omitempty"`
+	Signal   string `json:"signal,omitempty"`
+	Health   string `json:"health,omitempty"`
+}
+
+type eventLog struct {
+	mu    sync.Mutex
+	ring  []ContainerEvent
+	dirty bool
+	path  string
+}
+
+func (e *eventLog) load(path string) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.path = path
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return
+	}
+	var list []ContainerEvent
+	if json.Unmarshal(b, &list) == nil {
+		e.ring = list
+	}
+}
+
+func (e *eventLog) add(ev ContainerEvent) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.ring = append(e.ring, ev)
+	if len(e.ring) > eventsKept {
+		e.ring = append([]ContainerEvent(nil), e.ring[len(e.ring)-eventsKept:]...)
+	}
+	e.dirty = true
+}
+
+// forName is one container's events, newest first.
+func (e *eventLog) forName(name string, since time.Time, limit int) []ContainerEvent {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	out := []ContainerEvent{}
+	for _, ev := range e.ring {
+		if ev.Name == name && ev.T >= since.UnixMilli() {
+			out = append(out, ev)
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].T > out[j].T })
+	if len(out) > limit {
+		out = out[:limit]
+	}
+	return out
+}
+
+func (e *eventLog) save() {
+	e.mu.Lock()
+	if !e.dirty || e.path == "" {
+		e.mu.Unlock()
+		return
+	}
+	b, err := json.Marshal(e.ring)
+	e.dirty = false
+	path := e.path
+	e.mu.Unlock()
+	if err != nil {
+		return
+	}
+	_ = os.MkdirAll(filepath.Dir(path), 0o700)
+	tmp := path + ".tmp"
+	if os.WriteFile(tmp, b, 0o600) == nil {
+		_ = os.Rename(tmp, path)
+	}
+}
+
+// watchDockerEvents follows the stream, restarting if Docker goes away. It is the only
+// thing that writes the log, so nothing contends with it.
+func (s *Server) watchDockerEvents(ctx context.Context) {
+	s.events.load(filepath.Join(s.cfg.DataDir(), "docker-events.json"))
+	go func() { // a periodic flush, so a crash loses at most half a minute of history
+		t := time.NewTicker(eventsSaveEeem)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				s.events.save()
+				return
+			case <-t.C:
+				s.events.save()
+			}
+		}
+	}()
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+		if dockerTool() == "" {
+			time.Sleep(time.Minute) // no Docker here; look again later in case it is installed
+			continue
+		}
+		s.followEvents(ctx)
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(10 * time.Second): // the daemon restarted, or the pipe broke
+		}
+	}
+}
+
+func (s *Server) followEvents(ctx context.Context) {
+	args := []string{"events", "--format", "{{json .}}", "--filter", "type=container"}
+	for _, e := range []string{"create", "start", "restart", "stop", "kill", "die", "destroy", "oom", "pause", "unpause", "rename", "update", "health_status"} {
+		args = append(args, "--filter", "event="+e)
+	}
+	tool := dockerTool()
+	full := append([]string{tool}, args...)
+	if !dockerReachable() && priv.Available() {
+		full = priv.Wrap(full...)
+	}
+	cmd := exec.CommandContext(ctx, full[0], full[1:]...)
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	if err := cmd.Start(); err != nil {
+		return
+	}
+	sc := bufio.NewScanner(out)
+	sc.Buffer(make([]byte, 64<<10), 1<<20)
+	n := 0
+	for sc.Scan() {
+		var e struct {
+			Action   string `json:"Action"`
+			Time     int64  `json:"time"`
+			TimeNano int64  `json:"timeNano"`
+			Actor    struct {
+				Attributes map[string]string `json:"Attributes"`
+			} `json:"Actor"`
+		}
+		if json.Unmarshal(sc.Bytes(), &e) != nil || e.Action == "" {
+			continue
+		}
+		at := e.Actor.Attributes
+		ev := ContainerEvent{
+			T: e.Time * 1000, Name: at["name"], Image: at["image"],
+			Action: e.Action, ExitCode: at["exitCode"], Signal: at["signal"],
+		}
+		if e.TimeNano > 0 {
+			ev.T = e.TimeNano / 1e6
+		}
+		// health_status comes through as "health_status: healthy"
+		if a, h, ok := strings.Cut(e.Action, ": "); ok {
+			ev.Action, ev.Health = a, h
+		}
+		if ev.Name == "" {
+			continue
+		}
+		s.events.add(ev)
+		n++
+	}
+	_ = cmd.Wait()
+	if n > 0 {
+		log.Printf("docker events: stream ended after %d events; reconnecting", n)
+	}
+}
+
+// dockerEvents answers from Ham Lab's own history, not Docker's short memory.
+func (s *Server) dockerEvents(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if !dockerRef.MatchString(id) {
+		jsonErr(w, 400, "bad container")
+		return
+	}
+	since := r.URL.Query().Get("since")
+	d := 7 * 24 * time.Hour
+	switch since {
+	case "24h":
+		d = 24 * time.Hour
+	case "30d":
+		d = 30 * 24 * time.Hour
+	default:
+		since = "7d"
+	}
+	ev := s.events.forName(id, time.Now().Add(-d), 500)
+	s.events.mu.Lock()
+	total := len(s.events.ring)
+	s.events.mu.Unlock()
+	jsonOK(w, 200, map[string]any{"events": ev, "since": since, "kept": total, "capacity": eventsKept})
+}

internal/api/server.go

@@ -69,6 +69,7 @@ type Server struct {
 	storage     *storage.Manager
 	space       *space.Manager
 	update      updateState
+	events      eventLog
 	kvms        kvmProxies
 	fsLocal     files.FS
 	fsSudo      files.FS // nil without passwordless sudo
@@ -138,6 +139,7 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
 		go s.keepFleetInStep()
 	}
 	go s.watchForUpdates(context.Background())
+	go s.watchDockerEvents(context.Background())
 
 	m := http.NewServeMux()
 	m.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(204) })
@@ -206,6 +208,10 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
 	m.HandleFunc("POST /api/docker/containers/{id}/{action}", s.auth(s.dockerAction))
 	m.HandleFunc("GET /api/docker/containers/{id}/logs", s.auth(s.dockerLogs))
 	m.HandleFunc("GET /api/docker/containers/{id}/inspect", s.auth(s.dockerInspect))
+	m.HandleFunc("GET /api/docker/containers/{id}/health", s.auth(s.dockerHealth))
+	m.HandleFunc("GET /api/docker/containers/{id}/events", s.auth(s.dockerEvents))
+	m.HandleFunc("POST /api/docker/containers/{id}/exec", s.auth(s.dockerExec))
+	m.HandleFunc("POST /api/docker/containers/{id}/limits", s.auth(s.dockerUpdate))
 	m.HandleFunc("GET /api/alerts", s.auth(s.alertsGet))
 	m.HandleFunc("PUT /api/alerts", s.auth(s.alertsPut))
 	m.HandleFunc("POST /api/alerts/test", s.auth(s.alertsTest))

internal/term/manager.go

@@ -84,10 +84,21 @@ func newID() string {
 
 // Create starts a new login shell. id may be empty.
 func (m *Manager) Create(id, name string) (*Session, error) {
+	return m.CreateCmd(id, name, nil)
+}
+
+// CreateCmd is Create with something other than a login shell: argv nil is the shell, and
+// anything else runs that instead — `docker exec -it …` for a container's console. Every
+// session is the same thing afterwards, so attach, resize, the scrollback ring and the tab
+// strip work without knowing which it is.
+func (m *Manager) CreateCmd(id, name string, argv []string) (*Session, error) {
 	if id == "" {
 		id = newID()
 	}
 	cmd := exec.Command(m.shell, "-l")
+	if len(argv) > 0 {
+		cmd = exec.Command(argv[0], argv[1:]...)
+	}
 	cmd.Dir = m.home
 	cmd.Env = append(os.Environ(), "TERM=xterm-256color", "COLORTERM=truecolor", "HAMLAB=1")
 	ptmx, err := pty.StartWithSize(cmd, &pty.Winsize{Cols: defaultCol, Rows: defaultRow})

web/src/Docker.tsx

@@ -1,14 +1,16 @@
 // Docker: containers grouped by the stack that deployed them, with start / stop / restart,
 // logs that follow, and inspect — then images, volumes and networks. Live CPU and memory
 // come from the same container sensors everything else uses.
-import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
-import { api, DockerContainer, DockerInfo, ImageCheck, Stack, StackDiff } from './api'
+import { FormEvent, useCallback, useEffect, useMemo, useRef, useState } from 'react'
+import { api, DockerContainer, DockerEvent, DockerHealth, DockerInfo, ImageCheck, Stack, StackDiff } from './api'
 import TaskOutput from './TaskOutput'
 import StackNew, { DupFrom } from './StackNew'
-import { Sensor, fmtValue, useSensorStream } from './sensors'
+import { Sensor, fetchHistories, fmtValue, useSensorStream } from './sensors'
 import { useHashSel } from './hashsel'
-import { useRefreshMs } from './prefs'
-import { askText, askYes, Btn, CloseBtn, Loading, setUnsaved, toast } from './ui'
+import { fmtWindowSec, useHistorySec, useRefreshMs } from './prefs'
+import Sparkline from './Sparkline'
+import { fmtBytes } from './fmt'
+import { askText, askYes, Btn, CloseBtn, Loading, setUnsaved, toast, useEscape } from './ui'
 
 type Tab = 'containers' | 'stacks' | 'images' | 'volumes' | 'networks'
 const dot = (c: DockerContainer) => (c.state !== 'running' ? (c.state === 'exited' || c.state === 'dead' ? 'bad' : 'warn') : c.health === 'unhealthy' ? 'bad' : c.health === 'starting' ? 'warn' : 'ok')
@@ -50,18 +52,26 @@ function LogView({ id }: { id: string }) {
   )
 }
 
+// What a container is doing, and what can be done to it: its numbers over time, its health
+// checks, what has happened to it lately, a shell inside it, and its limits.
 function Detail({ c, live, onBack, onChanged }: { c: DockerContainer; live: Record<string, Sensor>; onBack: () => void; onChanged: () => void }) {
-  const [tab, setTab] = useState<'logs' | 'inspect'>('logs')
+  const [tab, setTab] = useState<'logs' | 'stats' | 'health' | 'events' | 'inspect'>('logs')
   const [info, setInfo] = useState<Record<string, unknown> | null>(null)
-  useEffect(() => { setInfo(null); if (tab === 'inspect') api.dockerInspect(c.name).then(setInfo).catch((e) => toast(e instanceof Error ? e.message : 'failed', 'err')) }, [tab, c.name])
+  const [limits, setLimits] = useState(false)
+  useEffect(() => { setInfo(null); if (tab === 'inspect' || limits) api.dockerInspect(c.name).then(setInfo).catch((e) => toast(e instanceof Error ? e.message : 'failed', 'err')) }, [tab, limits, c.name])
   const act = (a: 'start' | 'stop' | 'restart' | 'remove') => async () => { try { await api.dockerAction(c.name, a); toast(`${c.name}: ${a} ok`); onChanged() } catch (e) { toast(`${c.name}: ${e instanceof Error ? e.message : 'failed'}`, 'err') } }
   const running = c.state === 'running'
-  const s = (k: string) => live[`container.${c.name.toLowerCase().replace(/[^a-z0-9-]+/g, '_').replace(/^_|_$/g, '')}.${k}`]
+  const seg = c.name.toLowerCase().replace(/[^a-z0-9-]+/g, '_').replace(/^_|_$/g, '')
+  const s = (k: string) => live[`container.${seg}.${k}`]
   const mounts = (info?.mounts as { Source: string; Destination: string; RW: boolean }[] | undefined) ?? []
+  const shell = async () => {
+    try { const t = await api.dockerExec(c.name); toast(`shell open as “${t.name}” on the Terminal tab`); window.dispatchEvent(new CustomEvent('hamlab:terminal', { detail: t.id })) }
+    catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') }
+  }
   return (
     <div className="svc-detail">
       <CloseBtn onClick={onBack} />
-      <div className="svc-head"><button className="btn back" onClick={onBack}>‹ Back</button><span className={'sdot ' + dot(c)} /><span className="dname">{c.name}</span></div>
+      <div className="svc-head"><button className="btn back" onClick={onBack}>‹ Back</button><span className={'sdot ' + dot(c)} /><span className="dname">{c.name}</span>{c.health && c.health !== 'none' && <span className={'badge' + (c.health === 'healthy' ? '' : ' bad')}>{c.health}</span>}</div>
       <div className="props">
         <div><span className="k">Image</span><span className="mono small">{c.image}</span></div>
         <div><span className="k">Status</span><span>{c.status}</span></div>
@@ -72,11 +82,22 @@ function Detail({ c, live, onBack, onChanged }: { c: DockerContainer; live: Reco
       <div className="svc-actions">
         {!running && <Btn className="btn on" onClick={act('start')}>Start</Btn>}
         {!running && <Btn className="btn danger" confirm={`Remove the container ${c.name}? Its image, volumes and bind-mounted data stay; anything written inside the container itself is lost.${c.stack ? ` Deploying the ${c.stack} stack makes it again.` : ''}`} onClick={async () => { await act('remove')(); onBack() }}>Remove</Btn>}
+        {running && <Btn className="btn" onClick={shell}>Shell</Btn>}
         {running && <Btn className="btn" confirm={`Restart ${c.name}?`} onClick={act('restart')}>Restart</Btn>}
+        <Btn className="btn" onClick={() => setLimits(true)}>Limits</Btn>
         {running && <Btn className="btn danger" confirm={`Stop ${c.name}? It stays stopped until something starts it.`} onClick={act('stop')}>Stop</Btn>}
       </div>
-      <div className="tabs"><button className={tab === 'logs' ? 'on' : ''} onClick={() => setTab('logs')}>Logs</button><button className={tab === 'inspect' ? 'on' : ''} onClick={() => setTab('inspect')}>Inspect</button></div>
+      <div className="tabs">
+        <button className={tab === 'logs' ? 'on' : ''} onClick={() => setTab('logs')}>Logs</button>
+        <button className={tab === 'stats' ? 'on' : ''} onClick={() => setTab('stats')}>Stats</button>
+        <button className={tab === 'health' ? 'on' : ''} onClick={() => setTab('health')}>Health</button>
+        <button className={tab === 'events' ? 'on' : ''} onClick={() => setTab('events')}>Events</button>
+        <button className={tab === 'inspect' ? 'on' : ''} onClick={() => setTab('inspect')}>Inspect</button>
+      </div>
       {tab === 'logs' && <LogView id={c.name} />}
+      {tab === 'stats' && <Stats seg={seg} live={live} />}
+      {tab === 'health' && <Health id={c.name} />}
+      {tab === 'events' && <Events id={c.name} />}
       {tab === 'inspect' && (!info ? <Loading label="Inspecting…" /> : (
         <div className="props">
           <div><span className="k">Restarts</span><span>{String(info.restart || 'no')} policy · restarted {String(info.restarts ?? 0)}×</span></div>
@@ -88,11 +109,154 @@ function Detail({ c, live, onBack, onChanged }: { c: DockerContainer; live: Reco
           <div><span className="k">Env</span><span className="mono small muted">{((info.env_names as string[]) ?? []).join('  ')} <span className="muted">(names only)</span></span></div>
         </div>
       ))}
+      {limits && <LimitsDialog c={c} info={info} onClose={() => setLimits(false)} onSaved={() => { setLimits(false); onChanged() }} />}
+    </div>
+  )
+}
+
+// CPU and memory over time, from the container sensors the sampler already keeps.
+function Stats({ seg, live }: { seg: string; live: Record<string, Sensor> }) {
+  const [hist, setHist] = useState<{ id: string; t: number[]; v: number[] }[] | null>(null)
+  const historySec = useHistorySec()
+  useEffect(() => {
+    let alive = true
+    fetchHistories(`container.${seg}.cpu,container.${seg}.mem`)
+      .then((h) => { if (alive) setHist(h) })
+      .catch(() => { if (alive) setHist([]) })
+    return () => { alive = false }
+  }, [seg])
+  const css = (n: string) => getComputedStyle(document.documentElement).getPropertyValue(n).trim() || '#3987e5'
+  const of = (k: string) => hist?.find((h) => h.id === `container.${seg}.${k}`)?.v ?? []
+  const cpu = of('cpu'), mem = of('mem')
+  if (hist === null) return <Loading label="Reading the history…" />
+  if (cpu.length < 2 && mem.length < 2) return <div className="notice">No history yet for this container — it is sampled while it runs, so give it a moment.</div>
+  return (
+    <div className="cards">
+      <section className="card">
+        <div className="sub-head"><span className="dname">CPU</span><span className="spacer" /><span className="num">{fmtValue(live[`container.${seg}.cpu`]?.value, '%')} of a core</span></div>
+        <Sparkline series={[{ name: 'cpu', values: cpu, color: css('--s1') }]} format={(v) => `${v.toFixed(1)}%`} height={64} />
+        <div className="muted small">the last {fmtWindowSec(historySec)}</div>
+      </section>
+      <section className="card">
+        <div className="sub-head"><span className="dname">Memory</span><span className="spacer" /><span className="num">{fmtValue(live[`container.${seg}.mem`]?.value, 'B')}</span></div>
+        <Sparkline series={[{ name: 'mem', values: mem, color: css('--s2') }]} format={(v) => fmtBytes(v)} height={64} />
+        <div className="muted small">the last {fmtWindowSec(historySec)}</div>
+      </section>
+    </div>
+  )
+}
+
+// The health check's verdict, and what its last few probes actually said.
+function Health({ id }: { id: string }) {
+  const [h, setH] = useState<DockerHealth | null>(null)
+  const [err, setErr] = useState('')
+  useEffect(() => { setH(null); api.dockerHealth(id).then(setH).catch((e) => setErr(e instanceof Error ? e.message : 'failed')) }, [id])
+  if (err) return <div className="error">{err}</div>
+  if (!h) return <Loading label="Asking…" />
+  if (!h.configured) return <div className="notice">This image defines no health check, so Docker only knows whether the process is running — which it is ({h.state}). A health check is a <span className="mono">healthcheck:</span> block in the compose file.</div>
+  return (
+    <>
+      <div className="props">
+        <div><span className="k">Health</span><span className={h.status === 'healthy' ? '' : 'warn-text'}>{h.status}</span></div>
+        <div><span className="k">State</span><span>{h.state}</span></div>
+        {!!h.failing_streak && <div><span className="k">Failing</span><span className="warn-text">{h.failing_streak} probe{h.failing_streak === 1 ? '' : 's'} in a row</span></div>}
+      </div>
+      <div className="muted small" style={{ marginTop: 8 }}>The last {h.log?.length ?? 0} probes — Docker keeps five.</div>
+      {(h.log ?? []).slice().reverse().map((p, i) => (
+        <section className="card" key={i} style={{ marginTop: 6 }}>
+          <div className="sub-head">
+            <span className={'sdot ' + (p.ExitCode === 0 ? 'ok' : 'bad')} />
+            <span>{new Date(p.Start).toLocaleString()}</span>
+            <span className="spacer" />
+            <span className="muted small">exit {p.ExitCode} · {Math.max(0, new Date(p.End).getTime() - new Date(p.Start).getTime())} ms</span>
+          </div>
+          {p.Output.trim() && <pre className="run-out" style={{ maxHeight: 140 }}>{p.Output.trim()}</pre>}
+        </section>
+      ))}
+    </>
+  )
+}
+
+// What has happened to this container lately, asked of Docker rather than kept here.
+function Events({ id }: { id: string }) {
+  const [ev, setEv] = useState<DockerEvent[] | null>(null)
+  const [kept, setKept] = useState(0)
+  const [since, setSince] = useState('7d')
+  const [err, setErr] = useState('')
+  useEffect(() => { setEv(null); setErr(''); api.dockerEvents(id, since).then((r) => { setEv(r.events); setKept(r.kept) }).catch((e) => setErr(e instanceof Error ? e.message : 'failed')) }, [id, since])
+  const bad = (a: string, h?: string) => /die|kill|oom|destroy/.test(a) || (h !== undefined && h !== 'healthy')
+  return (
+    <>
+      <div className="chips" style={{ marginBottom: 8 }}>
+        {['24h', '7d', '30d'].map((x) => <button key={x} className={'chip' + (since === x ? ' on' : '')} onClick={() => setSince(x)}>{x}</button>)}
+      </div>
+      {err && <div className="error">{err}</div>}
+      {!ev && !err && <Loading label="Asking Docker…" />}
+      {ev && ev.length === 0 && <div className="notice">Nothing for this container in the last {since}. Ham Lab keeps its own record — Docker's own only reaches back a few minutes — so history starts from when this version was installed. {kept} events kept across every container.</div>}
+      {ev && ev.length > 0 && (
+        <div className="snap-list">
+          {ev.map((e, i) => (
+            <div className="snap" key={i}>
+              <span className={bad(e.action, e.health) ? 'warn-text' : ''}>{e.action}{e.health ? `: ${e.health}` : ''}</span>
+              <span className="muted">{new Date(e.t).toLocaleString()}</span>
+              <span className="muted small">{e.exit_code ? `exit ${e.exit_code}` : ''}{e.signal ? ` signal ${e.signal}` : ''}</span>
+              <span />
+              <span />
+            </div>
+          ))}
+        </div>
+      )}
+    </>
+  )
+}
+
+// Memory, CPU and restart policy, changed on the running container. Docker does this live,
+// which is the point — and also the catch, said plainly below.
+function LimitsDialog({ c, info, onClose, onSaved }: { c: DockerContainer; info: Record<string, unknown> | null; onClose: () => void; onSaved: () => void }) {
+  const curMem = Number(info?.memory_limit ?? 0)
+  const [mem, setMem] = useState(curMem > 0 ? String(Math.round(curMem / (1 << 20))) + 'm' : '')
+  const [cpus, setCpus] = useState('')
+  const [restart, setRestart] = useState(String(info?.restart ?? ''))
+  const [err, setErr] = useState('')
+  const [busy, setBusy] = useState(false)
+  useEscape(onClose)
+  const save = async (e: FormEvent) => {
+    e.preventDefault(); setErr(''); setBusy(true)
+    try { await api.dockerLimits(c.name, { memory: mem, cpus, restart }); toast(`${c.name} updated`); onSaved() }
+    catch (e) { setErr(e instanceof Error ? e.message : 'failed'); setBusy(false) }
+  }
+  return (
+    <div className="modal-back" onClick={onClose}>
+      <form className="modal" onClick={(e) => e.stopPropagation()} onSubmit={save}>
+        <h3>Limits for {c.name}</h3>
+        <label className="muted">Memory — 512m, 2g, or 0 for no limit{curMem > 0 ? ` (now ${fmtBytes(curMem)})` : ' (no limit now)'}
+          <input className="mono" value={mem} onChange={(e) => setMem(e.target.value)} placeholder="leave blank to keep" spellCheck={false} />
+        </label>
+        <label className="muted">CPUs — 1.5 means one and a half cores, 0 for no limit
+          <input className="mono" value={cpus} onChange={(e) => setCpus(e.target.value)} placeholder="leave blank to keep" spellCheck={false} />
+        </label>
+        <label className="muted">Restart policy
+          <select value={restart} onChange={(e) => setRestart(e.target.value)}>
+            <option value="">leave as it is</option>
+            {['no', 'on-failure', 'unless-stopped', 'always'].map((p) => <option key={p} value={p}>{p}</option>)}
+          </select>
+        </label>
+        <div className="notice attn">
+          This changes the running container now, without recreating it. It does <b>not</b> edit the
+          compose file{c.stack ? <> — so deploying <b>{c.stack}</b> puts back whatever the file says</> : ''}.
+          Edit the stack to make it stick.
+        </div>
+        {err && <div className="error">{err}</div>}
+        <div className="svc-actions">
+          <button type="submit" className="btn on" disabled={busy || (!mem && !cpus && !restart)}>Apply</button>
+          <span className="spacer" />
+          <button type="button" className="btn" onClick={onClose}>Cancel</button>
+        </div>
+      </form>
     </div>
   )
 }
 
-// A plain line diff (longest common subsequence): compose files are short, so the simple way is fine.
 function lineDiff(a: string, b: string): { s: ' ' | '+' | '-'; t: string }[] {
   const x = a.split('\n'), y = b.split('\n'); const n = x.length, m = y.length
   const L = Array.from({ length: n + 1 }, () => new Uint16Array(m + 1))
@@ -120,6 +284,7 @@ const maskEnv = (text: string) => text.split('\n').map((l) => { const i = l.inde
 
 // One stack: its compose file to edit, and the four things to do with it. Saving validates
 // with compose itself and keeps the previous file beside it; deploying is a separate, asked-for act.
+
 function StackDetail({ name, startTask, onBack, onChanged, onDuplicate }: { name: string; startTask?: string; onBack: () => void; onChanged: () => void; onDuplicate: (from: DupFrom) => void }) {
   const [st, setSt] = useState<Stack | null>(null)
   const [text, setText] = useState(''); const [orig, setOrig] = useState('')

web/src/Shell.tsx

@@ -55,18 +55,18 @@ const GROUPS: Group[] = [
   { id: 'procs', label: 'Processes', icon: '▚' },
   { id: 'terminal', label: 'Terminal', icon: '>_' },
   { id: 'services', label: 'Services', icon: '◈' },
-  { id: 'docker', label: 'Docker', icon: DockerIcon },
   { id: 'jobs', label: 'Jobs', icon: ClockIcon },
-  { id: 'updates', label: 'Updates', icon: '⇪' },
   { id: 'disks', label: 'Disks', icon: '▤' },
-  { id: 'zfs', label: 'ZFS', icon: '⛁' },
-  { id: 'usage', label: 'Usage', icon: '▧' },
   { id: 'files', label: 'Files', icon: FolderIcon },
-  { id: 'alerts', label: 'Alerts', icon: BellIcon },
+  { id: 'usage', label: 'Usage', icon: '▧' },
+  { id: 'zfs', label: 'ZFS', icon: '⛁' },
+  { id: 'docker', label: 'Docker', icon: DockerIcon },
   { id: 'kvm', label: 'KVM', icon: '⌨' },
+  { id: 'updates', label: 'Updates', icon: '⇪' },
+  { id: 'alerts', label: 'Alerts', icon: BellIcon },
   { id: 'machines', label: 'Machines', icon: '⌂' },
-  { id: 'users', label: 'Users', icon: '⚇' },
   { id: 'widgets', label: 'Widgets', icon: '▦' },
+  { id: 'users', label: 'Users', icon: '⚇' },
   { id: 'settings', label: 'Settings', icon: '⚙' },
 ]
 
@@ -125,6 +125,20 @@ export default function Shell({ me: hubMe, onLoggedOut }: { me: Me; onLoggedOut:
 
   useEffect(() => { reload() }, [reload])
 
+  // A shell opened inside a container from the Docker tab is a terminal like any other:
+  // pick it up, show the Terminal tab, and make it the active one.
+  useEffect(() => {
+    const h = async (e: Event) => {
+      const id = (e as CustomEvent<string>).detail
+      if (!id) return
+      await reload()
+      setActive(id)
+      setView('terminal')
+    }
+    window.addEventListener('hamlab:terminal', h)
+    return () => window.removeEventListener('hamlab:terminal', h)
+  }, [reload])
+
   // Machine list for the switcher (hub-local), refreshed while the console is open.
   const loadMachines = useCallback(async () => { try { setMachines((await api.machines()).nodes) } catch { /* not a hub or offline */ } }, [])
   useEffect(() => { loadMachines(); const t = setInterval(loadMachines, listRate); return () => clearInterval(t) }, [loadMachines, listRate])

web/src/api.ts

@@ -109,6 +109,10 @@ export interface StorageDevice { name: string; kind: string; model?: string; ven
 export interface SmartAttr { id: number; name: string; value: number; worst: number; thresh: number; raw: string; failed?: string; warn: boolean }
 export interface SmartInfo { device: string; available: boolean; supported: boolean; standby: boolean; passed: boolean | null; temp: number; power_on_hours: number; power_cycles: number; model?: string; serial?: string; firmware?: string; attrs: SmartAttr[]; nvme?: { critical_warning: number; percentage_used: number; available_spare: number; spare_threshold: number; media_errors: number; error_log_entries: number; data_read_bytes: number; data_written_bytes: number; unsafe_shutdowns: number }; mmc?: { life_a: number; life_b: number; pre_eol: number }; self_test?: string; self_test_remaining?: number; self_test_supported: boolean; self_test_note?: string; warnings: string[]; error?: string; fetched_at: number }
 export interface DockerContainer { id: string; name: string; image: string; state: string; status: string; health: string; ports: string; stack: string; service: string; created: string; mem: number }
+export interface DockerHealthProbe { Start: string; End: string; ExitCode: number; Output: string }
+export interface DockerHealth { configured: boolean; state: string; status?: string; failing_streak?: number; log?: DockerHealthProbe[] }
+export interface DockerEvent { t: number; name: string; action: string; image?: string; exit_code?: string; signal?: string; health?: string }
+
 export interface DockerInfo { available: boolean; message?: string; tool?: string; containers: DockerContainer[]; images: Record<string, unknown>[]; volumes: Record<string, unknown>[]; networks: Record<string, unknown>[] }
 export interface Stack { name: string; containers: string[]; running: number; file: string; host_file: string; files: string[]; host_files: string[] | null; images: string[] | null; via: string; remote: string; held_by: string; editable: boolean; note?: string; has_env: boolean }
 export interface StackHome { dir: string; inner: string; via: string; remote: string; held_by: string; stacks: number; default: boolean }
@@ -248,6 +252,10 @@ export const api = {
   dockerAction: (id: string, action: 'start' | 'stop' | 'restart' | 'remove') => req<void>('POST', `/api/docker/containers/${encodeURIComponent(id)}/${action}`),
   dockerLogs: (id: string, since = '') => req<{ lines: { t: string; m: string }[] }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/logs${since ? `?since=${encodeURIComponent(since)}` : '?n=300'}`),
   dockerInspect: (id: string) => req<Record<string, unknown>>('GET', `/api/docker/containers/${encodeURIComponent(id)}/inspect`),
+  dockerHealth: (id: string) => req<DockerHealth>('GET', `/api/docker/containers/${encodeURIComponent(id)}/health`),
+  dockerEvents: (id: string, since = '7d') => req<{ events: DockerEvent[]; since: string; kept: number; capacity: number }>('GET', `/api/docker/containers/${encodeURIComponent(id)}/events?since=${since}`),
+  dockerExec: (id: string) => req<{ id: string; name: string }>('POST', `/api/docker/containers/${encodeURIComponent(id)}/exec`, {}),
+  dockerLimits: (id: string, p: { memory?: string; cpus?: string; restart?: string }) => req<void>('POST', `/api/docker/containers/${encodeURIComponent(id)}/limits`, p),
   stacks: () => req<{ stacks: Stack[] }>('GET', '/api/docker/stacks'),
   stackHomes: () => req<{ homes: StackHome[] }>('GET', '/api/docker/stacks/homes'),
   createStack: (p: { name: string; text: string; env: string; home: string; env_from?: string }) => req<{ stack: Stack }>('POST', '/api/docker/stacks', p),