From 08fb9d6e38048d8e8f1d2bc36f1f4e06c1ddd3db Mon Sep 17 00:00:00 2001 From: Mitchell-Johnson Date: Sat, 19 Sep 2026 15:10:17 -0700 Subject: [PATCH] =?UTF-8?q?container..down:=20should=20be=20up=20and?= =?UTF-8?q?=20isn't=20=E2=80=94=20not=20merely=20stopped;=20the=20default?= =?UTF-8?q?=20alert=20watches=20that?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- SENSORS.md | 3 +++ internal/api/alerts.go | 7 ++++- internal/api/status_sensors.go | 47 ++++++++++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/SENSORS.md b/SENSORS.md index b404c8c..a9c46f2 100644 --- a/SENSORS.md +++ b/SENSORS.md @@ -136,6 +136,9 @@ services.failed|failed_names|running every 30 s service..active|state one pair per service that is enabled, up or failed containers.total|running|unhealthy every 15 s (Docker or Podman, if present) container..running|state|health|mem|cpu mem and cpu read from the container's cgroup +container..down should be up and isn't: Docker couldn't start it, it is + stuck restarting, or it died non-zero with no policy to + restart it — *not* a container someone stopped updates.pending|security|reboot_required every 5 min zfs..health|healthy|size|used|used_pct|frag every 1 min smart..passed|warnings|power_on|wear every 10 min, never waking a sleeping disk diff --git a/internal/api/alerts.go b/internal/api/alerts.go index 6011f12..8894bf3 100644 --- a/internal/api/alerts.go +++ b/internal/api/alerts.go @@ -90,7 +90,7 @@ type alertEngine struct { var defaultRules = []AlertRule{ {ID: "machine-offline", Name: "Machine not answering", Sensor: "hamlab.reachable", Op: "==", Value: false, ForSec: 120, Severity: "crit", Enabled: true}, - {ID: "container-down", Name: "Container not running", Sensor: "container.*.running", Op: "==", Value: false, ForSec: 120, Severity: "crit", Enabled: true}, + {ID: "container-down", Name: "Container down", Sensor: "container.*.down", Op: "==", Value: true, ForSec: 120, Severity: "crit", Enabled: true}, {ID: "container-unhealthy", Name: "Container unhealthy", Sensor: "container.*.health", Op: "==", Value: "unhealthy", ForSec: 180, Severity: "warn", Enabled: true}, {ID: "service-failed", Name: "A service has failed", Sensor: "services.failed", Op: ">", Value: 0.0, ForSec: 300, Severity: "warn", Enabled: true}, {ID: "disk-full", Name: "Filesystem nearly full", Sensor: "fs.*.used_pct", Op: ">=", Value: 90.0, ForSec: 300, Severity: "warn", Enabled: true}, @@ -109,6 +109,11 @@ func newAlertEngine(s *Server) *alertEngine { if !e.saved.Seeded { e.saved.Rules, e.saved.Seeded = append([]AlertRule{}, defaultRules...), true } + for i, r := range e.saved.Rules { // the first day's default watched "not running", which a container stopped on purpose also is + if r.ID == "container-down" && r.Sensor == "container.*.running" { + e.saved.Rules[i].Name, e.saved.Rules[i].Sensor, e.saved.Rules[i].Value = "Container down", "container.*.down", true + } + } go e.run() return e } diff --git a/internal/api/status_sensors.go b/internal/api/status_sensors.go index f1f5ba3..ae8dffc 100644 --- a/internal/api/status_sensors.go +++ b/internal/api/status_sensors.go @@ -155,10 +155,12 @@ func (p *statusSensors) services(ctx context.Context, t int64) []sensors.Sensor func (p *statusSensors) containers(ctx context.Context, t int64) []sensors.Sensor { var out []byte var err error - for _, tool := range []string{"docker", "podman"} { - if _, e := exec.LookPath(tool); e != nil { + tool := "" + for _, t := range []string{"docker", "podman"} { + if _, e := exec.LookPath(t); e != nil { continue } + tool = t args := []string{tool, "ps", "-a", "--no-trunc", "--format", "{{json .}}"} if out, err = exec.CommandContext(ctx, args[0], args[1:]...).Output(); err != nil && priv.Available() { full := priv.Wrap(args...) // not in the docker group: ask as root @@ -169,6 +171,46 @@ func (p *statusSensors) containers(ctx context.Context, t int64) []sensors.Senso if err != nil || len(out) == 0 { return nil } + // "Not running" is not "down": servarr stopped by hand in June is off, not broken. What is + // broken is a container that should be up and isn't — one inspect of the stopped ones tells + // them apart: Docker refused to start it (State.Error: yesterday's Gitea), it is stuck + // restarting, or it died with a non-zero code that no restart policy picked up. Under + // `always` / `unless-stopped` a crash is restarted by Docker, so "exited" there means a + // person stopped it. + type stopped struct { + Name string + State struct { + Status string + ExitCode int + Error string + } + HostConfig struct{ RestartPolicy struct{ Name string } } + } + down := map[string]bool{} + var ids []string + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + var c struct{ ID, State string } + if json.Unmarshal([]byte(line), &c) == nil && c.ID != "" && c.State != "running" { + ids = append(ids, c.ID) + } + } + if len(ids) > 0 { + args := append([]string{"inspect"}, ids...) + insp, ierr := exec.CommandContext(ctx, tool, args...).Output() + if ierr != nil && priv.Available() { + full := priv.Wrap(append([]string{tool}, args...)...) + insp, _ = exec.CommandContext(ctx, full[0], full[1:]...).Output() + } + var list []stopped + if json.Unmarshal(insp, &list) == nil { + for _, c := range list { + pol := c.HostConfig.RestartPolicy.Name + managed := pol == "always" || pol == "unless-stopped" + down[strings.TrimPrefix(c.Name, "/")] = c.State.Status == "restarting" || c.State.Status == "dead" || c.State.Error != "" || + (!managed && c.State.Status == "exited" && c.State.ExitCode != 0) + } + } + } var res []sensors.Sensor var running, unhealthy, total float64 seenCPU := map[string]cpuMark{} @@ -196,6 +238,7 @@ func (p *statusSensors) containers(ctx context.Context, t int64) []sensors.Senso } res = append(res, sensors.Sensor{ID: id + ".running", Name: name, Kind: "state", Value: up, T: t, Labels: l}, + sensors.Sensor{ID: id + ".down", Name: name + " down", Kind: "state", Value: down[name], T: t, Labels: l}, sensors.Sensor{ID: id + ".state", Name: name + " state", Kind: "state", Value: c.State, T: t, Labels: l}, sensors.Sensor{ID: id + ".health", Name: name + " health", Kind: "state", Value: health, T: t, Labels: l}) if !up {