container.<name>.down: should be up and isn't — not merely stopped; the default alert watches that

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

SENSORS.md

@@ -136,6 +136,9 @@ services.failed|failed_names|running            every 30 s
 service.<unit>.active|state                     one pair per service that is enabled, up or failed
 containers.total|running|unhealthy              every 15 s (Docker or Podman, if present)
 container.<name>.running|state|health|mem|cpu   mem and cpu read from the container's cgroup
+container.<name>.down                           should be up and isn't: Docker couldn't start it, it is
+                                                stuck restarting, or it died non-zero with no policy to
+                                                restart it — *not* a container someone stopped
 updates.pending|security|reboot_required        every 5 min
 zfs.<pool>.health|healthy|size|used|used_pct|frag   every 1 min
 smart.<dev>.passed|warnings|power_on|wear       every 10 min, never waking a sleeping disk

internal/api/alerts.go

@@ -90,7 +90,7 @@ type alertEngine struct {
 
 var defaultRules = []AlertRule{
 	{ID: "machine-offline", Name: "Machine not answering", Sensor: "hamlab.reachable", Op: "==", Value: false, ForSec: 120, Severity: "crit", Enabled: true},
-	{ID: "container-down", Name: "Container not running", Sensor: "container.*.running", Op: "==", Value: false, ForSec: 120, Severity: "crit", Enabled: true},
+	{ID: "container-down", Name: "Container down", Sensor: "container.*.down", Op: "==", Value: true, ForSec: 120, Severity: "crit", Enabled: true},
 	{ID: "container-unhealthy", Name: "Container unhealthy", Sensor: "container.*.health", Op: "==", Value: "unhealthy", ForSec: 180, Severity: "warn", Enabled: true},
 	{ID: "service-failed", Name: "A service has failed", Sensor: "services.failed", Op: ">", Value: 0.0, ForSec: 300, Severity: "warn", Enabled: true},
 	{ID: "disk-full", Name: "Filesystem nearly full", Sensor: "fs.*.used_pct", Op: ">=", Value: 90.0, ForSec: 300, Severity: "warn", Enabled: true},
@@ -109,6 +109,11 @@ func newAlertEngine(s *Server) *alertEngine {
 	if !e.saved.Seeded {
 		e.saved.Rules, e.saved.Seeded = append([]AlertRule{}, defaultRules...), true
 	}
+	for i, r := range e.saved.Rules { // the first day's default watched "not running", which a container stopped on purpose also is
+		if r.ID == "container-down" && r.Sensor == "container.*.running" {
+			e.saved.Rules[i].Name, e.saved.Rules[i].Sensor, e.saved.Rules[i].Value = "Container down", "container.*.down", true
+		}
+	}
 	go e.run()
 	return e
 }

internal/api/status_sensors.go

@@ -155,10 +155,12 @@ func (p *statusSensors) services(ctx context.Context, t int64) []sensors.Sensor
 func (p *statusSensors) containers(ctx context.Context, t int64) []sensors.Sensor {
 	var out []byte
 	var err error
-	for _, tool := range []string{"docker", "podman"} {
+	tool := ""
-		if _, e := exec.LookPath(tool); e != nil {
+	for _, t := range []string{"docker", "podman"} {
+		if _, e := exec.LookPath(t); e != nil {
 			continue
 		}
+		tool = t
 		args := []string{tool, "ps", "-a", "--no-trunc", "--format", "{{json .}}"}
 		if out, err = exec.CommandContext(ctx, args[0], args[1:]...).Output(); err != nil && priv.Available() {
 			full := priv.Wrap(args...) // not in the docker group: ask as root
@@ -169,6 +171,46 @@ func (p *statusSensors) containers(ctx context.Context, t int64) []sensors.Senso
 	if err != nil || len(out) == 0 {
 		return nil
 	}
+	// "Not running" is not "down": servarr stopped by hand in June is off, not broken. What is
+	// broken is a container that should be up and isn't — one inspect of the stopped ones tells
+	// them apart: Docker refused to start it (State.Error: yesterday's Gitea), it is stuck
+	// restarting, or it died with a non-zero code that no restart policy picked up. Under
+	// `always` / `unless-stopped` a crash is restarted by Docker, so "exited" there means a
+	// person stopped it.
+	type stopped struct {
+		Name  string
+		State struct {
+			Status   string
+			ExitCode int
+			Error    string
+		}
+		HostConfig struct{ RestartPolicy struct{ Name string } }
+	}
+	down := map[string]bool{}
+	var ids []string
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		var c struct{ ID, State string }
+		if json.Unmarshal([]byte(line), &c) == nil && c.ID != "" && c.State != "running" {
+			ids = append(ids, c.ID)
+		}
+	}
+	if len(ids) > 0 {
+		args := append([]string{"inspect"}, ids...)
+		insp, ierr := exec.CommandContext(ctx, tool, args...).Output()
+		if ierr != nil && priv.Available() {
+			full := priv.Wrap(append([]string{tool}, args...)...)
+			insp, _ = exec.CommandContext(ctx, full[0], full[1:]...).Output()
+		}
+		var list []stopped
+		if json.Unmarshal(insp, &list) == nil {
+			for _, c := range list {
+				pol := c.HostConfig.RestartPolicy.Name
+				managed := pol == "always" || pol == "unless-stopped"
+				down[strings.TrimPrefix(c.Name, "/")] = c.State.Status == "restarting" || c.State.Status == "dead" || c.State.Error != "" ||
+					(!managed && c.State.Status == "exited" && c.State.ExitCode != 0)
+			}
+		}
+	}
 	var res []sensors.Sensor
 	var running, unhealthy, total float64
 	seenCPU := map[string]cpuMark{}
@@ -196,6 +238,7 @@ func (p *statusSensors) containers(ctx context.Context, t int64) []sensors.Senso
 		}
 		res = append(res,
 			sensors.Sensor{ID: id + ".running", Name: name, Kind: "state", Value: up, T: t, Labels: l},
+			sensors.Sensor{ID: id + ".down", Name: name + " down", Kind: "state", Value: down[name], T: t, Labels: l},
 			sensors.Sensor{ID: id + ".state", Name: name + " state", Kind: "state", Value: c.State, T: t, Labels: l},
 			sensors.Sensor{ID: id + ".health", Name: name + " health", Kind: "state", Value: health, T: t, Labels: l})
 		if !up {