mitchell/hamlab
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 14 Wiki Activity

container.<name>.down: should be up and isn't — not merely stopped; the default alert watches that

Browse Source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
2026-09-19 15:10:17 -07:00
co-authored by Claude Fable 5.1
parent cb1617c2b8
commit 08fb9d6e38
3 changed files with 54 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/status_sensors.go
+45 -2
View File
@@ -155,10 +155,12 @@ func (p *statusSensors) services(ctx context.Context, t int64) []sensors.Sensor
func (p *statusSensors) containers(ctx context.Context, t int64) []sensors.Sensor {
var out []byte
var err error
for _, tool := range []string{"docker", "podman"} {
if _, e := exec.LookPath(tool); e != nil {
tool := ""
for _, t := range []string{"docker", "podman"} {
if _, e := exec.LookPath(t); e != nil {
continue
}
tool = t
args := []string{tool, "ps", "-a", "--no-trunc", "--format", "{{json .}}"}
if out, err = exec.CommandContext(ctx, args[0], args[1:]...).Output(); err != nil && priv.Available() {
full := priv.Wrap(args...) // not in the docker group: ask as root
@@ -169,6 +171,46 @@ func (p *statusSensors) containers(ctx context.Context, t int64) []sensors.Senso
if err != nil || len(out) == 0 {
return nil
}
// "Not running" is not "down": servarr stopped by hand in June is off, not broken. What is
// broken is a container that should be up and isn't — one inspect of the stopped ones tells
// them apart: Docker refused to start it (State.Error: yesterday's Gitea), it is stuck
// restarting, or it died with a non-zero code that no restart policy picked up. Under
// `always` / `unless-stopped` a crash is restarted by Docker, so "exited" there means a
// person stopped it.
type stopped struct {
Name string
State struct {
Status string
ExitCode int
Error string
}
HostConfig struct{ RestartPolicy struct{ Name string } }
}
down := map[string]bool{}
var ids []string
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
var c struct{ ID, State string }
if json.Unmarshal([]byte(line), &c) == nil && c.ID != "" && c.State != "running" {
ids = append(ids, c.ID)
}
}
if len(ids) > 0 {
args := append([]string{"inspect"}, ids...)
insp, ierr := exec.CommandContext(ctx, tool, args...).Output()
if ierr != nil && priv.Available() {
full := priv.Wrap(append([]string{tool}, args...)...)
insp, _ = exec.CommandContext(ctx, full[0], full[1:]...).Output()
}
var list []stopped
if json.Unmarshal(insp, &list) == nil {
for _, c := range list {
pol := c.HostConfig.RestartPolicy.Name
managed := pol == "always" || pol == "unless-stopped"
down[strings.TrimPrefix(c.Name, "/")] = c.State.Status == "restarting" || c.State.Status == "dead" || c.State.Error != "" ||
(!managed && c.State.Status == "exited" && c.State.ExitCode != 0)
}
}
}
var res []sensors.Sensor
var running, unhealthy, total float64
seenCPU := map[string]cpuMark{}
@@ -196,6 +238,7 @@ func (p *statusSensors) containers(ctx context.Context, t int64) []sensors.Senso
}
res = append(res,
sensors.Sensor{ID: id + ".running", Name: name, Kind: "state", Value: up, T: t, Labels: l},
sensors.Sensor{ID: id + ".down", Name: name + " down", Kind: "state", Value: down[name], T: t, Labels: l},
sensors.Sensor{ID: id + ".state", Name: name + " state", Kind: "state", Value: c.State, T: t, Labels: l},
sensors.Sensor{ID: id + ".health", Name: name + " health", Kind: "state", Value: health, T: t, Labels: l})
if !up {