Alerts tab, Docker tab, nine new skins including HUD, a colour for every machine
Alerts: rules over sensors, watched fleet-wide from the hub; ntfy and webhook channels; firing count on the rail. Docker: containers by stack with live use, start/stop/restart, following logs, inspect, images/volumes/networks. Skins: HUD, Nord, Gruvbox, Catppuccin (+Latte), Dracula, Tokyo Night, Solarized (+Light). Accents: per-machine colour. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
63fcba6077
commit
cb1617c2b8
@@ -0,0 +1,444 @@
|
||||
package api
|
||||
|
||||
// Alerts: rules over sensors, watched by the hub for itself and every machine it manages.
|
||||
//
|
||||
// A rule is a selector, a comparison and a patience: `container.*.running == false for
|
||||
// 120 s`. Every alertEvery the hub asks each machine one shaped query for all the rules'
|
||||
// selectors, compares, and keeps a little state per (rule, machine, sensor): ok, pending
|
||||
// since, or firing since. Going to firing, and coming back, is an event — kept, shown on
|
||||
// the Alerts tab, and sent to whatever channels are set (ntfy, or a webhook: Home
|
||||
// Assistant, Slack, anything that takes a POST). `hamlab.reachable` is a sensor the hub
|
||||
// supplies itself, so "that machine stopped answering" is a rule like any other.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"hamlab/internal/sensors"
|
||||
"hamlab/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
alertEvery = 15 * time.Second
|
||||
maxEvents = 200
|
||||
)
|
||||
|
||||
type AlertRule struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Sensor string `json:"sensor"` // an ID or a pattern
|
||||
Op string `json:"op"` // > >= < <= == !=
|
||||
Value any `json:"value"` // number, bool or string, to match the sensor
|
||||
ForSec int `json:"for_sec"`
|
||||
Severity string `json:"severity"` // info | warn | crit
|
||||
Enabled bool `json:"enabled"`
|
||||
Machines string `json:"machines"` // "*" or comma-separated names; "" is "*"
|
||||
}
|
||||
|
||||
type AlertChannel struct {
|
||||
Type string `json:"type"` // ntfy | webhook
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type AlertEvent struct {
|
||||
T int64 `json:"t"`
|
||||
Kind string `json:"kind"` // firing | resolved
|
||||
Rule string `json:"rule"`
|
||||
RuleID string `json:"rule_id"`
|
||||
Machine string `json:"machine"`
|
||||
Sensor string `json:"sensor"`
|
||||
Value any `json:"value"`
|
||||
Severity string `json:"severity"`
|
||||
}
|
||||
|
||||
type alertFiring struct {
|
||||
AlertEvent
|
||||
Since int64 `json:"since"`
|
||||
}
|
||||
|
||||
type alertSaved struct {
|
||||
Rules []AlertRule `json:"rules"`
|
||||
Channels []AlertChannel `json:"channels"`
|
||||
Events []AlertEvent `json:"events"`
|
||||
Seeded bool `json:"seeded"`
|
||||
}
|
||||
|
||||
type alertState struct {
|
||||
pending time.Time
|
||||
firing bool
|
||||
since time.Time
|
||||
value any
|
||||
}
|
||||
|
||||
type alertEngine struct {
|
||||
s *Server
|
||||
mu sync.Mutex
|
||||
saved alertSaved
|
||||
state map[string]*alertState // ruleID|machine|sensor
|
||||
last time.Time
|
||||
}
|
||||
|
||||
var defaultRules = []AlertRule{
|
||||
{ID: "machine-offline", Name: "Machine not answering", Sensor: "hamlab.reachable", Op: "==", Value: false, ForSec: 120, Severity: "crit", Enabled: true},
|
||||
{ID: "container-down", Name: "Container not running", Sensor: "container.*.running", Op: "==", Value: false, ForSec: 120, Severity: "crit", Enabled: true},
|
||||
{ID: "container-unhealthy", Name: "Container unhealthy", Sensor: "container.*.health", Op: "==", Value: "unhealthy", ForSec: 180, Severity: "warn", Enabled: true},
|
||||
{ID: "service-failed", Name: "A service has failed", Sensor: "services.failed", Op: ">", Value: 0.0, ForSec: 300, Severity: "warn", Enabled: true},
|
||||
{ID: "disk-full", Name: "Filesystem nearly full", Sensor: "fs.*.used_pct", Op: ">=", Value: 90.0, ForSec: 300, Severity: "warn", Enabled: true},
|
||||
{ID: "pool-unhealthy", Name: "ZFS pool not healthy", Sensor: "zfs.*.healthy", Op: "==", Value: false, ForSec: 60, Severity: "crit", Enabled: true},
|
||||
{ID: "drive-failing", Name: "Drive failing SMART", Sensor: "smart.*.passed", Op: "==", Value: false, ForSec: 0, Severity: "crit", Enabled: true},
|
||||
{ID: "on-battery", Name: "UPS on battery", Sensor: "ups.on_battery", Op: "==", Value: true, ForSec: 30, Severity: "crit", Enabled: true},
|
||||
{ID: "hot", Name: "CPU running hot", Sensor: "cpu.temp", Op: ">=", Value: 90.0, ForSec: 300, Severity: "warn", Enabled: true},
|
||||
{ID: "memory", Name: "Memory nearly exhausted", Sensor: "mem.used_pct", Op: ">=", Value: 95.0, ForSec: 300, Severity: "warn", Enabled: true},
|
||||
}
|
||||
|
||||
func newAlertEngine(s *Server) *alertEngine {
|
||||
e := &alertEngine{s: s, state: map[string]*alertState{}}
|
||||
var raw json.RawMessage
|
||||
s.store.View(func(st *store.State) { raw = st.Alerts })
|
||||
_ = json.Unmarshal(raw, &e.saved)
|
||||
if !e.saved.Seeded {
|
||||
e.saved.Rules, e.saved.Seeded = append([]AlertRule{}, defaultRules...), true
|
||||
}
|
||||
go e.run()
|
||||
return e
|
||||
}
|
||||
|
||||
func (e *alertEngine) persist() {
|
||||
b, _ := json.Marshal(e.saved)
|
||||
_ = e.s.store.Update(func(st *store.State) { st.Alerts = b })
|
||||
}
|
||||
|
||||
func (e *alertEngine) run() {
|
||||
time.Sleep(20 * time.Second) // let the status sensors take their first readings
|
||||
for {
|
||||
e.evaluate()
|
||||
time.Sleep(alertEvery)
|
||||
}
|
||||
}
|
||||
|
||||
func wants(r AlertRule, machine string) bool {
|
||||
if r.Machines == "" || r.Machines == "*" {
|
||||
return true
|
||||
}
|
||||
for _, m := range strings.Split(r.Machines, ",") {
|
||||
if strings.TrimSpace(m) == machine {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// holds reports whether `got <op> want`, comparing like with like.
|
||||
func holds(got any, op string, want any) bool {
|
||||
switch g := got.(type) {
|
||||
case float64:
|
||||
w, ok := want.(float64)
|
||||
if !ok {
|
||||
if s, isStr := want.(string); isStr {
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
w = f
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
switch op {
|
||||
case ">":
|
||||
return g > w
|
||||
case ">=":
|
||||
return g >= w
|
||||
case "<":
|
||||
return g < w
|
||||
case "<=":
|
||||
return g <= w
|
||||
case "==":
|
||||
return g == w
|
||||
case "!=":
|
||||
return g != w
|
||||
}
|
||||
case bool:
|
||||
w, ok := want.(bool)
|
||||
if !ok {
|
||||
w = fmt.Sprint(want) == "true"
|
||||
}
|
||||
return (op == "==" && g == w) || (op == "!=" && g != w)
|
||||
case string:
|
||||
w := fmt.Sprint(want)
|
||||
return (op == "==" && g == w) || (op == "!=" && g != w)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *alertEngine) evaluate() {
|
||||
e.mu.Lock()
|
||||
rules := append([]AlertRule{}, e.saved.Rules...)
|
||||
e.mu.Unlock()
|
||||
query := map[string]string{}
|
||||
for _, r := range rules {
|
||||
if r.Enabled && r.Sensor != "hamlab.reachable" {
|
||||
query[r.ID] = r.Sensor
|
||||
}
|
||||
}
|
||||
qb, _ := json.Marshal(query)
|
||||
machines := append([]string{e.s.host}, e.s.nodes.Names()...)
|
||||
results := map[string]map[string]any{} // machine → ruleID → value | {id: value}
|
||||
reach := map[string]bool{}
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
for _, m := range machines {
|
||||
wg.Add(1)
|
||||
go func(m string) {
|
||||
defer wg.Done()
|
||||
var res map[string]any
|
||||
ok := false
|
||||
if m == e.s.host {
|
||||
if out, err := sensors.Shape(e.s.sensors.All(), qb, false); err == nil {
|
||||
res, ok = out.(map[string]any)
|
||||
}
|
||||
} else {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
code, b, err := e.s.nodes.Do(ctx, m, "POST", "/api/sensors/query", qb, http.Header{"Content-Type": {"application/json"}})
|
||||
cancel()
|
||||
ok = err == nil && code == 200 && json.Unmarshal(b, &res) == nil
|
||||
}
|
||||
mu.Lock()
|
||||
results[m], reach[m] = res, ok
|
||||
mu.Unlock()
|
||||
}(m)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
now := time.Now()
|
||||
seen := map[string]bool{}
|
||||
var events []AlertEvent
|
||||
e.mu.Lock()
|
||||
for _, r := range rules {
|
||||
if !r.Enabled {
|
||||
continue
|
||||
}
|
||||
for _, m := range machines {
|
||||
if !wants(r, m) {
|
||||
continue
|
||||
}
|
||||
readings := map[string]any{}
|
||||
if r.Sensor == "hamlab.reachable" {
|
||||
readings["hamlab.reachable"] = reach[m]
|
||||
} else if !reach[m] {
|
||||
continue // no reading is not a reading: an unreachable machine is the offline rule's business
|
||||
} else if v, ok := results[m][r.ID]; ok {
|
||||
if many, isMap := v.(map[string]any); isMap {
|
||||
readings = many
|
||||
} else if v != nil {
|
||||
readings[r.Sensor] = v
|
||||
}
|
||||
}
|
||||
for id, val := range readings {
|
||||
key := r.ID + "|" + m + "|" + id
|
||||
seen[key] = true
|
||||
st := e.state[key]
|
||||
if st == nil {
|
||||
st = &alertState{}
|
||||
e.state[key] = st
|
||||
}
|
||||
st.value = val
|
||||
if holds(val, r.Op, r.Value) {
|
||||
if st.pending.IsZero() {
|
||||
st.pending = now
|
||||
}
|
||||
if !st.firing && now.Sub(st.pending) >= time.Duration(r.ForSec)*time.Second {
|
||||
st.firing, st.since = true, now
|
||||
events = append(events, AlertEvent{T: now.UnixMilli(), Kind: "firing", Rule: r.Name, RuleID: r.ID, Machine: m, Sensor: id, Value: val, Severity: r.Severity})
|
||||
}
|
||||
} else {
|
||||
if st.firing {
|
||||
events = append(events, AlertEvent{T: now.UnixMilli(), Kind: "resolved", Rule: r.Name, RuleID: r.ID, Machine: m, Sensor: id, Value: val, Severity: r.Severity})
|
||||
}
|
||||
st.pending, st.firing = time.Time{}, false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for key := range e.state { // a sensor that is gone (a removed container, a deleted rule) takes its state with it
|
||||
if !seen[key] {
|
||||
delete(e.state, key)
|
||||
}
|
||||
}
|
||||
e.last = now
|
||||
if len(events) > 0 {
|
||||
e.saved.Events = append(e.saved.Events, events...)
|
||||
if len(e.saved.Events) > maxEvents {
|
||||
e.saved.Events = e.saved.Events[len(e.saved.Events)-maxEvents:]
|
||||
}
|
||||
e.persist()
|
||||
}
|
||||
channels := append([]AlertChannel{}, e.saved.Channels...)
|
||||
e.mu.Unlock()
|
||||
for _, ev := range events {
|
||||
e.notify(channels, ev)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *alertEngine) firing() []alertFiring {
|
||||
out := []alertFiring{}
|
||||
names := map[string]AlertRule{}
|
||||
for _, r := range e.saved.Rules {
|
||||
names[r.ID] = r
|
||||
}
|
||||
for key, st := range e.state {
|
||||
if !st.firing {
|
||||
continue
|
||||
}
|
||||
p := strings.SplitN(key, "|", 3)
|
||||
r := names[p[0]]
|
||||
out = append(out, alertFiring{AlertEvent: AlertEvent{T: st.since.UnixMilli(), Kind: "firing", Rule: r.Name, RuleID: r.ID, Machine: p[1], Sensor: p[2], Value: st.value, Severity: r.Severity}, Since: st.since.UnixMilli()})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Since > out[j].Since })
|
||||
return out
|
||||
}
|
||||
|
||||
func describe(ev AlertEvent) (title, body string) {
|
||||
mark := map[string]string{"firing": "▲", "resolved": "✓"}[ev.Kind]
|
||||
title = fmt.Sprintf("%s %s — %s", mark, ev.Rule, ev.Machine)
|
||||
body = fmt.Sprintf("%s = %v", ev.Sensor, ev.Value)
|
||||
if ev.Kind == "resolved" {
|
||||
body = "back to normal: " + body
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (e *alertEngine) notify(channels []AlertChannel, ev AlertEvent) {
|
||||
title, body := describe(ev)
|
||||
for _, c := range channels {
|
||||
var req *http.Request
|
||||
switch c.Type {
|
||||
case "ntfy":
|
||||
req, _ = http.NewRequest("POST", c.URL, strings.NewReader(body))
|
||||
req.Header.Set("Title", title)
|
||||
req.Header.Set("Priority", map[string]string{"crit": "urgent", "warn": "high"}[ev.Severity])
|
||||
req.Header.Set("Tags", map[string]string{"firing": "warning", "resolved": "white_check_mark"}[ev.Kind])
|
||||
if ev.Kind == "resolved" {
|
||||
req.Header.Set("Priority", "default")
|
||||
}
|
||||
case "webhook":
|
||||
b, _ := json.Marshal(map[string]any{"title": title, "message": body, "event": ev})
|
||||
req, _ = http.NewRequest("POST", c.URL, bytes.NewReader(b))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if req == nil {
|
||||
continue
|
||||
}
|
||||
go func(req *http.Request, url string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
|
||||
if err != nil {
|
||||
log.Printf("alert: notifying %s: %v", url, err)
|
||||
return
|
||||
}
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode >= 300 {
|
||||
log.Printf("alert: %s answered %d", url, resp.StatusCode)
|
||||
}
|
||||
}(req, c.URL)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- API ----
|
||||
|
||||
func (s *Server) alertsGet(w http.ResponseWriter, _ *http.Request) {
|
||||
e := s.alerts
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
ev := append([]AlertEvent{}, e.saved.Events...)
|
||||
for i, j := 0, len(ev)-1; i < j; i, j = i+1, j-1 {
|
||||
ev[i], ev[j] = ev[j], ev[i]
|
||||
}
|
||||
rules, channels := e.saved.Rules, e.saved.Channels
|
||||
if rules == nil { // an empty list is a list: never hand the page a null to take the length of
|
||||
rules = []AlertRule{}
|
||||
}
|
||||
if channels == nil {
|
||||
channels = []AlertChannel{}
|
||||
}
|
||||
jsonOK(w, 200, map[string]any{"rules": rules, "channels": channels, "events": ev, "firing": e.firing(), "checked": e.last.UnixMilli(), "every_sec": int(alertEvery.Seconds())})
|
||||
}
|
||||
|
||||
func (s *Server) alertsPut(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Rules *[]AlertRule `json:"rules"`
|
||||
Channels *[]AlertChannel `json:"channels"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 256<<10)).Decode(&body); err != nil {
|
||||
jsonErr(w, 400, "bad request")
|
||||
return
|
||||
}
|
||||
if body.Rules != nil {
|
||||
ids := map[string]bool{}
|
||||
for i := range *body.Rules {
|
||||
ru := &(*body.Rules)[i]
|
||||
ru.Name, ru.Sensor = strings.TrimSpace(ru.Name), strings.TrimSpace(ru.Sensor)
|
||||
if ru.ID == "" || ids[ru.ID] || ru.Name == "" || len(ru.Name) > maxNameLen || ru.ForSec < 0 || ru.ForSec > 86400 {
|
||||
jsonErr(w, 400, "every rule needs a unique id, a name (64 characters at most) and a patience between 0 and 86400 s")
|
||||
return
|
||||
}
|
||||
ids[ru.ID] = true
|
||||
if _, err := sensors.Matcher(ru.Sensor); err != nil || ru.Sensor == "" {
|
||||
jsonErr(w, 400, fmt.Sprintf("rule %q: %q is not a sensor ID or pattern", ru.Name, ru.Sensor))
|
||||
return
|
||||
}
|
||||
if !strings.Contains(" > >= < <= == != ", " "+ru.Op+" ") {
|
||||
jsonErr(w, 400, fmt.Sprintf("rule %q: the comparison must be one of > >= < <= == !=", ru.Name))
|
||||
return
|
||||
}
|
||||
if ru.Severity != "info" && ru.Severity != "crit" {
|
||||
ru.Severity = "warn"
|
||||
}
|
||||
}
|
||||
}
|
||||
if body.Channels != nil {
|
||||
for _, c := range *body.Channels {
|
||||
if (c.Type != "ntfy" && c.Type != "webhook") || !(strings.HasPrefix(c.URL, "https://") || strings.HasPrefix(c.URL, "http://")) {
|
||||
jsonErr(w, 400, "a channel is {type: ntfy|webhook, url: http(s)://…}")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
e := s.alerts
|
||||
e.mu.Lock()
|
||||
if body.Rules != nil {
|
||||
e.saved.Rules = *body.Rules
|
||||
}
|
||||
if body.Channels != nil {
|
||||
e.saved.Channels = *body.Channels
|
||||
}
|
||||
e.persist()
|
||||
e.mu.Unlock()
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
// alertsTest sends one clearly-marked test notification through every channel.
|
||||
func (s *Server) alertsTest(w http.ResponseWriter, _ *http.Request) {
|
||||
e := s.alerts
|
||||
e.mu.Lock()
|
||||
channels := append([]AlertChannel{}, e.saved.Channels...)
|
||||
e.mu.Unlock()
|
||||
if len(channels) == 0 {
|
||||
jsonErr(w, 400, "no channels to test")
|
||||
return
|
||||
}
|
||||
e.notify(channels, AlertEvent{T: time.Now().UnixMilli(), Kind: "firing", Rule: "Test alert", Machine: s.host, Sensor: "hamlab.test", Value: "this is only a test", Severity: "info"})
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
package api
|
||||
|
||||
// The Docker tab: what is running, grouped the way it was deployed (compose stacks), with
|
||||
// start / stop / restart, logs and inspect — plus images, volumes and networks to look at.
|
||||
// It drives the docker CLI (or podman), as the user first and as root if that is refused.
|
||||
// Creating, removing and pruning are deliberately absent: whatever deployed the stacks
|
||||
// (Dockhand, compose) stays the one place that changes what exists.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"hamlab/internal/priv"
|
||||
)
|
||||
|
||||
type dockerContainer struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Image string `json:"image"`
|
||||
State string `json:"state"` // running | exited | paused | restarting | created | dead
|
||||
Status string `json:"status"` // "Up 3 hours (healthy)"
|
||||
Health string `json:"health"` // healthy | unhealthy | starting | none
|
||||
Ports string `json:"ports"`
|
||||
Stack string `json:"stack"` // compose project, "" when started by hand
|
||||
Service string `json:"service"` // compose service
|
||||
Created string `json:"created"`
|
||||
Mem float64 `json:"mem"` // bytes, from the container's cgroup; 0 when unknown
|
||||
}
|
||||
|
||||
func dockerTool() string {
|
||||
for _, t := range []string{"docker", "podman"} {
|
||||
if _, err := exec.LookPath(t); err == nil {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// dockerRun runs the tool as this user, and again as root if the socket said no.
|
||||
func dockerRun(ctx context.Context, args ...string) ([]byte, error) {
|
||||
tool := dockerTool()
|
||||
if tool == "" {
|
||||
return nil, errors.New("neither docker nor podman is installed here")
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, tool, args...).CombinedOutput()
|
||||
if err != nil && priv.Available() && strings.Contains(strings.ToLower(string(out)), "permission denied") {
|
||||
full := priv.Wrap(append([]string{tool}, args...)...)
|
||||
out, err = exec.CommandContext(ctx, full[0], full[1:]...).CombinedOutput()
|
||||
}
|
||||
if err != nil {
|
||||
msg := strings.TrimSpace(string(out))
|
||||
if i := strings.LastIndex(msg, "\n"); i >= 0 {
|
||||
msg = msg[i+1:]
|
||||
}
|
||||
if msg == "" {
|
||||
msg = err.Error()
|
||||
}
|
||||
return out, errors.New(msg)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func jsonLines(out []byte, each func(map[string]any)) {
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
var m map[string]any
|
||||
if json.Unmarshal([]byte(line), &m) == nil {
|
||||
each(m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func dstr(m map[string]any, k string) string { s, _ := m[k].(string); return s }
|
||||
|
||||
func labelOf(labels, key string) string {
|
||||
for _, kv := range strings.Split(labels, ",") {
|
||||
if v, ok := strings.CutPrefix(kv, key+"="); ok {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Server) dockerOverview(w http.ResponseWriter, r *http.Request) {
|
||||
if dockerTool() == "" {
|
||||
jsonOK(w, 200, map[string]any{"available": false, "message": "Docker isn't installed on this machine."})
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
|
||||
defer cancel()
|
||||
out, err := dockerRun(ctx, "ps", "-a", "--no-trunc", "--format", "{{json .}}")
|
||||
if err != nil {
|
||||
jsonOK(w, 200, map[string]any{"available": false, "message": err.Error()})
|
||||
return
|
||||
}
|
||||
containers := []dockerContainer{}
|
||||
jsonLines(out, func(m map[string]any) {
|
||||
c := dockerContainer{ID: dstr(m, "ID"), Name: strings.Split(dstr(m, "Names"), ",")[0], Image: dstr(m, "Image"), State: dstr(m, "State"), Status: dstr(m, "Status"),
|
||||
Ports: dstr(m, "Ports"), Created: dstr(m, "CreatedAt"), Health: "none", Stack: labelOf(dstr(m, "Labels"), "com.docker.compose.project"), Service: labelOf(dstr(m, "Labels"), "com.docker.compose.service")}
|
||||
for _, h := range []string{"unhealthy", "healthy", "starting"} {
|
||||
if strings.Contains(c.Status, "("+h+")") || strings.Contains(c.Status, "health: "+h) {
|
||||
c.Health = h
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, dir := range []string{"/sys/fs/cgroup/system.slice/docker-" + c.ID + ".scope", "/sys/fs/cgroup/docker/" + c.ID, "/sys/fs/cgroup/machine.slice/libpod-" + c.ID + ".scope"} {
|
||||
if b, err := os.ReadFile(dir + "/memory.current"); err == nil {
|
||||
c.Mem, _ = strconv.ParseFloat(strings.TrimSpace(string(b)), 64)
|
||||
break
|
||||
}
|
||||
}
|
||||
containers = append(containers, c)
|
||||
})
|
||||
sort.Slice(containers, func(i, j int) bool {
|
||||
if containers[i].Stack != containers[j].Stack {
|
||||
return containers[i].Stack < containers[j].Stack
|
||||
}
|
||||
return containers[i].Name < containers[j].Name
|
||||
})
|
||||
list := func(args ...string) []map[string]any {
|
||||
rows := []map[string]any{}
|
||||
if out, err := dockerRun(ctx, args...); err == nil {
|
||||
jsonLines(out, func(m map[string]any) { rows = append(rows, m) })
|
||||
}
|
||||
return rows
|
||||
}
|
||||
jsonOK(w, 200, map[string]any{"available": true, "tool": dockerTool(), "containers": containers,
|
||||
"images": list("images", "--format", "{{json .}}"),
|
||||
"volumes": list("volume", "ls", "--format", "{{json .}}"),
|
||||
"networks": list("network", "ls", "--format", "{{json .}}")})
|
||||
}
|
||||
|
||||
var dockerRef = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$`)
|
||||
|
||||
func (s *Server) dockerAction(w http.ResponseWriter, r *http.Request) {
|
||||
id, action := r.PathValue("id"), r.PathValue("action")
|
||||
if !dockerRef.MatchString(id) || (action != "start" && action != "stop" && action != "restart") {
|
||||
jsonErr(w, 400, "start, stop or restart a container by name or ID")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 90*time.Second)
|
||||
defer cancel()
|
||||
if _, err := dockerRun(ctx, action, id); err != nil {
|
||||
jsonErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
// dockerLogs is the tail of a container's log; ?since=<RFC3339> asks only for what is new,
|
||||
// which is how the page follows a log without holding a connection open.
|
||||
func (s *Server) dockerLogs(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if !dockerRef.MatchString(id) {
|
||||
jsonErr(w, 400, "bad container")
|
||||
return
|
||||
}
|
||||
args := []string{"logs", "--timestamps"}
|
||||
if since := r.URL.Query().Get("since"); since != "" {
|
||||
if _, err := time.Parse(time.RFC3339Nano, since); err != nil {
|
||||
jsonErr(w, 400, "since is an RFC 3339 time")
|
||||
return
|
||||
}
|
||||
args = append(args, "--since", since)
|
||||
} else {
|
||||
n, _ := strconv.Atoi(r.URL.Query().Get("n"))
|
||||
if n <= 0 || n > 5000 {
|
||||
n = 300
|
||||
}
|
||||
args = append(args, "--tail", strconv.Itoa(n))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
|
||||
defer cancel()
|
||||
out, err := dockerRun(ctx, append(args, id)...)
|
||||
if err != nil {
|
||||
jsonErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
type line struct {
|
||||
T string `json:"t"`
|
||||
M string `json:"m"`
|
||||
}
|
||||
lines := []line{}
|
||||
for _, l := range strings.Split(strings.TrimRight(string(out), "\n"), "\n") {
|
||||
if l == "" {
|
||||
continue
|
||||
}
|
||||
t, m, ok := strings.Cut(l, " ")
|
||||
if _, err := time.Parse(time.RFC3339Nano, t); !ok || err != nil {
|
||||
t, m = "", l
|
||||
}
|
||||
lines = append(lines, line{t, m})
|
||||
}
|
||||
jsonOK(w, 200, map[string]any{"lines": lines})
|
||||
}
|
||||
|
||||
// dockerInspect is the parts of `docker inspect` worth reading on a phone.
|
||||
func (s *Server) dockerInspect(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if !dockerRef.MatchString(id) {
|
||||
jsonErr(w, 400, "bad container")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
||||
defer cancel()
|
||||
out, err := dockerRun(ctx, "inspect", id)
|
||||
var all []map[string]any
|
||||
if err != nil || json.Unmarshal(out, &all) != nil || len(all) == 0 {
|
||||
jsonErr(w, 500, fmt.Sprint("inspect failed: ", err))
|
||||
return
|
||||
}
|
||||
c := all[0]
|
||||
dig := func(m map[string]any, path ...string) any {
|
||||
var cur any = m
|
||||
for _, p := range path {
|
||||
mm, ok := cur.(map[string]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
cur = mm[p]
|
||||
}
|
||||
return cur
|
||||
}
|
||||
env := []string{}
|
||||
if list, ok := dig(c, "Config", "Env").([]any); ok {
|
||||
for _, e := range list {
|
||||
k, _, _ := strings.Cut(fmt.Sprint(e), "=")
|
||||
env = append(env, k) // names only: values are where the secrets live
|
||||
}
|
||||
}
|
||||
sort.Strings(env)
|
||||
jsonOK(w, 200, map[string]any{
|
||||
"image": dig(c, "Config", "Image"), "command": dig(c, "Config", "Cmd"), "entrypoint": dig(c, "Config", "Entrypoint"),
|
||||
"restart": dig(c, "HostConfig", "RestartPolicy", "Name"), "network_mode": dig(c, "HostConfig", "NetworkMode"),
|
||||
"memory_limit": dig(c, "HostConfig", "Memory"), "started": dig(c, "State", "StartedAt"), "exit_code": dig(c, "State", "ExitCode"),
|
||||
"error": dig(c, "State", "Error"), "restarts": c["RestartCount"], "mounts": c["Mounts"], "ports": dig(c, "NetworkSettings", "Ports"),
|
||||
"networks": dig(c, "NetworkSettings", "Networks"), "env_names": env,
|
||||
"compose_file": dig(c, "Config", "Labels", "com.docker.compose.project.config_files"),
|
||||
})
|
||||
}
|
||||
@@ -59,6 +59,7 @@ type Server struct {
|
||||
metrics *metrics.Collector
|
||||
sensors *sensors.Registry
|
||||
streams streamGate
|
||||
alerts *alertEngine
|
||||
store *store.Store
|
||||
svc *services.Manager
|
||||
cron *cron.Manager
|
||||
@@ -88,6 +89,7 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
|
||||
s.user = "?"
|
||||
}
|
||||
s.sensors.Add(newStatusSensors(s))
|
||||
s.alerts = newAlertEngine(s)
|
||||
// The saved dashboard layout carries the sampling interval; apply it on boot.
|
||||
var raw json.RawMessage
|
||||
st.View(func(x *store.State) { raw = x.Dashboard })
|
||||
@@ -126,6 +128,13 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
|
||||
m.HandleFunc("GET /api/nodes/{name}/sensors/{rest...}", s.sensorAuth(s.proxyNodeSensors))
|
||||
m.HandleFunc("POST /api/nodes/{name}/sensors/query", s.sensorAuth(s.proxyNodeSensors))
|
||||
m.HandleFunc("POST /api/tokens/read", s.auth(s.issueReadToken))
|
||||
m.HandleFunc("GET /api/docker", s.auth(s.dockerOverview))
|
||||
m.HandleFunc("POST /api/docker/containers/{id}/{action}", s.auth(s.dockerAction))
|
||||
m.HandleFunc("GET /api/docker/containers/{id}/logs", s.auth(s.dockerLogs))
|
||||
m.HandleFunc("GET /api/docker/containers/{id}/inspect", s.auth(s.dockerInspect))
|
||||
m.HandleFunc("GET /api/alerts", s.auth(s.alertsGet))
|
||||
m.HandleFunc("PUT /api/alerts", s.auth(s.alertsPut))
|
||||
m.HandleFunc("POST /api/alerts/test", s.auth(s.alertsTest))
|
||||
m.HandleFunc("GET /api/fleet/sensors/query", s.sensorAuth(s.fleetQuery))
|
||||
m.HandleFunc("POST /api/fleet/sensors/query", s.sensorAuth(s.fleetQuery))
|
||||
m.HandleFunc("GET /api/services", s.auth(s.listServices))
|
||||
|
||||
Reference in New Issue
Block a user