Snapshot schedules are real cron jobs, written and owned by Ham Lab
A schedule is now a line in /etc/cron.d/hamlab-zfs, which Ham Lab owns and rewrites whole whenever the schedules change; the line calls `hamlab zfs-snapshot <id>` as the account Ham Lab runs as, reaching zfs through sudo exactly as the daemon does. The daemon's own ticker is gone. Snapshots are backup-shaped and should not stop because a web console is down, being upgraded, or uninstalled — and a cron line is greppable by anyone looking at the machine the normal way, which a schedule hidden inside a daemon is not. A cron expression is now the single description of when a schedule fires: the four common shapes render to one, a custom schedule states it outright, and the next run is read back with the same evaluator the Jobs tab uses on everyone else's lines. So the page, the file and the prediction cannot disagree. Schedules gain a custom cron expression and an optional snapshot name prefix. What a run did moves out of the policy into zfs-runs/<id>.json beside the config, because the cron job is a separate process and must never write state.json behind the daemon's back. The detected external job can be adopted: its schedule, retention and dataset are read out of the cron line into a new Ham Lab schedule, ready to check rather than retype. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
co-authored by
Claude Opus 5
Claude Opus 5
parent
4dc25fdd13
commit
09b7558d44
+182
-57
@@ -18,13 +18,21 @@ package storage
|
||||
// to it. And every policy can be asked what it would delete, before it deletes anything.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"hamlab/internal/cron"
|
||||
"hamlab/internal/priv"
|
||||
)
|
||||
|
||||
// stampLayout is zfSnap's timestamp: 2026-09-24_02.30.01.
|
||||
@@ -41,12 +49,15 @@ type Policy struct {
|
||||
Dataset string `json:"dataset"`
|
||||
// Recursive takes one atomic snapshot of the dataset and everything under it (zfs -r).
|
||||
Recursive bool `json:"recursive"`
|
||||
// Every is hourly | daily | weekly | monthly. At is "HH:MM" (the minute alone is used
|
||||
// for hourly). Weekday is 0=Sunday, Day is 1–28.
|
||||
// Every is hourly | daily | weekly | monthly | custom. At is "HH:MM" (the minute alone
|
||||
// is used for hourly). Weekday is 0=Sunday, Day is 1–28. With "custom", Cron is a plain
|
||||
// cron expression and the rest is ignored — the escape hatch for anything the four
|
||||
// common shapes cannot say.
|
||||
Every string `json:"every"`
|
||||
At string `json:"at"`
|
||||
Weekday int `json:"weekday"`
|
||||
Day int `json:"day"`
|
||||
Cron string `json:"cron,omitempty"`
|
||||
// KeepFor is how long a snapshot lives: "6w", "30d", "12h". It is written into the
|
||||
// snapshot's own name, zfSnap-style, so each one carries the rule it was taken under.
|
||||
// KeepMin is a floor — that many newest snapshots survive however old they are, so a
|
||||
@@ -56,10 +67,6 @@ type Policy struct {
|
||||
Prefix string `json:"prefix"`
|
||||
Enabled bool `json:"enabled"`
|
||||
|
||||
LastRun time.Time `json:"last_run,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
LastTaken string `json:"last_taken,omitempty"` // the snapshot the last run made
|
||||
Pruned int `json:"pruned,omitempty"` // how many the last run destroyed
|
||||
}
|
||||
|
||||
func (p *Policy) prefix() string { return strings.TrimSpace(p.Prefix) }
|
||||
@@ -74,8 +81,15 @@ func (p *Policy) Validate() error {
|
||||
}
|
||||
switch p.Every {
|
||||
case "hourly", "daily", "weekly", "monthly":
|
||||
case "custom":
|
||||
if strings.TrimSpace(p.Cron) == "" {
|
||||
return errors.New("a custom schedule needs a cron expression")
|
||||
}
|
||||
default:
|
||||
return errors.New("every: hourly, daily, weekly or monthly")
|
||||
return errors.New("every: hourly, daily, weekly, monthly or custom")
|
||||
}
|
||||
if _, err := p.CronExpr(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, _, err := p.clock(); err != nil {
|
||||
return err
|
||||
@@ -144,67 +158,50 @@ func ParseAge(s string) (time.Duration, error) {
|
||||
return 0, fmt.Errorf("%q: the unit is m, h, d or w", s)
|
||||
}
|
||||
|
||||
// lastDue is the most recent moment this policy was supposed to run, at or before now.
|
||||
// A policy whose LastRun is older than that is due — so a machine that was off through
|
||||
// its window takes one snapshot when it comes back, rather than skipping silently.
|
||||
func (p *Policy) lastDue(now time.Time) time.Time {
|
||||
hour, min, _ := p.clock()
|
||||
y, mo, d := now.Date()
|
||||
loc := now.Location()
|
||||
// CronExpr renders the schedule as a cron expression — which is what actually runs it. The
|
||||
// four common shapes are sugar over this; a custom policy states it outright. One expression
|
||||
// is the only description of when a schedule fires, so the page, the cron file and the next
|
||||
// run can never disagree.
|
||||
func (p *Policy) CronExpr() (string, error) {
|
||||
if p.Every == "custom" {
|
||||
expr := strings.Join(strings.Fields(p.Cron), " ")
|
||||
if err := cron.ValidSchedule(expr); err != nil {
|
||||
return "", fmt.Errorf("cron expression: %w", err)
|
||||
}
|
||||
if strings.HasPrefix(expr, "@reboot") {
|
||||
return "", errors.New("@reboot is not a schedule for snapshots")
|
||||
}
|
||||
return expr, nil
|
||||
}
|
||||
hour, min, err := p.clock()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
switch p.Every {
|
||||
case "hourly":
|
||||
t := time.Date(y, mo, d, now.Hour(), min, 0, 0, loc)
|
||||
if t.After(now) {
|
||||
t = t.Add(-time.Hour)
|
||||
}
|
||||
return t
|
||||
return fmt.Sprintf("%d * * * *", min), nil
|
||||
case "daily":
|
||||
t := time.Date(y, mo, d, hour, min, 0, 0, loc)
|
||||
if t.After(now) {
|
||||
t = t.AddDate(0, 0, -1)
|
||||
}
|
||||
return t
|
||||
return fmt.Sprintf("%d %d * * *", min, hour), nil
|
||||
case "weekly":
|
||||
t := time.Date(y, mo, d, hour, min, 0, 0, loc)
|
||||
back := (int(t.Weekday()) - p.Weekday + 7) % 7
|
||||
t = t.AddDate(0, 0, -back)
|
||||
if t.After(now) {
|
||||
t = t.AddDate(0, 0, -7)
|
||||
}
|
||||
return t
|
||||
return fmt.Sprintf("%d %d * * %d", min, hour, p.Weekday), nil
|
||||
case "monthly":
|
||||
t := time.Date(y, mo, p.Day, hour, min, 0, 0, loc)
|
||||
if t.After(now) {
|
||||
t = t.AddDate(0, -1, 0)
|
||||
}
|
||||
return t
|
||||
return fmt.Sprintf("%d %d %d * *", min, hour, p.Day), nil
|
||||
}
|
||||
return time.Time{}
|
||||
return "", errors.New("unknown schedule")
|
||||
}
|
||||
|
||||
// NextRun is when this policy runs next, for the UI to show.
|
||||
// NextRun is when cron will next fire this schedule, read with the same evaluator the Jobs
|
||||
// tab uses on everyone else's lines.
|
||||
func (p *Policy) NextRun(now time.Time) time.Time {
|
||||
last := p.lastDue(now)
|
||||
switch p.Every {
|
||||
case "hourly":
|
||||
return last.Add(time.Hour)
|
||||
case "daily":
|
||||
return last.AddDate(0, 0, 1)
|
||||
case "weekly":
|
||||
return last.AddDate(0, 0, 7)
|
||||
case "monthly":
|
||||
return last.AddDate(0, 1, 0)
|
||||
expr, err := p.CronExpr()
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// Due reports whether the policy should run now.
|
||||
func (p *Policy) Due(now time.Time) bool {
|
||||
if !p.Enabled {
|
||||
return false
|
||||
t, ok := cron.NextRun(expr, now)
|
||||
if !ok {
|
||||
return time.Time{}
|
||||
}
|
||||
d := p.lastDue(now)
|
||||
return !d.IsZero() && p.LastRun.Before(d)
|
||||
return t
|
||||
}
|
||||
|
||||
// SnapTime reads a snapshot's name in zfSnap's convention and returns when it was taken,
|
||||
@@ -349,3 +346,131 @@ func (m *Manager) Run(ctx context.Context, p *Policy) (RunResult, error) {
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// ---- what happened when it ran ----
|
||||
//
|
||||
// The policies are a definition the daemon owns; what a run did is written by whatever ran
|
||||
// it, which is a cron job in its own process. Keeping the two apart means the CLI never has
|
||||
// to write state.json behind the daemon's back — the last writer would win and one of them
|
||||
// would lose its work.
|
||||
|
||||
// RunRecord is the outcome of one run of one schedule.
|
||||
type RunRecord struct {
|
||||
At time.Time `json:"at"`
|
||||
Taken string `json:"taken,omitempty"`
|
||||
Pruned []string `json:"pruned,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func runPath(dir, id string) string { return filepath.Join(dir, "zfs-runs", id+".json") }
|
||||
|
||||
// WriteRun records what a run did, beside the config.
|
||||
func WriteRun(dir, id string, r RunRecord) error {
|
||||
if err := os.MkdirAll(filepath.Join(dir, "zfs-runs"), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p := runPath(dir, id)
|
||||
tmp := p + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, p)
|
||||
}
|
||||
|
||||
// ReadRun is the last thing a schedule did, or the zero value if it has never run.
|
||||
func ReadRun(dir, id string) RunRecord {
|
||||
var r RunRecord
|
||||
b, err := os.ReadFile(runPath(dir, id))
|
||||
if err == nil {
|
||||
_ = json.Unmarshal(b, &r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// ForgetRun drops a schedule's record, for when the schedule itself goes.
|
||||
func ForgetRun(dir, id string) { _ = os.Remove(runPath(dir, id)) }
|
||||
|
||||
// ---- the cron file ----
|
||||
|
||||
// CronFile is where Ham Lab writes the snapshot schedules. One file it owns entirely and
|
||||
// rewrites whole, so there is never a half-updated set of lines, and so a schedule removed
|
||||
// in the UI is removed from the machine.
|
||||
const CronFile = "/etc/cron.d/hamlab-zfs"
|
||||
|
||||
// CronFileText renders every schedule as cron lines. user is who they run as — the account
|
||||
// Ham Lab itself runs as, which reaches zfs through sudo exactly as the daemon does, so the
|
||||
// files it writes stay owned by that account rather than by root.
|
||||
func CronFileText(policies []Policy, user, exe, config string) (string, error) {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Ham Lab snapshot schedules — written by the ZFS tab, rewritten whole on every change.\n")
|
||||
b.WriteString("# Edit them there, not here: anything added by hand is lost the next time a schedule changes.\n")
|
||||
b.WriteString("SHELL=/bin/sh\n")
|
||||
b.WriteString("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n")
|
||||
for i := range policies {
|
||||
p := &policies[i]
|
||||
if !p.Enabled {
|
||||
b.WriteString("# " + p.Name + " — paused\n")
|
||||
continue
|
||||
}
|
||||
expr, err := p.CronExpr()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %w", p.Name, err)
|
||||
}
|
||||
name := strings.ReplaceAll(p.Name, "\n", " ")
|
||||
fmt.Fprintf(&b, "# %s — %s, keep %s\n", name, p.Dataset, p.KeepFor)
|
||||
fmt.Fprintf(&b, "%s %s %s -config %s zfs-snapshot %s\n\n", expr, user, exe, config, p.ID)
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// WriteCronFile puts the schedules on the machine, or removes the file when there are none.
|
||||
func (m *Manager) WriteCronFile(ctx context.Context, policies []Policy, user, exe, config string) error {
|
||||
live := false
|
||||
for i := range policies {
|
||||
if policies[i].Enabled {
|
||||
live = true
|
||||
}
|
||||
}
|
||||
if !live {
|
||||
out, code, err := m.run(ctx, "rm", "-f", CronFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errors.New(strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
text, err := CronFileText(policies, user, exe, config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.writeRoot(ctx, CronFile, text)
|
||||
}
|
||||
|
||||
// writeRoot puts text at a root-owned path, through the same privilege escalation every
|
||||
// other write here uses.
|
||||
func (m *Manager) writeRoot(ctx context.Context, path, text string) error {
|
||||
args := []string{"tee", path}
|
||||
if m.sudo {
|
||||
args = priv.Wrap(args...)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, args[0], args[1:]...)
|
||||
cmd.Stdin = strings.NewReader(text)
|
||||
var errb bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = nil, &errb
|
||||
if err := cmd.Run(); err != nil {
|
||||
msg := strings.TrimSpace(errb.String())
|
||||
if msg == "" {
|
||||
msg = err.Error()
|
||||
}
|
||||
return errors.New(msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user