ZFS gets its own tab, with scheduled snapshots and a retention period
Pools, datasets and snapshots move out of Disks — Disks is the hardware, this is the filesystem on top of it. New: snapshot schedules, hourly to monthly, with a retention period and a floor that keeps the newest few whatever their age. The naming is zfSnap's, deliberately. grid has run zfSnap from root cron since before this repo existed and it works, so rather than invent a second convention this writes and reads exactly what zfSnap does: 2026-09-24_02.30.01--6w, with the retention in the name. Ham Lab can therefore show and expire the snapshots zfSnap already took, the two cannot disagree about what a snapshot is, and changing a schedule never re-dates old ones — each carries the rule it was taken under. An existing cron job is detected and shown on the page, with a warning if a Ham Lab schedule is running as well. Nothing is destroyed unseen. Every schedule answers what it would delete, with the list, before Run now touches anything, and only names that parse as a stamp and a TTL are ever candidates. A snapshot taken by hand is invisible to it. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
co-authored by
Claude Opus 5
Claude Opus 5
parent
a07a444170
commit
7d0ec5e018
@@ -0,0 +1,351 @@
|
||||
package storage
|
||||
|
||||
// Scheduled snapshots with a retention period: take a snapshot of a dataset on a schedule,
|
||||
// and drop the ones that have aged out.
|
||||
//
|
||||
// **The naming is zfSnap's, deliberately.** grid has run zfSnap v1.11.1 from root cron since
|
||||
// before this repo existed and it works well, so rather than invent a second convention and
|
||||
// have two tools disagree about what a snapshot is, this writes and reads exactly what
|
||||
// zfSnap does: `<dataset>@2026-09-24_02.30.01--6w`, with the retention encoded in the name.
|
||||
// That has three consequences worth knowing. Ham Lab can show and manage the snapshots
|
||||
// zfSnap already took. The two can run side by side without fighting, because both only
|
||||
// ever destroy snapshots that have passed the expiry written into their own name. And
|
||||
// changing a policy's retention does not retroactively re-date old snapshots — each one
|
||||
// carries the rule it was taken under, which is the behaviour you want from a backup.
|
||||
//
|
||||
// Two rules make this safe to run unattended. Only a snapshot whose name parses as a stamp
|
||||
// and a TTL is ever a candidate — anything taken by hand or named anything else is invisible
|
||||
// to it. And every policy can be asked what it would delete, before it deletes anything.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// stampLayout is zfSnap's timestamp: 2026-09-24_02.30.01.
|
||||
const stampLayout = "2006-01-02_15.04.05"
|
||||
|
||||
// ttlSep separates the stamp from the retention in a snapshot name, as zfSnap writes it.
|
||||
const ttlSep = "--"
|
||||
|
||||
// Policy is a standing instruction: snapshot this dataset on this schedule, keep the
|
||||
// snapshots this long.
|
||||
type Policy struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Dataset string `json:"dataset"`
|
||||
// Recursive takes one atomic snapshot of the dataset and everything under it (zfs -r).
|
||||
Recursive bool `json:"recursive"`
|
||||
// Every is hourly | daily | weekly | monthly. At is "HH:MM" (the minute alone is used
|
||||
// for hourly). Weekday is 0=Sunday, Day is 1–28.
|
||||
Every string `json:"every"`
|
||||
At string `json:"at"`
|
||||
Weekday int `json:"weekday"`
|
||||
Day int `json:"day"`
|
||||
// KeepFor is how long a snapshot lives: "6w", "30d", "12h". It is written into the
|
||||
// snapshot's own name, zfSnap-style, so each one carries the rule it was taken under.
|
||||
// KeepMin is a floor — that many newest snapshots survive however old they are, so a
|
||||
// dataset is never left with none.
|
||||
KeepFor string `json:"keep_for"`
|
||||
KeepMin int `json:"keep_min"`
|
||||
Prefix string `json:"prefix"`
|
||||
Enabled bool `json:"enabled"`
|
||||
|
||||
LastRun time.Time `json:"last_run,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
LastTaken string `json:"last_taken,omitempty"` // the snapshot the last run made
|
||||
Pruned int `json:"pruned,omitempty"` // how many the last run destroyed
|
||||
}
|
||||
|
||||
func (p *Policy) prefix() string { return strings.TrimSpace(p.Prefix) }
|
||||
|
||||
// Validate reports what is wrong with a policy, or nil.
|
||||
func (p *Policy) Validate() error {
|
||||
if !datasetRe.MatchString(p.Dataset) {
|
||||
return errors.New("dataset: a pool or pool/dataset name")
|
||||
}
|
||||
if pre := p.prefix(); pre != "" && (!snapRe.MatchString(pre) || strings.Contains(pre, ttlSep)) {
|
||||
return errors.New("prefix: letters, digits, . _ : and - only, and no --")
|
||||
}
|
||||
switch p.Every {
|
||||
case "hourly", "daily", "weekly", "monthly":
|
||||
default:
|
||||
return errors.New("every: hourly, daily, weekly or monthly")
|
||||
}
|
||||
if _, _, err := p.clock(); err != nil {
|
||||
return err
|
||||
}
|
||||
if p.Every == "weekly" && (p.Weekday < 0 || p.Weekday > 6) {
|
||||
return errors.New("weekday: 0 (Sunday) to 6")
|
||||
}
|
||||
if p.Every == "monthly" && (p.Day < 1 || p.Day > 28) {
|
||||
return errors.New("day: 1 to 28, so every month has one")
|
||||
}
|
||||
if p.KeepFor != "" {
|
||||
if _, err := ParseAge(p.KeepFor); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if p.KeepMin < 0 {
|
||||
return errors.New("keep at least: 0 or more")
|
||||
}
|
||||
if p.KeepFor == "" {
|
||||
return errors.New("a retention period is required — it is written into each snapshot's name, and is what lets it be cleaned up later")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Policy) clock() (hour, min int, err error) {
|
||||
at := strings.TrimSpace(p.At)
|
||||
if at == "" {
|
||||
return 0, 0, nil
|
||||
}
|
||||
h, m, ok := strings.Cut(at, ":")
|
||||
if !ok {
|
||||
return 0, 0, errors.New("at: HH:MM")
|
||||
}
|
||||
hour, err = strconv.Atoi(h)
|
||||
if err == nil {
|
||||
min, err = strconv.Atoi(m)
|
||||
}
|
||||
if err != nil || hour < 0 || hour > 23 || min < 0 || min > 59 {
|
||||
return 0, 0, errors.New("at: HH:MM, 00:00 to 23:59")
|
||||
}
|
||||
return hour, min, nil
|
||||
}
|
||||
|
||||
// ParseAge reads "6w", "30d", "12h", "90m" as a duration. Weeks and days are what a
|
||||
// retention period is actually written in, and time.ParseDuration knows neither.
|
||||
func ParseAge(s string) (time.Duration, error) {
|
||||
s = strings.TrimSpace(strings.ToLower(s))
|
||||
if s == "" {
|
||||
return 0, errors.New("empty period")
|
||||
}
|
||||
unit := s[len(s)-1]
|
||||
n, err := strconv.Atoi(s[:len(s)-1])
|
||||
if err != nil || n <= 0 {
|
||||
return 0, fmt.Errorf("%q: a number and a unit, like 6w, 30d or 12h", s)
|
||||
}
|
||||
switch unit {
|
||||
case 'm':
|
||||
return time.Duration(n) * time.Minute, nil
|
||||
case 'h':
|
||||
return time.Duration(n) * time.Hour, nil
|
||||
case 'd':
|
||||
return time.Duration(n) * 24 * time.Hour, nil
|
||||
case 'w':
|
||||
return time.Duration(n) * 7 * 24 * time.Hour, nil
|
||||
}
|
||||
return 0, fmt.Errorf("%q: the unit is m, h, d or w", s)
|
||||
}
|
||||
|
||||
// lastDue is the most recent moment this policy was supposed to run, at or before now.
|
||||
// A policy whose LastRun is older than that is due — so a machine that was off through
|
||||
// its window takes one snapshot when it comes back, rather than skipping silently.
|
||||
func (p *Policy) lastDue(now time.Time) time.Time {
|
||||
hour, min, _ := p.clock()
|
||||
y, mo, d := now.Date()
|
||||
loc := now.Location()
|
||||
switch p.Every {
|
||||
case "hourly":
|
||||
t := time.Date(y, mo, d, now.Hour(), min, 0, 0, loc)
|
||||
if t.After(now) {
|
||||
t = t.Add(-time.Hour)
|
||||
}
|
||||
return t
|
||||
case "daily":
|
||||
t := time.Date(y, mo, d, hour, min, 0, 0, loc)
|
||||
if t.After(now) {
|
||||
t = t.AddDate(0, 0, -1)
|
||||
}
|
||||
return t
|
||||
case "weekly":
|
||||
t := time.Date(y, mo, d, hour, min, 0, 0, loc)
|
||||
back := (int(t.Weekday()) - p.Weekday + 7) % 7
|
||||
t = t.AddDate(0, 0, -back)
|
||||
if t.After(now) {
|
||||
t = t.AddDate(0, 0, -7)
|
||||
}
|
||||
return t
|
||||
case "monthly":
|
||||
t := time.Date(y, mo, p.Day, hour, min, 0, 0, loc)
|
||||
if t.After(now) {
|
||||
t = t.AddDate(0, -1, 0)
|
||||
}
|
||||
return t
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// NextRun is when this policy runs next, for the UI to show.
|
||||
func (p *Policy) NextRun(now time.Time) time.Time {
|
||||
last := p.lastDue(now)
|
||||
switch p.Every {
|
||||
case "hourly":
|
||||
return last.Add(time.Hour)
|
||||
case "daily":
|
||||
return last.AddDate(0, 0, 1)
|
||||
case "weekly":
|
||||
return last.AddDate(0, 0, 7)
|
||||
case "monthly":
|
||||
return last.AddDate(0, 1, 0)
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// Due reports whether the policy should run now.
|
||||
func (p *Policy) Due(now time.Time) bool {
|
||||
if !p.Enabled {
|
||||
return false
|
||||
}
|
||||
d := p.lastDue(now)
|
||||
return !d.IsZero() && p.LastRun.Before(d)
|
||||
}
|
||||
|
||||
// SnapTime reads a snapshot's name in zfSnap's convention and returns when it was taken,
|
||||
// when it expires, and whether the name parsed at all. A name without a TTL never expires.
|
||||
// An optional prefix in front of the stamp is allowed, so a policy can label its own.
|
||||
func SnapTime(name string) (taken time.Time, expires time.Time, ok bool) {
|
||||
_, snap, cut := strings.Cut(name, "@")
|
||||
if !cut {
|
||||
return
|
||||
}
|
||||
body, ttl, hasTTL := strings.Cut(snap, ttlSep)
|
||||
// an optional "<prefix>-" before the stamp
|
||||
stamp := body
|
||||
if len(body) > len(stampLayout) {
|
||||
stamp = body[len(body)-len(stampLayout):]
|
||||
}
|
||||
t, err := time.ParseInLocation(stampLayout, stamp, time.Local)
|
||||
if err != nil {
|
||||
return time.Time{}, time.Time{}, false
|
||||
}
|
||||
taken, ok = t, true
|
||||
if hasTTL {
|
||||
if d, err := ParseAge(ttl); err == nil {
|
||||
expires = t.Add(d)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Doomed is which of this dataset's snapshots have passed the expiry written into their own
|
||||
// names, oldest first — exactly what zfSnap -d would remove. This is what the UI shows
|
||||
// before anything is deleted, and what Run then acts on. KeepMin holds back that many of the
|
||||
// newest whatever their names say, so a dataset is never left with none.
|
||||
func (m *Manager) Doomed(ctx context.Context, p *Policy) ([]Snapshot, error) {
|
||||
if err := p.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all, err := m.Snapshots(ctx, p.Dataset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var dated []Snapshot
|
||||
expiry := map[string]time.Time{}
|
||||
for _, s := range all {
|
||||
taken, exp, ok := SnapTime(s.Name)
|
||||
if !ok || exp.IsZero() {
|
||||
continue // hand-made, or kept forever: not ours to remove
|
||||
}
|
||||
_ = taken
|
||||
dated = append(dated, s)
|
||||
expiry[s.Name] = exp
|
||||
}
|
||||
sort.Slice(dated, func(i, j int) bool { return dated[i].Created > dated[j].Created }) // newest first
|
||||
now := time.Now()
|
||||
var doomed []Snapshot
|
||||
for i, s := range dated {
|
||||
if i < p.KeepMin {
|
||||
continue // the floor: these survive however old they are
|
||||
}
|
||||
if expiry[s.Name].Before(now) {
|
||||
doomed = append(doomed, s)
|
||||
}
|
||||
}
|
||||
sort.Slice(doomed, func(i, j int) bool { return doomed[i].Created < doomed[j].Created }) // oldest first
|
||||
return doomed, nil
|
||||
}
|
||||
|
||||
// DestroySnapshot removes one snapshot. recursive also removes the same-named snapshot on
|
||||
// every dataset beneath it, which is what a recursive policy created.
|
||||
func (m *Manager) DestroySnapshot(ctx context.Context, full string, recursive bool) error {
|
||||
dataset, snap, ok := strings.Cut(full, "@")
|
||||
if !ok || !datasetRe.MatchString(dataset) || !snapRe.MatchString(snap) {
|
||||
return errors.New("bad snapshot name")
|
||||
}
|
||||
args := []string{"zfs", "destroy"}
|
||||
if recursive {
|
||||
args = append(args, "-r")
|
||||
}
|
||||
out, code, err := m.run(ctx, append(args, full)...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errors.New(strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SnapshotRecursive takes dataset@name, optionally of everything beneath it too.
|
||||
func (m *Manager) SnapshotRecursive(ctx context.Context, dataset, name string, recursive bool) error {
|
||||
if !datasetRe.MatchString(dataset) || !snapRe.MatchString(name) {
|
||||
return errors.New("bad snapshot name")
|
||||
}
|
||||
args := []string{"zfs", "snapshot"}
|
||||
if recursive {
|
||||
args = append(args, "-r")
|
||||
}
|
||||
out, code, err := m.run(ctx, append(args, dataset+"@"+name)...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errors.New(strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunResult is what one pass of a policy did.
|
||||
type RunResult struct {
|
||||
Taken string `json:"taken"`
|
||||
Pruned []string `json:"pruned"`
|
||||
Skipped int `json:"skipped"` // snapshots that are not this policy's, left alone
|
||||
}
|
||||
|
||||
// Run takes the snapshot and prunes what has aged out. It is the same code the scheduler
|
||||
// calls, so "Run now" and the timer cannot drift apart.
|
||||
func (m *Manager) Run(ctx context.Context, p *Policy) (RunResult, error) {
|
||||
var res RunResult
|
||||
if err := p.Validate(); err != nil {
|
||||
return res, err
|
||||
}
|
||||
name := time.Now().Format(stampLayout)
|
||||
if pre := strings.TrimSpace(p.Prefix); pre != "" {
|
||||
name = pre + "-" + name
|
||||
}
|
||||
if p.KeepFor != "" {
|
||||
name += ttlSep + strings.TrimSpace(strings.ToLower(p.KeepFor)) // zfSnap's convention: the retention lives in the name
|
||||
}
|
||||
if err := m.SnapshotRecursive(ctx, p.Dataset, name, p.Recursive); err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Taken = p.Dataset + "@" + name
|
||||
doomed, err := m.Doomed(ctx, p)
|
||||
if err != nil {
|
||||
return res, err // the snapshot was taken; the prune is what failed
|
||||
}
|
||||
for _, s := range doomed {
|
||||
if err := m.DestroySnapshot(ctx, s.Name, p.Recursive); err != nil {
|
||||
return res, fmt.Errorf("destroying %s: %w", s.Name, err)
|
||||
}
|
||||
res.Pruned = append(res.Pruned, s.Name)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSnapTimeReadsZfSnapNames(t *testing.T) {
|
||||
// the real thing off grid
|
||||
taken, exp, ok := SnapTime("tank@2026-09-24_02.30.01--6w")
|
||||
if !ok {
|
||||
t.Fatal("zfSnap's own name must parse")
|
||||
}
|
||||
if taken.Format("2006-01-02 15:04:05") != "2026-09-24 02:30:01" {
|
||||
t.Fatalf("taken = %v", taken)
|
||||
}
|
||||
if got := exp.Sub(taken); got != 6*7*24*time.Hour {
|
||||
t.Fatalf("ttl = %v, want 6w", got)
|
||||
}
|
||||
// a labelled one from a policy with a prefix
|
||||
if _, e, ok := SnapTime("tank/apps@nightly-2026-09-24_02.30.01--30d"); !ok || e.IsZero() {
|
||||
t.Fatal("a prefixed name must parse and expire")
|
||||
}
|
||||
// no TTL: never expires
|
||||
if _, e, ok := SnapTime("tank@2026-09-24_02.30.01"); !ok || !e.IsZero() {
|
||||
t.Fatal("a name without a TTL must parse and never expire")
|
||||
}
|
||||
// things that are not ours
|
||||
for _, n := range []string{"tank@before-upgrade", "tank@manual", "tank", "tank@2026-13-45_99.99.99--6w"} {
|
||||
if _, _, ok := SnapTime(n); ok {
|
||||
t.Errorf("%q must not parse as a scheduled snapshot", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleIsDueAndNext(t *testing.T) {
|
||||
now := time.Date(2026, 9, 24, 9, 15, 0, 0, time.Local)
|
||||
daily := &Policy{Every: "daily", At: "02:30", Enabled: true}
|
||||
if d := daily.lastDue(now); d.Hour() != 2 || d.Day() != 24 {
|
||||
t.Fatalf("daily lastDue = %v", d)
|
||||
}
|
||||
if !daily.Due(now) {
|
||||
t.Fatal("never run: must be due")
|
||||
}
|
||||
daily.LastRun = now.Add(-time.Hour) // ran at 08:15 today, after 02:30
|
||||
if daily.Due(now) {
|
||||
t.Fatal("already ran since the window: not due")
|
||||
}
|
||||
if n := daily.NextRun(now); n.Day() != 25 || n.Hour() != 2 {
|
||||
t.Fatalf("daily NextRun = %v", n)
|
||||
}
|
||||
// a machine that was off through its window runs once when it comes back, not never
|
||||
daily.LastRun = now.AddDate(0, 0, -2)
|
||||
if !daily.Due(now) {
|
||||
t.Fatal("a missed window must still be due")
|
||||
}
|
||||
// disabled never runs
|
||||
daily.Enabled = false
|
||||
if daily.Due(now) {
|
||||
t.Fatal("disabled must never be due")
|
||||
}
|
||||
weekly := &Policy{Every: "weekly", At: "04:00", Weekday: int(time.Sunday), Enabled: true}
|
||||
if d := weekly.lastDue(now); d.Weekday() != time.Sunday || d.After(now) {
|
||||
t.Fatalf("weekly lastDue = %v (%v)", d, d.Weekday())
|
||||
}
|
||||
hourly := &Policy{Every: "hourly", At: "00:30", Enabled: true}
|
||||
if d := hourly.lastDue(now); d.Minute() != 30 || d.Hour() != 8 {
|
||||
t.Fatalf("hourly lastDue at 09:15 with minute 30 = %v, want 08:30", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRefusesNonsense(t *testing.T) {
|
||||
ok := Policy{Dataset: "tank/apps", Every: "daily", At: "02:30", KeepFor: "6w"}
|
||||
if err := ok.Validate(); err != nil {
|
||||
t.Fatalf("a sane policy: %v", err)
|
||||
}
|
||||
bad := []Policy{
|
||||
{Dataset: "", Every: "daily", At: "02:30", KeepFor: "6w"},
|
||||
{Dataset: "tank", Every: "yearly", At: "02:30", KeepFor: "6w"},
|
||||
{Dataset: "tank", Every: "daily", At: "25:00", KeepFor: "6w"},
|
||||
{Dataset: "tank", Every: "daily", At: "02:30"}, // no retention
|
||||
{Dataset: "tank", Every: "daily", At: "02:30", KeepFor: "6 weeks"}, // unparseable
|
||||
{Dataset: "tank", Every: "monthly", At: "02:30", Day: 31, KeepFor: "6w"}, // not every month has one
|
||||
{Dataset: "tank", Every: "daily", At: "02:30", KeepFor: "6w", Prefix: "a--b"},
|
||||
}
|
||||
for i, p := range bad {
|
||||
if err := p.Validate(); err == nil {
|
||||
t.Errorf("bad policy %d was accepted: %+v", i, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAge(t *testing.T) {
|
||||
for s, want := range map[string]time.Duration{
|
||||
"6w": 6 * 7 * 24 * time.Hour, "30d": 30 * 24 * time.Hour,
|
||||
"12h": 12 * time.Hour, "90m": 90 * time.Minute,
|
||||
} {
|
||||
if got, err := ParseAge(s); err != nil || got != want {
|
||||
t.Errorf("ParseAge(%q) = %v, %v; want %v", s, got, err, want)
|
||||
}
|
||||
}
|
||||
for _, s := range []string{"", "6", "w", "-1d", "0w", "6y", "six weeks"} {
|
||||
if _, err := ParseAge(s); err == nil {
|
||||
t.Errorf("ParseAge(%q) should fail", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user