ZFS tab: landing tier checks — stale files, verify passes against the mover's ledger, verify schedules

A verify pass reads the pool's copy of every file the mover moved back against the SHA-256 the
mover took of the landing copy (its ledger, named in its status file): match, mismatch, edited
since, gone, waiting, unreadable, unhashed. 'New moves' checks what moved since the last pass,
'everything' the whole ledger. Passes run as 'hamlab landing-verify' in their own session — the
same command a schedule's line in /etc/cron.d/hamlab-landing runs — so a console restart does
not cut one short. Stale files: anything waiting on the landing branch over an hour, listed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/storage/landing_test.go

@@ -0,0 +1,137 @@
+package storage
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestVerifyClassifiesEveryCase(t *testing.T) {
+	root := t.TempDir()
+	src, dst := filepath.Join(root, "landing"), filepath.Join(root, "pool")
+	for _, d := range []string{src, dst + "/a", src + "/a"} {
+		if err := os.MkdirAll(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	mtime := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC)
+	put := func(p, body string) LedgerEntry {
+		if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		_ = os.Chtimes(p, mtime, mtime)
+		sum := sha256.Sum256([]byte(body))
+		return LedgerEntry{Size: int64(len(body)), MTimeNs: mtime.UnixNano(), SHA256: hex.EncodeToString(sum[:])}
+	}
+	var ledger []LedgerEntry
+	add := func(rel string, e LedgerEntry, at int64) { e.Path, e.T = rel, at; ledger = append(ledger, e) }
+
+	add("a/match.bin", put(dst+"/a/match.bin", "intact"), 10)
+	e := put(dst+"/a/mismatch.bin", "garbled") // same size and mtime as what was hashed, other bytes
+	sum := sha256.Sum256([]byte("GARBLED"))
+	e.SHA256 = hex.EncodeToString(sum[:])
+	add("a/mismatch.bin", e, 11)
+	e = put(dst+"/a/changed.bin", "edited since, longer now")
+	e.Size = 3
+	add("a/changed.bin", e, 12)
+	add("a/gone.bin", LedgerEntry{Size: 4, MTimeNs: mtime.UnixNano(), SHA256: "00"}, 13)
+	put(src+"/a/waiting.bin", "back again")
+	add("a/waiting.bin", LedgerEntry{Size: 10, MTimeNs: mtime.UnixNano(), SHA256: "00"}, 14)
+	e = put(dst+"/a/unhashed", "x")
+	e.SHA256, e.Note = "", "not a regular file"
+	add("a/unhashed", e, 15)
+	add("../escape", LedgerEntry{Size: 1, SHA256: "00"}, 16)
+	// moved twice: only the newer entry counts, and it matches
+	add("a/twice.bin", LedgerEntry{Size: 99, SHA256: "stale"}, 5)
+	add("a/twice.bin", put(dst+"/a/twice.bin", "second time"), 17)
+
+	lp := filepath.Join(root, "ledger.jsonl")
+	f, _ := os.Create(lp)
+	for _, e := range ledger {
+		b, _ := json.Marshal(e)
+		f.Write(append(b, '\n'))
+	}
+	f.WriteString("not json\n")
+	f.Close()
+
+	tg := LandingTarget{Src: src, Dst: dst, Ledger: lp}
+	r := Verify(tg, "manual", "all", 0, nil)
+	want := map[string]int{"checked": 8, "matched": 2, "mismatched": 1, "changed": 1, "gone": 1, "waiting": 1, "unhashed": 1, "unreadable": 1}
+	got := map[string]int{"checked": r.Checked, "matched": r.Matched, "mismatched": r.Mismatched, "changed": r.Changed,
+		"gone": r.Gone, "waiting": r.Waiting, "unhashed": r.Unhashed, "unreadable": r.Unreadable}
+	for k, v := range want {
+		if got[k] != v {
+			t.Errorf("%s: got %d want %d (%+v)", k, got[k], v, r)
+		}
+	}
+	if r.OK() || r.Cursor != 17 || r.Error != "" {
+		t.Errorf("ok=%v cursor=%d err=%q", r.OK(), r.Cursor, r.Error)
+	}
+	kinds := map[string]string{}
+	for _, i := range r.Issues {
+		kinds[i.Path] = i.Kind
+	}
+	if kinds["a/mismatch.bin"] != "mismatch" || kinds["a/gone.bin"] != "gone" || kinds["../escape"] != "unreadable" {
+		t.Errorf("issues %+v", r.Issues)
+	}
+
+	// "new" after a pass at cursor 14 only looks at what moved later
+	r = Verify(tg, "manual", "new", 14, nil)
+	if r.Checked != 3 || r.Matched != 1 || r.Unhashed != 1 || r.Unreadable != 1 {
+		t.Errorf("new since 14: %+v", r)
+	}
+
+	// records: the cursor only advances on a pass that read the ledger
+	dir := t.TempDir()
+	if err := RecordVerify(dir, tg, Verify(tg, "manual", "all", 0, nil)); err != nil {
+		t.Fatal(err)
+	}
+	if c := VerifyCursor(dir, tg); c != 17 {
+		t.Errorf("cursor %d", c)
+	}
+	bad := Verify(LandingTarget{Src: src, Dst: dst, Ledger: filepath.Join(root, "missing")}, "manual", "new", 0, nil)
+	if !strings.Contains(bad.Error, "no ledger yet") {
+		t.Errorf("missing ledger: %q", bad.Error)
+	}
+	if len(VerifyHistory(dir)) != 1 || LastVerify(dir, "manual") == nil {
+		t.Errorf("history/last not kept")
+	}
+}
+
+func TestVerifyPolicy(t *testing.T) {
+	p := VerifyPolicy{ID: "abc", Name: "nightly", Every: "daily", At: "05:10", Scope: "new", Enabled: true}
+	if err := p.Validate(); err != nil {
+		t.Fatal(err)
+	}
+	if x, _ := p.CronExpr(); x != "10 5 * * *" {
+		t.Fatalf("cron %q", x)
+	}
+	txt, err := VerifyCronText([]VerifyPolicy{p, {ID: "off", Name: "paused one", Every: "hourly", Scope: "all"}}, "core", "/usr/local/bin/hamlab", "/cfg.json")
+	if err != nil || !strings.Contains(txt, "10 5 * * * core /usr/local/bin/hamlab -config /cfg.json landing-verify abc") || !strings.Contains(txt, "# paused one — paused") {
+		t.Fatalf("%v\n%s", err, txt)
+	}
+	for _, b := range []VerifyPolicy{{Every: "daily", Scope: "some"}, {Every: "sometimes", Scope: "new"}, {Every: "custom", Scope: "new"}} {
+		if b.Validate() == nil {
+			t.Errorf("accepted %+v", b)
+		}
+	}
+}
+
+func TestStaleFiles(t *testing.T) {
+	d := t.TempDir()
+	now := time.Now()
+	os.MkdirAll(d+"/x", 0o755)
+	for name, age := range map[string]time.Duration{"x/old.bin": 3 * time.Hour, "x/older.bin": 5 * time.Hour, "fresh.bin": time.Minute} {
+		os.WriteFile(filepath.Join(d, name), []byte("12345"), 0o644)
+		os.Chtimes(filepath.Join(d, name), now.Add(-age), now.Add(-age))
+	}
+	files, n, b := StaleFiles(d, 1, now)
+	if n != 2 || b != 10 || len(files) != 1 || files[0].Path != "x/older.bin" {
+		t.Fatalf("%d %d %+v", n, b, files)
+	}
+}