mitchell/hamlab
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 14 Wiki Activity

Space keeps its scans and refreshes them: zfs diff, folder mtimes, resume after cancel

Browse Source 
A scan is saved beside the config and read back after a restart. The next scan starts
from it: on a pool with snapshots zfs diff names what changed and only those folders are
read; elsewhere a folder whose mtime is unchanged keeps its list and only its kept files
are stat'ed again. A cancelled scan keeps what it walked and Resume reads the rest.
Forget drops the saved result. The treemap is every file a tile, grouped by folder, with
no nested boxes: the folder chains of a Linux system made a staircase of empty headers.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
2026-09-23 00:43:30 -07:00
co-authored by Claude Fable 5.1
parent c96d6facad
commit 53d1da08a5
11 changed files with 869 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/space/walk.go
+180 -22
View File
@@ -9,6 +9,14 @@
// Files below KeepMin are not kept one by one: a directory of a million small files is
// one "smaller files" entry with their count and total, which is all a treemap can show
// of them anyway, and what keeps a scan of a few million files within a few hundred MB.
//
// A walk can start from the last one. Given the previous tree, a directory is read again
// only when something says it changed: on ZFS, `zfs diff` against a snapshot older than
// the last scan names every path that changed, and directories it does not name are kept
// as they were without so much as a stat; elsewhere a directory whose mtime is what it was
// keeps its list and only its kept files are stat'ed again, since a file can grow without
// its directory noticing. A directory the last walk did not finish (it was cancelled) is
// read again whatever else is true, which is what makes a cancelled scan resumable.
package space
import (
@@ -34,34 +42,59 @@ type Node struct {
Size int64 `json:"size"`
Apparent int64 `json:"apparent"`
MTime int64 `json:"mtime"` // unix seconds
MTimeNs int64 `json:"-"` // directories: the same to the nanosecond, so a change in the same second is not missed
Files uint32 `json:"files"` // beneath a directory (kept and small alike)
Dirs uint32 `json:"dirs"`
SmallN uint32 `json:"small_n,omitempty"` // files here below KeepMin, not listed
SmallSize int64 `json:"small_size,omitempty"` // their allocated bytes
Mount bool `json:"mount,omitempty"` // another filesystem starts here (not this disk's, or a root of its own)
Denied bool `json:"denied,omitempty"` // could not be read
Partial bool `json:"partial,omitempty"` // the walk was cancelled before this directory was read
Children []*Node `json:"children,omitempty"` // directories: sorted by Size, largest first
}
// Progress is how far a running walk has got.
type Progress struct {
Files int64 `json:"files"`
Dirs int64 `json:"dirs"`
Dirs int64 `json:"dirs"` // directories read
Reused int64 `json:"reused"` // directories kept from the last scan without reading
Bytes int64 `json:"bytes"`
Denied int64 `json:"denied"`
Current string `json:"current"`
}
// Mode is how a root is walked when there is a previous tree.
type Mode int
const (
ModeFull Mode = iota // read everything
ModeMTime // read a directory again only if its mtime changed; stat its kept files
ModeDiff // read only the directories a change list names
)
// Options is what a walk starts from.
type Options struct {
Prev *Node // the last scan's tree, or nil
// Modes says how each root is walked (by root path); a root not listed is ModeFull.
Modes map[string]Mode
// Dirty is every directory a change list named (ModeDiff roots): those are read again.
// Touched is every ancestor of a dirty directory: those are kept but looked into.
Dirty, Touched map[string]bool
}
type walker struct {
ctx context.Context
opts Options
mu sync.Mutex
queue []job
cond *sync.Cond
pending int // dirs queued or being read
files atomic.Int64
dirs atomic.Int64
reused atomic.Int64
bytes atomic.Int64
denied atomic.Int64
partial atomic.Int64
curMu sync.Mutex
current string
stopping bool
@@ -70,13 +103,25 @@ type walker struct {
type job struct {
path string
node *Node
prev *Node // the same directory in the last scan, if it had one
dev uint64 // the filesystem this root is on; a child on another is a mountpoint, not descended
mode Mode
}
// Result is what a walk returns besides the tree.
type Result struct {
Root *Node
Partial int64 // directories not read because the walk was cancelled
Reused int64
Read int64
Complete bool
}
// Walk measures roots (mountpoints, absolute) and returns them grafted into one tree whose
// root is "/". progress, if set, is called about four times a second.
func Walk(ctx context.Context, roots []string, progress func(Progress)) (*Node, error) {
w := &walker{ctx: ctx}
// root is "/". progress, if set, is called about four times a second. A cancelled walk
// still returns the tree, with the directories it did not get to marked Partial.
func Walk(ctx context.Context, roots []string, opts Options, progress func(Progress)) (Result, error) {
w := &walker{ctx: ctx, opts: opts}
w.cond = sync.NewCond(&w.mu)
root := &Node{Name: "/", Dir: true}
sort.Strings(roots)
@@ -90,15 +135,23 @@ func Walk(ctx context.Context, roots []string, progress func(Progress)) (*Node,
if err := syscall.Lstat(r, &st); err != nil || st.Mode&syscall.S_IFMT != syscall.S_IFDIR {
continue
}
n := &Node{Name: filepath.Base(r), Dir: true, MTime: int64(st.Mtim.Sec)}
n := &Node{Name: filepath.Base(r), Dir: true, MTime: int64(st.Mtim.Sec), MTimeNs: st.Mtim.Nano()}
if r == "/" {
n = root
}
trees[r] = n
w.push(job{path: r, node: n, dev: uint64(st.Dev)})
var prev *Node
if opts.Prev != nil {
prev = Find(opts.Prev, r)
}
mode := opts.Modes[r]
if prev == nil {
mode = ModeFull
}
w.push(job{path: r, node: n, prev: prev, dev: uint64(st.Dev), mode: mode})
}
if len(trees) == 0 {
return root, os.ErrNotExist
return Result{Root: root}, os.ErrNotExist
}
workers := 32
var wg sync.WaitGroup
@@ -110,6 +163,7 @@ func Walk(ctx context.Context, roots []string, progress func(Progress)) (*Node,
go func() { wg.Wait(); close(done) }()
tick := time.NewTicker(250 * time.Millisecond)
defer tick.Stop()
cancelled := false
for waiting := true; waiting; {
select {
case <-done:
@@ -119,13 +173,18 @@ func Walk(ctx context.Context, roots []string, progress func(Progress)) (*Node,
progress(w.snapshot())
}
case <-ctx.Done():
cancelled = true
w.mu.Lock()
w.stopping = true
for _, j := range w.queue { // never read: say so, so the next walk reads them
j.node.Partial = true
w.partial.Add(1)
}
w.queue = nil
w.cond.Broadcast()
w.mu.Unlock()
<-done
return root, ctx.Err()
waiting = false
}
}
if progress != nil {
@@ -140,18 +199,28 @@ func Walk(ctx context.Context, roots []string, progress func(Progress)) (*Node,
graft(root, filepath.Clean(r), n)
}
total(root)
return root, nil
res := Result{Root: root, Partial: w.partial.Load(), Reused: w.reused.Load(), Read: w.dirs.Load(), Complete: !cancelled}
if cancelled {
return res, ctx.Err()
}
return res, nil
}
func (w *walker) snapshot() Progress {
w.curMu.Lock()
cur := w.current
w.curMu.Unlock()
return Progress{Files: w.files.Load(), Dirs: w.dirs.Load(), Bytes: w.bytes.Load(), Denied: w.denied.Load(), Current: cur}
return Progress{Files: w.files.Load(), Dirs: w.dirs.Load(), Reused: w.reused.Load(), Bytes: w.bytes.Load(), Denied: w.denied.Load(), Current: cur}
}
func (w *walker) push(j job) {
w.mu.Lock()
if w.stopping {
j.node.Partial = true
w.partial.Add(1)
w.mu.Unlock()
return
}
w.queue = append(w.queue, j)
w.pending++
w.cond.Signal()
@@ -173,7 +242,7 @@ func (w *walker) run() {
j := w.queue[len(w.queue)-1] // depth first: keeps the queue short and the disk head local
w.queue = w.queue[:len(w.queue)-1]
w.mu.Unlock()
w.readDir(j)
w.visit(j)
w.mu.Lock()
w.pending--
if w.pending == 0 {
@@ -183,6 +252,92 @@ func (w *walker) run() {
}
}
func join(dir, name string) string {
if dir == "/" {
return "/" + name
}
return dir + "/" + name
}
// visit decides whether a directory must be read or can be taken from the last scan.
func (w *walker) visit(j job) {
if j.prev == nil || j.prev.Partial || j.prev.Denied || j.mode == ModeFull {
w.readDir(j)
return
}
switch j.mode {
case ModeDiff:
if w.opts.Dirty[j.path] {
w.readDir(j)
return
}
if !w.opts.Touched[j.path] {
w.reuse(j, false) // nothing under here changed: keep the whole subtree, no stat at all
return
}
w.reuse(j, true) // something deeper changed: keep this list, look into the children
case ModeMTime:
var st syscall.Stat_t
if err := syscall.Lstat(j.path, &st); err != nil {
w.readDir(j)
return
}
if st.Mtim.Nano() != j.prev.MTimeNs {
w.readDir(j) // an entry was added, removed or renamed
return
}
w.reuse(j, true) // same entries; files may have grown, and subdirectories are their own question
}
}
// reuse keeps a directory as the last scan saw it. With descend, its subdirectories are
// visited (each decides for itself) and, in mtime mode, its kept files are stat'ed again.
func (w *walker) reuse(j job, descend bool) {
w.reused.Add(1)
n, p := j.node, j.prev
n.MTime, n.MTimeNs, n.SmallN, n.SmallSize, n.Denied, n.Mount = p.MTime, p.MTimeNs, p.SmallN, p.SmallSize, p.Denied, p.Mount
if !descend {
n.Children = p.Children
n.Apparent = p.Apparent - childApparent(p) // total() adds the children back
w.files.Add(int64(p.Files))
w.bytes.Add(p.Size)
w.reused.Add(int64(p.Dirs))
return
}
w.files.Add(int64(p.SmallN))
w.bytes.Add(p.SmallSize)
n.Apparent = p.Apparent - childApparent(p) // the small files' lengths; total() adds the children back
var st syscall.Stat_t
for _, c := range p.Children {
if c.Dir {
d := &Node{Name: c.Name, Dir: true, MTime: c.MTime, MTimeNs: c.MTimeNs, Mount: c.Mount}
n.Children = append(n.Children, d)
if c.Mount {
continue
}
w.push(job{path: join(j.path, c.Name), node: d, prev: c, dev: j.dev, mode: j.mode})
continue
}
if j.mode == ModeMTime {
if err := syscall.Lstat(join(j.path, c.Name), &st); err != nil {
continue // gone since (the directory's mtime would have said so, but be safe)
}
c = &Node{Name: c.Name, Size: int64(st.Blocks) * 512, Apparent: int64(st.Size), MTime: int64(st.Mtim.Sec)}
}
n.Children = append(n.Children, c)
w.files.Add(1)
w.bytes.Add(c.Size)
}
}
func childApparent(p *Node) int64 {
var a int64
for _, c := range p.Children {
a += c.Apparent
}
return a
}
func (w *walker) readDir(j job) {
w.curMu.Lock()
w.current = j.path
@@ -200,24 +355,30 @@ func (w *walker) readDir(j job) {
j.node.Denied = true
w.denied.Add(1)
}
var prevDirs map[string]*Node
if j.prev != nil {
prevDirs = map[string]*Node{}
for _, c := range j.prev.Children {
if c.Dir {
prevDirs[c.Name] = c
}
}
}
var st syscall.Stat_t
for _, name := range names {
p := j.path + "/" + name
if j.path == "/" {
p = "/" + name
}
p := join(j.path, name)
if err := syscall.Lstat(p, &st); err != nil {
continue
}
switch st.Mode & syscall.S_IFMT {
case syscall.S_IFDIR:
c := &Node{Name: name, Dir: true, MTime: int64(st.Mtim.Sec)}
c := &Node{Name: name, Dir: true, MTime: int64(st.Mtim.Sec), MTimeNs: st.Mtim.Nano()}
j.node.Children = append(j.node.Children, c)
if uint64(st.Dev) != j.dev {
c.Mount = true // something else is mounted here; if it is this disk's too, it is a root of its own
continue
}
w.push(job{path: p, node: c, dev: j.dev})
w.push(job{path: p, node: c, prev: prevDirs[name], dev: j.dev, mode: j.mode})
case syscall.S_IFREG, syscall.S_IFLNK:
alloc := int64(st.Blocks) * 512
w.files.Add(1)
@@ -250,7 +411,7 @@ func graft(root *Node, path string, tree *Node) {
cur.Children = append(cur.Children, tree)
} else {
// the parent's walk saw this directory as a mountpoint; the walk of the mount fills it
next.Children, next.SmallN, next.SmallSize, next.Apparent, next.Denied = tree.Children, tree.SmallN, tree.SmallSize, tree.Apparent, tree.Denied
next.Children, next.SmallN, next.SmallSize, next.Apparent, next.Denied, next.Partial = tree.Children, tree.SmallN, tree.SmallSize, tree.Apparent, tree.Denied, tree.Partial
next.Mount = true
}
return
@@ -324,10 +485,7 @@ func Largest(root *Node, n int) []File {
var walk func(n *Node, path string)
walk = func(node *Node, path string) {
for _, c := range node.Children {
p := path + "/" + c.Name
if path == "/" {
p = "/" + c.Name
}
p := join(path, c.Name)
if c.Dir {
if c.Size > 0 && (len(out) < n || c.Size > out[minIdx].Size) {
walk(c, p)