Space keeps its scans and refreshes them: zfs diff, folder mtimes, resume after cancel
A scan is saved beside the config and read back after a restart. The next scan starts from it: on a pool with snapshots zfs diff names what changed and only those folders are read; elsewhere a folder whose mtime is unchanged keeps its list and only its kept files are stat'ed again. A cancelled scan keeps what it walked and Resume reads the rest. Forget drops the saved result. The treemap is every file a tile, grouped by folder, with no nested boxes: the folder chains of a Linux system made a staircase of empty headers. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
c96d6facad
commit
53d1da08a5
@@ -346,10 +346,27 @@ something long in one, switch away and back, refresh — it's all still there.
|
||||
databases & logs, other; the validated categorical set, dark and light) with folders as
|
||||
labelled boxes; hover for path, size and share; click a folder to drill, breadcrumbs and ↑
|
||||
to come back. The map asks the server for the folder pruned to what a tile could show
|
||||
(`min` = 1/4000 of the folder, depth 12), the list one level at a time. **Largest files**:
|
||||
the 1,000 biggest with a path filter. Works on every machine through the hub. Not kept
|
||||
across a restart; not done: delete from here, a size-over-time history, per-user column
|
||||
choices.
|
||||
(`min` = 1/6000 of the folder, 24 deep; deeper folders come back as one lump), the list
|
||||
one level at a time. **Largest files**: the 1,000 biggest with a path filter. Works on
|
||||
every machine through the hub.
|
||||
**Smarter and faster (2026-09-23, at Mitchell's request):** a scan is *kept* — in memory
|
||||
and in `<data>/space/<target>.scan` (the same varint stream) + `.json`, read back after a
|
||||
restart — and the next scan *starts from it*: the helper is handed the last tree on stdin
|
||||
and reads a directory again only when something says it changed. On a ZFS pool with
|
||||
snapshots, the scan records the newest snapshot of each dataset at start (`bases`), and the
|
||||
next one runs `zfs diff -H -F <base>` per dataset: only the directories it names are
|
||||
read, their ancestors kept but looked into, everything else kept wholesale without a
|
||||
stat. Elsewhere (ext4, or a dataset with no snapshot) a directory whose mtime — to the
|
||||
nanosecond — is unchanged keeps its list and only its kept files are stat'ed again, since
|
||||
a file can grow without its directory noticing. A **cancelled** scan keeps what it walked,
|
||||
marks the folders it never got to (`partial`, "not yet read" in the tree) and **Resume**
|
||||
reads just those. core: full 1.2 s → refresh 0.3 s (2 folders read, 84,755 kept). The
|
||||
status line says which (`method`, `by_root`, read/kept counts); **Forget** drops the saved
|
||||
result so the next scan is full. The old result stays browsable while a refresh runs.
|
||||
**Treemap rethought** the same day: every file a tile grouped by folder — folders are only
|
||||
a thin line and a name strip on the first level — instead of nested boxes with headers,
|
||||
which on Linux's single-child folder chains was a staircase of headers around nothing.
|
||||
Not done: delete from here, a size-over-time history, per-user column choices.
|
||||
- **Auth hardening** — WebAuthn passkeys (phone Face ID), TOTP fallback, optional NPM
|
||||
access list restricting to WireGuard/LAN.
|
||||
- **Files tab** — **shipped 2026-09-18** (`internal/files`, `Files.tsx`): browse from
|
||||
|
||||
+1
-1
@@ -77,7 +77,7 @@ func main() {
|
||||
case "fs": // privileged helper for the Files tab (run as root by the daemon)
|
||||
os.Exit(files.Helper(flag.Args()[1:], os.Stdin, os.Stdout, os.Stderr))
|
||||
case "space-walk": // helper for the Space tab: measure mountpoints, stream the tree back
|
||||
os.Exit(space.Helper(flag.Args()[1:], os.Stdout, os.Stderr))
|
||||
os.Exit(space.Helper(flag.Args()[1:], os.Stdin, os.Stdout, os.Stderr))
|
||||
case "install", "uninstall":
|
||||
var err error
|
||||
if cmd == "install" {
|
||||
|
||||
@@ -85,11 +85,11 @@ type Server struct {
|
||||
func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *metrics.Collector, st *store.Store, svc *services.Manager, cr *cron.Manager, tm *tasks.Manager, up *updates.Manager, sto *storage.Manager, fsLocal, fsSudo files.FS, nm *nodes.Manager, fingerprint string, ui fs.FS, version string) *Server {
|
||||
s := &Server{cfg: cfg, sessions: sessions, mgr: mgr, metrics: col, sensors: sensors.NewRegistry(col), store: st, svc: svc, cron: cr, tasks: tm, updates: up, storage: sto, fsLocal: fsLocal, fsSudo: fsSudo, nodes: nm, fingerprint: fingerprint, ui: ui, version: version,
|
||||
started: time.Now(), user: os.Getenv("USER")}
|
||||
if self, err := os.Executable(); err == nil {
|
||||
s.space = space.New(self)
|
||||
} else {
|
||||
s.space = space.New("hamlab")
|
||||
self, err := os.Executable()
|
||||
if err != nil {
|
||||
self = "hamlab"
|
||||
}
|
||||
s.space = space.New(self, filepath.Join(cfg.DataDir(), "space"))
|
||||
s.hostName() // read it once now so the first request doesn't have to
|
||||
if u, err := user.Current(); err == nil && u.Username != "" {
|
||||
s.user = u.Username
|
||||
@@ -260,6 +260,7 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me
|
||||
m.HandleFunc("GET /api/space/{disk}", s.auth(s.spaceStatus))
|
||||
m.HandleFunc("POST /api/space/{disk}/scan", s.auth(s.spaceScan))
|
||||
m.HandleFunc("POST /api/space/{disk}/cancel", s.auth(s.spaceCancel))
|
||||
m.HandleFunc("DELETE /api/space/{disk}", s.auth(s.spaceForget))
|
||||
m.HandleFunc("GET /api/space/{disk}/tree", s.auth(s.spaceTree))
|
||||
m.HandleFunc("GET /api/space/{disk}/largest", s.auth(s.spaceLargest))
|
||||
m.HandleFunc("GET /api/procs", s.auth(s.procList))
|
||||
|
||||
+22
-11
@@ -35,6 +35,8 @@ type spaceTarget struct {
|
||||
Disks []string `json:"disks,omitempty"` // pool: its member disks
|
||||
Roots []string `json:"roots"` // the mountpoints a scan walks
|
||||
Scan *space.Scan `json:"scan,omitempty"` // the last scan, if any
|
||||
|
||||
datasets map[string]string // root → ZFS dataset, for the ones that are
|
||||
}
|
||||
|
||||
var layoutRe = regexp.MustCompile(`\b(draid\d*|raidz[123]?|mirror)\b`)
|
||||
@@ -54,9 +56,10 @@ func (s *Server) spaceTargets(ctx context.Context) []spaceTarget {
|
||||
inPool = true
|
||||
pt := pools[p.Label]
|
||||
if pt == nil {
|
||||
pt = &spaceTarget{ID: "pool:" + p.Label, Kind: "pool", Name: p.Label, Roots: zfsMounts(ctx, p.Label)}
|
||||
if pt.Roots == nil {
|
||||
pt.Roots = []string{}
|
||||
pt = &spaceTarget{ID: "pool:" + p.Label, Kind: "pool", Name: p.Label, Roots: []string{}, datasets: map[string]string{}}
|
||||
for _, zm := range zfsMounts(ctx, p.Label) {
|
||||
pt.Roots = append(pt.Roots, zm.mount)
|
||||
pt.datasets[zm.mount] = zm.dataset
|
||||
}
|
||||
pools[p.Label] = pt
|
||||
poolOrder = append(poolOrder, p.Label)
|
||||
@@ -101,19 +104,21 @@ func (s *Server) spaceTargets(ctx context.Context) []spaceTarget {
|
||||
return out
|
||||
}
|
||||
|
||||
// zfsMounts is every mounted filesystem of a pool.
|
||||
func zfsMounts(ctx context.Context, pool string) []string {
|
||||
type zfsMount struct{ mount, dataset string }
|
||||
|
||||
// zfsMounts is every mounted filesystem of a pool, with its dataset.
|
||||
func zfsMounts(ctx context.Context, pool string) []zfsMount {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, "zfs", "list", "-H", "-t", "filesystem", "-r", "-o", "mountpoint,mounted", pool).Output()
|
||||
out, err := exec.CommandContext(ctx, "zfs", "list", "-H", "-t", "filesystem", "-r", "-o", "name,mountpoint,mounted", pool).Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var mounts []string
|
||||
var mounts []zfsMount
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
f := strings.Split(line, "\t")
|
||||
if len(f) == 2 && f[1] == "yes" && strings.HasPrefix(f[0], "/") {
|
||||
mounts = append(mounts, f[0])
|
||||
if len(f) == 3 && f[2] == "yes" && strings.HasPrefix(f[1], "/") {
|
||||
mounts = append(mounts, zfsMount{mount: f[1], dataset: f[0]})
|
||||
}
|
||||
}
|
||||
return mounts
|
||||
@@ -152,7 +157,7 @@ func (s *Server) spaceScan(w http.ResponseWriter, r *http.Request) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
st, err := s.space.Start(t.ID, t.Roots)
|
||||
st, err := s.space.Start(t.ID, t.Roots, t.datasets)
|
||||
if err != nil {
|
||||
jsonErr(w, 409, err.Error())
|
||||
return
|
||||
@@ -161,6 +166,12 @@ func (s *Server) spaceScan(w http.ResponseWriter, r *http.Request) {
|
||||
jsonOK(w, 202, st)
|
||||
}
|
||||
|
||||
func (s *Server) spaceForget(w http.ResponseWriter, r *http.Request) {
|
||||
s.space.Forget(r.PathValue("disk"))
|
||||
log.Printf("space: scan of %s forgotten by %s", r.PathValue("disk"), clientIP(r))
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
func (s *Server) spaceCancel(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.space.Cancel(r.PathValue("disk")) {
|
||||
jsonErr(w, 404, "no scan running")
|
||||
@@ -176,7 +187,7 @@ func (s *Server) spaceTree(w http.ResponseWriter, r *http.Request) {
|
||||
path = "/"
|
||||
}
|
||||
depth, _ := strconv.Atoi(q.Get("depth"))
|
||||
if depth < 1 || depth > 16 {
|
||||
if depth < 1 || depth > 32 {
|
||||
depth = 1
|
||||
}
|
||||
limit, _ := strconv.Atoi(q.Get("limit"))
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// a few bytes per node, decoded without reflection, so a million nodes take well under a
|
||||
// second either way.
|
||||
|
||||
const codecMagic = "HLSPACE1"
|
||||
const codecMagic = "HLSPACE2"
|
||||
|
||||
func Encode(w io.Writer, root *Node) error {
|
||||
bw := bufio.NewWriterSize(w, 1<<20)
|
||||
@@ -39,11 +39,15 @@ func Encode(w io.Writer, root *Node) error {
|
||||
if n.Denied {
|
||||
flags |= 4
|
||||
}
|
||||
if n.Partial {
|
||||
flags |= 8
|
||||
}
|
||||
for _, f := range []func() error{
|
||||
func() error { return putU(flags) },
|
||||
func() error { return putI(n.Size) },
|
||||
func() error { return putI(n.Apparent) },
|
||||
func() error { return putI(n.MTime) },
|
||||
func() error { return putI(n.MTimeNs) },
|
||||
func() error { return putU(uint64(n.Files)) },
|
||||
func() error { return putU(uint64(n.Dirs)) },
|
||||
func() error { return putU(uint64(n.SmallN)) },
|
||||
@@ -91,7 +95,7 @@ func Decode(r io.Reader) (*Node, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n.Dir, n.Mount, n.Denied = flags&1 != 0, flags&2 != 0, flags&4 != 0
|
||||
n.Dir, n.Mount, n.Denied, n.Partial = flags&1 != 0, flags&2 != 0, flags&4 != 0, flags&8 != 0
|
||||
if n.Size, err = binary.ReadVarint(br); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -101,6 +105,9 @@ func Decode(r io.Reader) (*Node, error) {
|
||||
if n.MTime, err = binary.ReadVarint(br); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if n.MTimeNs, err = binary.ReadVarint(br); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u, err := binary.ReadUvarint(br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
+467
-61
@@ -11,6 +11,9 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
@@ -23,9 +26,26 @@ import (
|
||||
//
|
||||
// The walk runs in its own process so that root can read every directory, so that the
|
||||
// memory of the walk is given back when it ends, and so that a runaway scan can be killed
|
||||
// without touching the console. Progress goes to stderr as JSON lines; the tree to stdout.
|
||||
// without touching the console. The daemon writes a header line to the helper's stdin
|
||||
// (how each root is to be walked, and what changed) followed by the last scan's tree when
|
||||
// there is one, then holds stdin open: EOF means the daemon has gone or wants the walk
|
||||
// stopped. Progress goes to stderr as JSON lines; a result line and the tree to stdout.
|
||||
|
||||
func Helper(args []string, stdout, stderr io.Writer) int {
|
||||
type header struct {
|
||||
Prev bool `json:"prev"`
|
||||
Modes map[string]Mode `json:"modes,omitempty"`
|
||||
Dirty []string `json:"dirty,omitempty"`
|
||||
Touched []string `json:"touched,omitempty"`
|
||||
}
|
||||
|
||||
type resultLine struct {
|
||||
Complete bool `json:"complete"`
|
||||
Partial int64 `json:"partial"`
|
||||
Reused int64 `json:"reused"`
|
||||
Read int64 `json:"read"`
|
||||
}
|
||||
|
||||
func Helper(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprintln(stderr, "usage: hamlab space-walk <mountpoint>...")
|
||||
return 2
|
||||
@@ -35,24 +55,57 @@ func Helper(args []string, stdout, stderr io.Writer) int {
|
||||
sig := make(chan os.Signal, 1)
|
||||
signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
|
||||
go func() { <-sig; cancel() }()
|
||||
go func() { _, _ = io.Copy(io.Discard, os.Stdin); cancel() }() // the daemon went away
|
||||
in := bufio.NewReaderSize(stdin, 1<<20)
|
||||
line, err := in.ReadBytes('\n')
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "error: no header:", err)
|
||||
return 2
|
||||
}
|
||||
var h header
|
||||
if err := json.Unmarshal(line, &h); err != nil {
|
||||
fmt.Fprintln(stderr, "error: bad header:", err)
|
||||
return 2
|
||||
}
|
||||
opts := Options{Modes: h.Modes, Dirty: set(h.Dirty), Touched: set(h.Touched)}
|
||||
if h.Prev {
|
||||
if opts.Prev, err = Decode(in); err != nil {
|
||||
fmt.Fprintln(stderr, "error: last tree:", err)
|
||||
return 2
|
||||
}
|
||||
}
|
||||
go func() { _, _ = io.Copy(io.Discard, in); cancel() }() // the daemon went away, or asked us to stop
|
||||
enc := json.NewEncoder(stderr)
|
||||
root, err := Walk(ctx, args, func(p Progress) { _ = enc.Encode(p) })
|
||||
res, err := Walk(ctx, args, opts, func(p Progress) { _ = enc.Encode(p) })
|
||||
if err != nil && !errors.Is(err, context.Canceled) {
|
||||
fmt.Fprintln(stderr, "error:", err)
|
||||
return 1
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return 3
|
||||
rl, _ := json.Marshal(resultLine{Complete: res.Complete, Partial: res.Partial, Reused: res.Reused, Read: res.Read})
|
||||
if _, err := stdout.Write(append(rl, '\n')); err != nil {
|
||||
return 1
|
||||
}
|
||||
if err := Encode(stdout, root); err != nil {
|
||||
if err := Encode(stdout, res.Root); err != nil {
|
||||
fmt.Fprintln(stderr, "error:", err)
|
||||
return 1
|
||||
}
|
||||
if !res.Complete {
|
||||
return 3
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ---- the daemon side: one scan per disk, the last result kept ----
|
||||
func set(list []string) map[string]bool {
|
||||
if len(list) == 0 {
|
||||
return nil
|
||||
}
|
||||
m := make(map[string]bool, len(list))
|
||||
for _, s := range list {
|
||||
m[s] = true
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// ---- the daemon side: one scan per target, the last result kept, on disk too ----
|
||||
|
||||
type Scan struct {
|
||||
Disk string `json:"disk"`
|
||||
@@ -60,14 +113,26 @@ type Scan struct {
|
||||
Started time.Time `json:"started"`
|
||||
Finished time.Time `json:"finished,omitempty"`
|
||||
Running bool `json:"running"`
|
||||
Loading bool `json:"loading,omitempty"` // read back from disk after a restart, not yet decoded
|
||||
Error string `json:"error,omitempty"`
|
||||
Progress Progress `json:"progress"`
|
||||
AsRoot bool `json:"as_root"`
|
||||
// How this scan was done: "full", or "refresh" / "resume" from the one before, and by what
|
||||
// means per root ("zfs diff" or "mtime").
|
||||
Method string `json:"method,omitempty"`
|
||||
ByRoot map[string]string `json:"by_root,omitempty"`
|
||||
Partial bool `json:"partial,omitempty"` // cancelled before it finished; Scan again resumes it
|
||||
// once finished:
|
||||
Files uint32 `json:"files"`
|
||||
Dirs uint32 `json:"dirs"`
|
||||
Size int64 `json:"size"`
|
||||
Denied int64 `json:"denied"`
|
||||
Files uint32 `json:"files"`
|
||||
Dirs uint32 `json:"dirs"`
|
||||
Size int64 `json:"size"`
|
||||
Denied int64 `json:"denied"`
|
||||
Read int64 `json:"read"` // directories read this time
|
||||
Reused int64 `json:"reused"` // directories kept from the scan before
|
||||
PartialDirs int64 `json:"partial_dirs"` // directories not read (cancelled)
|
||||
// Bases is, per root that is a ZFS dataset, the newest snapshot that existed when the
|
||||
// scan started: the next scan asks `zfs diff` what changed since it.
|
||||
Bases map[string]string `json:"bases,omitempty"`
|
||||
|
||||
root *Node
|
||||
largest []File
|
||||
@@ -76,17 +141,105 @@ type Scan struct {
|
||||
|
||||
type Manager struct {
|
||||
self string // this binary, for the helper
|
||||
dir string // where results are kept between restarts
|
||||
mu sync.Mutex
|
||||
scans map[string]*Scan
|
||||
}
|
||||
|
||||
func New(self string) *Manager { return &Manager{self: self, scans: map[string]*Scan{}} }
|
||||
// New makes a manager that keeps results under dir and reads back the ones there.
|
||||
func New(self, dir string) *Manager {
|
||||
m := &Manager{self: self, dir: dir, scans: map[string]*Scan{}}
|
||||
m.loadAll()
|
||||
return m
|
||||
}
|
||||
|
||||
// Status is the scan of disk (a copy without the tree), or nil.
|
||||
func (m *Manager) Status(disk string) *Scan {
|
||||
func (m *Manager) fileFor(id, ext string) string {
|
||||
safe := strings.NewReplacer(":", "_", "/", "_", "\\", "_").Replace(id)
|
||||
return filepath.Join(m.dir, safe+ext)
|
||||
}
|
||||
|
||||
// loadAll reads the saved scans' metadata now and their trees in the background, one at a
|
||||
// time, so a restart does not double the memory of the last scans all at once.
|
||||
func (m *Manager) loadAll() {
|
||||
metas, _ := filepath.Glob(filepath.Join(m.dir, "*.json"))
|
||||
var pending []*Scan
|
||||
for _, mp := range metas {
|
||||
b, err := os.ReadFile(mp)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s := &Scan{}
|
||||
if json.Unmarshal(b, s) != nil || s.Disk == "" {
|
||||
continue
|
||||
}
|
||||
s.Running, s.cancel = false, nil
|
||||
s.Loading = true
|
||||
m.scans[s.Disk] = s
|
||||
pending = append(pending, s)
|
||||
}
|
||||
if len(pending) == 0 {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
for _, s := range pending {
|
||||
root, err := m.readTree(s.Disk)
|
||||
m.mu.Lock()
|
||||
if err != nil {
|
||||
log.Printf("space: saved scan of %s unreadable (%v); dropped", s.Disk, err)
|
||||
if m.scans[s.Disk] == s {
|
||||
delete(m.scans, s.Disk)
|
||||
}
|
||||
} else if m.scans[s.Disk] == s {
|
||||
s.root, s.Loading = root, false
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (m *Manager) readTree(id string) (*Node, error) {
|
||||
f, err := os.Open(m.fileFor(id, ".scan"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
br := bufio.NewReaderSize(f, 1<<20)
|
||||
if _, err := br.ReadBytes('\n'); err != nil { // the result line
|
||||
return nil, err
|
||||
}
|
||||
return Decode(br)
|
||||
}
|
||||
|
||||
func (m *Manager) saveMeta(s *Scan) {
|
||||
if m.dir == "" {
|
||||
return
|
||||
}
|
||||
_ = os.MkdirAll(m.dir, 0o700)
|
||||
b, err := json.MarshalIndent(s, "", " ")
|
||||
if err == nil {
|
||||
err = writeAtomic(m.fileFor(s.Disk, ".json"), b)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("space: save %s: %v", s.Disk, err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeAtomic(path string, b []byte) error {
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
// Status is the scan of id (a copy without the tree), or nil.
|
||||
func (m *Manager) Status(id string) *Scan {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
s := m.scans[disk]
|
||||
return m.copyOf(m.scans[id])
|
||||
}
|
||||
|
||||
func (m *Manager) copyOf(s *Scan) *Scan {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -100,40 +253,122 @@ func (m *Manager) All() map[string]*Scan {
|
||||
defer m.mu.Unlock()
|
||||
out := map[string]*Scan{}
|
||||
for k, s := range m.scans {
|
||||
c := *s
|
||||
c.root, c.largest, c.cancel = nil, nil, nil
|
||||
out[k] = &c
|
||||
out[k] = m.copyOf(s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Start scans the roots of disk in the background. One scan per disk at a time.
|
||||
func (m *Manager) Start(disk string, roots []string) (*Scan, error) {
|
||||
// Start scans the roots of id in the background, from the last scan of it when there is
|
||||
// one. datasets names the ZFS dataset behind each root that has one.
|
||||
func (m *Manager) Start(id string, roots []string, datasets map[string]string) (*Scan, error) {
|
||||
if len(roots) == 0 {
|
||||
return nil, errors.New("nothing is mounted from this disk")
|
||||
}
|
||||
m.mu.Lock()
|
||||
if s := m.scans[disk]; s != nil && s.Running {
|
||||
prev := m.scans[id]
|
||||
if prev != nil && prev.Running {
|
||||
m.mu.Unlock()
|
||||
return nil, errors.New("a scan of this disk is already running")
|
||||
}
|
||||
if prev != nil && (prev.root == nil || prev.Loading) {
|
||||
prev = nil // nothing to start from
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s := &Scan{Disk: disk, Roots: roots, Started: time.Now(), Running: true, AsRoot: priv.Available(), cancel: cancel}
|
||||
m.scans[disk] = s
|
||||
s := &Scan{Disk: id, Roots: roots, Started: time.Now(), Running: true, AsRoot: priv.Available(), cancel: cancel, Bases: map[string]string{}}
|
||||
if prev != nil {
|
||||
// the new scan takes the old one's place in the list while it runs, but keeps its
|
||||
// tree, so the pages still answer until the new one is in
|
||||
s.root, s.largest = prev.root, prev.largest
|
||||
s.Files, s.Dirs, s.Size = prev.Files, prev.Dirs, prev.Size
|
||||
}
|
||||
m.scans[id] = s
|
||||
m.mu.Unlock()
|
||||
go m.run(ctx, s)
|
||||
return m.Status(disk), nil
|
||||
go m.run(ctx, s, prev, datasets)
|
||||
return m.Status(id), nil
|
||||
}
|
||||
|
||||
func (m *Manager) run(ctx context.Context, s *Scan) {
|
||||
args := append([]string{m.self, "space-walk"}, s.Roots...)
|
||||
if s.AsRoot {
|
||||
args = priv.Wrap(args...)
|
||||
// plan decides how each root is walked this time, given the last scan.
|
||||
func (m *Manager) plan(ctx context.Context, s, prev *Scan, datasets map[string]string) (header, Options) {
|
||||
h := header{Modes: map[string]Mode{}}
|
||||
opts := Options{Modes: h.Modes}
|
||||
if prev == nil || prev.root == nil {
|
||||
s.Method = "full"
|
||||
return h, opts
|
||||
}
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
stdin, _ := cmd.StdinPipe() // held open: closing it tells the helper the daemon is gone
|
||||
stdout, _ := cmd.StdoutPipe()
|
||||
stderr, _ := cmd.StderrPipe()
|
||||
h.Prev = true
|
||||
opts.Prev = prev.root
|
||||
s.ByRoot = map[string]string{}
|
||||
dirty, touched := map[string]bool{}, map[string]bool{}
|
||||
for _, r := range s.Roots {
|
||||
mode := ModeMTime
|
||||
how := "mtime"
|
||||
if base := prev.Bases[r]; base != "" && datasets[r] != "" {
|
||||
changed, err := zfsDiff(ctx, base, r)
|
||||
if err == nil {
|
||||
mode, how = ModeDiff, fmt.Sprintf("zfs diff since %s: %d changes", strings.TrimPrefix(base, datasets[r]), len(changed))
|
||||
for _, c := range changed {
|
||||
dirty[c.parent] = true
|
||||
if c.dir {
|
||||
dirty[c.path] = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
how = "mtime (" + err.Error() + ")"
|
||||
}
|
||||
}
|
||||
if Find(prev.root, r) == nil {
|
||||
mode, how = ModeFull, "full (new root)"
|
||||
}
|
||||
h.Modes[r] = mode
|
||||
s.ByRoot[r] = how
|
||||
}
|
||||
// what the last walk never got to is read whatever the change list says
|
||||
for _, p := range partialPaths(prev.root, "/") {
|
||||
dirty[p] = true
|
||||
}
|
||||
for p := range dirty {
|
||||
for a := filepath.Dir(p); ; a = filepath.Dir(a) {
|
||||
touched[a] = true
|
||||
if a == "/" {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for p := range dirty {
|
||||
h.Dirty = append(h.Dirty, p)
|
||||
}
|
||||
for p := range touched {
|
||||
h.Touched = append(h.Touched, p)
|
||||
}
|
||||
sort.Strings(h.Dirty)
|
||||
sort.Strings(h.Touched)
|
||||
opts.Dirty, opts.Touched = dirty, touched
|
||||
if prev.Partial {
|
||||
s.Method = "resume"
|
||||
} else {
|
||||
s.Method = "refresh"
|
||||
}
|
||||
return h, opts
|
||||
}
|
||||
|
||||
func partialPaths(n *Node, path string) []string {
|
||||
var out []string
|
||||
var walk func(n *Node, path string)
|
||||
walk = func(n *Node, path string) {
|
||||
if n.Partial {
|
||||
out = append(out, path)
|
||||
}
|
||||
for _, c := range n.Children {
|
||||
if c.Dir {
|
||||
walk(c, join(path, c.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(n, path)
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Manager) run(ctx context.Context, s *Scan, prev *Scan, datasets map[string]string) {
|
||||
finish := func(err error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
@@ -142,12 +377,42 @@ func (m *Manager) run(ctx context.Context, s *Scan) {
|
||||
if err != nil {
|
||||
s.Error = err.Error()
|
||||
}
|
||||
m.saveMeta(s)
|
||||
}
|
||||
// the newest snapshot of each dataset, so the next scan can ask what changed since
|
||||
for _, r := range s.Roots {
|
||||
if ds := datasets[r]; ds != "" {
|
||||
if snap := newestSnapshot(ctx, ds); snap != "" {
|
||||
s.Bases[r] = snap
|
||||
}
|
||||
}
|
||||
}
|
||||
h, _ := m.plan(ctx, s, prev, datasets)
|
||||
args := append([]string{m.self, "space-walk"}, s.Roots...)
|
||||
if s.AsRoot {
|
||||
args = priv.Wrap(args...)
|
||||
}
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
stdin, _ := cmd.StdinPipe() // held open: closing it tells the helper to stop
|
||||
stdout, _ := cmd.StdoutPipe()
|
||||
stderr, _ := cmd.StderrPipe()
|
||||
if err := cmd.Start(); err != nil {
|
||||
finish(err)
|
||||
return
|
||||
}
|
||||
go func() { <-ctx.Done(); _ = cmd.Process.Signal(syscall.SIGTERM); stdin.Close() }()
|
||||
go func() {
|
||||
hb, _ := json.Marshal(h)
|
||||
_, err := stdin.Write(append(hb, '\n'))
|
||||
if err == nil && h.Prev {
|
||||
err = Encode(stdin, prev.root)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("space: %s: sending the last tree: %v", s.Disk, err)
|
||||
}
|
||||
<-ctx.Done()
|
||||
_ = cmd.Process.Signal(syscall.SIGTERM)
|
||||
stdin.Close()
|
||||
}()
|
||||
var lastErr string
|
||||
go func() {
|
||||
sc := bufio.NewScanner(stderr)
|
||||
@@ -166,36 +431,72 @@ func (m *Manager) run(ctx context.Context, s *Scan) {
|
||||
}
|
||||
}
|
||||
}()
|
||||
root, decErr := Decode(stdout)
|
||||
waitErr := cmd.Wait()
|
||||
if ctx.Err() != nil {
|
||||
finish(errors.New("cancelled"))
|
||||
return
|
||||
// the result streams to disk as it is decoded, so a restart finds it
|
||||
_ = os.MkdirAll(m.dir, 0o700)
|
||||
tmp := m.fileFor(s.Disk, ".scan.tmp")
|
||||
out, ferr := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
||||
var src io.Reader = stdout
|
||||
if ferr == nil {
|
||||
src = io.TeeReader(stdout, out)
|
||||
}
|
||||
if waitErr != nil {
|
||||
if lastErr != "" {
|
||||
br := bufio.NewReaderSize(src, 1<<20)
|
||||
var rl resultLine
|
||||
line, rerr := br.ReadBytes('\n')
|
||||
if rerr == nil {
|
||||
rerr = json.Unmarshal(line, &rl)
|
||||
}
|
||||
var root *Node
|
||||
var decErr error
|
||||
if rerr == nil {
|
||||
root, decErr = Decode(br)
|
||||
} else {
|
||||
decErr = rerr
|
||||
_, _ = io.Copy(io.Discard, br)
|
||||
}
|
||||
if ferr == nil {
|
||||
out.Close()
|
||||
}
|
||||
waitErr := cmd.Wait()
|
||||
cancelled := ctx.Err() != nil
|
||||
if root == nil {
|
||||
_ = os.Remove(tmp)
|
||||
switch {
|
||||
case cancelled:
|
||||
finish(errors.New("cancelled"))
|
||||
case waitErr != nil && lastErr != "":
|
||||
finish(errors.New(strings.TrimPrefix(lastErr, "error: ")))
|
||||
} else {
|
||||
case waitErr != nil:
|
||||
finish(waitErr)
|
||||
default:
|
||||
finish(decErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if decErr != nil {
|
||||
finish(decErr)
|
||||
return
|
||||
if ferr == nil {
|
||||
_ = os.Rename(tmp, m.fileFor(s.Disk, ".scan"))
|
||||
}
|
||||
m.mu.Lock()
|
||||
s.root = root
|
||||
s.root, s.largest = root, nil
|
||||
s.Files, s.Dirs, s.Size, s.Denied = root.Files, root.Dirs, root.Size, s.Progress.Denied
|
||||
s.Read, s.Reused, s.PartialDirs = rl.Read, rl.Reused, rl.Partial
|
||||
s.Partial = !rl.Complete
|
||||
s.Running = false
|
||||
s.Finished = time.Now()
|
||||
if s.Partial {
|
||||
s.Error = "cancelled"
|
||||
}
|
||||
m.saveMeta(s)
|
||||
m.mu.Unlock()
|
||||
log.Printf("space: %s scanned: %d files, %d dirs, %d bytes in %s%s", s.Disk, root.Files, root.Dirs, root.Size, s.Finished.Sub(s.Started).Round(time.Millisecond), map[bool]string{true: " (as root)", false: ""}[s.AsRoot])
|
||||
what := "scanned"
|
||||
if s.Partial {
|
||||
what = "partly scanned (cancelled)"
|
||||
}
|
||||
log.Printf("space: %s %s: %d files, %d dirs, %d bytes in %s (%s; %d dirs read, %d kept%s)", s.Disk, what, root.Files, root.Dirs, root.Size, s.Finished.Sub(s.Started).Round(time.Millisecond), s.Method, rl.Read, rl.Reused, map[bool]string{true: ", as root", false: ""}[s.AsRoot])
|
||||
}
|
||||
|
||||
func (m *Manager) Cancel(disk string) bool {
|
||||
func (m *Manager) Cancel(id string) bool {
|
||||
m.mu.Lock()
|
||||
s := m.scans[disk]
|
||||
s := m.scans[id]
|
||||
m.mu.Unlock()
|
||||
if s == nil || !s.Running || s.cancel == nil {
|
||||
return false
|
||||
@@ -204,6 +505,106 @@ func (m *Manager) Cancel(disk string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Forget drops the scan of id, in memory and on disk.
|
||||
func (m *Manager) Forget(id string) {
|
||||
m.mu.Lock()
|
||||
if s := m.scans[id]; s != nil && s.cancel != nil {
|
||||
s.cancel()
|
||||
}
|
||||
delete(m.scans, id)
|
||||
m.mu.Unlock()
|
||||
_ = os.Remove(m.fileFor(id, ".scan"))
|
||||
_ = os.Remove(m.fileFor(id, ".json"))
|
||||
}
|
||||
|
||||
// ---- ZFS: what changed since the last scan ----
|
||||
|
||||
type change struct {
|
||||
path, parent string
|
||||
dir bool
|
||||
}
|
||||
|
||||
// newestSnapshot is the most recent snapshot of a dataset, "" when it has none.
|
||||
func newestSnapshot(ctx context.Context, dataset string) string {
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, "zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-s", "creation", "-d", "1", dataset).Output()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
|
||||
last := lines[len(lines)-1]
|
||||
if !strings.Contains(last, "@") {
|
||||
return ""
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
// zfsDiff lists what changed in a dataset since a snapshot: every changed, added, removed
|
||||
// or renamed path, with the directory whose listing that touches.
|
||||
func zfsDiff(ctx context.Context, snapshot, mount string) ([]change, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer cancel()
|
||||
args := priv.Wrap("zfs", "diff", "-H", "-F", snapshot)
|
||||
cmd := exec.CommandContext(ctx, args[0], args[1:]...)
|
||||
var stderr strings.Builder
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var changes []change
|
||||
sc := bufio.NewScanner(out)
|
||||
sc.Buffer(make([]byte, 64<<10), 4<<20)
|
||||
for sc.Scan() {
|
||||
f := strings.Split(sc.Text(), "\t")
|
||||
if len(f) < 3 {
|
||||
continue
|
||||
}
|
||||
isDir := f[1] == "/"
|
||||
for _, p := range f[2:] { // a rename has the old and the new path
|
||||
p = unescape(p)
|
||||
if !strings.HasPrefix(p, mount) {
|
||||
continue
|
||||
}
|
||||
changes = append(changes, change{path: p, parent: filepath.Dir(p), dir: isDir})
|
||||
}
|
||||
}
|
||||
if err := cmd.Wait(); err != nil {
|
||||
msg := strings.TrimSpace(stderr.String())
|
||||
if msg == "" {
|
||||
msg = err.Error()
|
||||
}
|
||||
if i := strings.IndexByte(msg, '\n'); i > 0 {
|
||||
msg = msg[:i]
|
||||
}
|
||||
return nil, errors.New(msg)
|
||||
}
|
||||
return changes, nil
|
||||
}
|
||||
|
||||
// unescape undoes zfs diff's \ooo escapes for spaces and odd bytes in names.
|
||||
func unescape(s string) string {
|
||||
if !strings.Contains(s, "\\") {
|
||||
return s
|
||||
}
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '\\' && i+3 < len(s) {
|
||||
if v, err := strconv.ParseUint(s[i+1:i+4], 8, 8); err == nil {
|
||||
b.WriteByte(byte(v))
|
||||
i += 3
|
||||
continue
|
||||
}
|
||||
}
|
||||
b.WriteByte(s[i])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Tree is the subtree at path, cut to depth levels of at most limit children each, leaving
|
||||
// out children smaller than min bytes; what is left out is summed into MoreN/MoreSize. A
|
||||
// treemap asks deep with a min the size of the smallest tile it could draw; a folder list
|
||||
@@ -217,15 +618,18 @@ type Tree struct {
|
||||
Total int64 `json:"total"` // the whole scan's size, for percentages
|
||||
}
|
||||
|
||||
func (m *Manager) Tree(disk, path string, depth, limit int, min int64) (*Tree, error) {
|
||||
func (m *Manager) Tree(id, path string, depth, limit int, min int64) (*Tree, error) {
|
||||
m.mu.Lock()
|
||||
s := m.scans[disk]
|
||||
s := m.scans[id]
|
||||
var root *Node
|
||||
if s != nil {
|
||||
root = s.root
|
||||
}
|
||||
m.mu.Unlock()
|
||||
if root == nil {
|
||||
if s != nil && s.Loading {
|
||||
return nil, errors.New("the last scan is still being read back")
|
||||
}
|
||||
return nil, errors.New("no scan of this disk yet")
|
||||
}
|
||||
n := Find(root, path)
|
||||
@@ -243,7 +647,13 @@ func (m *Manager) Tree(disk, path string, depth, limit int, min int64) (*Tree, e
|
||||
func cut(n *Node, path string, depth, limit int, min int64) *Tree {
|
||||
t := &Tree{Node: *n, Path: path}
|
||||
t.Node.Children = nil
|
||||
if depth <= 0 || !n.Dir {
|
||||
if !n.Dir {
|
||||
return t
|
||||
}
|
||||
if depth <= 0 { // as deep as was asked: what is inside is one lump, not nothing
|
||||
if len(n.Children) > 0 {
|
||||
t.MoreN, t.MoreSize = len(n.Children), n.Size-n.SmallSize
|
||||
}
|
||||
return t
|
||||
}
|
||||
for i, c := range n.Children {
|
||||
@@ -252,19 +662,15 @@ func cut(n *Node, path string, depth, limit int, min int64) *Tree {
|
||||
t.MoreSize += c.Size
|
||||
continue
|
||||
}
|
||||
p := path + "/" + c.Name
|
||||
if path == "/" {
|
||||
p = "/" + c.Name
|
||||
}
|
||||
t.Children = append(t.Children, cut(c, p, depth-1, limit, min))
|
||||
t.Children = append(t.Children, cut(c, join(path, c.Name), depth-1, limit, min))
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// Largest is the n biggest files of the last scan of disk.
|
||||
func (m *Manager) Largest(disk string, n int) ([]File, error) {
|
||||
// Largest is the n biggest files of the last scan of id.
|
||||
func (m *Manager) Largest(id string, n int) ([]File, error) {
|
||||
m.mu.Lock()
|
||||
s := m.scans[disk]
|
||||
s := m.scans[id]
|
||||
var root *Node
|
||||
var have []File
|
||||
if s != nil {
|
||||
|
||||
+180
-22
@@ -9,6 +9,14 @@
|
||||
// Files below KeepMin are not kept one by one: a directory of a million small files is
|
||||
// one "smaller files" entry with their count and total, which is all a treemap can show
|
||||
// of them anyway, and what keeps a scan of a few million files within a few hundred MB.
|
||||
//
|
||||
// A walk can start from the last one. Given the previous tree, a directory is read again
|
||||
// only when something says it changed: on ZFS, `zfs diff` against a snapshot older than
|
||||
// the last scan names every path that changed, and directories it does not name are kept
|
||||
// as they were without so much as a stat; elsewhere a directory whose mtime is what it was
|
||||
// keeps its list and only its kept files are stat'ed again, since a file can grow without
|
||||
// its directory noticing. A directory the last walk did not finish (it was cancelled) is
|
||||
// read again whatever else is true, which is what makes a cancelled scan resumable.
|
||||
package space
|
||||
|
||||
import (
|
||||
@@ -34,34 +42,59 @@ type Node struct {
|
||||
Size int64 `json:"size"`
|
||||
Apparent int64 `json:"apparent"`
|
||||
MTime int64 `json:"mtime"` // unix seconds
|
||||
MTimeNs int64 `json:"-"` // directories: the same to the nanosecond, so a change in the same second is not missed
|
||||
Files uint32 `json:"files"` // beneath a directory (kept and small alike)
|
||||
Dirs uint32 `json:"dirs"`
|
||||
SmallN uint32 `json:"small_n,omitempty"` // files here below KeepMin, not listed
|
||||
SmallSize int64 `json:"small_size,omitempty"` // their allocated bytes
|
||||
Mount bool `json:"mount,omitempty"` // another filesystem starts here (not this disk's, or a root of its own)
|
||||
Denied bool `json:"denied,omitempty"` // could not be read
|
||||
Partial bool `json:"partial,omitempty"` // the walk was cancelled before this directory was read
|
||||
Children []*Node `json:"children,omitempty"` // directories: sorted by Size, largest first
|
||||
}
|
||||
|
||||
// Progress is how far a running walk has got.
|
||||
type Progress struct {
|
||||
Files int64 `json:"files"`
|
||||
Dirs int64 `json:"dirs"`
|
||||
Dirs int64 `json:"dirs"` // directories read
|
||||
Reused int64 `json:"reused"` // directories kept from the last scan without reading
|
||||
Bytes int64 `json:"bytes"`
|
||||
Denied int64 `json:"denied"`
|
||||
Current string `json:"current"`
|
||||
}
|
||||
|
||||
// Mode is how a root is walked when there is a previous tree.
|
||||
type Mode int
|
||||
|
||||
const (
|
||||
ModeFull Mode = iota // read everything
|
||||
ModeMTime // read a directory again only if its mtime changed; stat its kept files
|
||||
ModeDiff // read only the directories a change list names
|
||||
)
|
||||
|
||||
// Options is what a walk starts from.
|
||||
type Options struct {
|
||||
Prev *Node // the last scan's tree, or nil
|
||||
// Modes says how each root is walked (by root path); a root not listed is ModeFull.
|
||||
Modes map[string]Mode
|
||||
// Dirty is every directory a change list named (ModeDiff roots): those are read again.
|
||||
// Touched is every ancestor of a dirty directory: those are kept but looked into.
|
||||
Dirty, Touched map[string]bool
|
||||
}
|
||||
|
||||
type walker struct {
|
||||
ctx context.Context
|
||||
opts Options
|
||||
mu sync.Mutex
|
||||
queue []job
|
||||
cond *sync.Cond
|
||||
pending int // dirs queued or being read
|
||||
files atomic.Int64
|
||||
dirs atomic.Int64
|
||||
reused atomic.Int64
|
||||
bytes atomic.Int64
|
||||
denied atomic.Int64
|
||||
partial atomic.Int64
|
||||
curMu sync.Mutex
|
||||
current string
|
||||
stopping bool
|
||||
@@ -70,13 +103,25 @@ type walker struct {
|
||||
type job struct {
|
||||
path string
|
||||
node *Node
|
||||
prev *Node // the same directory in the last scan, if it had one
|
||||
dev uint64 // the filesystem this root is on; a child on another is a mountpoint, not descended
|
||||
mode Mode
|
||||
}
|
||||
|
||||
// Result is what a walk returns besides the tree.
|
||||
type Result struct {
|
||||
Root *Node
|
||||
Partial int64 // directories not read because the walk was cancelled
|
||||
Reused int64
|
||||
Read int64
|
||||
Complete bool
|
||||
}
|
||||
|
||||
// Walk measures roots (mountpoints, absolute) and returns them grafted into one tree whose
|
||||
// root is "/". progress, if set, is called about four times a second.
|
||||
func Walk(ctx context.Context, roots []string, progress func(Progress)) (*Node, error) {
|
||||
w := &walker{ctx: ctx}
|
||||
// root is "/". progress, if set, is called about four times a second. A cancelled walk
|
||||
// still returns the tree, with the directories it did not get to marked Partial.
|
||||
func Walk(ctx context.Context, roots []string, opts Options, progress func(Progress)) (Result, error) {
|
||||
w := &walker{ctx: ctx, opts: opts}
|
||||
w.cond = sync.NewCond(&w.mu)
|
||||
root := &Node{Name: "/", Dir: true}
|
||||
sort.Strings(roots)
|
||||
@@ -90,15 +135,23 @@ func Walk(ctx context.Context, roots []string, progress func(Progress)) (*Node,
|
||||
if err := syscall.Lstat(r, &st); err != nil || st.Mode&syscall.S_IFMT != syscall.S_IFDIR {
|
||||
continue
|
||||
}
|
||||
n := &Node{Name: filepath.Base(r), Dir: true, MTime: int64(st.Mtim.Sec)}
|
||||
n := &Node{Name: filepath.Base(r), Dir: true, MTime: int64(st.Mtim.Sec), MTimeNs: st.Mtim.Nano()}
|
||||
if r == "/" {
|
||||
n = root
|
||||
}
|
||||
trees[r] = n
|
||||
w.push(job{path: r, node: n, dev: uint64(st.Dev)})
|
||||
var prev *Node
|
||||
if opts.Prev != nil {
|
||||
prev = Find(opts.Prev, r)
|
||||
}
|
||||
mode := opts.Modes[r]
|
||||
if prev == nil {
|
||||
mode = ModeFull
|
||||
}
|
||||
w.push(job{path: r, node: n, prev: prev, dev: uint64(st.Dev), mode: mode})
|
||||
}
|
||||
if len(trees) == 0 {
|
||||
return root, os.ErrNotExist
|
||||
return Result{Root: root}, os.ErrNotExist
|
||||
}
|
||||
workers := 32
|
||||
var wg sync.WaitGroup
|
||||
@@ -110,6 +163,7 @@ func Walk(ctx context.Context, roots []string, progress func(Progress)) (*Node,
|
||||
go func() { wg.Wait(); close(done) }()
|
||||
tick := time.NewTicker(250 * time.Millisecond)
|
||||
defer tick.Stop()
|
||||
cancelled := false
|
||||
for waiting := true; waiting; {
|
||||
select {
|
||||
case <-done:
|
||||
@@ -119,13 +173,18 @@ func Walk(ctx context.Context, roots []string, progress func(Progress)) (*Node,
|
||||
progress(w.snapshot())
|
||||
}
|
||||
case <-ctx.Done():
|
||||
cancelled = true
|
||||
w.mu.Lock()
|
||||
w.stopping = true
|
||||
for _, j := range w.queue { // never read: say so, so the next walk reads them
|
||||
j.node.Partial = true
|
||||
w.partial.Add(1)
|
||||
}
|
||||
w.queue = nil
|
||||
w.cond.Broadcast()
|
||||
w.mu.Unlock()
|
||||
<-done
|
||||
return root, ctx.Err()
|
||||
waiting = false
|
||||
}
|
||||
}
|
||||
if progress != nil {
|
||||
@@ -140,18 +199,28 @@ func Walk(ctx context.Context, roots []string, progress func(Progress)) (*Node,
|
||||
graft(root, filepath.Clean(r), n)
|
||||
}
|
||||
total(root)
|
||||
return root, nil
|
||||
res := Result{Root: root, Partial: w.partial.Load(), Reused: w.reused.Load(), Read: w.dirs.Load(), Complete: !cancelled}
|
||||
if cancelled {
|
||||
return res, ctx.Err()
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (w *walker) snapshot() Progress {
|
||||
w.curMu.Lock()
|
||||
cur := w.current
|
||||
w.curMu.Unlock()
|
||||
return Progress{Files: w.files.Load(), Dirs: w.dirs.Load(), Bytes: w.bytes.Load(), Denied: w.denied.Load(), Current: cur}
|
||||
return Progress{Files: w.files.Load(), Dirs: w.dirs.Load(), Reused: w.reused.Load(), Bytes: w.bytes.Load(), Denied: w.denied.Load(), Current: cur}
|
||||
}
|
||||
|
||||
func (w *walker) push(j job) {
|
||||
w.mu.Lock()
|
||||
if w.stopping {
|
||||
j.node.Partial = true
|
||||
w.partial.Add(1)
|
||||
w.mu.Unlock()
|
||||
return
|
||||
}
|
||||
w.queue = append(w.queue, j)
|
||||
w.pending++
|
||||
w.cond.Signal()
|
||||
@@ -173,7 +242,7 @@ func (w *walker) run() {
|
||||
j := w.queue[len(w.queue)-1] // depth first: keeps the queue short and the disk head local
|
||||
w.queue = w.queue[:len(w.queue)-1]
|
||||
w.mu.Unlock()
|
||||
w.readDir(j)
|
||||
w.visit(j)
|
||||
w.mu.Lock()
|
||||
w.pending--
|
||||
if w.pending == 0 {
|
||||
@@ -183,6 +252,92 @@ func (w *walker) run() {
|
||||
}
|
||||
}
|
||||
|
||||
func join(dir, name string) string {
|
||||
if dir == "/" {
|
||||
return "/" + name
|
||||
}
|
||||
return dir + "/" + name
|
||||
}
|
||||
|
||||
// visit decides whether a directory must be read or can be taken from the last scan.
|
||||
func (w *walker) visit(j job) {
|
||||
if j.prev == nil || j.prev.Partial || j.prev.Denied || j.mode == ModeFull {
|
||||
w.readDir(j)
|
||||
return
|
||||
}
|
||||
switch j.mode {
|
||||
case ModeDiff:
|
||||
if w.opts.Dirty[j.path] {
|
||||
w.readDir(j)
|
||||
return
|
||||
}
|
||||
if !w.opts.Touched[j.path] {
|
||||
w.reuse(j, false) // nothing under here changed: keep the whole subtree, no stat at all
|
||||
return
|
||||
}
|
||||
w.reuse(j, true) // something deeper changed: keep this list, look into the children
|
||||
case ModeMTime:
|
||||
var st syscall.Stat_t
|
||||
if err := syscall.Lstat(j.path, &st); err != nil {
|
||||
w.readDir(j)
|
||||
return
|
||||
}
|
||||
if st.Mtim.Nano() != j.prev.MTimeNs {
|
||||
w.readDir(j) // an entry was added, removed or renamed
|
||||
return
|
||||
}
|
||||
w.reuse(j, true) // same entries; files may have grown, and subdirectories are their own question
|
||||
}
|
||||
}
|
||||
|
||||
// reuse keeps a directory as the last scan saw it. With descend, its subdirectories are
|
||||
// visited (each decides for itself) and, in mtime mode, its kept files are stat'ed again.
|
||||
func (w *walker) reuse(j job, descend bool) {
|
||||
w.reused.Add(1)
|
||||
n, p := j.node, j.prev
|
||||
n.MTime, n.MTimeNs, n.SmallN, n.SmallSize, n.Denied, n.Mount = p.MTime, p.MTimeNs, p.SmallN, p.SmallSize, p.Denied, p.Mount
|
||||
if !descend {
|
||||
n.Children = p.Children
|
||||
n.Apparent = p.Apparent - childApparent(p) // total() adds the children back
|
||||
w.files.Add(int64(p.Files))
|
||||
w.bytes.Add(p.Size)
|
||||
w.reused.Add(int64(p.Dirs))
|
||||
return
|
||||
}
|
||||
w.files.Add(int64(p.SmallN))
|
||||
w.bytes.Add(p.SmallSize)
|
||||
n.Apparent = p.Apparent - childApparent(p) // the small files' lengths; total() adds the children back
|
||||
var st syscall.Stat_t
|
||||
for _, c := range p.Children {
|
||||
if c.Dir {
|
||||
d := &Node{Name: c.Name, Dir: true, MTime: c.MTime, MTimeNs: c.MTimeNs, Mount: c.Mount}
|
||||
n.Children = append(n.Children, d)
|
||||
if c.Mount {
|
||||
continue
|
||||
}
|
||||
w.push(job{path: join(j.path, c.Name), node: d, prev: c, dev: j.dev, mode: j.mode})
|
||||
continue
|
||||
}
|
||||
if j.mode == ModeMTime {
|
||||
if err := syscall.Lstat(join(j.path, c.Name), &st); err != nil {
|
||||
continue // gone since (the directory's mtime would have said so, but be safe)
|
||||
}
|
||||
c = &Node{Name: c.Name, Size: int64(st.Blocks) * 512, Apparent: int64(st.Size), MTime: int64(st.Mtim.Sec)}
|
||||
}
|
||||
n.Children = append(n.Children, c)
|
||||
w.files.Add(1)
|
||||
w.bytes.Add(c.Size)
|
||||
}
|
||||
}
|
||||
|
||||
func childApparent(p *Node) int64 {
|
||||
var a int64
|
||||
for _, c := range p.Children {
|
||||
a += c.Apparent
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func (w *walker) readDir(j job) {
|
||||
w.curMu.Lock()
|
||||
w.current = j.path
|
||||
@@ -200,24 +355,30 @@ func (w *walker) readDir(j job) {
|
||||
j.node.Denied = true
|
||||
w.denied.Add(1)
|
||||
}
|
||||
var prevDirs map[string]*Node
|
||||
if j.prev != nil {
|
||||
prevDirs = map[string]*Node{}
|
||||
for _, c := range j.prev.Children {
|
||||
if c.Dir {
|
||||
prevDirs[c.Name] = c
|
||||
}
|
||||
}
|
||||
}
|
||||
var st syscall.Stat_t
|
||||
for _, name := range names {
|
||||
p := j.path + "/" + name
|
||||
if j.path == "/" {
|
||||
p = "/" + name
|
||||
}
|
||||
p := join(j.path, name)
|
||||
if err := syscall.Lstat(p, &st); err != nil {
|
||||
continue
|
||||
}
|
||||
switch st.Mode & syscall.S_IFMT {
|
||||
case syscall.S_IFDIR:
|
||||
c := &Node{Name: name, Dir: true, MTime: int64(st.Mtim.Sec)}
|
||||
c := &Node{Name: name, Dir: true, MTime: int64(st.Mtim.Sec), MTimeNs: st.Mtim.Nano()}
|
||||
j.node.Children = append(j.node.Children, c)
|
||||
if uint64(st.Dev) != j.dev {
|
||||
c.Mount = true // something else is mounted here; if it is this disk's too, it is a root of its own
|
||||
continue
|
||||
}
|
||||
w.push(job{path: p, node: c, dev: j.dev})
|
||||
w.push(job{path: p, node: c, prev: prevDirs[name], dev: j.dev, mode: j.mode})
|
||||
case syscall.S_IFREG, syscall.S_IFLNK:
|
||||
alloc := int64(st.Blocks) * 512
|
||||
w.files.Add(1)
|
||||
@@ -250,7 +411,7 @@ func graft(root *Node, path string, tree *Node) {
|
||||
cur.Children = append(cur.Children, tree)
|
||||
} else {
|
||||
// the parent's walk saw this directory as a mountpoint; the walk of the mount fills it
|
||||
next.Children, next.SmallN, next.SmallSize, next.Apparent, next.Denied = tree.Children, tree.SmallN, tree.SmallSize, tree.Apparent, tree.Denied
|
||||
next.Children, next.SmallN, next.SmallSize, next.Apparent, next.Denied, next.Partial = tree.Children, tree.SmallN, tree.SmallSize, tree.Apparent, tree.Denied, tree.Partial
|
||||
next.Mount = true
|
||||
}
|
||||
return
|
||||
@@ -324,10 +485,7 @@ func Largest(root *Node, n int) []File {
|
||||
var walk func(n *Node, path string)
|
||||
walk = func(node *Node, path string) {
|
||||
for _, c := range node.Children {
|
||||
p := path + "/" + c.Name
|
||||
if path == "/" {
|
||||
p = "/" + c.Name
|
||||
}
|
||||
p := join(path, c.Name)
|
||||
if c.Dir {
|
||||
if c.Size > 0 && (len(out) < n || c.Size > out[minIdx].Size) {
|
||||
walk(c, p)
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package space
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func mk(t *testing.T, dir string, files map[string]int) {
|
||||
for p, n := range files {
|
||||
full := filepath.Join(dir, p)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(full, make([]byte, n), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshAndResume(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
mk(t, dir, map[string]int{"a/big1": 300 << 10, "a/small": 100, "b/c/big2": 500 << 10, "b/d/big3": 200 << 10})
|
||||
res, err := Walk(context.Background(), []string{dir}, Options{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := Find(res.Root, dir)
|
||||
if root == nil || root.Files != 4 {
|
||||
t.Fatalf("full walk: %+v", root)
|
||||
}
|
||||
want := root.Size
|
||||
// grow a file in place (its directory's mtime does not change) and add a file elsewhere
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if err := os.WriteFile(filepath.Join(dir, "a/big1"), make([]byte, 600<<10), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mk(t, dir, map[string]int{"b/d/big4": 400 << 10})
|
||||
res2, err := Walk(context.Background(), []string{dir}, Options{Prev: res.Root, Modes: map[string]Mode{dir: ModeMTime}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r2 := Find(res2.Root, dir)
|
||||
if r2.Files != 5 || r2.Size != want+(300<<10)+(400<<10) {
|
||||
t.Fatalf("mtime refresh: files %d size %d want %d (reused %d read %d)", r2.Files, r2.Size, want+(300<<10)+(400<<10), res2.Reused, res2.Read)
|
||||
}
|
||||
if res2.Reused == 0 || res2.Read == 0 {
|
||||
t.Fatalf("mtime refresh should reuse some and read some: reused %d read %d", res2.Reused, res2.Read)
|
||||
}
|
||||
// diff mode: only b/d named dirty; a's growth must NOT be seen (that is the contract), b/d's new file must
|
||||
dirty := map[string]bool{filepath.Join(dir, "b/d"): true}
|
||||
touched := map[string]bool{dir: true, filepath.Join(dir, "b"): true}
|
||||
res3, err := Walk(context.Background(), []string{dir}, Options{Prev: res.Root, Modes: map[string]Mode{dir: ModeDiff}, Dirty: dirty, Touched: touched}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r3 := Find(res3.Root, dir)
|
||||
if r3.Files != 5 || r3.Size != want+(400<<10) || res3.Read != 1 {
|
||||
t.Fatalf("diff refresh: files %d size %d want %d read %d", r3.Files, r3.Size, want+(400<<10), res3.Read)
|
||||
}
|
||||
// a cancelled walk marks what it did not read; resuming reads those
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
res4, err := Walk(ctx, []string{dir}, Options{}, nil)
|
||||
if err == nil || res4.Complete {
|
||||
t.Fatal("cancelled walk should say so")
|
||||
}
|
||||
if len(partialPaths(res4.Root, "/")) == 0 {
|
||||
t.Fatal("cancelled walk should leave partial directories")
|
||||
}
|
||||
res5, err := Walk(context.Background(), []string{dir}, Options{Prev: res4.Root, Modes: map[string]Mode{dir: ModeMTime}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r5 := Find(res5.Root, dir); r5.Files != 5 || len(partialPaths(res5.Root, "/")) != 0 {
|
||||
t.Fatalf("resume: files %d partial %v", r5.Files, partialPaths(res5.Root, "/"))
|
||||
}
|
||||
// the codec round-trips the partial flag
|
||||
pr, pw, _ := os.Pipe()
|
||||
go func() { _ = Encode(pw, res4.Root); pw.Close() }()
|
||||
back, err := Decode(pr)
|
||||
if err != nil || len(partialPaths(back, "/")) != len(partialPaths(res4.Root, "/")) {
|
||||
t.Fatalf("codec: %v", err)
|
||||
}
|
||||
}
|
||||
+72
-37
@@ -7,7 +7,7 @@ import { api, SpaceFile, SpaceTarget, SpaceTree } from './api'
|
||||
import { fmtBytes } from './Dashboard'
|
||||
import { useHashSel } from './hashsel'
|
||||
import { useRefreshMs } from './prefs'
|
||||
import { Btn, Loading, Spinner, toast, useEscape } from './ui'
|
||||
import { Btn, Loading, Spinner, askYes, toast, useEscape } from './ui'
|
||||
|
||||
// ---- what kind of file a tile is: the colour it wears (the validated categorical set, dark and light) ----
|
||||
|
||||
@@ -75,10 +75,17 @@ export default function Space() {
|
||||
|
||||
const start = async () => { if (!target) return; try { await api.spaceScan(target.id); await load() } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } }
|
||||
const cancel = async () => { if (!target) return; try { await api.spaceCancel(target.id); await load() } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } }
|
||||
const forget = async () => { if (!target || !await askYes(`Forget the saved scan of ${target.name}? The next scan reads everything again.`)) return; try { await api.spaceForget(target.id); await load() } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } }
|
||||
// the target list is not polled during a scan; when one ends it is read again, so the
|
||||
// picker's "scanned … ago" and the new tree come in together
|
||||
const wasRunning = useRef(false)
|
||||
useEffect(() => { if (wasRunning.current && !scan?.running) load(); wasRunning.current = !!scan?.running }, [scan?.running, load])
|
||||
|
||||
if (err && !targets) return <div className="dash"><div className="error">{err}</div></div>
|
||||
if (!targets) return <div className="dash"><Loading label="Looking at the disks…" /></div>
|
||||
const done = !!scan && !scan.running && !scan.error && !!scan.finished
|
||||
// a result to show: the last finished one, kept while a refresh of it runs
|
||||
const have = !!scan && !scan.loading && scan.files > 0 && (!scan.error || scan.error === 'cancelled')
|
||||
const done = have && !scan.running
|
||||
return (
|
||||
<div className="space">
|
||||
<div className="space-bar">
|
||||
@@ -88,14 +95,21 @@ export default function Space() {
|
||||
<div className="space-status">
|
||||
{scan?.running ? (
|
||||
<>
|
||||
<Spinner /><span className="mono small">{fmtN(scan.progress.files)} files · {fmtBytes(scan.progress.bytes)} · {fmtDur(scan.started)}</span>
|
||||
<Spinner /><span className="mono small">{fmtN(scan.progress.files)} files · {fmtBytes(scan.progress.bytes)} · {fmtDur(scan.started)}{scan.progress.reused ? ` · ${fmtN(scan.progress.reused)} folders kept, ${fmtN(scan.progress.dirs)} read` : ''}</span>
|
||||
<Btn className="btn danger" onClick={cancel}>Cancel</Btn>
|
||||
</>
|
||||
) : scan?.loading ? (
|
||||
<><Spinner /><span className="muted small">reading the last scan back…</span></>
|
||||
) : (
|
||||
<>
|
||||
{done && <span className="small"><b>{fmtBytes(scan.size)}</b> in {fmtN(scan.files)} files, {fmtN(scan.dirs)} folders · scanned {ago(scan.finished)} in {fmtDur(scan.started, scan.finished)}{scan.as_root ? '' : ' · not as root, so some folders were skipped'}{scan.denied ? ` · ${fmtN(scan.denied)} unreadable` : ''}</span>}
|
||||
{scan?.error && <span className={(scan.error === 'cancelled' ? 'muted' : 'error') + ' small'}>{scan.error === 'cancelled' ? 'the last scan was cancelled' : scan.error}</span>}
|
||||
<Btn className="btn on" disabled={target.roots.length === 0} onClick={start}>{done ? 'Rescan' : 'Scan'}</Btn>
|
||||
{done && <span className="small" title={scan.by_root ? Object.entries(scan.by_root).map(([r, how]) => `${r}: ${how}`).join('\n') : undefined}>
|
||||
<b>{fmtBytes(scan.size)}</b> in {fmtN(scan.files)} files, {fmtN(scan.dirs)} folders · {scan.partial ? <span className="warn">partly scanned, {fmtN(scan.partial_dirs)} folders not yet read</span> : `scanned ${ago(scan.finished)}`} in {fmtDur(scan.started, scan.finished)}
|
||||
{scan.method && scan.method !== 'full' ? ` · ${scan.method === 'resume' ? 'resumed' : 'refreshed'}: ${fmtN(scan.read)} folders read, ${fmtN(scan.reused)} kept` : ''}
|
||||
{scan.as_root ? '' : ' · not as root, so some folders were skipped'}{scan.denied ? ` · ${fmtN(scan.denied)} unreadable` : ''}
|
||||
</span>}
|
||||
{scan?.error && scan.error !== 'cancelled' && <span className="error small">{scan.error}</span>}
|
||||
<Btn className="btn on" disabled={target.roots.length === 0} onClick={start} title={done ? (scan.partial ? 'reads the folders the cancelled scan did not get to' : 'reads only what changed since the last scan (zfs diff on a pool with snapshots; folder times elsewhere)') : undefined}>{done ? (scan.partial ? 'Resume' : 'Rescan') : 'Scan'}</Btn>
|
||||
{done && <Btn className="btn" onClick={forget} title="drop the saved result; the next scan reads everything">Forget</Btn>}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
@@ -103,7 +117,7 @@ export default function Space() {
|
||||
</div>
|
||||
{target && <div className="muted small space-about">{describe(target)}{scan?.running && scan.progress.current && <span className="mono cur" title={scan.progress.current}> · {scan.progress.current}</span>}</div>}
|
||||
</div>
|
||||
{target && done && (
|
||||
{target && have && (
|
||||
<>
|
||||
<div className="chips space-tabs">
|
||||
<button className={'chip' + (tab === 'tree' ? ' on' : '')} onClick={() => setTab('tree')}>Folders</button>
|
||||
@@ -112,8 +126,9 @@ export default function Space() {
|
||||
{tab === 'tree' ? <Explorer key={target.id + (scan?.finished ?? '')} disk={target.id} /> : <LargestFiles key={target.id + (scan?.finished ?? '')} disk={target.id} />}
|
||||
</>
|
||||
)}
|
||||
{target && !done && !scan?.running && <div className="svc-empty muted">{target.roots.length === 0 ? 'Nothing is mounted from it, so there is nothing to scan.' : scan?.error && scan.error !== 'cancelled' ? 'The last scan failed. Scan again.' : `Scan the ${target.kind} to see what is on it.`}</div>}
|
||||
{target && scan?.running && <div className="svc-empty muted">Scanning…</div>}
|
||||
{target && !have && !scan?.running && !scan?.loading && <div className="svc-empty muted">{target.roots.length === 0 ? 'Nothing is mounted from it, so there is nothing to scan.' : scan?.error && scan.error !== 'cancelled' ? 'The last scan failed. Scan again.' : `Scan the ${target.kind} to see what is on it.`}</div>}
|
||||
{target && !have && scan?.running && <div className="svc-empty muted">Scanning…</div>}
|
||||
{target && scan?.loading && <div className="svc-empty muted">Reading the last scan back…</div>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -195,7 +210,7 @@ function Explorer({ disk }: { disk: string }) {
|
||||
const key = 'map:' + path
|
||||
if (pending.current.has(key)) return
|
||||
pending.current.add(key)
|
||||
try { const t = await api.spaceTree(disk, path, 12, 200, Math.max(1, Math.floor(size / 4000))); setMaps((m) => ({ ...m, [path]: t })) }
|
||||
try { const t = await api.spaceTree(disk, path, 24, 300, Math.max(1, Math.floor(size / 6000))); setMaps((m) => ({ ...m, [path]: t })) }
|
||||
catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') }
|
||||
finally { pending.current.delete(key) }
|
||||
}, [disk])
|
||||
@@ -251,7 +266,7 @@ function Explorer({ disk }: { disk: string }) {
|
||||
<span className="n">
|
||||
{t.dir && r.kind === 'node' ? <button className="tw" onClick={(e) => { e.stopPropagation(); toggle(t.path) }}>{isOpen ? '▾' : '▸'}</button> : <span className="tw" />}
|
||||
<span className="ico" style={r.kind === 'node' && !t.dir ? { background: lightTheme() ? kindOf(t.name).light : kindOf(t.name).dark } : undefined}>{t.dir && r.kind === 'node' ? '▰' : ''}</span>
|
||||
<span className="nm" title={t.path}>{t.name}{t.mount && t.dir && r.kind === 'node' ? <span className="muted small"> mount</span> : ''}{t.denied ? <span className="muted small"> unreadable</span> : ''}</span>
|
||||
<span className="nm" title={t.path}>{t.name}{t.mount && t.dir && r.kind === 'node' ? <span className="muted small"> mount</span> : ''}{t.denied ? <span className="muted small"> unreadable</span> : ''}{t.partial ? <span className="warn small"> not yet read</span> : ''}</span>
|
||||
</span>
|
||||
<span className="r mono">{fmtBytes(t.size)}</span>
|
||||
<span className="pct"><span className="bar"><span style={{ width: `${Math.min(100, pct)}%` }} /></span><span className="muted small">{pct >= 10 ? pct.toFixed(0) : pct.toFixed(1)}%</span></span>
|
||||
@@ -287,9 +302,15 @@ function crumbs(path: string): { name: string; path: string }[] {
|
||||
return out
|
||||
}
|
||||
|
||||
// ---- the treemap: squarified, nested three deep, drawn on a canvas ----
|
||||
// ---- the treemap: every file a tile, grouped by folder, drawn on a canvas ----
|
||||
//
|
||||
// WizTree's picture: the whole area is files, sized by what they take and coloured by kind,
|
||||
// and folders are only the grouping — a thin line, and a name on the biggest ones directly
|
||||
// under the folder being looked at. No folder gets a box of its own deeper down, because a
|
||||
// box costs a header and padding at every level, and a chain of single-child folders (as
|
||||
// every Linux system has) turned the picture into a staircase of headers with nothing inside.
|
||||
|
||||
interface Tile { x: number; y: number; w: number; h: number; t: SpaceTree; agg?: 'small' | 'more'; depth: number; head?: number; inner?: boolean }
|
||||
interface Tile { x: number; y: number; w: number; h: number; t: SpaceTree; agg?: 'small' | 'more'; depth: number; head: number }
|
||||
|
||||
// squarify lays items (largest first) into rect, keeping tiles as square as it can.
|
||||
function squarify(items: { size: number; t: SpaceTree; agg?: 'small' | 'more' }[], x: number, y: number, w: number, h: number, depth: number, out: Tile[]) {
|
||||
@@ -319,7 +340,7 @@ function squarify(items: { size: number; t: SpaceTree; agg?: 'small' | 'more' }[
|
||||
let off = 0
|
||||
for (const r of row) {
|
||||
const len = (r.size * scale) / thick
|
||||
const tile: Tile = vertical ? { x: cx, y: cy + off, w: thick, h: len, t: r.t, agg: r.agg, depth } : { x: cx + off, y: cy, w: len, h: thick, t: r.t, agg: r.agg, depth }
|
||||
const tile: Tile = vertical ? { x: cx, y: cy + off, w: thick, h: len, t: r.t, agg: r.agg, depth, head: 0 } : { x: cx + off, y: cy, w: len, h: thick, t: r.t, agg: r.agg, depth, head: 0 }
|
||||
out.push(tile)
|
||||
off += len
|
||||
}
|
||||
@@ -338,12 +359,12 @@ function layout(tree: SpaceTree, x: number, y: number, w: number, h: number, dep
|
||||
for (const t of tiles) {
|
||||
out.push(t)
|
||||
const kids = (t.t.children?.length ?? 0) + (t.t.small_size ? 1 : 0) + (t.t.more_size ? 1 : 0)
|
||||
if (t.t.dir && kids > 0 && depth < 12 && t.w > 20 && t.h > 20) {
|
||||
// a title strip when there is room for one; a tiny folder is just a box of its children
|
||||
t.head = t.h > 40 && t.w > 40 ? 15 : 0
|
||||
t.inner = true
|
||||
layout(t.t, t.x + 2, t.y + 2 + t.head, t.w - 4, t.h - 4 - t.head, depth + 1, out)
|
||||
}
|
||||
if (!t.t.dir || kids === 0 || t.w < 3 || t.h < 3) continue
|
||||
// the folders directly under the one being looked at get a name strip when there is room;
|
||||
// everything deeper is grouping only
|
||||
t.head = depth === 0 && t.h >= 34 && t.w >= 50 ? 14 : 0
|
||||
const pad = depth === 0 ? 1 : 0
|
||||
layout(t.t, t.x + pad, t.y + pad + t.head, t.w - 2 * pad, t.h - 2 * pad - t.head, depth + 1, out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -379,29 +400,41 @@ function Treemap({ tree, onOpen }: { tree: SpaceTree; onOpen: (path: string) =>
|
||||
layout(tree, 0, 0, size.w, size.h, 0, out)
|
||||
tiles.current = out
|
||||
const font = cssVar('--font') || 'system-ui, sans-serif'
|
||||
const dirFill = light ? '#dfe3e8' : '#1c2229'
|
||||
for (const t of out) {
|
||||
if (t.w < 1 || t.h < 1) continue
|
||||
if (t.w < 0.5 || t.h < 0.5) continue
|
||||
const isDir = t.t.dir && !t.agg
|
||||
if (isDir) {
|
||||
// the folder's own colour shows only where its children don't cover it: the strip,
|
||||
// the padding, and anything too small to have been asked for
|
||||
g.fillStyle = dirFill
|
||||
g.fillRect(t.x, t.y, t.w, t.h)
|
||||
if (t.depth <= 1 && t.w >= 4 && t.h >= 4) { g.strokeStyle = border; g.lineWidth = 1; g.strokeRect(t.x + 0.5, t.y + 0.5, t.w - 1, t.h - 1) }
|
||||
if (t.head) {
|
||||
g.save()
|
||||
g.beginPath(); g.rect(t.x + 1, t.y + 1, t.w - 2, t.head - 1); g.clip()
|
||||
g.font = `600 11px ${font}`; g.fillStyle = fg; g.textBaseline = 'top'
|
||||
const label = t.t.name
|
||||
g.fillText(label, t.x + 4, t.y + 1, t.w - 8)
|
||||
const sz = fmtBytes(t.t.size)
|
||||
if (t.w > g.measureText(label).width + g.measureText(sz).width + 20) { g.font = `10px ${font}`; g.fillStyle = muted; g.textAlign = 'right'; g.fillText(sz, t.x + t.w - 4, t.y + 2) }
|
||||
g.restore()
|
||||
}
|
||||
continue
|
||||
}
|
||||
let fill: string
|
||||
if (t.agg) fill = light ? '#c9ced4' : '#3a414b'
|
||||
else if (isDir) fill = light ? (t.depth % 2 ? '#eef0f3' : '#e3e6ea') : (t.depth % 2 ? '#262d36' : '#1e242c')
|
||||
if (t.agg) fill = t.agg === 'small' ? (light ? '#c4cad1' : '#39424c') : (light ? '#d3d8de' : '#2f3740')
|
||||
else { const k = kindOf(t.t.name); fill = light ? k.light : k.dark }
|
||||
g.fillStyle = fill
|
||||
g.fillRect(t.x, t.y, t.w, t.h)
|
||||
g.strokeStyle = isDir ? border : bg
|
||||
g.lineWidth = 1
|
||||
g.strokeRect(t.x + 0.5, t.y + 0.5, t.w - 1, t.h - 1)
|
||||
// a label when there is room and it would not sit on the folder's own children: the
|
||||
// name, and the size on a second line
|
||||
const labelled = !isDir || !t.inner || (t.head ?? 0) > 0
|
||||
if (labelled && t.w > 36 && t.h > 14) {
|
||||
if (t.w >= 3 && t.h >= 3) { g.strokeStyle = bg; g.lineWidth = 1; g.strokeRect(t.x + 0.5, t.y + 0.5, t.w - 1, t.h - 1) }
|
||||
if (t.w >= 46 && t.h >= 14) {
|
||||
g.save()
|
||||
g.beginPath(); g.rect(t.x + 1, t.y + 1, t.w - 2, t.h - 2); g.clip()
|
||||
g.font = `${isDir ? '600 ' : ''}11px ${font}`
|
||||
g.fillStyle = isDir || t.agg ? (isDir ? fg : muted) : (light ? '#111' : '#fff')
|
||||
g.textBaseline = 'top'
|
||||
g.font = `11px ${font}`; g.textBaseline = 'top'
|
||||
g.fillStyle = t.agg ? muted : (light ? '#111' : '#fff')
|
||||
g.fillText(t.t.name, t.x + 4, t.y + 2, t.w - 8)
|
||||
if (t.h > 30 && !t.inner) { g.font = `10px ${font}`; g.fillStyle = isDir || t.agg ? muted : (light ? 'rgba(0,0,0,.7)' : 'rgba(255,255,255,.8)'); g.fillText(fmtBytes(t.t.size), t.x + 4, t.y + 16, t.w - 8) }
|
||||
if (t.h >= 28) { g.font = `10px ${font}`; g.fillStyle = t.agg ? muted : (light ? 'rgba(0,0,0,.7)' : 'rgba(255,255,255,.8)'); g.fillText(fmtBytes(t.t.size), t.x + 4, t.y + 15, t.w - 8) }
|
||||
g.restore()
|
||||
}
|
||||
}
|
||||
@@ -415,6 +448,8 @@ function Treemap({ tree, onOpen }: { tree: SpaceTree; onOpen: (path: string) =>
|
||||
for (const t of tiles.current) if (x >= t.x && x < t.x + t.w && y >= t.y && y < t.y + t.h && (!best || t.depth > best.depth)) best = t
|
||||
return { x, y, tile: best }
|
||||
}
|
||||
// a path relative to the folder being looked at, for the tooltip
|
||||
const within = (p: string) => { const base = tree.path === '/' ? '' : tree.path; return p.startsWith(base + '/') ? p.slice(base.length + 1) : p }
|
||||
return (
|
||||
<div className="space-canvas" ref={box}>
|
||||
<canvas ref={canvas} style={{ width: size.w, height: size.h }}
|
||||
@@ -422,10 +457,10 @@ function Treemap({ tree, onOpen }: { tree: SpaceTree; onOpen: (path: string) =>
|
||||
onMouseLeave={() => setHover(null)}
|
||||
onClick={(e) => { const h = at(e); if (!h.tile) return; const t = h.tile.t; onOpen(t.dir && !h.tile.agg ? t.path : parentOf(t.path)) }} />
|
||||
{hover && (
|
||||
<div className="space-tip" style={{ left: Math.min(hover.x + 12, size.w - 260), top: hover.y + 12 }}>
|
||||
<div className="mono small" style={{ wordBreak: 'break-all' }}>{hover.tile.agg ? hover.tile.t.name + ' in ' + parentOf(hover.tile.t.path) : hover.tile.t.path}</div>
|
||||
<div className="space-tip" style={{ left: Math.min(hover.x + 12, size.w - 280), top: Math.min(hover.y + 12, size.h - 70) }}>
|
||||
<div className="mono small" style={{ wordBreak: 'break-all' }}>{hover.tile.agg ? hover.tile.t.name + ' in ' + (within(parentOf(hover.tile.t.path)) || '.') : within(hover.tile.t.path)}</div>
|
||||
<div><b>{fmtBytes(hover.tile.t.size)}</b> <span className="muted">· {tree.size > 0 ? ((hover.tile.t.size / tree.size) * 100).toFixed(1) : 0}% of {tree.name === '/' ? 'the disk' : tree.name}</span></div>
|
||||
{hover.tile.t.dir && !hover.tile.agg && <div className="muted small">{fmtN(hover.tile.t.files)} files · {fmtN(hover.tile.t.dirs)} folders</div>}
|
||||
{hover.tile.t.dir && !hover.tile.agg && <div className="muted small">{fmtN(hover.tile.t.files)} files · {fmtN(hover.tile.t.dirs)} folders · click to open</div>}
|
||||
{!hover.tile.t.dir && !hover.tile.agg && <div className="muted small">{kindOf(hover.tile.t.name).label} · {fmtWhen(hover.tile.t.mtime)}{hover.tile.t.apparent !== hover.tile.t.size ? ` · ${fmtBytes(hover.tile.t.apparent)} apparent` : ''}</div>}
|
||||
</div>
|
||||
)}
|
||||
|
||||
+4
-3
@@ -126,10 +126,10 @@ export interface User { name: string; created: string; last_seen?: string; sessi
|
||||
export interface Session { id: string; created: string; last_seen: string; expires: string; ip?: string; agent?: string; current: boolean }
|
||||
export interface TokenInfo { name: string; created: string; scope?: string; sensors?: string[]; issued_by?: string }
|
||||
|
||||
export interface SpaceProgress { files: number; dirs: number; bytes: number; denied: number; current: string }
|
||||
export interface SpaceScan { disk: string; roots: string[]; started: string; finished?: string; running: boolean; error?: string; progress: SpaceProgress; as_root: boolean; files: number; dirs: number; size: number; denied: number }
|
||||
export interface SpaceProgress { files: number; dirs: number; reused: number; bytes: number; denied: number; current: string }
|
||||
export interface SpaceScan { disk: string; roots: string[]; started: string; finished?: string; running: boolean; loading?: boolean; error?: string; progress: SpaceProgress; as_root: boolean; method?: string; by_root?: Record<string, string>; partial?: boolean; files: number; dirs: number; size: number; denied: number; read: number; reused: number; partial_dirs: number; bases?: Record<string, string> }
|
||||
export interface SpaceTarget { id: string; kind: 'disk' | 'pool'; name: string; model?: string; layout?: string; health?: string; size: number; used?: number; disks?: string[]; roots: string[]; scan?: SpaceScan }
|
||||
export interface SpaceNode { name: string; dir: boolean; size: number; apparent: number; mtime: number; files: number; dirs: number; small_n?: number; small_size?: number; mount?: boolean; denied?: boolean }
|
||||
export interface SpaceNode { name: string; dir: boolean; size: number; apparent: number; mtime: number; files: number; dirs: number; small_n?: number; small_size?: number; mount?: boolean; denied?: boolean; partial?: boolean }
|
||||
export interface SpaceTree extends SpaceNode { path: string; children?: SpaceTree[]; more_n?: number; more_size?: number; total: number }
|
||||
export interface SpaceFile { path: string; size: number; apparent: number; mtime: number }
|
||||
|
||||
@@ -144,6 +144,7 @@ export const api = {
|
||||
spaceStatus: (disk: string) => req<SpaceScan>('GET', `/api/space/${encodeURIComponent(disk)}`),
|
||||
spaceScan: (disk: string) => req<SpaceScan>('POST', `/api/space/${encodeURIComponent(disk)}/scan`),
|
||||
spaceCancel: (disk: string) => req<void>('POST', `/api/space/${encodeURIComponent(disk)}/cancel`),
|
||||
spaceForget: (disk: string) => req<void>('DELETE', `/api/space/${encodeURIComponent(disk)}`),
|
||||
spaceTree: (disk: string, path: string, depth = 1, limit = 200, min = 0) => req<SpaceTree>('GET', `/api/space/${encodeURIComponent(disk)}/tree?path=${encodeURIComponent(path)}&depth=${depth}&limit=${limit}&min=${min}`),
|
||||
spaceLargest: (disk: string, n = 500) => req<SpaceFile[]>('GET', `/api/space/${encodeURIComponent(disk)}/largest?n=${n}`),
|
||||
procSignal: (pid: number, signal: 'TERM' | 'KILL') => req<void>('POST', `/api/procs/${pid}/signal`, { signal }),
|
||||
|
||||
@@ -687,6 +687,7 @@ pre.diff .add { color: var(--ok); background: color-mix(in srgb, var(--ok) 12%,
|
||||
.titem .tmain { display: flex; flex-direction: column; min-width: 0; flex: 1; } .titem .tname b { color: var(--fg); }
|
||||
.titem .tsub { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-family: ui-monospace, monospace; }
|
||||
.titem .tstate { flex-shrink: 0; white-space: nowrap; }
|
||||
.space .warn { color: var(--s2); }
|
||||
.space-tabs { padding: 8px 12px 0; }
|
||||
.space-split { flex: 1; display: flex; flex-direction: column; min-height: 0; padding: 8px 12px 12px; gap: 8px; }
|
||||
.space-tree { flex: 1 1 55%; min-height: 120px; display: flex; flex-direction: column; border: 1px solid var(--border); border-radius: 6px; background: var(--bg2); overflow: hidden; }
|
||||
|
||||
Reference in New Issue
Block a user