diff --git a/PLAN.md b/PLAN.md index 4a3305c..cbdacb0 100644 --- a/PLAN.md +++ b/PLAN.md @@ -326,6 +326,28 @@ something long in one, switch away and back, refresh — it's all still there. `hamlab passwd [user]` (makes the account if new; reads stdin when piped), `hamlab users`, `hamlab user-rm `; `hamlab install` names the first account after the service user. Not done: a read-only viewer role. +- **Space tab** — **shipped 2026-09-22** (`internal/space`, `internal/api/space.go`, + `Space.tsx`), asked for as "exactly WizTree": what is using a disk. A **disk** is picked at + the top, not a partition: the scan covers every filesystem the disk backs — its partitions' + mountpoints, and for a disk in a ZFS pool every mounted dataset of the pool (any of `tank`'s + eight disks shows `tank`). The walk runs as a helper process, `hamlab space-walk `, + as root through `priv` when the host allows (so `/var/lib/docker` and `/root` count), 16 + workers reading directories depth-first, never crossing a filesystem boundary by itself; + each root is grafted into one tree at its path. Files under 128 KiB are not kept one by one + but summed per folder as "N smaller files" — what keeps a scan of millions of files to tens + of MB in the daemon; the tree crosses back as a varint stream (`codec.go`), and the helper's + memory goes with it. core's root: 646k files in ~1.2 s. Progress (files, bytes, current + folder) every 250 ms; Cancel sends SIGTERM. The page: disk chips → Scan/Rescan → **Folders** + (a tree largest-first: size, share of parent as a bar, files, folders, modified; expand in + place) over a **treemap** of the selected folder — squarified, nested to the file, tiles + coloured by kind (video, images, audio, archives & disk images, documents, code & config, + databases & logs, other; the validated categorical set, dark and light) with folders as + labelled boxes; hover for path, size and share; click a folder to drill, breadcrumbs and ↑ + to come back. The map asks the server for the folder pruned to what a tile could show + (`min` = 1/4000 of the folder, depth 12), the list one level at a time. **Largest files**: + the 1,000 biggest with a path filter. Works on every machine through the hub. Not kept + across a restart; not done: delete from here, a size-over-time history, per-user column + choices. - **Auth hardening** — WebAuthn passkeys (phone Face ID), TOTP fallback, optional NPM access list restricting to WireGuard/LAN. - **Files tab** — **shipped 2026-09-18** (`internal/files`, `Files.tsx`): browse from diff --git a/cmd/hamlab/main.go b/cmd/hamlab/main.go index 7894bf3..d0bb5a0 100644 --- a/cmd/hamlab/main.go +++ b/cmd/hamlab/main.go @@ -33,6 +33,7 @@ import ( "hamlab/internal/nodes" "hamlab/internal/services" "hamlab/internal/setup" + "hamlab/internal/space" "hamlab/internal/storage" "hamlab/internal/store" "hamlab/internal/tasks" @@ -75,6 +76,8 @@ func main() { switch cmd { // commands that don't read the config case "fs": // privileged helper for the Files tab (run as root by the daemon) os.Exit(files.Helper(flag.Args()[1:], os.Stdin, os.Stdout, os.Stderr)) + case "space-walk": // helper for the Space tab: measure mountpoints, stream the tree back + os.Exit(space.Helper(flag.Args()[1:], os.Stdout, os.Stderr)) case "install", "uninstall": var err error if cmd == "install" { diff --git a/internal/api/server.go b/internal/api/server.go index d9bb032..34e3397 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -39,6 +39,7 @@ import ( "hamlab/internal/sensors" "hamlab/internal/services" "hamlab/internal/setup" + "hamlab/internal/space" "hamlab/internal/storage" "hamlab/internal/store" "hamlab/internal/tasks" @@ -66,6 +67,7 @@ type Server struct { tasks *tasks.Manager updates *updates.Manager storage *storage.Manager + space *space.Manager fsLocal files.FS fsSudo files.FS // nil without passwordless sudo nodes *nodes.Manager @@ -83,6 +85,11 @@ type Server struct { func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *metrics.Collector, st *store.Store, svc *services.Manager, cr *cron.Manager, tm *tasks.Manager, up *updates.Manager, sto *storage.Manager, fsLocal, fsSudo files.FS, nm *nodes.Manager, fingerprint string, ui fs.FS, version string) *Server { s := &Server{cfg: cfg, sessions: sessions, mgr: mgr, metrics: col, sensors: sensors.NewRegistry(col), store: st, svc: svc, cron: cr, tasks: tm, updates: up, storage: sto, fsLocal: fsLocal, fsSudo: fsSudo, nodes: nm, fingerprint: fingerprint, ui: ui, version: version, started: time.Now(), user: os.Getenv("USER")} + if self, err := os.Executable(); err == nil { + s.space = space.New(self) + } else { + s.space = space.New("hamlab") + } s.hostName() // read it once now so the first request doesn't have to if u, err := user.Current(); err == nil && u.Username != "" { s.user = u.Username @@ -249,6 +256,12 @@ func New(cfg *config.Config, sessions *auth.Sessions, mgr *term.Manager, col *me m.HandleFunc("GET /api/storage/zfs/snapshots", s.auth(s.zfsSnapshots)) m.HandleFunc("POST /api/storage/zfs/scrub/{pool}/{action}", s.auth(s.zfsScrub)) m.HandleFunc("POST /api/storage/zfs/snapshot", s.auth(s.zfsSnapshot)) + m.HandleFunc("GET /api/space", s.auth(s.spaceOverview)) + m.HandleFunc("GET /api/space/{disk}", s.auth(s.spaceStatus)) + m.HandleFunc("POST /api/space/{disk}/scan", s.auth(s.spaceScan)) + m.HandleFunc("POST /api/space/{disk}/cancel", s.auth(s.spaceCancel)) + m.HandleFunc("GET /api/space/{disk}/tree", s.auth(s.spaceTree)) + m.HandleFunc("GET /api/space/{disk}/largest", s.auth(s.spaceLargest)) m.HandleFunc("GET /api/procs", s.auth(s.procList)) m.HandleFunc("POST /api/procs/{pid}/signal", s.auth(s.procSignal)) m.HandleFunc("POST /api/password", s.auth(s.changePassword)) diff --git a/internal/api/space.go b/internal/api/space.go new file mode 100644 index 0000000..3c4d6ba --- /dev/null +++ b/internal/api/space.go @@ -0,0 +1,179 @@ +package api + +// The Space tab: what is using a disk. A disk is picked, not a partition: the scan covers +// every filesystem the disk backs — its partitions' mountpoints, and for a disk in a ZFS +// pool, every mounted dataset of that pool (so any of the pool's disks shows the pool). + +import ( + "context" + "log" + "net/http" + "os/exec" + "sort" + "strconv" + "strings" + "time" + + "hamlab/internal/space" + "hamlab/internal/storage" +) + +type spaceDisk struct { + Name string `json:"name"` + Kind string `json:"kind"` + Model string `json:"model,omitempty"` + Size uint64 `json:"size"` + Pool string `json:"pool,omitempty"` // the ZFS pool this disk is part of + Roots []string `json:"roots"` // the mountpoints a scan walks + Scan *space.Scan `json:"scan,omitempty"` // the last scan, if any +} + +// spaceDisks maps every disk to the mountpoints it backs. +func (s *Server) spaceDisks(ctx context.Context) []spaceDisk { + devs, _ := storage.Devices(ctx) + var out []spaceDisk + poolMounts := map[string][]string{} + for _, d := range devs { + sd := spaceDisk{Name: d.Name, Kind: d.Kind, Model: d.Model, Size: d.Size, Roots: []string{}} + seen := map[string]bool{} + for _, p := range d.Parts { + if p.Mount != "" && p.Mount != "[SWAP]" && !seen[p.Mount] { + seen[p.Mount] = true + sd.Roots = append(sd.Roots, p.Mount) + } + if p.FSType == "zfs_member" && p.Label != "" && sd.Pool == "" { + sd.Pool = p.Label + } + } + if sd.Pool != "" { + mounts, ok := poolMounts[sd.Pool] + if !ok { + mounts = zfsMounts(ctx, sd.Pool) + poolMounts[sd.Pool] = mounts + } + for _, m := range mounts { + if !seen[m] { + seen[m] = true + sd.Roots = append(sd.Roots, m) + } + } + } + sort.Strings(sd.Roots) + out = append(out, sd) + } + if out == nil { + out = []spaceDisk{} + } + return out +} + +// zfsMounts is every mounted filesystem of a pool. +func zfsMounts(ctx context.Context, pool string) []string { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, "zfs", "list", "-H", "-t", "filesystem", "-r", "-o", "mountpoint,mounted", pool).Output() + if err != nil { + return nil + } + var mounts []string + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + f := strings.Split(line, "\t") + if len(f) == 2 && f[1] == "yes" && strings.HasPrefix(f[0], "/") { + mounts = append(mounts, f[0]) + } + } + return mounts +} + +func (s *Server) spaceOverview(w http.ResponseWriter, r *http.Request) { + disks := s.spaceDisks(r.Context()) + scans := s.space.All() + for i := range disks { + disks[i].Scan = scans[disks[i].Name] + } + jsonOK(w, 200, map[string]any{"disks": disks}) +} + +func (s *Server) spaceDisk(w http.ResponseWriter, r *http.Request, name string) (spaceDisk, bool) { + for _, d := range s.spaceDisks(r.Context()) { + if d.Name == name { + return d, true + } + } + jsonErr(w, 404, "no such disk") + return spaceDisk{}, false +} + +func (s *Server) spaceStatus(w http.ResponseWriter, r *http.Request) { + st := s.space.Status(r.PathValue("disk")) + if st == nil { + jsonErr(w, 404, "not scanned") + return + } + jsonOK(w, 200, st) +} + +func (s *Server) spaceScan(w http.ResponseWriter, r *http.Request) { + d, ok := s.spaceDisk(w, r, r.PathValue("disk")) + if !ok { + return + } + st, err := s.space.Start(d.Name, d.Roots) + if err != nil { + jsonErr(w, 409, err.Error()) + return + } + log.Printf("space: scan of %s (%s) started by %s", d.Name, strings.Join(d.Roots, " "), clientIP(r)) + jsonOK(w, 202, st) +} + +func (s *Server) spaceCancel(w http.ResponseWriter, r *http.Request) { + if !s.space.Cancel(r.PathValue("disk")) { + jsonErr(w, 404, "no scan running") + return + } + w.WriteHeader(204) +} + +func (s *Server) spaceTree(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + path := q.Get("path") + if path == "" { + path = "/" + } + depth, _ := strconv.Atoi(q.Get("depth")) + if depth < 1 || depth > 16 { + depth = 1 + } + limit, _ := strconv.Atoi(q.Get("limit")) + if limit < 1 || limit > 2000 { + limit = 300 + } + min, _ := strconv.ParseInt(q.Get("min"), 10, 64) + if min < 0 { + min = 0 + } + t, err := s.space.Tree(r.PathValue("disk"), path, depth, limit, min) + if err != nil { + jsonErr(w, 404, err.Error()) + return + } + w.Header().Set("Cache-Control", "no-store") + jsonOK(w, 200, t) +} + +func (s *Server) spaceLargest(w http.ResponseWriter, r *http.Request) { + n, _ := strconv.Atoi(r.URL.Query().Get("n")) + if n < 1 || n > 5000 { + n = 500 + } + files, err := s.space.Largest(r.PathValue("disk"), n) + if err != nil { + jsonErr(w, 404, err.Error()) + return + } + if files == nil { + files = []space.File{} + } + jsonOK(w, 200, files) +} diff --git a/internal/space/codec.go b/internal/space/codec.go new file mode 100644 index 0000000..a8eecbe --- /dev/null +++ b/internal/space/codec.go @@ -0,0 +1,140 @@ +package space + +import ( + "bufio" + "encoding/binary" + "errors" + "io" +) + +// The tree crosses from the helper process to the daemon as a preorder stream of varints: +// a few bytes per node, decoded without reflection, so a million nodes take well under a +// second either way. + +const codecMagic = "HLSPACE1" + +func Encode(w io.Writer, root *Node) error { + bw := bufio.NewWriterSize(w, 1<<20) + if _, err := bw.WriteString(codecMagic); err != nil { + return err + } + var buf [binary.MaxVarintLen64]byte + putU := func(v uint64) error { n := binary.PutUvarint(buf[:], v); _, err := bw.Write(buf[:n]); return err } + putI := func(v int64) error { n := binary.PutVarint(buf[:], v); _, err := bw.Write(buf[:n]); return err } + var enc func(n *Node) error + enc = func(n *Node) error { + if err := putU(uint64(len(n.Name))); err != nil { + return err + } + if _, err := bw.WriteString(n.Name); err != nil { + return err + } + flags := uint64(0) + if n.Dir { + flags |= 1 + } + if n.Mount { + flags |= 2 + } + if n.Denied { + flags |= 4 + } + for _, f := range []func() error{ + func() error { return putU(flags) }, + func() error { return putI(n.Size) }, + func() error { return putI(n.Apparent) }, + func() error { return putI(n.MTime) }, + func() error { return putU(uint64(n.Files)) }, + func() error { return putU(uint64(n.Dirs)) }, + func() error { return putU(uint64(n.SmallN)) }, + func() error { return putI(n.SmallSize) }, + func() error { return putU(uint64(len(n.Children))) }, + } { + if err := f(); err != nil { + return err + } + } + for _, c := range n.Children { + if err := enc(c); err != nil { + return err + } + } + return nil + } + if err := enc(root); err != nil { + return err + } + return bw.Flush() +} + +func Decode(r io.Reader) (*Node, error) { + br := bufio.NewReaderSize(r, 1<<20) + magic := make([]byte, len(codecMagic)) + if _, err := io.ReadFull(br, magic); err != nil || string(magic) != codecMagic { + return nil, errors.New("space: not a scan stream") + } + var dec func() (*Node, error) + dec = func() (*Node, error) { + l, err := binary.ReadUvarint(br) + if err != nil { + return nil, err + } + if l > 1<<16 { + return nil, errors.New("space: bad name length") + } + name := make([]byte, l) + if _, err := io.ReadFull(br, name); err != nil { + return nil, err + } + n := &Node{Name: string(name)} + flags, err := binary.ReadUvarint(br) + if err != nil { + return nil, err + } + n.Dir, n.Mount, n.Denied = flags&1 != 0, flags&2 != 0, flags&4 != 0 + if n.Size, err = binary.ReadVarint(br); err != nil { + return nil, err + } + if n.Apparent, err = binary.ReadVarint(br); err != nil { + return nil, err + } + if n.MTime, err = binary.ReadVarint(br); err != nil { + return nil, err + } + u, err := binary.ReadUvarint(br) + if err != nil { + return nil, err + } + n.Files = uint32(u) + if u, err = binary.ReadUvarint(br); err != nil { + return nil, err + } + n.Dirs = uint32(u) + if u, err = binary.ReadUvarint(br); err != nil { + return nil, err + } + n.SmallN = uint32(u) + if n.SmallSize, err = binary.ReadVarint(br); err != nil { + return nil, err + } + cnt, err := binary.ReadUvarint(br) + if err != nil { + return nil, err + } + if cnt > 1<<24 { + return nil, errors.New("space: bad child count") + } + if cnt > 0 { + n.Children = make([]*Node, 0, cnt) + for i := uint64(0); i < cnt; i++ { + c, err := dec() + if err != nil { + return nil, err + } + n.Children = append(n.Children, c) + } + } + return n, nil + } + return dec() +} diff --git a/internal/space/manager.go b/internal/space/manager.go new file mode 100644 index 0000000..00a0a40 --- /dev/null +++ b/internal/space/manager.go @@ -0,0 +1,289 @@ +package space + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + "os/exec" + "os/signal" + "strings" + "sync" + "syscall" + "time" + + "hamlab/internal/priv" +) + +// ---- the helper: `hamlab space-walk …`, run by the daemon (as root when it can) ---- +// +// The walk runs in its own process so that root can read every directory, so that the +// memory of the walk is given back when it ends, and so that a runaway scan can be killed +// without touching the console. Progress goes to stderr as JSON lines; the tree to stdout. + +func Helper(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprintln(stderr, "usage: hamlab space-walk ...") + return 2 + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) + go func() { <-sig; cancel() }() + go func() { _, _ = io.Copy(io.Discard, os.Stdin); cancel() }() // the daemon went away + enc := json.NewEncoder(stderr) + root, err := Walk(ctx, args, func(p Progress) { _ = enc.Encode(p) }) + if err != nil && !errors.Is(err, context.Canceled) { + fmt.Fprintln(stderr, "error:", err) + return 1 + } + if ctx.Err() != nil { + return 3 + } + if err := Encode(stdout, root); err != nil { + fmt.Fprintln(stderr, "error:", err) + return 1 + } + return 0 +} + +// ---- the daemon side: one scan per disk, the last result kept ---- + +type Scan struct { + Disk string `json:"disk"` + Roots []string `json:"roots"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished,omitempty"` + Running bool `json:"running"` + Error string `json:"error,omitempty"` + Progress Progress `json:"progress"` + AsRoot bool `json:"as_root"` + // once finished: + Files uint32 `json:"files"` + Dirs uint32 `json:"dirs"` + Size int64 `json:"size"` + Denied int64 `json:"denied"` + + root *Node + largest []File + cancel context.CancelFunc +} + +type Manager struct { + self string // this binary, for the helper + mu sync.Mutex + scans map[string]*Scan +} + +func New(self string) *Manager { return &Manager{self: self, scans: map[string]*Scan{}} } + +// Status is the scan of disk (a copy without the tree), or nil. +func (m *Manager) Status(disk string) *Scan { + m.mu.Lock() + defer m.mu.Unlock() + s := m.scans[disk] + if s == nil { + return nil + } + c := *s + c.root, c.largest, c.cancel = nil, nil, nil + return &c +} + +func (m *Manager) All() map[string]*Scan { + m.mu.Lock() + defer m.mu.Unlock() + out := map[string]*Scan{} + for k, s := range m.scans { + c := *s + c.root, c.largest, c.cancel = nil, nil, nil + out[k] = &c + } + return out +} + +// Start scans the roots of disk in the background. One scan per disk at a time. +func (m *Manager) Start(disk string, roots []string) (*Scan, error) { + if len(roots) == 0 { + return nil, errors.New("nothing is mounted from this disk") + } + m.mu.Lock() + if s := m.scans[disk]; s != nil && s.Running { + m.mu.Unlock() + return nil, errors.New("a scan of this disk is already running") + } + ctx, cancel := context.WithCancel(context.Background()) + s := &Scan{Disk: disk, Roots: roots, Started: time.Now(), Running: true, AsRoot: priv.Available(), cancel: cancel} + m.scans[disk] = s + m.mu.Unlock() + go m.run(ctx, s) + return m.Status(disk), nil +} + +func (m *Manager) run(ctx context.Context, s *Scan) { + args := append([]string{m.self, "space-walk"}, s.Roots...) + if s.AsRoot { + args = priv.Wrap(args...) + } + cmd := exec.Command(args[0], args[1:]...) + stdin, _ := cmd.StdinPipe() // held open: closing it tells the helper the daemon is gone + stdout, _ := cmd.StdoutPipe() + stderr, _ := cmd.StderrPipe() + finish := func(err error) { + m.mu.Lock() + defer m.mu.Unlock() + s.Running = false + s.Finished = time.Now() + if err != nil { + s.Error = err.Error() + } + } + if err := cmd.Start(); err != nil { + finish(err) + return + } + go func() { <-ctx.Done(); _ = cmd.Process.Signal(syscall.SIGTERM); stdin.Close() }() + var lastErr string + go func() { + sc := bufio.NewScanner(stderr) + sc.Buffer(make([]byte, 64<<10), 1<<20) + for sc.Scan() { + line := sc.Text() + var p Progress + if strings.HasPrefix(line, "{") && json.Unmarshal([]byte(line), &p) == nil { + m.mu.Lock() + s.Progress = p + m.mu.Unlock() + continue + } + if line != "" { + lastErr = line + } + } + }() + root, decErr := Decode(stdout) + waitErr := cmd.Wait() + if ctx.Err() != nil { + finish(errors.New("cancelled")) + return + } + if waitErr != nil { + if lastErr != "" { + finish(errors.New(strings.TrimPrefix(lastErr, "error: "))) + } else { + finish(waitErr) + } + return + } + if decErr != nil { + finish(decErr) + return + } + m.mu.Lock() + s.root = root + s.Files, s.Dirs, s.Size, s.Denied = root.Files, root.Dirs, root.Size, s.Progress.Denied + s.Running = false + s.Finished = time.Now() + m.mu.Unlock() + log.Printf("space: %s scanned: %d files, %d dirs, %d bytes in %s%s", s.Disk, root.Files, root.Dirs, root.Size, s.Finished.Sub(s.Started).Round(time.Millisecond), map[bool]string{true: " (as root)", false: ""}[s.AsRoot]) +} + +func (m *Manager) Cancel(disk string) bool { + m.mu.Lock() + s := m.scans[disk] + m.mu.Unlock() + if s == nil || !s.Running || s.cancel == nil { + return false + } + s.cancel() + return true +} + +// Tree is the subtree at path, cut to depth levels of at most limit children each, leaving +// out children smaller than min bytes; what is left out is summed into MoreN/MoreSize. A +// treemap asks deep with a min the size of the smallest tile it could draw; a folder list +// asks one level with no min. +type Tree struct { + Node + Path string `json:"path"` + Children []*Tree `json:"children,omitempty"` + MoreN int `json:"more_n,omitempty"` + MoreSize int64 `json:"more_size,omitempty"` + Total int64 `json:"total"` // the whole scan's size, for percentages +} + +func (m *Manager) Tree(disk, path string, depth, limit int, min int64) (*Tree, error) { + m.mu.Lock() + s := m.scans[disk] + var root *Node + if s != nil { + root = s.root + } + m.mu.Unlock() + if root == nil { + return nil, errors.New("no scan of this disk yet") + } + n := Find(root, path) + if n == nil { + return nil, errors.New("no such path in the scan") + } + if path == "" { + path = "/" + } + t := cut(n, path, depth, limit, min) + t.Total = root.Size + return t, nil +} + +func cut(n *Node, path string, depth, limit int, min int64) *Tree { + t := &Tree{Node: *n, Path: path} + t.Node.Children = nil + if depth <= 0 || !n.Dir { + return t + } + for i, c := range n.Children { + if i >= limit || (c.Size < min && i > 0) { // children are sorted largest first: the first is always kept + t.MoreN++ + t.MoreSize += c.Size + continue + } + p := path + "/" + c.Name + if path == "/" { + p = "/" + c.Name + } + t.Children = append(t.Children, cut(c, p, depth-1, limit, min)) + } + return t +} + +// Largest is the n biggest files of the last scan of disk. +func (m *Manager) Largest(disk string, n int) ([]File, error) { + m.mu.Lock() + s := m.scans[disk] + var root *Node + var have []File + if s != nil { + root, have = s.root, s.largest + } + m.mu.Unlock() + if root == nil { + return nil, errors.New("no scan of this disk yet") + } + if len(have) < n { + have = Largest(root, n) // the tree is never changed once made, so no lock while walking it + m.mu.Lock() + if s.root == root { + s.largest = have + } + m.mu.Unlock() + } + if len(have) > n { + return have[:n], nil + } + return have, nil +} diff --git a/internal/space/walk.go b/internal/space/walk.go new file mode 100644 index 0000000..c63d748 --- /dev/null +++ b/internal/space/walk.go @@ -0,0 +1,356 @@ +// Package space measures what is using a disk: a parallel walk of the filesystems a disk +// backs, kept as a tree of directories and the files worth seeing, the way WizTree does it. +// +// The walk never crosses a filesystem boundary by itself — /proc, /sys, another disk's +// mount — but every mountpoint the disk backs is a root of the same scan, and the roots +// are grafted into one tree at their paths, so a ZFS pool of many datasets, or a disk with +// / and /home on separate partitions, reads as one picture. +// +// Files below KeepMin are not kept one by one: a directory of a million small files is +// one "smaller files" entry with their count and total, which is all a treemap can show +// of them anyway, and what keeps a scan of a few million files within a few hundred MB. +package space + +import ( + "context" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" +) + +// KeepMin is the smallest file kept as its own node. +const KeepMin = 128 << 10 + +// Node is a directory or a file. Sizes are bytes allocated on disk (what fills the disk); +// Apparent is the length. For a directory both include everything beneath it. +type Node struct { + Name string `json:"name"` + Dir bool `json:"dir"` + Size int64 `json:"size"` + Apparent int64 `json:"apparent"` + MTime int64 `json:"mtime"` // unix seconds + Files uint32 `json:"files"` // beneath a directory (kept and small alike) + Dirs uint32 `json:"dirs"` + SmallN uint32 `json:"small_n,omitempty"` // files here below KeepMin, not listed + SmallSize int64 `json:"small_size,omitempty"` // their allocated bytes + Mount bool `json:"mount,omitempty"` // another filesystem starts here (not this disk's, or a root of its own) + Denied bool `json:"denied,omitempty"` // could not be read + Children []*Node `json:"children,omitempty"` // directories: sorted by Size, largest first +} + +// Progress is how far a running walk has got. +type Progress struct { + Files int64 `json:"files"` + Dirs int64 `json:"dirs"` + Bytes int64 `json:"bytes"` + Denied int64 `json:"denied"` + Current string `json:"current"` +} + +type walker struct { + ctx context.Context + mu sync.Mutex + queue []job + cond *sync.Cond + pending int // dirs queued or being read + files atomic.Int64 + dirs atomic.Int64 + bytes atomic.Int64 + denied atomic.Int64 + curMu sync.Mutex + current string + stopping bool +} + +type job struct { + path string + node *Node + dev uint64 // the filesystem this root is on; a child on another is a mountpoint, not descended +} + +// Walk measures roots (mountpoints, absolute) and returns them grafted into one tree whose +// root is "/". progress, if set, is called about four times a second. +func Walk(ctx context.Context, roots []string, progress func(Progress)) (*Node, error) { + w := &walker{ctx: ctx} + w.cond = sync.NewCond(&w.mu) + root := &Node{Name: "/", Dir: true} + sort.Strings(roots) + trees := map[string]*Node{} + for _, r := range roots { + r = filepath.Clean(r) + if !filepath.IsAbs(r) { + continue + } + var st syscall.Stat_t + if err := syscall.Lstat(r, &st); err != nil || st.Mode&syscall.S_IFMT != syscall.S_IFDIR { + continue + } + n := &Node{Name: filepath.Base(r), Dir: true, MTime: st.Mtim.Sec} + if r == "/" { + n = root + } + trees[r] = n + w.push(job{path: r, node: n, dev: uint64(st.Dev)}) + } + if len(trees) == 0 { + return root, os.ErrNotExist + } + workers := 16 + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { defer wg.Done(); w.run() }() + } + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + tick := time.NewTicker(250 * time.Millisecond) + defer tick.Stop() + for waiting := true; waiting; { + select { + case <-done: + waiting = false + case <-tick.C: + if progress != nil { + progress(w.snapshot()) + } + case <-ctx.Done(): + w.mu.Lock() + w.stopping = true + w.queue = nil + w.cond.Broadcast() + w.mu.Unlock() + <-done + return root, ctx.Err() + } + } + if progress != nil { + progress(w.snapshot()) + } + // graft each root at its path; deepest last so parents exist first + for _, r := range roots { + n, ok := trees[filepath.Clean(r)] + if !ok || n == root { + continue + } + graft(root, filepath.Clean(r), n) + } + total(root) + return root, nil +} + +func (w *walker) snapshot() Progress { + w.curMu.Lock() + cur := w.current + w.curMu.Unlock() + return Progress{Files: w.files.Load(), Dirs: w.dirs.Load(), Bytes: w.bytes.Load(), Denied: w.denied.Load(), Current: cur} +} + +func (w *walker) push(j job) { + w.mu.Lock() + w.queue = append(w.queue, j) + w.pending++ + w.cond.Signal() + w.mu.Unlock() +} + +// run takes directories off the queue until there are none left and none being read. +func (w *walker) run() { + for { + w.mu.Lock() + for len(w.queue) == 0 && w.pending > 0 && !w.stopping { + w.cond.Wait() + } + if w.stopping || (len(w.queue) == 0 && w.pending == 0) { + w.cond.Broadcast() + w.mu.Unlock() + return + } + j := w.queue[len(w.queue)-1] // depth first: keeps the queue short and the disk head local + w.queue = w.queue[:len(w.queue)-1] + w.mu.Unlock() + w.readDir(j) + w.mu.Lock() + w.pending-- + if w.pending == 0 { + w.cond.Broadcast() + } + w.mu.Unlock() + } +} + +func (w *walker) readDir(j job) { + w.curMu.Lock() + w.current = j.path + w.curMu.Unlock() + w.dirs.Add(1) + f, err := os.Open(j.path) + if err != nil { + j.node.Denied = true + w.denied.Add(1) + return + } + names, err := f.Readdirnames(-1) + f.Close() + if err != nil { + j.node.Denied = true + w.denied.Add(1) + } + var st syscall.Stat_t + for _, name := range names { + p := j.path + "/" + name + if j.path == "/" { + p = "/" + name + } + if err := syscall.Lstat(p, &st); err != nil { + continue + } + switch st.Mode & syscall.S_IFMT { + case syscall.S_IFDIR: + c := &Node{Name: name, Dir: true, MTime: st.Mtim.Sec} + j.node.Children = append(j.node.Children, c) + if uint64(st.Dev) != j.dev { + c.Mount = true // something else is mounted here; if it is this disk's too, it is a root of its own + continue + } + w.push(job{path: p, node: c, dev: j.dev}) + case syscall.S_IFREG, syscall.S_IFLNK: + alloc := st.Blocks * 512 + w.files.Add(1) + w.bytes.Add(alloc) + if alloc >= KeepMin || st.Size >= KeepMin { + j.node.Children = append(j.node.Children, &Node{Name: name, Size: alloc, Apparent: st.Size, MTime: st.Mtim.Sec}) + } else { + j.node.SmallN++ + j.node.SmallSize += alloc + j.node.Apparent += st.Size // added to the directory's own apparent total below + } + } + } +} + +// graft puts a root's tree at path under root, making any directories in between. +func graft(root *Node, path string, tree *Node) { + parts := strings.Split(strings.Trim(path, "/"), "/") + cur := root + for i, part := range parts { + var next *Node + for _, c := range cur.Children { + if c.Name == part && c.Dir { + next = c + break + } + } + if i == len(parts)-1 { + if next == nil { + cur.Children = append(cur.Children, tree) + } else { + // the parent's walk saw this directory as a mountpoint; the walk of the mount fills it + next.Children, next.SmallN, next.SmallSize, next.Apparent, next.Denied = tree.Children, tree.SmallN, tree.SmallSize, tree.Apparent, tree.Denied + next.Mount = true + } + return + } + if next == nil { + next = &Node{Name: part, Dir: true} + cur.Children = append(cur.Children, next) + } + cur = next + } +} + +// total sums every directory from its children and sorts them largest first. +func total(n *Node) { + if !n.Dir { + return + } + n.Size = n.SmallSize + n.Files = n.SmallN + n.Dirs = 0 + apparent := n.Apparent // the small files' lengths + for _, c := range n.Children { + total(c) + n.Size += c.Size + apparent += c.Apparent + if c.Dir { + n.Dirs += 1 + c.Dirs + n.Files += c.Files + } else { + n.Files++ + } + } + n.Apparent = apparent + sort.Slice(n.Children, func(i, j int) bool { return n.Children[i].Size > n.Children[j].Size }) +} + +// Find follows path (absolute, "/" for the root) down the tree. +func Find(root *Node, path string) *Node { + cur := root + for _, part := range strings.Split(strings.Trim(filepath.Clean(path), "/"), "/") { + if part == "" { + continue + } + var next *Node + for _, c := range cur.Children { + if c.Name == part { + next = c + break + } + } + if next == nil { + return nil + } + cur = next + } + return cur +} + +// File is one of the largest files, with its full path. +type File struct { + Path string `json:"path"` + Size int64 `json:"size"` + Apparent int64 `json:"apparent"` + MTime int64 `json:"mtime"` +} + +// Largest is the n biggest files in the tree, largest first. +func Largest(root *Node, n int) []File { + var out []File + minIdx := -1 + var walk func(n *Node, path string) + walk = func(node *Node, path string) { + for _, c := range node.Children { + p := path + "/" + c.Name + if path == "/" { + p = "/" + c.Name + } + if c.Dir { + if c.Size > 0 && (len(out) < n || c.Size > out[minIdx].Size) { + walk(c, p) + } + continue + } + if len(out) < n { + out = append(out, File{Path: p, Size: c.Size, Apparent: c.Apparent, MTime: c.MTime}) + if minIdx < 0 || c.Size < out[minIdx].Size { + minIdx = len(out) - 1 + } + } else if c.Size > out[minIdx].Size { + out[minIdx] = File{Path: p, Size: c.Size, Apparent: c.Apparent, MTime: c.MTime} + minIdx = 0 + for i := range out { + if out[i].Size < out[minIdx].Size { + minIdx = i + } + } + } + } + } + walk(root, "/") + sort.Slice(out, func(i, j int) bool { return out[i].Size > out[j].Size }) + return out +} diff --git a/web/src/Shell.tsx b/web/src/Shell.tsx index bfccd76..b0f3935 100644 --- a/web/src/Shell.tsx +++ b/web/src/Shell.tsx @@ -17,6 +17,7 @@ import Procs from './Procs' import Top from './Top' import Machines from './Machines' import Settings from './Settings' +import Space from './Space' import Users from './Users' import MachineSwitcher from './MachineSwitcher' import { MachineStatus, setMachine } from './api' @@ -60,6 +61,7 @@ const GROUPS: Group[] = [ { id: 'cron', label: 'Cron', icon: ClockIcon }, { id: 'updates', label: 'Updates', icon: '⇪' }, { id: 'disks', label: 'Disks', icon: '▤' }, + { id: 'space', label: 'Space', icon: '▧' }, { id: 'files', label: 'Files', icon: FolderIcon }, { id: 'alerts', label: 'Alerts', icon: BellIcon }, { id: 'machines', label: 'Machines', icon: '⌂' }, @@ -68,8 +70,8 @@ const GROUPS: Group[] = [ { id: 'settings', label: 'Settings', icon: '⚙' }, ] -type View = 'terminal' | 'dashboard' | 'top' | 'widgets' | 'docker' | 'alerts' | 'services' | 'cron' | 'updates' | 'disks' | 'files' | 'procs' | 'machines' | 'users' | 'settings' -const VIEWS: View[] = ['terminal', 'dashboard', 'top', 'widgets', 'docker', 'alerts', 'services', 'cron', 'updates', 'disks', 'files', 'procs', 'machines', 'users', 'settings'] +type View = 'terminal' | 'dashboard' | 'top' | 'widgets' | 'docker' | 'alerts' | 'services' | 'cron' | 'updates' | 'disks' | 'space' | 'files' | 'procs' | 'machines' | 'users' | 'settings' +const VIEWS: View[] = ['terminal', 'dashboard', 'top', 'widgets', 'docker', 'alerts', 'services', 'cron', 'updates', 'disks', 'space', 'files', 'procs', 'machines', 'users', 'settings'] const HUB_ONLY: View[] = ['machines', 'users', 'settings', 'alerts'] // URL hash is the source of truth for where you are: #//. Back and @@ -256,7 +258,7 @@ export default function Shell({ me: hubMe, onLoggedOut }: { me: Me; onLoggedOut:
- !HUB_ONLY.includes(g.id as View)) : GROUPS} view={view} badges={{ alerts: firing }} header={ { setView('machines'); setDrawer(false) }} />} onView={async (id) => { if (id !== view && !(await okToLeave())) return; setView(id as View); setDrawer(false) }} />
setDrawer(false)} /> @@ -264,6 +266,7 @@ export default function Shell({ me: hubMe, onLoggedOut }: { me: Me; onLoggedOut:
{view === 'settings' && } {view === 'users' && } + {view === 'space' && } {view === 'machines' && { switchMachine(n); setView('dashboard') }} />} {view === 'dashboard' && } {view === 'widgets' && } @@ -301,7 +304,7 @@ export default function Shell({ me: hubMe, onLoggedOut }: { me: Me; onLoggedOut: {machine ? `${title} · ` : ''}{me.version.startsWith('v') ? me.version : 'v' + me.version} up {fmtUptime(me.uptime)} - {view === 'top' ? 'Top' : view === 'dashboard' ? 'Dashboard' : view === 'widgets' ? 'Widgets' : view === 'docker' ? 'Docker' : view === 'alerts' ? 'Alerts' : view === 'services' ? 'Services' : view === 'cron' ? 'Cron' : view === 'updates' ? 'Updates' : view === 'disks' ? 'Disks' : view === 'files' ? 'Files' : view === 'procs' ? 'Processes' : view === 'machines' ? 'Machines' : view === 'users' ? 'Users' : view === 'settings' ? 'Settings' : activeTerm ? activeTerm.name : ''} + {view === 'top' ? 'Top' : view === 'dashboard' ? 'Dashboard' : view === 'widgets' ? 'Widgets' : view === 'docker' ? 'Docker' : view === 'alerts' ? 'Alerts' : view === 'services' ? 'Services' : view === 'cron' ? 'Cron' : view === 'updates' ? 'Updates' : view === 'disks' ? 'Disks' : view === 'space' ? 'Space' : view === 'files' ? 'Files' : view === 'procs' ? 'Processes' : view === 'machines' ? 'Machines' : view === 'users' ? 'Users' : view === 'settings' ? 'Settings' : activeTerm ? activeTerm.name : ''}
) diff --git a/web/src/Space.tsx b/web/src/Space.tsx new file mode 100644 index 0000000..3968495 --- /dev/null +++ b/web/src/Space.tsx @@ -0,0 +1,414 @@ +// Space: what is using a disk, WizTree-style. Pick a disk, scan it (a parallel walk of every +// filesystem the disk backs, as root when the host allows), then read the result two ways: +// a tree of folders largest-first, and a treemap of whatever folder is selected, where every +// file is a tile sized by what it takes on disk and coloured by what kind of thing it is. +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { api, SpaceDisk, SpaceFile, SpaceScan, SpaceTree } from './api' +import { fmtBytes } from './Dashboard' +import { useHashSel } from './hashsel' +import { useRefreshMs } from './prefs' +import { Btn, Loading, Spinner, toast } from './ui' + +// ---- what kind of file a tile is: the colour it wears (the validated categorical set, dark and light) ---- + +interface Kind { id: string; label: string; dark: string; light: string; ext: string[] } +const KINDS: Kind[] = [ + { id: 'video', label: 'Video', dark: '#9085e9', light: '#4a3aa7', ext: ['mp4', 'mkv', 'avi', 'mov', 'webm', 'm4v', 'ts', 'wmv', 'flv', 'mpg', 'mpeg', 'vob', 'm2ts'] }, + { id: 'image', label: 'Images', dark: '#d55181', light: '#e87ba4', ext: ['jpg', 'jpeg', 'png', 'gif', 'heic', 'heif', 'webp', 'raw', 'dng', 'cr2', 'nef', 'arw', 'tif', 'tiff', 'bmp', 'svg', 'psd'] }, + { id: 'audio', label: 'Audio', dark: '#199e70', light: '#1baf7a', ext: ['mp3', 'flac', 'm4a', 'aac', 'ogg', 'opus', 'wav', 'wma', 'aiff'] }, + { id: 'archive', label: 'Archives & images', dark: '#d95926', light: '#eb6834', ext: ['zip', 'tar', 'gz', 'tgz', 'xz', 'zst', 'bz2', '7z', 'rar', 'iso', 'img', 'qcow2', 'vmdk', 'vdi', 'vhd', 'vhdx', 'deb', 'rpm', 'apk', 'jar', 'whl'] }, + { id: 'doc', label: 'Documents', dark: '#c98500', light: '#eda100', ext: ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'odt', 'ods', 'txt', 'md', 'epub', 'mobi', 'rtf', 'csv'] }, + { id: 'code', label: 'Code & config', dark: '#3987e5', light: '#2a78d6', ext: ['js', 'mjs', 'ts', 'tsx', 'jsx', 'go', 'py', 'rs', 'c', 'h', 'cpp', 'java', 'sh', 'css', 'html', 'json', 'yaml', 'yml', 'toml', 'xml', 'sql', 'wasm', 'so', 'a', 'o', 'node'] }, + { id: 'data', label: 'Databases & logs', dark: '#e66767', light: '#e34948', ext: ['db', 'sqlite', 'sqlite3', 'log', 'ldb', 'sst', 'wal', 'journal', 'bin', 'dat', 'idx', 'pack', 'ibd', 'frm', 'mdb', 'rdb', 'aof'] }, + { id: 'other', label: 'Other', dark: '#008300', light: '#008300', ext: [] }, +] +const BY_EXT = new Map() +for (const k of KINDS) for (const e of k.ext) BY_EXT.set(e, k) +function kindOf(name: string): Kind { + const dot = name.lastIndexOf('.') + const ext = dot > 0 ? name.slice(dot + 1).toLowerCase() : '' + return (ext && BY_EXT.get(ext)) || KINDS[KINDS.length - 1] +} +function lightTheme(): boolean { + const bg = getComputedStyle(document.documentElement).getPropertyValue('--bg').trim() + return /^#[0-9a-f]{6}$/i.test(bg) && parseInt(bg.slice(1, 3), 16) > 128 +} +function cssVar(name: string): string { return getComputedStyle(document.documentElement).getPropertyValue(name).trim() } + +const fmtN = (n: number) => n.toLocaleString() +const fmtWhen = (t: number) => (t ? new Date(t * 1000).toLocaleDateString() : '') +const ago = (iso?: string) => { + if (!iso) return '' + const s = Math.max(0, (Date.now() - new Date(iso).getTime()) / 1000) + return s < 60 ? 'just now' : s < 3600 ? `${Math.floor(s / 60)} min ago` : s < 86400 ? `${Math.floor(s / 3600)} h ago` : `${Math.floor(s / 86400)} d ago` +} +const fmtDur = (a: string, b?: string) => { const ms = (b ? new Date(b).getTime() : Date.now()) - new Date(a).getTime(); return ms < 1000 ? `${ms} ms` : ms < 60000 ? `${(ms / 1000).toFixed(1)} s` : `${Math.floor(ms / 60000)} min ${Math.round((ms % 60000) / 1000)} s` } +const parentOf = (p: string) => (p === '/' ? '/' : p.slice(0, p.lastIndexOf('/')) || '/') + +export default function Space() { + const [disks, setDisks] = useState(null) + const [err, setErr] = useState('') + const [sel, setSel] = useHashSel('space') + const [tab, setTab] = useState<'tree' | 'files'>('tree') + const rate = useRefreshMs() + + const load = useCallback(async () => { + try { const r = await api.space(); setDisks(r.disks); setErr('') } catch (e) { setErr(e instanceof Error ? e.message : 'failed') } + }, []) + useEffect(() => { load() }, [load]) + const disk = disks?.find((d) => d.name === sel) ?? disks?.find((d) => d.roots.length > 0) ?? disks?.[0] + useEffect(() => { if (disk && disk.name !== sel) setSel(disk.name) }, [disk, sel, setSel]) + const scan = disk?.scan + // while a scan runs its status alone is polled at the refresh rate (the disk list means + // lsblk and zfs list, which need not run twice a second); the list is read again when it ends + useEffect(() => { + if (!scan?.running || !disk) return + const name = disk.name + const t = setInterval(async () => { + try { + const st = await api.spaceStatus(name) + setDisks((ds) => ds?.map((d) => (d.name === name ? { ...d, scan: st } : d)) ?? ds) + } catch { /* the next tick */ } + }, Math.max(250, rate)) + return () => clearInterval(t) + }, [scan?.running, disk, rate]) + + const start = async () => { if (!disk) return; try { await api.spaceScan(disk.name); await load() } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } } + const cancel = async () => { if (!disk) return; try { await api.spaceCancel(disk.name); await load() } catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } } + + if (err && !disks) return
{err}
+ if (!disks) return
+ const done = !!scan && !scan.running && !scan.error && !!scan.finished + return ( +
+
+
+ {disks.map((d) => ( + + ))} + {disks.length === 0 && No disks found.} +
+ {disk && ( +
+ {scan?.running ? ( + <> + {fmtN(scan.progress.files)} files · {fmtBytes(scan.progress.bytes)} · {fmtDur(scan.started)} + {scan.progress.current} + Cancel + + ) : ( + <> + {done && {fmtBytes(scan.size)} in {fmtN(scan.files)} files, {fmtN(scan.dirs)} folders · scanned {ago(scan.finished)} in {fmtDur(scan.started, scan.finished)}{scan.as_root ? '' : ' · not as root, so some folders were skipped'}{scan.denied ? ` · ${fmtN(scan.denied)} unreadable` : ''}} + {scan?.error && {scan.error}} + {!scan && {disk.roots.length ? `Scans ${disk.roots.join(', ')}` : 'Nothing is mounted from this disk.'}} + {done ? 'Rescan' : 'Scan'} + + )} +
+ )} +
+ {disk && done && ( + <> +
+ + +
+ {tab === 'tree' ? : } + + )} + {disk && !done && !scan?.running &&
{scan?.error ? 'The last scan failed. Scan again.' : 'Scan the disk to see what is on it.'}
} + {disk && scan?.running &&
Scanning…
} +
+ ) +} + +// ---- the folder tree and the treemap of the selected folder ---- + +interface Row { t: SpaceTree; depth: number; kind: 'node' | 'small' | 'more'; parent: number } + +function Explorer({ disk }: { disk: string }) { + const [cache, setCache] = useState>({}) // one level per folder, for the rows + const [maps, setMaps] = useState>({}) // deep, pruned to what a tile could show, for the map + const [open, setOpen] = useState>(() => new Set(['/'])) + const [focus, setFocus] = useState('/') + const pending = useRef(new Set()) + + const fetch = useCallback(async (path: string) => { + if (pending.current.has(path)) return + pending.current.add(path) + try { const t = await api.spaceTree(disk, path, 1, 200); setCache((c) => ({ ...c, [path]: t })) } + catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } + finally { pending.current.delete(path) } + }, [disk]) + // The map of a folder reaches as deep as there is room to draw: everything under a + // four-thousandth of the folder would be a tile smaller than a few pixels, so it is left out. + const fetchMap = useCallback(async (path: string, size: number) => { + const key = 'map:' + path + if (pending.current.has(key)) return + pending.current.add(key) + try { const t = await api.spaceTree(disk, path, 12, 200, Math.max(1, Math.floor(size / 4000))); setMaps((m) => ({ ...m, [path]: t })) } + catch (e) { toast(e instanceof Error ? e.message : 'failed', 'err') } + finally { pending.current.delete(key) } + }, [disk]) + useEffect(() => { if (!cache['/']) fetch('/') }, [cache, fetch]) + useEffect(() => { if (!cache[focus]) fetch(focus) }, [focus, cache, fetch]) + useEffect(() => { const c = cache[focus]; if (c && !maps[focus]) fetchMap(focus, c.size) }, [focus, cache, maps, fetchMap]) + + const toggle = (path: string) => { + setOpen((o) => { const n = new Set(o); if (n.has(path)) n.delete(path); else { n.add(path); if (!cache[path]) fetch(path) } return n }) + } + const select = (path: string) => { + setFocus(path) + // open every folder down to it so the row is on screen + setOpen((o) => { const n = new Set(o); let p = path; while (p !== '/') { p = parentOf(p); n.add(p) } return n }) + if (!cache[path]) fetch(path) + } + + const rows = useMemo(() => { + const out: Row[] = [] + const walk = (t: SpaceTree, depth: number) => { + out.push({ t, depth, kind: 'node', parent: t.size }) + if (!t.dir || !open.has(t.path)) return + const full = cache[t.path] ?? t // the row's own fetch has the full list; the parent's cut copy does until then + for (const c of full.children ?? []) walk(c, depth + 1) + if (full.small_n) out.push({ t: { ...full, name: `${fmtN(full.small_n)} smaller files`, dir: false, size: full.small_size ?? 0, files: full.small_n, path: full.path + '/*' }, depth: depth + 1, kind: 'small', parent: full.size }) + if (full.more_n) out.push({ t: { ...full, name: `${fmtN(full.more_n)} more items`, dir: false, size: full.more_size ?? 0, path: full.path + '/…' }, depth: depth + 1, kind: 'more', parent: full.size }) + } + if (cache['/']) walk(cache['/'], 0) + // parent sizes for the bars: each row's share of the folder it sits in + const stack: number[] = [] + for (const r of out) { stack.length = r.depth; r.parent = stack[r.depth - 1] ?? r.t.size; stack[r.depth] = r.t.size } + return out + }, [cache, open]) + + const root = cache['/'] + if (!root) return + const focused = cache[focus] + const map = maps[focus] + return ( +
+
+
+ NameSizeof parentFilesFoldersModified +
+
+ {rows.map((r) => { + const t = r.t + const pct = r.parent > 0 ? (t.size / r.parent) * 100 : 0 + const isOpen = open.has(t.path) + return ( +
{ if (r.kind === 'node') { if (t.dir) { select(t.path); if (!isOpen) toggle(t.path) } else select(parentOf(t.path)) } }}> + + {t.dir && r.kind === 'node' ? : } + {t.dir && r.kind === 'node' ? '▰' : ''} + {t.name}{t.mount && t.dir && r.kind === 'node' ? mount : ''}{t.denied ? unreadable : ''} + + {fmtBytes(t.size)} + {pct >= 10 ? pct.toFixed(0) : pct.toFixed(1)}% + {t.dir && r.kind === 'node' ? fmtN(t.files) : r.kind === 'small' ? fmtN(t.files) : ''} + {t.dir && r.kind === 'node' ? fmtN(t.dirs) : ''} + {r.kind === 'node' ? fmtWhen(t.mtime) : ''} +
+ ) + })} +
+
+
+
+ + {crumbs(focus).map((c) => )} + + {focused && {fmtBytes(focused.size)} · {root.size > 0 ? ((focused.size / root.size) * 100).toFixed(1) : 0}% of the disk's {fmtBytes(root.size)}} +
+ {map ? : } +
+ {KINDS.map((k) => {k.label})} + FolderSmaller files +
+
+
+ ) +} + +function crumbs(path: string): { name: string; path: string }[] { + const out = [{ name: '/', path: '/' }] + let p = '' + for (const part of path.split('/').filter(Boolean)) { p += '/' + part; out.push({ name: part, path: p }) } + return out +} + +// ---- the treemap: squarified, nested three deep, drawn on a canvas ---- + +interface Tile { x: number; y: number; w: number; h: number; t: SpaceTree; agg?: 'small' | 'more'; depth: number; head?: number; inner?: boolean } + +// squarify lays items (largest first) into rect, keeping tiles as square as it can. +function squarify(items: { size: number; t: SpaceTree; agg?: 'small' | 'more' }[], x: number, y: number, w: number, h: number, depth: number, out: Tile[]) { + const total = items.reduce((a, b) => a + b.size, 0) + if (total <= 0 || w <= 1 || h <= 1) return + let i = 0 + let cx = x, cy = y, cw = w, ch = h + while (i < items.length) { + const vertical = cw < ch // lay the next row along the shorter side + const side = vertical ? cw : ch + const scale = (cw * ch) / items.slice(i).reduce((a, b) => a + b.size, 0) + let row: typeof items = [] + let rowSum = 0 + let worst = Infinity + for (let j = i; j < items.length; j++) { + const s = items[j].size * scale + if (s <= 0) { i = j + 1; continue } + const sum = rowSum + s + const thick = sum / side + let wst = 0 + for (const r of [...row, items[j]]) { const len = (r.size * scale) / thick; wst = Math.max(wst, len / thick, thick / len) } + if (row.length && wst > worst) break + row.push(items[j]); rowSum = sum; worst = wst + } + if (row.length === 0) break + const thick = rowSum / side + let off = 0 + for (const r of row) { + const len = (r.size * scale) / thick + const tile: Tile = vertical ? { x: cx, y: cy + off, w: thick, h: len, t: r.t, agg: r.agg, depth } : { x: cx + off, y: cy, w: len, h: thick, t: r.t, agg: r.agg, depth } + out.push(tile) + off += len + } + if (vertical) { cy += thick; ch -= thick } else { cx += thick; cw -= thick } + i += row.length + } +} + +function layout(tree: SpaceTree, x: number, y: number, w: number, h: number, depth: number, out: Tile[]) { + const items: { size: number; t: SpaceTree; agg?: 'small' | 'more' }[] = (tree.children ?? []).filter((c) => c.size > 0).map((c) => ({ size: c.size, t: c })) + if (tree.small_n && tree.small_size) items.push({ size: tree.small_size, t: { ...tree, name: `${fmtN(tree.small_n)} smaller files`, dir: false, path: tree.path + '/*', children: undefined }, agg: 'small' }) + if (tree.more_n && tree.more_size) items.push({ size: tree.more_size, t: { ...tree, name: `${fmtN(tree.more_n)} more items`, dir: false, path: tree.path + '/…', children: undefined }, agg: 'more' }) + items.sort((a, b) => b.size - a.size) + const tiles: Tile[] = [] + squarify(items, x, y, w, h, depth, tiles) + for (const t of tiles) { + out.push(t) + const kids = (t.t.children?.length ?? 0) + (t.t.small_size ? 1 : 0) + (t.t.more_size ? 1 : 0) + if (t.t.dir && kids > 0 && depth < 12 && t.w > 20 && t.h > 20) { + // a title strip when there is room for one; a tiny folder is just a box of its children + t.head = t.h > 40 && t.w > 40 ? 15 : 0 + t.inner = true + layout(t.t, t.x + 2, t.y + 2 + t.head, t.w - 4, t.h - 4 - t.head, depth + 1, out) + } + } +} + +function Treemap({ tree, onOpen }: { tree: SpaceTree; onOpen: (path: string) => void }) { + const box = useRef(null) + const canvas = useRef(null) + const tiles = useRef([]) + const [size, setSize] = useState({ w: 0, h: 0 }) + const [hover, setHover] = useState<{ x: number; y: number; tile: Tile } | null>(null) + + useEffect(() => { + const el = box.current + if (!el) return + const ro = new ResizeObserver(() => setSize({ w: el.clientWidth, h: el.clientHeight })) + ro.observe(el) + setSize({ w: el.clientWidth, h: el.clientHeight }) + return () => ro.disconnect() + }, []) + + useEffect(() => { + const c = canvas.current + if (!c || size.w === 0 || size.h === 0) return + const dpr = window.devicePixelRatio || 1 + c.width = Math.round(size.w * dpr); c.height = Math.round(size.h * dpr) + const g = c.getContext('2d') + if (!g) return + g.scale(dpr, dpr) + const light = lightTheme() + const bg = cssVar('--bg2') || '#171b21', border = cssVar('--border') || '#2a313b', fg = cssVar('--fg') || '#d0d7de', muted = cssVar('--muted') || '#7d8590' + g.fillStyle = bg + g.fillRect(0, 0, size.w, size.h) + const out: Tile[] = [] + layout(tree, 0, 0, size.w, size.h, 0, out) + tiles.current = out + const font = cssVar('--font') || 'system-ui, sans-serif' + for (const t of out) { + if (t.w < 1 || t.h < 1) continue + const isDir = t.t.dir && !t.agg + let fill: string + if (t.agg) fill = light ? '#c9ced4' : '#3a414b' + else if (isDir) fill = light ? (t.depth % 2 ? '#eef0f3' : '#e3e6ea') : (t.depth % 2 ? '#262d36' : '#1e242c') + else { const k = kindOf(t.t.name); fill = light ? k.light : k.dark } + g.fillStyle = fill + g.fillRect(t.x, t.y, t.w, t.h) + g.strokeStyle = isDir ? border : bg + g.lineWidth = 1 + g.strokeRect(t.x + 0.5, t.y + 0.5, t.w - 1, t.h - 1) + // a label when there is room and it would not sit on the folder's own children: the + // name, and the size on a second line + const labelled = !isDir || !t.inner || (t.head ?? 0) > 0 + if (labelled && t.w > 36 && t.h > 14) { + g.save() + g.beginPath(); g.rect(t.x + 1, t.y + 1, t.w - 2, t.h - 2); g.clip() + g.font = `${isDir ? '600 ' : ''}11px ${font}` + g.fillStyle = isDir || t.agg ? (isDir ? fg : muted) : (light ? '#111' : '#fff') + g.textBaseline = 'top' + g.fillText(t.t.name, t.x + 4, t.y + 2, t.w - 8) + if (t.h > 30 && !t.inner) { g.font = `10px ${font}`; g.fillStyle = isDir || t.agg ? muted : (light ? 'rgba(0,0,0,.7)' : 'rgba(255,255,255,.8)'); g.fillText(fmtBytes(t.t.size), t.x + 4, t.y + 16, t.w - 8) } + g.restore() + } + } + }, [tree, size]) + + const at = (e: React.MouseEvent) => { + const r = canvas.current!.getBoundingClientRect() + const x = e.clientX - r.left, y = e.clientY - r.top + // the deepest tile under the pointer + let best: Tile | null = null + for (const t of tiles.current) if (x >= t.x && x < t.x + t.w && y >= t.y && y < t.y + t.h && (!best || t.depth > best.depth)) best = t + return { x, y, tile: best } + } + return ( +
+ { const h = at(e); setHover(h.tile ? { x: h.x, y: h.y, tile: h.tile } : null) }} + onMouseLeave={() => setHover(null)} + onClick={(e) => { const h = at(e); if (!h.tile) return; const t = h.tile.t; onOpen(t.dir && !h.tile.agg ? t.path : parentOf(t.path)) }} /> + {hover && ( +
+
{hover.tile.agg ? hover.tile.t.name + ' in ' + parentOf(hover.tile.t.path) : hover.tile.t.path}
+
{fmtBytes(hover.tile.t.size)} · {tree.size > 0 ? ((hover.tile.t.size / tree.size) * 100).toFixed(1) : 0}% of {tree.name === '/' ? 'the disk' : tree.name}
+ {hover.tile.t.dir && !hover.tile.agg &&
{fmtN(hover.tile.t.files)} files · {fmtN(hover.tile.t.dirs)} folders
} + {!hover.tile.t.dir && !hover.tile.agg &&
{kindOf(hover.tile.t.name).label} · {fmtWhen(hover.tile.t.mtime)}{hover.tile.t.apparent !== hover.tile.t.size ? ` · ${fmtBytes(hover.tile.t.apparent)} apparent` : ''}
} +
+ )} +
+ ) +} + +// ---- the largest files on the disk ---- + +function LargestFiles({ disk }: { disk: string }) { + const [files, setFiles] = useState(null) + const [filter, setFilter] = useState('') + useEffect(() => { api.spaceLargest(disk, 1000).then(setFiles).catch((e) => toast(e instanceof Error ? e.message : 'failed', 'err')) }, [disk]) + if (!files) return + const q = filter.toLowerCase() + const shown = q ? files.filter((f) => f.path.toLowerCase().includes(q)) : files + const top = shown[0]?.size ?? 1 + return ( +
+
setFilter(e.target.value)} spellCheck={false} />the {fmtN(files.length)} largest files{q ? ` · ${fmtN(shown.length)} match` : ''}
+
+ {shown.map((f) => ( +
+ {f.path} + {fmtBytes(f.size)} + + {fmtWhen(f.mtime)} +
+ ))} +
+
+ ) +} diff --git a/web/src/api.ts b/web/src/api.ts index c77e552..961de1b 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -126,6 +126,13 @@ export interface User { name: string; created: string; last_seen?: string; sessi export interface Session { id: string; created: string; last_seen: string; expires: string; ip?: string; agent?: string; current: boolean } export interface TokenInfo { name: string; created: string; scope?: string; sensors?: string[]; issued_by?: string } +export interface SpaceProgress { files: number; dirs: number; bytes: number; denied: number; current: string } +export interface SpaceScan { disk: string; roots: string[]; started: string; finished?: string; running: boolean; error?: string; progress: SpaceProgress; as_root: boolean; files: number; dirs: number; size: number; denied: number } +export interface SpaceDisk { name: string; kind: string; model?: string; size: number; pool?: string; roots: string[]; scan?: SpaceScan } +export interface SpaceNode { name: string; dir: boolean; size: number; apparent: number; mtime: number; files: number; dirs: number; small_n?: number; small_size?: number; mount?: boolean; denied?: boolean } +export interface SpaceTree extends SpaceNode { path: string; children?: SpaceTree[]; more_n?: number; more_size?: number; total: number } +export interface SpaceFile { path: string; size: number; apparent: number; mtime: number } + export interface FileEntry { name: string; type: 'dir' | 'file' | 'link' | 'other'; size: number; mode: string; perm: string; mtime: number; user: string; group: string; target?: string; broken?: boolean } export interface FileInfo extends FileEntry { path: string; text: boolean } @@ -133,6 +140,12 @@ const q = (path: string, sudo: boolean, extra = '') => `path=${encodeURIComponen export const api = { procs: () => req('GET', '/api/procs'), + space: () => req<{ disks: SpaceDisk[] }>('GET', '/api/space'), + spaceStatus: (disk: string) => req('GET', `/api/space/${encodeURIComponent(disk)}`), + spaceScan: (disk: string) => req('POST', `/api/space/${encodeURIComponent(disk)}/scan`), + spaceCancel: (disk: string) => req('POST', `/api/space/${encodeURIComponent(disk)}/cancel`), + spaceTree: (disk: string, path: string, depth = 1, limit = 200, min = 0) => req('GET', `/api/space/${encodeURIComponent(disk)}/tree?path=${encodeURIComponent(path)}&depth=${depth}&limit=${limit}&min=${min}`), + spaceLargest: (disk: string, n = 500) => req('GET', `/api/space/${encodeURIComponent(disk)}/largest?n=${n}`), procSignal: (pid: number, signal: 'TERM' | 'KILL') => req('POST', `/api/procs/${pid}/signal`, { signal }), files: (path: string, sudo: boolean) => req<{ path: string; parent?: string; entries: FileEntry[] }>('GET', `/api/files?${q(path, sudo)}`), fileStat: (path: string, sudo: boolean) => req('GET', `/api/files/stat?${q(path, sudo)}`), diff --git a/web/src/styles.css b/web/src/styles.css index 59157ed..a6b79ac 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -668,3 +668,43 @@ pre.diff .add { color: var(--ok); background: color-mix(in srgb, var(--ok) 12%, .t-net { flex-direction: column; } .t-net-nums { width: auto; } .t-net-graphs { flex: 0 0 130px; } .top-proc { height: 70vh; min-height: 420px; } } + +/* ---- Space: what is using a disk ---- */ +.space { position: absolute; inset: 0; display: flex; flex-direction: column; background: var(--bg); min-height: 0; } +.space-bar { display: flex; flex-direction: column; gap: 8px; padding: 10px 12px; border-bottom: 1px solid var(--border); background: var(--bg2); } +.space-bar .chip.disk b { color: var(--fg); font-weight: 600; } .space-bar .chip.disk:disabled { opacity: .45; cursor: default; } +.space-bar .chip.disk .spin { margin-left: 6px; margin-right: 0; } +.space-status { display: flex; align-items: center; gap: 10px; min-height: 28px; flex-wrap: wrap; } +.space-status .cur { flex: 1 1 200px; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; direction: rtl; text-align: left; } +.space-tabs { padding: 8px 12px 0; } +.space-split { flex: 1; display: flex; flex-direction: column; min-height: 0; padding: 8px 12px 12px; gap: 8px; } +.space-tree { flex: 1 1 55%; min-height: 120px; display: flex; flex-direction: column; border: 1px solid var(--border); border-radius: 6px; background: var(--bg2); overflow: hidden; } +.space-rows { flex: 1; overflow: auto; min-height: 0; } +.space-row { display: grid; grid-template-columns: minmax(160px, 1fr) 84px 120px 70px 64px 88px; gap: 8px; align-items: center; padding: 3px 8px; font-size: 13px; border-bottom: 1px solid var(--border); cursor: pointer; white-space: nowrap; } +.space-row.head { position: sticky; top: 0; z-index: 1; background: var(--bg3); color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: .04em; cursor: default; } +.space-row:hover:not(.head) { background: var(--bg3); } .space-row.active { background: var(--bg3); box-shadow: inset 3px 0 0 var(--accent); } +.space-row.agg { color: var(--muted); font-style: italic; } +.space-row .n { display: flex; align-items: center; gap: 4px; min-width: 0; } .space-row .nm { overflow: hidden; text-overflow: ellipsis; } +.space-row .tw { width: 16px; flex-shrink: 0; color: var(--muted); padding: 0; font-size: 11px; } .space-row .tw:hover { color: var(--fg); } +.space-row .ico { width: 10px; height: 10px; flex-shrink: 0; border-radius: 2px; font-size: 9px; line-height: 10px; color: var(--accent2); text-align: center; } +.space-row .r { text-align: right; } .space-row .pct { display: flex; align-items: center; gap: 6px; } +.space-row .bar { flex: 1; height: 8px; background: var(--track); border-radius: 4px; overflow: hidden; } .space-row .bar > span { display: block; height: 100%; background: var(--s1); border-radius: 4px; } +.space-row.file { grid-template-columns: minmax(160px, 1fr) 84px 120px 88px; } +.space-map { flex: 1 1 45%; min-height: 200px; display: flex; flex-direction: column; gap: 6px; } +.space-crumbs { display: flex; align-items: center; gap: 2px; flex-wrap: wrap; font-size: 13px; } +.space-crumbs .crumb { padding: 2px 6px; border-radius: 4px; color: var(--muted); } .space-crumbs .crumb:hover { color: var(--fg); background: var(--bg3); } .space-crumbs .crumb.on { color: var(--fg); font-weight: 600; } +.space-crumbs .crumb + .crumb::before { content: '›'; color: var(--muted); margin-right: 4px; } +.space-canvas { flex: 1; position: relative; min-height: 160px; border: 1px solid var(--border); border-radius: 6px; overflow: hidden; background: var(--bg2); } +.space-canvas canvas { display: block; cursor: pointer; } +.space-tip { position: absolute; z-index: 2; max-width: 260px; padding: 6px 8px; background: var(--bg); border: 1px solid var(--border); border-radius: 6px; font-size: 12px; pointer-events: none; box-shadow: 0 4px 16px rgba(0,0,0,.35); } +.space-legend { display: flex; flex-wrap: wrap; gap: 4px 12px; font-size: 11px; color: var(--muted); } +.space-legend i { display: inline-block; width: 10px; height: 10px; border-radius: 2px; margin-right: 4px; vertical-align: -1px; } +.space-legend i.dir { background: var(--bg3); border: 1px solid var(--border); } .space-legend i.small { background: #3a414b; } +.space-files { flex: 1; display: flex; flex-direction: column; min-height: 0; padding: 8px 12px 12px; gap: 8px; } +.space-files .space-rows { border: 1px solid var(--border); border-radius: 6px; background: var(--bg2); } +.space-files input.filter { padding: 6px 10px; background: var(--bg2); border: 1px solid var(--border); border-radius: 6px; color: var(--fg); font-size: 13px; width: min(320px, 100%); } +@media (max-width: 800px) { + .space-row { grid-template-columns: minmax(120px, 1fr) 72px 80px; } .space-row .c, .space-row .d { display: none; } + .space-row.file { grid-template-columns: minmax(120px, 1fr) 72px 80px; } + .space-split { padding: 8px; } .space-tree { flex: 1 1 45%; } .space-map { flex: 1 1 55%; } +}