From b44faf3ac98adff8869055d34971ac21a5b70ec9 Mon Sep 17 00:00:00 2001 From: Mitchell-Johnson Date: Fri, 2 Oct 2026 18:30:02 -0700 Subject: [PATCH] The ZFS tab shows the ARC, and names every caching tier a pool has or lacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Cache section between the pools and the schedules. One card for the ARC: size against its ceiling with the target and floor, sparklines of size and hit rate from the sampler's history, what it holds (MRU/MFU/dirty, data vs metadata, compression, the ghost lists), the prefetcher and dbuf cache, the memory-pressure counters, and the since-boot breakdown by demand/prefetch and data/metadata. Then a card per pool for L2ARC, SLOG, special vdev, dedup table and spares — the devices and their fill where they exist, and where they don't, plainly none and what that means (the ZIL writing to the pool's own vdevs, with how much it has written). Live numbers come from the sampler, which reads arcstats every tick and publishes zfs.arc.* sensors; the fuller picture is read on request, with zpool list -Hpv and zfs get run without sudo and cached for thirty seconds. Co-Authored-By: Claude Fable 5.1 --- PLAN.md | 14 ++ SENSORS.md | 5 + internal/api/server.go | 1 + internal/api/zfs.go | 11 ++ internal/metrics/arc.go | 113 ++++++++++++++ internal/metrics/gpu_test.go | 18 +++ internal/metrics/metrics.go | 3 + internal/metrics/sensors.go | 27 ++++ internal/storage/arc.go | 292 +++++++++++++++++++++++++++++++++++ internal/storage/arc_test.go | 71 +++++++++ web/src/ZFS.tsx | 150 +++++++++++++++++- web/src/api.ts | 23 +++ web/src/styles.css | 10 ++ 13 files changed, 736 insertions(+), 2 deletions(-) create mode 100644 internal/metrics/arc.go create mode 100644 internal/storage/arc.go create mode 100644 internal/storage/arc_test.go diff --git a/PLAN.md b/PLAN.md index daaa6a1..513e5af 100644 --- a/PLAN.md +++ b/PLAN.md @@ -314,6 +314,20 @@ something long in one, switch away and back, refresh — it's all still there. GPU shows the CPU package temperature marked `*`. Asahi (the Mac mini) has neither a busy counter nor fdinfo in 6.19, so it lists the device and says so. No NVIDIA path: nothing here has one to test against. +- **The ARC on the ZFS tab** — **shipped 2026-10-02** (`internal/metrics/arc.go`, + `internal/storage/arc.go`, `GET /api/zfs/arc`, the Cache section in `ZFS.tsx`, `zfs.arc.*` + sensors). Asked for as "current ARC usage, including any types of caching services in the + pool". One card for the ARC: size against its ceiling with the target and floor, sparklines + of size and hit rate from the sampler's history, what it holds (MRU/MFU/dirty, data vs + metadata, compression, the ghost lists), the prefetcher and dbuf cache, memory-pressure + counters, and the since-boot breakdown by demand/prefetch × data/metadata. Then a card per + pool naming every caching tier — L2ARC, SLOG, special vdev, dedup table, spares — with its + devices and fill, **or saying plainly that there is none and what that means** (the ZIL on + the pool's own vdevs, with how much it has written), plus the properties that steer caching + (`primarycache`, `secondarycache`, `sync`, `logbias`). Read from `/proc/spl/kstat/zfs/*` + and `/sys/module/zfs/parameters`, with `zpool list -Hpv` and `zfs get` run without sudo + (they need none) every 30 s at most. The day it shipped it showed `tank`'s ARC pinned at + its 2 GiB floor under `arc_no_grow`, with a 53 GiB python3 process holding the RAM. - **Dashboard retired** — **2026-09-23**, at Mitchell's instruction, the same reasoning that merged Top into Processes: the app repeated itself. The customisable module list is gone (`Dashboard.tsx` deleted), and with it `/api/prefs/dashboard`, `applyPrefs`, and the diff --git a/SENSORS.md b/SENSORS.md index a991418..18fec25 100644 --- a/SENSORS.md +++ b/SENSORS.md @@ -21,6 +21,7 @@ disk..read|write|util|size|temp fs..size|used|used_pct ups.battery|runtime|load|load_pct|in_v|out_v|on_battery|state|source gpu..usage|render|video|video-enhance|copy|compute|temp|power|fan|freq|mem_used|mem_total|clients +zfs.arc.size|target|min|max|mru|mfu|data|metadata|hit_pct|reads|misses|lifetime_hit_pct|no_grow|compress_ratio|l2_size|l2_hit_pct ``` - Segments are lower case `[a-z0-9_-]`: `Package id 0` → `package_id_0`, `/mnt/tank` → @@ -42,6 +43,10 @@ gpu..usage|render|video|video-enhance|copy|compute|temp|power|fan|freq|mem has it open. `temp` is the card's own sensor, or on an integrated GPU the CPU package it shares a die with (label `source` says which); an Arc on i915 has none and reports none. `freq` is the actual clock, 0 when power-gated. `clients` counts the processes using it. +- `zfs.arc.*` is the ZFS read cache in RAM, on a machine with the module loaded. `hit_pct` + is over the last sampling interval and appears once an interval has had any reads (an idle + interval repeats the last value); `lifetime_hit_pct` is since boot. `no_grow` is true + while memory pressure stops the cache growing. `l2_*` appear only with cache devices. ## A sensor diff --git a/internal/api/server.go b/internal/api/server.go index 8cf067d..f53fda9 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -295,6 +295,7 @@ func (s *Server) routes() *http.ServeMux { m.HandleFunc("POST /api/storage/zfs/scrub/{pool}/{action}", s.auth(s.zfsScrub)) m.HandleFunc("POST /api/storage/zfs/snapshot", s.auth(s.zfsSnapshot)) m.HandleFunc("GET /api/zfs", s.auth(s.zfsOverview)) + m.HandleFunc("GET /api/zfs/arc", s.auth(s.zfsARC)) m.HandleFunc("PUT /api/zfs/schedules", s.auth(s.zfsPolicyPut)) m.HandleFunc("DELETE /api/zfs/schedules/{id}", s.auth(s.zfsPolicyDelete)) m.HandleFunc("POST /api/zfs/schedules/{id}/run", s.auth(s.zfsRunNow)) diff --git a/internal/api/zfs.go b/internal/api/zfs.go index fcf41bb..009459b 100644 --- a/internal/api/zfs.go +++ b/internal/api/zfs.go @@ -136,6 +136,17 @@ func (s *Server) zfsOverview(w http.ResponseWriter, r *http.Request) { }) } +// zfsARC is the cache: the sampler's live ARC (size, the hit rate over the last interval) +// beside the storage package's fuller picture (since-boot counters, tunables, and each +// pool's cache/log/special devices or the lack of them). +func (s *Server) zfsARC(w http.ResponseWriter, r *http.Request) { + out := map[string]any{"detail": s.storage.ARC(r.Context()), "live": nil} + if smp, ok := s.metrics.Latest(); ok && smp.ARC != nil { + out["live"] = smp.ARC + } + jsonOK(w, 200, out) +} + func (s *Server) zfsPolicyPut(w http.ResponseWriter, r *http.Request) { var p storage.Policy if err := json.NewDecoder(io.LimitReader(r.Body, 16<<10)).Decode(&p); err != nil { diff --git a/internal/metrics/arc.go b/internal/metrics/arc.go new file mode 100644 index 0000000..caf0745 --- /dev/null +++ b/internal/metrics/arc.go @@ -0,0 +1,113 @@ +package metrics + +// The ZFS ARC — the read cache in RAM — read live from /proc/spl/kstat/zfs/arcstats on a +// machine that has the module loaded, and left out of the sample everywhere else. The kstat +// file is counters since boot; the hit rate here is over the sampling interval, which is the +// number that says whether the cache is doing its job *now*. The lifetime figures and the +// pool's caching tiers (L2ARC, SLOG, special vdevs) are the storage package's business. + +import ( + "strconv" + "strings" + "time" +) + +const arcstatsPath = "/proc/spl/kstat/zfs/arcstats" + +// ARC is the read cache as one sample sees it. Sizes are bytes. +type ARC struct { + Size uint64 `json:"size"` + Target uint64 `json:"target"` // c: what the ARC is currently allowed to hold + Min uint64 `json:"min"` // c_min: the floor it will not shrink below + Max uint64 `json:"max"` // c_max: the ceiling (zfs_arc_max, or half of RAM) + // what is in it + MRU uint64 `json:"mru"` // recently used once + MFU uint64 `json:"mfu"` // used more than once + Anon uint64 `json:"anon"` // in flight: dirty, not yet on disk + Data uint64 `json:"data"` // file contents + Metadata uint64 `json:"metadata"` // dnodes, indirect blocks, dbufs, bonus buffers + MetaUsed uint64 `json:"meta_used"` + Headers uint64 `json:"headers"` + Compressed uint64 `json:"compressed"` + Uncompressed uint64 `json:"uncompressed"` + MRUGhost uint64 `json:"mru_ghost"` // what it recently evicted and still remembers + MFUGhost uint64 `json:"mfu_ghost"` + // L2ARC, the cache devices (0 when there are none) + L2Size uint64 `json:"l2_size"` + L2ASize uint64 `json:"l2_asize"` // as stored on the device, after compression + // over this interval + ReadsPerSec float64 `json:"reads_per_sec"` + MissesPerSec float64 `json:"misses_per_sec"` + HitPct float64 `json:"hit_pct"` // of the interval's reads; an idle interval keeps the last one + HitPctKnown bool `json:"hit_pct_known"` // false until an interval has had any reads + L2HitPct float64 `json:"l2_hit_pct"` + // since boot + Hits uint64 `json:"hits"` + Misses uint64 `json:"misses"` + LifetimeHitPct float64 `json:"lifetime_hit_pct"` + // pressure + NoGrow bool `json:"no_grow"` // arc_no_grow: memory pressure is stopping it growing + Throttles uint64 `json:"throttles"` // memory_throttle_count since boot + Prunes uint64 `json:"prunes"` // arc_prune: metadata evictions asked of the filesystems + EvictSkip uint64 `json:"evict_skip"` +} + +type arcPrev struct { + hits, misses, l2hits, l2misses uint64 + hitPct float64 + known bool + at time.Time +} + +// readKstat parses a kstat table ("name type data" rows after two header lines). +func readKstat(p string) map[string]uint64 { + s := readStr(p) + if s == "" { + return nil + } + out := map[string]uint64{} + for _, line := range strings.Split(s, "\n") { + f := strings.Fields(line) + if len(f) != 3 || f[1] != "4" { // type 4 is KSTAT_DATA_UINT64 + continue + } + v, err := strconv.ParseUint(f[2], 10, 64) + if err == nil { + out[f[0]] = v + } + } + return out +} + +// arc reads the ARC; nil when this machine has no ZFS module loaded. +func (c *Collector) arc(now time.Time) *ARC { + k := readKstat(arcstatsPath) + if k == nil { + return nil + } + a := &ARC{Size: k["size"], Target: k["c"], Min: k["c_min"], Max: k["c_max"], + MRU: k["mru_size"], MFU: k["mfu_size"], Anon: k["anon_size"], Data: k["data_size"], Metadata: k["metadata_size"], + MetaUsed: k["arc_meta_used"], Headers: k["hdr_size"], Compressed: k["compressed_size"], Uncompressed: k["uncompressed_size"], + MRUGhost: k["mru_ghost_size"], MFUGhost: k["mfu_ghost_size"], L2Size: k["l2_size"], L2ASize: k["l2_asize"], + Hits: k["hits"], Misses: k["misses"], NoGrow: k["arc_no_grow"] != 0, Throttles: k["memory_throttle_count"], Prunes: k["arc_prune"], EvictSkip: k["evict_skip"]} + if n := a.Hits + a.Misses; n > 0 { + a.LifetimeHitPct = round1(float64(a.Hits) / float64(n) * 100) + } + p := c.prevARC + if !p.at.IsZero() && a.Hits >= p.hits && a.Misses >= p.misses { + if dt := now.Sub(p.at).Seconds(); dt > 0 { + hits, misses := a.Hits-p.hits, a.Misses-p.misses + a.ReadsPerSec = round1(float64(hits+misses) / dt) + a.MissesPerSec = round1(float64(misses) / dt) + if hits+misses > 0 { + p.hitPct, p.known = round1(float64(hits)/float64(hits+misses)*100), true + } + if l2 := (k["l2_hits"] - p.l2hits) + (k["l2_misses"] - p.l2misses); l2 > 0 && k["l2_hits"] >= p.l2hits { + a.L2HitPct = round1(float64(k["l2_hits"]-p.l2hits) / float64(l2) * 100) + } + } + } + a.HitPct, a.HitPctKnown = p.hitPct, p.known + c.prevARC = arcPrev{hits: a.Hits, misses: a.Misses, l2hits: k["l2_hits"], l2misses: k["l2_misses"], hitPct: p.hitPct, known: p.known, at: now} + return a +} diff --git a/internal/metrics/gpu_test.go b/internal/metrics/gpu_test.go index 61075f6..0ab1d26 100644 --- a/internal/metrics/gpu_test.go +++ b/internal/metrics/gpu_test.go @@ -3,6 +3,7 @@ package metrics import ( "encoding/json" "os" + "strings" "testing" "time" ) @@ -134,3 +135,20 @@ func TestLiveGPU(t *testing.T) { t.Logf("%s = %v %s", x.ID, x.Value, x.Unit) } } + +// TestLiveARC: HAMLAB_LIVE=1 go test -run TestLiveARC -v ./internal/metrics +func TestLiveARC(t *testing.T) { + if os.Getenv("HAMLAB_LIVE") == "" { + t.Skip("set HAMLAB_LIVE=1 to read the real machine") + } + c := NewCollector() + c.sample() + time.Sleep(1100 * time.Millisecond) + c.sample() + s, _ := c.Latest() + b, _ := json.MarshalIndent(s.ARC, "", " ") + t.Logf("%s", b) + for _, x := range flatten(s, func(id string) bool { return strings.HasPrefix(id, "zfs.arc.") }) { + t.Logf("%s = %v %s", x.ID, x.Value, x.Unit) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index f61bb42..5292302 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -147,6 +147,7 @@ type Sample struct { FS []FS `json:"fs"` UPS *UPS `json:"ups"` GPUs []GPU `json:"gpus"` + ARC *ARC `json:"arc"` // the ZFS read cache; nil without the module Uptime float64 `json:"uptime"` fsT, upsT int64 // when the slow sections were really read; sensors report these, not T @@ -176,6 +177,7 @@ type Collector struct { lastTable *ProcTable upsProbe func() *UPS // nil when no UPS daemon is installed gpu gpuState + prevARC arcPrev subMu sync.Mutex subs map[chan struct{}]struct{} // told after every sample; see Subscribe @@ -360,6 +362,7 @@ func (c *Collector) sample() { } s.UPS, s.upsT = c.lastUPS, c.upsAt.UnixMilli() s.GPUs = c.gpus(now, cpuTemp(s.Temps)) + s.ARC = c.arc(now) if first { return // deltas are meaningless on the priming pass diff --git a/internal/metrics/sensors.go b/internal/metrics/sensors.go index 5db1b4c..4b9c1cd 100644 --- a/internal/metrics/sensors.go +++ b/internal/metrics/sensors.go @@ -173,6 +173,33 @@ func flatten(s Sample, want func(id string) bool) []sensors.Sensor { f.add(p+".clients", g.Name+" clients", "count", "", float64(len(g.Clients)), t, none, lbl...) } + if a := s.ARC; a != nil { + p := "zfs.arc." + cap := upTo(float64(a.Max)) + f.add(p+"size", "ARC size", "bytes", "B", float64(a.Size), t, cap) + f.add(p+"target", "ARC target", "bytes", "B", float64(a.Target), t, cap) + f.add(p+"min", "ARC floor", "bytes", "B", float64(a.Min), t, none) + f.add(p+"max", "ARC ceiling", "bytes", "B", float64(a.Max), t, none) + f.add(p+"mru", "ARC recently used", "bytes", "B", float64(a.MRU), t, cap) + f.add(p+"mfu", "ARC frequently used", "bytes", "B", float64(a.MFU), t, cap) + f.add(p+"metadata", "ARC metadata", "bytes", "B", float64(a.Metadata), t, cap) + f.add(p+"data", "ARC data", "bytes", "B", float64(a.Data), t, cap) + if a.HitPctKnown { + f.add(p+"hit_pct", "ARC hit rate", "usage", "%", a.HitPct, t, pct) + } + f.add(p+"reads", "ARC reads", "rate", "/s", a.ReadsPerSec, t, none) + f.add(p+"misses", "ARC misses", "rate", "/s", a.MissesPerSec, t, none) + f.add(p+"lifetime_hit_pct", "ARC hit rate since boot", "usage", "%", a.LifetimeHitPct, t, pct) + f.add(p+"no_grow", "ARC held by memory pressure", "state", "", a.NoGrow, t, none) + if a.Compressed > 0 { + f.add(p+"compress_ratio", "ARC compression", "ratio", "x", round1(float64(a.Uncompressed)/float64(a.Compressed)*10)/10, t, none) + } + if a.L2Size > 0 { + f.add(p+"l2_size", "L2ARC size", "bytes", "B", float64(a.L2Size), t, none) + f.add(p+"l2_hit_pct", "L2ARC hit rate", "usage", "%", a.L2HitPct, t, pct) + } + } + if u := s.UPS; u != nil { ut := s.upsT f.add("ups.battery", "UPS battery", "usage", "%", u.BatteryPct, ut, pct, "model", u.Model) diff --git a/internal/storage/arc.go b/internal/storage/arc.go new file mode 100644 index 0000000..15df901 --- /dev/null +++ b/internal/storage/arc.go @@ -0,0 +1,292 @@ +package storage + +// The ZFS caching tiers, for the ZFS tab: what the ARC has done since boot and how it is +// tuned, and for each pool which cache devices it has — L2ARC, a separate log (SLOG), a +// special allocation class, a dedup table — or, as is common, none of them, which the tab +// should say plainly rather than leave a blank. +// +// The live numbers (size, the hit rate over the last interval) come from the sampler in +// internal/metrics, which also publishes them as zfs.arc.* sensors; this is the slower, +// fuller picture, read when the tab asks. + +import ( + "context" + "os" + "os/exec" + "strconv" + "strings" + "sync" + "time" +) + +// CacheDev is one member of a pool's cache, log, special, dedup or spare class. +type CacheDev struct { + Name string `json:"name"` + Size uint64 `json:"size,omitempty"` + Alloc uint64 `json:"alloc,omitempty"` + Health string `json:"health,omitempty"` +} + +// PoolCaching is what one pool has beyond its data vdevs, and the properties that steer +// what is cached. +type PoolCaching struct { + Pool string `json:"pool"` + Cache []CacheDev `json:"cache"` // L2ARC + Log []CacheDev `json:"log"` // SLOG + Special []CacheDev `json:"special"` // metadata / small blocks + Dedup []CacheDev `json:"dedup"` // dedup table vdevs + Spare []CacheDev `json:"spare"` + // dataset properties at the pool root + PrimaryCache string `json:"primarycache"` + SecondaryCache string `json:"secondarycache"` + Sync string `json:"sync"` + LogBias string `json:"logbias"` + DedupProp string `json:"dedup_prop"` + Compression string `json:"compression"` + RecordSize string `json:"recordsize"` + DedupRatio string `json:"dedup_ratio"` // zpool's, e.g. "1.00x" +} + +// ARCDetail is the full picture. +type ARCDetail struct { + Available bool `json:"available"` + MemTotal uint64 `json:"mem_total"` + Tunables map[string]string `json:"tunables"` // zfs module parameters that shape the cache; "" when unset + Lifetime struct { + Hits, Misses uint64 `json:",omitempty"` + DemandDataHits, DemandDataMisses uint64 + DemandMetaHits, DemandMetaMisses uint64 + PrefetchDataHits, PrefetchDataMisses uint64 + PrefetchMetaHits, PrefetchMetaMisses uint64 + L2Hits, L2Misses, L2ReadBytes, L2WriteBytes uint64 + L2Feeds, L2RebuildSize uint64 + EvictL2Eligible uint64 // bytes evicted from RAM that an L2ARC could have caught + Deleted, EvictSkip, MutexMiss, Prune, Throttle uint64 + } `json:"lifetime"` + Zfetch struct{ Hits, Misses, IOIssued uint64 } `json:"zfetch"` // the prefetcher + ZIL struct { + Commits uint64 `json:"commits"` + NormalBytes, NormalWrite uint64 // written to the pool's own vdevs + SlogBytes, SlogWrite uint64 // written to a separate log device + } `json:"zil"` + DBuf struct{ Size, Target, Hits, Misses uint64 } `json:"dbuf"` // the dbuf cache in front of the ARC + Pools []PoolCaching `json:"pools"` +} + +const kstatDir = "/proc/spl/kstat/zfs/" + +func readKstat(p string) map[string]uint64 { + b, err := os.ReadFile(p) + if err != nil { + return nil + } + out := map[string]uint64{} + for _, line := range strings.Split(string(b), "\n") { + f := strings.Fields(line) + if len(f) != 3 || f[1] != "4" { + continue + } + if v, err := strconv.ParseUint(f[2], 10, 64); err == nil { + out[f[0]] = v + } + } + return out +} + +var arcTunables = []string{"zfs_arc_max", "zfs_arc_min", "zfs_arc_sys_free", "zfs_arc_meta_balance", "zfs_prefetch_disable", "l2arc_write_max", "l2arc_write_boost", "l2arc_noprefetch", "l2arc_rebuild_enabled", "zfs_dirty_data_max"} + +const poolCachingEvery = 30 * time.Second // zpool list -v and zfs get are an exec each + +// ARC reads the kstats now and the pool layout from a short cache. +func (m *Manager) ARC(ctx context.Context) ARCDetail { + d := ARCDetail{Tunables: map[string]string{}, Pools: []PoolCaching{}} + k := readKstat(kstatDir + "arcstats") + if k == nil { + return d + } + d.Available = true + l := &d.Lifetime + l.Hits, l.Misses = k["hits"], k["misses"] + l.DemandDataHits, l.DemandDataMisses = k["demand_data_hits"], k["demand_data_misses"] + l.DemandMetaHits, l.DemandMetaMisses = k["demand_metadata_hits"], k["demand_metadata_misses"] + l.PrefetchDataHits, l.PrefetchDataMisses = k["prefetch_data_hits"], k["prefetch_data_misses"] + l.PrefetchMetaHits, l.PrefetchMetaMisses = k["prefetch_metadata_hits"], k["prefetch_metadata_misses"] + l.L2Hits, l.L2Misses, l.L2ReadBytes, l.L2WriteBytes = k["l2_hits"], k["l2_misses"], k["l2_read_bytes"], k["l2_write_bytes"] + l.L2Feeds, l.L2RebuildSize, l.EvictL2Eligible = k["l2_feeds"], k["l2_rebuild_size"], k["evict_l2_eligible"] + l.Deleted, l.EvictSkip, l.MutexMiss, l.Prune, l.Throttle = k["deleted"], k["evict_skip"], k["mutex_miss"], k["arc_prune"], k["memory_throttle_count"] + if z := readKstat(kstatDir + "zfetchstats"); z != nil { + d.Zfetch.Hits, d.Zfetch.Misses, d.Zfetch.IOIssued = z["hits"], z["misses"], z["io_issued"] + } + if z := readKstat(kstatDir + "zil"); z != nil { + d.ZIL.Commits = z["zil_commit_count"] + d.ZIL.NormalBytes, d.ZIL.NormalWrite = z["zil_itx_metaslab_normal_bytes"], z["zil_itx_metaslab_normal_write"] + d.ZIL.SlogBytes, d.ZIL.SlogWrite = z["zil_itx_metaslab_slog_bytes"], z["zil_itx_metaslab_slog_write"] + } + if z := readKstat(kstatDir + "dbufstats"); z != nil { + d.DBuf.Size, d.DBuf.Target, d.DBuf.Hits, d.DBuf.Misses = z["cache_size_bytes"], z["cache_target_bytes"], z["hash_hits"], z["hash_misses"] + } + for _, t := range arcTunables { + if b, err := os.ReadFile("/sys/module/zfs/parameters/" + t); err == nil { + d.Tunables[t] = strings.TrimSpace(string(b)) + } + } + if b, err := os.ReadFile("/proc/meminfo"); err == nil { + for _, line := range strings.Split(string(b), "\n") { + if strings.HasPrefix(line, "MemTotal:") { + f := strings.Fields(line) + if len(f) >= 2 { + d.MemTotal = u64(f[1]) << 10 + } + break + } + } + } + d.Pools = m.poolCaching(ctx) + return d +} + +var cachingMu sync.Mutex +var cachingAt time.Time +var cachingLast []PoolCaching + +func (m *Manager) poolCaching(ctx context.Context) []PoolCaching { + cachingMu.Lock() + defer cachingMu.Unlock() + if cachingLast != nil && time.Since(cachingAt) < poolCachingEvery { + return cachingLast + } + out := []PoolCaching{} + bin, ok := lookPath("zpool") + if !ok { + return out + } + // zpool list -v needs no root; the vdev tree under each pool, with the class headers + raw, code, err := m.runPlain(ctx, bin, "list", "-Hpv", "-o", "name,size,alloc,free,health") + if err != nil || code != 0 { + return out + } + out = parsePoolCaching(string(raw)) + if zfs, ok := lookPath("zfs"); ok { + for i := range out { + p := &out[i] + if props, code, err := m.runPlain(ctx, zfs, "get", "-Hp", "-o", "property,value", "primarycache,secondarycache,sync,logbias,dedup,compression,recordsize", p.Pool); err == nil && code == 0 { + for _, line := range strings.Split(strings.TrimSpace(string(props)), "\n") { + f := strings.Split(line, "\t") + if len(f) < 2 { + continue + } + switch f[0] { + case "primarycache": + p.PrimaryCache = f[1] + case "secondarycache": + p.SecondaryCache = f[1] + case "sync": + p.Sync = f[1] + case "logbias": + p.LogBias = f[1] + case "dedup": + p.DedupProp = f[1] + case "compression": + p.Compression = f[1] + case "recordsize": + p.RecordSize = humanSize(u64(f[1])) + } + } + } + } + } + for _, pool := range parsePools(func() []byte { + b, _, _ := m.runPlain(ctx, bin, "list", "-Hp", "-o", "name,size,alloc,free,frag,cap,health,dedup") + return b + }()) { + for i := range out { + if out[i].Pool == pool.Name { + out[i].DedupRatio = pool.Dedup + } + } + } + cachingLast, cachingAt = out, time.Now() + return out +} + +// runPlain runs a command as ourselves, never through sudo: these are read-only zpool/zfs +// queries that need no root on Linux, and a poll must not write to the auth log. +func (m *Manager) runPlain(ctx context.Context, bin string, args ...string) ([]byte, int, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, bin, args...).Output() + if ee, ok := err.(*exec.ExitError); ok { + return out, ee.ExitCode(), nil + } + return out, 0, err +} + +var cacheClasses = map[string]bool{"logs": true, "log": true, "cache": true, "special": true, "dedup": true, "spares": true, "spare": true} + +// parsePoolCaching reads `zpool list -Hpv -o name,size,alloc,free,health`: a pool line, then +// its vdevs indented, with a bare class name ("logs", "cache", "special", "dedup", "spares") +// introducing each extra class. Only those classes' members are kept; the data vdevs are the +// pool and are shown elsewhere. +func parsePoolCaching(text string) []PoolCaching { + var out []PoolCaching + var cur *PoolCaching + class := "" + for _, raw := range strings.Split(text, "\n") { + if strings.TrimSpace(raw) == "" { + continue + } + indented := strings.HasPrefix(raw, " ") || strings.HasPrefix(raw, "\t") + f := strings.Split(strings.TrimSpace(raw), "\t") + name := strings.TrimSpace(f[0]) + if !indented { + out = append(out, PoolCaching{Pool: name, Cache: []CacheDev{}, Log: []CacheDev{}, Special: []CacheDev{}, Dedup: []CacheDev{}, Spare: []CacheDev{}}) + cur, class = &out[len(out)-1], "" + continue + } + if cur == nil { + continue + } + if cacheClasses[name] && (len(f) < 2 || f[1] == "-" || f[1] == "") { + class = name + continue + } + if class == "" { + continue // a data vdev or one of its disks + } + // a mirror or raidz inside a class lists its members below it; keep them all, the + // grouping is visible in the vdev tree on the pool card + dev := CacheDev{Name: name} + if len(f) >= 5 { + dev.Size, dev.Alloc, dev.Health = u64(f[1]), u64(f[2]), strings.TrimSpace(f[4]) + if f[1] == "-" { + dev.Size = 0 + } + } + switch class { + case "logs", "log": + cur.Log = append(cur.Log, dev) + case "cache": + cur.Cache = append(cur.Cache, dev) + case "special": + cur.Special = append(cur.Special, dev) + case "dedup": + cur.Dedup = append(cur.Dedup, dev) + case "spares", "spare": + cur.Spare = append(cur.Spare, dev) + } + } + if out == nil { + out = []PoolCaching{} + } + return out +} + +func humanSize(b uint64) string { + switch { + case b >= 1<<20 && b%(1<<20) == 0: + return strconv.FormatUint(b>>20, 10) + "M" + case b >= 1<<10 && b%(1<<10) == 0: + return strconv.FormatUint(b>>10, 10) + "K" + } + return strconv.FormatUint(b, 10) +} diff --git a/internal/storage/arc_test.go b/internal/storage/arc_test.go new file mode 100644 index 0000000..e518571 --- /dev/null +++ b/internal/storage/arc_test.go @@ -0,0 +1,71 @@ +package storage + +import ( + "context" + "encoding/json" + "os" + "testing" +) + +// zpool list -Hpv on a pool with every extra class; the data raidz and its disks are skipped +const listV = "tank\t32000000000000\t23000000000000\t9000000000000\tONLINE\n" + + "\traidz2-0\t32000000000000\t23000000000000\t9000000000000\tONLINE\n" + + "\t\twwn-0x1\t4000000000000\t-\t-\tONLINE\n" + + "\t\twwn-0x2\t4000000000000\t-\t-\tONLINE\n" + + "\tspecial\t-\t-\t-\t-\n" + + "\t\tmirror-1\t500000000000\t120000000000\t380000000000\tONLINE\n" + + "\t\t\tnvme0n1p1\t500000000000\t-\t-\tONLINE\n" + + "\t\t\tnvme1n1p1\t500000000000\t-\t-\tONLINE\n" + + "\tlogs\t-\t-\t-\t-\n" + + "\t\tnvme2n1p1\t16000000000\t4000000\t15996000000\tONLINE\n" + + "\tcache\t-\t-\t-\t-\n" + + "\t\tnvme2n1p2\t400000000000\t210000000000\t190000000000\tONLINE\n" + + "\tspares\t-\t-\t-\t-\n" + + "\t\twwn-0x9\t4000000000000\t-\t-\tAVAIL\n" + + "scratch\t1000000000000\t1000000000\t999000000000\tONLINE\n" + + "\tsda\t1000000000000\t1000000000\t999000000000\tONLINE\n" + +func TestParsePoolCaching(t *testing.T) { + got := parsePoolCaching(listV) + if len(got) != 2 || got[0].Pool != "tank" || got[1].Pool != "scratch" { + t.Fatalf("pools: %+v", got) + } + tank := got[0] + if len(tank.Cache) != 1 || tank.Cache[0].Name != "nvme2n1p2" || tank.Cache[0].Alloc != 210000000000 { + t.Errorf("cache: %+v", tank.Cache) + } + if len(tank.Log) != 1 || tank.Log[0].Name != "nvme2n1p1" { + t.Errorf("log: %+v", tank.Log) + } + if len(tank.Special) != 3 || tank.Special[0].Name != "mirror-1" || tank.Special[0].Alloc != 120000000000 { + t.Errorf("special: %+v", tank.Special) + } + if len(tank.Spare) != 1 || tank.Spare[0].Health != "AVAIL" { + t.Errorf("spares: %+v", tank.Spare) + } + if len(tank.Dedup) != 0 { + t.Errorf("no dedup class: %+v", tank.Dedup) + } + s := got[1] + if len(s.Cache)+len(s.Log)+len(s.Special)+len(s.Spare) != 0 { + t.Errorf("a plain pool has no extra classes: %+v", s) + } +} + +func TestHumanSize(t *testing.T) { + for in, want := range map[uint64]string{131072: "128K", 1048576: "1M", 16384: "16K", 1000: "1000"} { + if got := humanSize(in); got != want { + t.Errorf("%d → %s, want %s", in, got, want) + } + } +} + +// TestLiveARC reads this machine's ZFS for real: HAMLAB_LIVE=1 go test -run TestLiveARC -v ./internal/storage +func TestLiveARC(t *testing.T) { + if os.Getenv("HAMLAB_LIVE") == "" { + t.Skip("set HAMLAB_LIVE=1 to read the real machine") + } + d := New(false).ARC(context.Background()) + b, _ := json.MarshalIndent(d, "", " ") + t.Logf("%s", b) +} diff --git a/web/src/ZFS.tsx b/web/src/ZFS.tsx index dc781f1..9578e24 100644 --- a/web/src/ZFS.tsx +++ b/web/src/ZFS.tsx @@ -8,10 +8,12 @@ // the retention in the name — rather than inventing a second convention for the two tools to // disagree over. An existing cron job is detected and shown, so the page tells the truth // about the machine whether Ham Lab or something else is doing the work. -import { FormEvent, useCallback, useEffect, useState } from 'react' -import { api, ZFSExternal, ZFSInfo, ZFSOverview, ZFSPolicy, ZFSRun, ZFSSnapshot } from './api' +import { FormEvent, Fragment, useCallback, useEffect, useState } from 'react' +import { api, ARCView, CacheDev, PoolCaching, ZFSExternal, ZFSInfo, ZFSOverview, ZFSPolicy, ZFSRun, ZFSSnapshot } from './api' import { fmtBytes } from './fmt' import { useRefreshMs } from './prefs' +import { fetchHistories } from './sensors' +import Sparkline from './Sparkline' import { askText, askYes, Btn, Loading, Meter, Spinner, toast, useEscape } from './ui' const ago = (t?: string | number) => { @@ -305,6 +307,145 @@ function adopt(x: ExternalJob, fallbackDataset: string): Partial { } } +// ---- the cache: the ARC in RAM, and what each pool has in front of its disks ---- +// +// Two kinds of number. The live ones (size, the hit rate over the last sampling interval) +// come from the sampler, which also keeps their history for the sparklines; the since-boot +// counters, the tunables and the pool layout come with the same request from the storage +// package. A tier a pool does not have is named and said to be absent, with what that means, +// because "nothing shown" and "none fitted" are different answers. + +const hitPct = (h: number, m: number) => (h + m > 0 ? (h / (h + m)) * 100 : NaN) +const fmtPct = (v: number) => (Number.isNaN(v) ? '—' : `${v.toFixed(1)}%`) +const fmtN = (n: number) => n.toLocaleString() +const cssVar = (n: string) => getComputedStyle(document.documentElement).getPropertyValue(n).trim() || '#3987e5' + +function Tier({ name, what, devs, none, note }: { name: string; what: string; devs: CacheDev[]; none: string; note?: string }) { + return ( +
+ {name} + + {devs.length === 0 ? {none} : devs.map((x) => ( + + {x.name}{x.size ? ` · ${fmtBytes(x.alloc ?? 0)} of ${fmtBytes(x.size)} used` : ''}{x.health && x.health !== 'ONLINE' ? ` · ${x.health}` : ''} + + ))} + {devs.length > 0 && note && {note}} + +
+ ) +} + +function PoolTiers({ p, v }: { p: PoolCaching; v: ARCView }) { + const L = v.detail.lifetime, z = v.detail.zil, a = v.live + return ( +
+
+ {p.pool} + caching tiers + + primarycache={p.primarycache || '?'} · secondarycache={p.secondarycache || '?'} +
+
+ + + + + {p.spare.length > 0 && } +
+
sync={p.sync || '?'} · logbias={p.logbias || '?'} · compression={p.compression || '?'} · recordsize={p.recordsize || '?'}
+
+ ) +} + +function ARCSection() { + const [v, setV] = useState(null) + const [hist, setHist] = useState<{ size: number[]; hit: number[] }>({ size: [], hit: [] }) + const [err, setErr] = useState('') + const rate = useRefreshMs() + useEffect(() => { + let stop = false + const poll = async () => { + try { const r = await api.zfsARC(); if (!stop) { setV(r); setErr('') } } catch (e) { if (!stop) setErr(e instanceof Error ? e.message : 'failed') } + try { + const hs = await fetchHistories('zfs.arc.size,zfs.arc.hit_pct') + if (!stop) setHist({ size: hs.find((h) => h.id === 'zfs.arc.size')?.v ?? [], hit: hs.find((h) => h.id === 'zfs.arc.hit_pct')?.v ?? [] }) + } catch { /* the sparklines stay empty; the numbers still show */ } + } + poll() + const t = setInterval(poll, Math.max(2000, rate)) + return () => { stop = true; clearInterval(t) } + }, [rate]) + + if (err && !v) return
{err}
+ if (!v) return
reading the cache…
+ const d = v.detail, a = v.live + if (!d.available || !a) return
The ZFS module reports no ARC statistics on this machine.
+ + const L = d.lifetime + const ratio = a.compressed > 0 ? a.uncompressed / a.compressed : 0 + const maxSet = d.tunables.zfs_arc_max && d.tunables.zfs_arc_max !== '0' + const atFloor = a.target <= a.min * 1.02 + const rows: [string, number, number][] = [ + ['Demand data', L.DemandDataHits, L.DemandDataMisses], ['Demand metadata', L.DemandMetaHits, L.DemandMetaMisses], + ['Prefetch data', L.PrefetchDataHits, L.PrefetchDataMisses], ['Prefetch metadata', L.PrefetchMetaHits, L.PrefetchMetaMisses], + ] + return ( + <> +
+
+ + ARC + in RAM + + {fmtBytes(a.size)} / {fmtBytes(a.max)} · {Math.round((a.size / a.max) * 100)}% +
+ + {a.no_grow && ( +
+ Held by memory pressure. ZFS is not letting the cache grow (arc_no_grow): the system is short of free memory + {atFloor ? <>, and the target sits at its floor of {fmtBytes(a.min)} against a ceiling of {fmtBytes(a.max)} : ''}. + {' '}Whatever no longer fits is read from the disks. +
+ )} +
+
+
Size · ceiling {fmtBytes(a.max)}
+ +
+
+
Hit rate · {a.hit_pct_known ? `${a.hit_pct.toFixed(1)}% now` : 'no reads yet'} · {fmtN(Math.round(a.reads_per_sec))} reads/s
+ `${x.toFixed(1)}%`} height={48} /> +
+
+
+
Target{fmtBytes(a.target)} · floor {fmtBytes(a.min)} · ceiling {fmtBytes(a.max)} {maxSet ? '(zfs_arc_max)' : '(default: half of RAM)'}
+
Hit rate{a.hit_pct_known ? `${a.hit_pct.toFixed(1)}% now` : 'idle'} · {a.lifetime_hit_pct.toFixed(1)}% since boot · {fmtN(a.hits)} hits, {fmtN(a.misses)} misses
+
HoldsMRU {fmtBytes(a.mru)} · MFU {fmtBytes(a.mfu)} · dirty {fmtBytes(a.anon)}
+
Of whichdata {fmtBytes(a.data)} · metadata {fmtBytes(a.metadata)} · headers {fmtBytes(a.headers)}
+
Compressed{ratio > 0 ? `${fmtBytes(a.uncompressed)} held in ${fmtBytes(a.compressed)} · ${ratio.toFixed(2)}x` : 'not compressed'}
+
GhostsMRU {fmtBytes(a.mru_ghost)} · MFU {fmtBytes(a.mfu_ghost)}
+
Prefetch{d.tunables.zfs_prefetch_disable === '1' ? 'disabled (zfs_prefetch_disable)' : `${fmtPct(hitPct(d.zfetch.Hits, d.zfetch.Misses))} of ${fmtN(d.zfetch.Hits + d.zfetch.Misses)} predictions since boot`}
+
dbuf cache{fmtBytes(d.dbuf.Size)} of {fmtBytes(d.dbuf.Target)} · {fmtPct(hitPct(d.dbuf.Hits, d.dbuf.Misses))} hits
+
Pressure{fmtN(a.throttles)} throttles · {fmtN(a.prunes)} prunes · {fmtN(a.evict_skip)} evict skips
+
+
+ Since boothit ratehitsmisses + {rows.map(([k, h, m]) => {k}{fmtPct(hitPct(h, m))}{fmtN(h)}{fmtN(m)})} + All reads{fmtPct(hitPct(L.Hits, L.Misses))}{fmtN(L.Hits)}{fmtN(L.Misses)} +
+
+ {d.pools.length > 0 &&
{d.pools.map((p) => )}
} + + ) +} + // ---- page ---- export default function ZFS() { @@ -333,6 +474,11 @@ export default function ZFS() { {zfs.pools.length === 0 &&
ZFS is present but no pools are imported.
}
{zfs.pools.map((p) => )}
+

Cache

+ the ARC in RAM, and what each pool has in front of its disks +
+ +

Snapshot schedules

setEditing({ ...BLANK, dataset: zfs.pools[0]?.name ?? '' })}>+ New schedule
diff --git a/web/src/api.ts b/web/src/api.ts index 3565c92..2d49623 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -226,6 +226,28 @@ export interface ZFSPolicy { id: string; name: string; dataset: string; recursiv export interface ZFSRun { at?: string; taken?: string; pruned?: string[]; error?: string } export interface ZFSExternal { tool: string; schedule: string; command: string; user: string; source: string; id: string; next?: number; last?: number } export interface ZFSOverview { zfs: ZFSInfo; policies: { policy: ZFSPolicy; cron: string; next?: string; last: ZFSRun }[]; external?: ZFSExternal | null } +// The ARC as the sampler sees it (sizes in bytes; hit_pct over the last sampling interval). +export interface ARCLive { + size: number; target: number; min: number; max: number + mru: number; mfu: number; anon: number; data: number; metadata: number; meta_used: number; headers: number + compressed: number; uncompressed: number; mru_ghost: number; mfu_ghost: number; l2_size: number; l2_asize: number + reads_per_sec: number; misses_per_sec: number; hit_pct: number; hit_pct_known: boolean; l2_hit_pct: number + hits: number; misses: number; lifetime_hit_pct: number; no_grow: boolean; throttles: number; prunes: number; evict_skip: number +} +export interface CacheDev { name: string; size?: number; alloc?: number; health?: string } +export interface PoolCaching { + pool: string; cache: CacheDev[]; log: CacheDev[]; special: CacheDev[]; dedup: CacheDev[]; spare: CacheDev[] + primarycache: string; secondarycache: string; sync: string; logbias: string; dedup_prop: string; compression: string; recordsize: string; dedup_ratio: string +} +export interface ARCDetail { + available: boolean; mem_total: number; tunables: Record + lifetime: { Hits: number; Misses: number; DemandDataHits: number; DemandDataMisses: number; DemandMetaHits: number; DemandMetaMisses: number; PrefetchDataHits: number; PrefetchDataMisses: number; PrefetchMetaHits: number; PrefetchMetaMisses: number; L2Hits: number; L2Misses: number; L2ReadBytes: number; L2WriteBytes: number; L2Feeds: number; L2RebuildSize: number; EvictL2Eligible: number; Deleted: number; EvictSkip: number; MutexMiss: number; Prune: number; Throttle: number } + zfetch: { Hits: number; Misses: number; IOIssued: number } + zil: { commits: number; NormalBytes: number; NormalWrite: number; SlogBytes: number; SlogWrite: number } + dbuf: { Size: number; Target: number; Hits: number; Misses: number } + pools: PoolCaching[] +} +export interface ARCView { detail: ARCDetail; live: ARCLive | null } export interface FileEntry { name: string; type: 'dir' | 'file' | 'link' | 'other'; size: number; mode: string; perm: string; mtime: number; user: string; group: string; target?: string; broken?: boolean } export interface FileInfo extends FileEntry { path: string; text: boolean } @@ -278,6 +300,7 @@ export const api = { addKVM: (name: string, url: string, machine: string) => req('POST', '/api/kvms', { name, url, machine }, true), removeKVM: (id: string) => req('DELETE', `/api/kvms/${encodeURIComponent(id)}`, undefined, true), zfsOverview: () => req('GET', '/api/zfs'), + zfsARC: () => req('GET', '/api/zfs/arc'), saveZfsSchedule: (p: Partial) => req('PUT', '/api/zfs/schedules', p), deleteZfsSchedule: (id: string) => req('DELETE', `/api/zfs/schedules/${encodeURIComponent(id)}`), runZfsSchedule: (id: string) => req<{ taken: string; pruned: string[] }>('POST', `/api/zfs/schedules/${encodeURIComponent(id)}/run`), diff --git a/web/src/styles.css b/web/src/styles.css index 1d9c93f..809215e 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -740,6 +740,16 @@ pre.diff .add { color: var(--ok); background: color-mix(in srgb, var(--ok) 12%, .top-proc { height: 70vh; min-height: 420px; } } +/* ---- ZFS: the ARC card and each pool's caching tiers ---- */ +.arc-sparks { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; margin-top: 8px; } .arc-sparks .small { margin-bottom: 2px; } +@media (max-width: 640px) { .arc-sparks { grid-template-columns: 1fr; } } +.arc-break { display: grid; grid-template-columns: minmax(140px, 1fr) 80px 110px 110px; gap: 2px 10px; font-size: 12px; margin-top: 10px; } +.arc-break .head { color: var(--muted); } .arc-break .num { text-align: right; } +.arc-tiers { display: flex; flex-direction: column; gap: 5px; font-size: 13px; margin-top: 4px; } +.arc-tier { display: flex; gap: 8px; min-width: 0; } .arc-tier .k { color: var(--muted); width: 80px; flex-shrink: 0; cursor: help; } +.arc-tier-v { display: flex; flex-direction: column; gap: 1px; min-width: 0; } .arc-tier-v > span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.arc-tier-v .none { color: var(--muted); white-space: normal; } + /* ---- Space: what is using a disk ---- */ .space { position: absolute; inset: 0; display: flex; flex-direction: column; background: var(--bg); min-height: 0; } .space-bar { display: flex; flex-direction: column; gap: 8px; padding: 10px 12px; border-bottom: 1px solid var(--border); background: var(--bg2); }