diff --git a/PLAN.md b/PLAN.md index daaa6a1..513e5af 100644 --- a/PLAN.md +++ b/PLAN.md @@ -314,6 +314,20 @@ something long in one, switch away and back, refresh — it's all still there. GPU shows the CPU package temperature marked `*`. Asahi (the Mac mini) has neither a busy counter nor fdinfo in 6.19, so it lists the device and says so. No NVIDIA path: nothing here has one to test against. +- **The ARC on the ZFS tab** — **shipped 2026-10-02** (`internal/metrics/arc.go`, + `internal/storage/arc.go`, `GET /api/zfs/arc`, the Cache section in `ZFS.tsx`, `zfs.arc.*` + sensors). Asked for as "current ARC usage, including any types of caching services in the + pool". One card for the ARC: size against its ceiling with the target and floor, sparklines + of size and hit rate from the sampler's history, what it holds (MRU/MFU/dirty, data vs + metadata, compression, the ghost lists), the prefetcher and dbuf cache, memory-pressure + counters, and the since-boot breakdown by demand/prefetch × data/metadata. Then a card per + pool naming every caching tier — L2ARC, SLOG, special vdev, dedup table, spares — with its + devices and fill, **or saying plainly that there is none and what that means** (the ZIL on + the pool's own vdevs, with how much it has written), plus the properties that steer caching + (`primarycache`, `secondarycache`, `sync`, `logbias`). Read from `/proc/spl/kstat/zfs/*` + and `/sys/module/zfs/parameters`, with `zpool list -Hpv` and `zfs get` run without sudo + (they need none) every 30 s at most. The day it shipped it showed `tank`'s ARC pinned at + its 2 GiB floor under `arc_no_grow`, with a 53 GiB python3 process holding the RAM. - **Dashboard retired** — **2026-09-23**, at Mitchell's instruction, the same reasoning that merged Top into Processes: the app repeated itself. The customisable module list is gone (`Dashboard.tsx` deleted), and with it `/api/prefs/dashboard`, `applyPrefs`, and the diff --git a/SENSORS.md b/SENSORS.md index a991418..18fec25 100644 --- a/SENSORS.md +++ b/SENSORS.md @@ -21,6 +21,7 @@ disk..read|write|util|size|temp fs..size|used|used_pct ups.battery|runtime|load|load_pct|in_v|out_v|on_battery|state|source gpu..usage|render|video|video-enhance|copy|compute|temp|power|fan|freq|mem_used|mem_total|clients +zfs.arc.size|target|min|max|mru|mfu|data|metadata|hit_pct|reads|misses|lifetime_hit_pct|no_grow|compress_ratio|l2_size|l2_hit_pct ``` - Segments are lower case `[a-z0-9_-]`: `Package id 0` → `package_id_0`, `/mnt/tank` → @@ -42,6 +43,10 @@ gpu..usage|render|video|video-enhance|copy|compute|temp|power|fan|freq|mem has it open. `temp` is the card's own sensor, or on an integrated GPU the CPU package it shares a die with (label `source` says which); an Arc on i915 has none and reports none. `freq` is the actual clock, 0 when power-gated. `clients` counts the processes using it. +- `zfs.arc.*` is the ZFS read cache in RAM, on a machine with the module loaded. `hit_pct` + is over the last sampling interval and appears once an interval has had any reads (an idle + interval repeats the last value); `lifetime_hit_pct` is since boot. `no_grow` is true + while memory pressure stops the cache growing. `l2_*` appear only with cache devices. ## A sensor diff --git a/internal/api/server.go b/internal/api/server.go index 8cf067d..f53fda9 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -295,6 +295,7 @@ func (s *Server) routes() *http.ServeMux { m.HandleFunc("POST /api/storage/zfs/scrub/{pool}/{action}", s.auth(s.zfsScrub)) m.HandleFunc("POST /api/storage/zfs/snapshot", s.auth(s.zfsSnapshot)) m.HandleFunc("GET /api/zfs", s.auth(s.zfsOverview)) + m.HandleFunc("GET /api/zfs/arc", s.auth(s.zfsARC)) m.HandleFunc("PUT /api/zfs/schedules", s.auth(s.zfsPolicyPut)) m.HandleFunc("DELETE /api/zfs/schedules/{id}", s.auth(s.zfsPolicyDelete)) m.HandleFunc("POST /api/zfs/schedules/{id}/run", s.auth(s.zfsRunNow)) diff --git a/internal/api/zfs.go b/internal/api/zfs.go index fcf41bb..009459b 100644 --- a/internal/api/zfs.go +++ b/internal/api/zfs.go @@ -136,6 +136,17 @@ func (s *Server) zfsOverview(w http.ResponseWriter, r *http.Request) { }) } +// zfsARC is the cache: the sampler's live ARC (size, the hit rate over the last interval) +// beside the storage package's fuller picture (since-boot counters, tunables, and each +// pool's cache/log/special devices or the lack of them). +func (s *Server) zfsARC(w http.ResponseWriter, r *http.Request) { + out := map[string]any{"detail": s.storage.ARC(r.Context()), "live": nil} + if smp, ok := s.metrics.Latest(); ok && smp.ARC != nil { + out["live"] = smp.ARC + } + jsonOK(w, 200, out) +} + func (s *Server) zfsPolicyPut(w http.ResponseWriter, r *http.Request) { var p storage.Policy if err := json.NewDecoder(io.LimitReader(r.Body, 16<<10)).Decode(&p); err != nil { diff --git a/internal/metrics/arc.go b/internal/metrics/arc.go new file mode 100644 index 0000000..caf0745 --- /dev/null +++ b/internal/metrics/arc.go @@ -0,0 +1,113 @@ +package metrics + +// The ZFS ARC — the read cache in RAM — read live from /proc/spl/kstat/zfs/arcstats on a +// machine that has the module loaded, and left out of the sample everywhere else. The kstat +// file is counters since boot; the hit rate here is over the sampling interval, which is the +// number that says whether the cache is doing its job *now*. The lifetime figures and the +// pool's caching tiers (L2ARC, SLOG, special vdevs) are the storage package's business. + +import ( + "strconv" + "strings" + "time" +) + +const arcstatsPath = "/proc/spl/kstat/zfs/arcstats" + +// ARC is the read cache as one sample sees it. Sizes are bytes. +type ARC struct { + Size uint64 `json:"size"` + Target uint64 `json:"target"` // c: what the ARC is currently allowed to hold + Min uint64 `json:"min"` // c_min: the floor it will not shrink below + Max uint64 `json:"max"` // c_max: the ceiling (zfs_arc_max, or half of RAM) + // what is in it + MRU uint64 `json:"mru"` // recently used once + MFU uint64 `json:"mfu"` // used more than once + Anon uint64 `json:"anon"` // in flight: dirty, not yet on disk + Data uint64 `json:"data"` // file contents + Metadata uint64 `json:"metadata"` // dnodes, indirect blocks, dbufs, bonus buffers + MetaUsed uint64 `json:"meta_used"` + Headers uint64 `json:"headers"` + Compressed uint64 `json:"compressed"` + Uncompressed uint64 `json:"uncompressed"` + MRUGhost uint64 `json:"mru_ghost"` // what it recently evicted and still remembers + MFUGhost uint64 `json:"mfu_ghost"` + // L2ARC, the cache devices (0 when there are none) + L2Size uint64 `json:"l2_size"` + L2ASize uint64 `json:"l2_asize"` // as stored on the device, after compression + // over this interval + ReadsPerSec float64 `json:"reads_per_sec"` + MissesPerSec float64 `json:"misses_per_sec"` + HitPct float64 `json:"hit_pct"` // of the interval's reads; an idle interval keeps the last one + HitPctKnown bool `json:"hit_pct_known"` // false until an interval has had any reads + L2HitPct float64 `json:"l2_hit_pct"` + // since boot + Hits uint64 `json:"hits"` + Misses uint64 `json:"misses"` + LifetimeHitPct float64 `json:"lifetime_hit_pct"` + // pressure + NoGrow bool `json:"no_grow"` // arc_no_grow: memory pressure is stopping it growing + Throttles uint64 `json:"throttles"` // memory_throttle_count since boot + Prunes uint64 `json:"prunes"` // arc_prune: metadata evictions asked of the filesystems + EvictSkip uint64 `json:"evict_skip"` +} + +type arcPrev struct { + hits, misses, l2hits, l2misses uint64 + hitPct float64 + known bool + at time.Time +} + +// readKstat parses a kstat table ("name type data" rows after two header lines). +func readKstat(p string) map[string]uint64 { + s := readStr(p) + if s == "" { + return nil + } + out := map[string]uint64{} + for _, line := range strings.Split(s, "\n") { + f := strings.Fields(line) + if len(f) != 3 || f[1] != "4" { // type 4 is KSTAT_DATA_UINT64 + continue + } + v, err := strconv.ParseUint(f[2], 10, 64) + if err == nil { + out[f[0]] = v + } + } + return out +} + +// arc reads the ARC; nil when this machine has no ZFS module loaded. +func (c *Collector) arc(now time.Time) *ARC { + k := readKstat(arcstatsPath) + if k == nil { + return nil + } + a := &ARC{Size: k["size"], Target: k["c"], Min: k["c_min"], Max: k["c_max"], + MRU: k["mru_size"], MFU: k["mfu_size"], Anon: k["anon_size"], Data: k["data_size"], Metadata: k["metadata_size"], + MetaUsed: k["arc_meta_used"], Headers: k["hdr_size"], Compressed: k["compressed_size"], Uncompressed: k["uncompressed_size"], + MRUGhost: k["mru_ghost_size"], MFUGhost: k["mfu_ghost_size"], L2Size: k["l2_size"], L2ASize: k["l2_asize"], + Hits: k["hits"], Misses: k["misses"], NoGrow: k["arc_no_grow"] != 0, Throttles: k["memory_throttle_count"], Prunes: k["arc_prune"], EvictSkip: k["evict_skip"]} + if n := a.Hits + a.Misses; n > 0 { + a.LifetimeHitPct = round1(float64(a.Hits) / float64(n) * 100) + } + p := c.prevARC + if !p.at.IsZero() && a.Hits >= p.hits && a.Misses >= p.misses { + if dt := now.Sub(p.at).Seconds(); dt > 0 { + hits, misses := a.Hits-p.hits, a.Misses-p.misses + a.ReadsPerSec = round1(float64(hits+misses) / dt) + a.MissesPerSec = round1(float64(misses) / dt) + if hits+misses > 0 { + p.hitPct, p.known = round1(float64(hits)/float64(hits+misses)*100), true + } + if l2 := (k["l2_hits"] - p.l2hits) + (k["l2_misses"] - p.l2misses); l2 > 0 && k["l2_hits"] >= p.l2hits { + a.L2HitPct = round1(float64(k["l2_hits"]-p.l2hits) / float64(l2) * 100) + } + } + } + a.HitPct, a.HitPctKnown = p.hitPct, p.known + c.prevARC = arcPrev{hits: a.Hits, misses: a.Misses, l2hits: k["l2_hits"], l2misses: k["l2_misses"], hitPct: p.hitPct, known: p.known, at: now} + return a +} diff --git a/internal/metrics/gpu_test.go b/internal/metrics/gpu_test.go index 61075f6..0ab1d26 100644 --- a/internal/metrics/gpu_test.go +++ b/internal/metrics/gpu_test.go @@ -3,6 +3,7 @@ package metrics import ( "encoding/json" "os" + "strings" "testing" "time" ) @@ -134,3 +135,20 @@ func TestLiveGPU(t *testing.T) { t.Logf("%s = %v %s", x.ID, x.Value, x.Unit) } } + +// TestLiveARC: HAMLAB_LIVE=1 go test -run TestLiveARC -v ./internal/metrics +func TestLiveARC(t *testing.T) { + if os.Getenv("HAMLAB_LIVE") == "" { + t.Skip("set HAMLAB_LIVE=1 to read the real machine") + } + c := NewCollector() + c.sample() + time.Sleep(1100 * time.Millisecond) + c.sample() + s, _ := c.Latest() + b, _ := json.MarshalIndent(s.ARC, "", " ") + t.Logf("%s", b) + for _, x := range flatten(s, func(id string) bool { return strings.HasPrefix(id, "zfs.arc.") }) { + t.Logf("%s = %v %s", x.ID, x.Value, x.Unit) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index f61bb42..5292302 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -147,6 +147,7 @@ type Sample struct { FS []FS `json:"fs"` UPS *UPS `json:"ups"` GPUs []GPU `json:"gpus"` + ARC *ARC `json:"arc"` // the ZFS read cache; nil without the module Uptime float64 `json:"uptime"` fsT, upsT int64 // when the slow sections were really read; sensors report these, not T @@ -176,6 +177,7 @@ type Collector struct { lastTable *ProcTable upsProbe func() *UPS // nil when no UPS daemon is installed gpu gpuState + prevARC arcPrev subMu sync.Mutex subs map[chan struct{}]struct{} // told after every sample; see Subscribe @@ -360,6 +362,7 @@ func (c *Collector) sample() { } s.UPS, s.upsT = c.lastUPS, c.upsAt.UnixMilli() s.GPUs = c.gpus(now, cpuTemp(s.Temps)) + s.ARC = c.arc(now) if first { return // deltas are meaningless on the priming pass diff --git a/internal/metrics/sensors.go b/internal/metrics/sensors.go index 5db1b4c..4b9c1cd 100644 --- a/internal/metrics/sensors.go +++ b/internal/metrics/sensors.go @@ -173,6 +173,33 @@ func flatten(s Sample, want func(id string) bool) []sensors.Sensor { f.add(p+".clients", g.Name+" clients", "count", "", float64(len(g.Clients)), t, none, lbl...) } + if a := s.ARC; a != nil { + p := "zfs.arc." + cap := upTo(float64(a.Max)) + f.add(p+"size", "ARC size", "bytes", "B", float64(a.Size), t, cap) + f.add(p+"target", "ARC target", "bytes", "B", float64(a.Target), t, cap) + f.add(p+"min", "ARC floor", "bytes", "B", float64(a.Min), t, none) + f.add(p+"max", "ARC ceiling", "bytes", "B", float64(a.Max), t, none) + f.add(p+"mru", "ARC recently used", "bytes", "B", float64(a.MRU), t, cap) + f.add(p+"mfu", "ARC frequently used", "bytes", "B", float64(a.MFU), t, cap) + f.add(p+"metadata", "ARC metadata", "bytes", "B", float64(a.Metadata), t, cap) + f.add(p+"data", "ARC data", "bytes", "B", float64(a.Data), t, cap) + if a.HitPctKnown { + f.add(p+"hit_pct", "ARC hit rate", "usage", "%", a.HitPct, t, pct) + } + f.add(p+"reads", "ARC reads", "rate", "/s", a.ReadsPerSec, t, none) + f.add(p+"misses", "ARC misses", "rate", "/s", a.MissesPerSec, t, none) + f.add(p+"lifetime_hit_pct", "ARC hit rate since boot", "usage", "%", a.LifetimeHitPct, t, pct) + f.add(p+"no_grow", "ARC held by memory pressure", "state", "", a.NoGrow, t, none) + if a.Compressed > 0 { + f.add(p+"compress_ratio", "ARC compression", "ratio", "x", round1(float64(a.Uncompressed)/float64(a.Compressed)*10)/10, t, none) + } + if a.L2Size > 0 { + f.add(p+"l2_size", "L2ARC size", "bytes", "B", float64(a.L2Size), t, none) + f.add(p+"l2_hit_pct", "L2ARC hit rate", "usage", "%", a.L2HitPct, t, pct) + } + } + if u := s.UPS; u != nil { ut := s.upsT f.add("ups.battery", "UPS battery", "usage", "%", u.BatteryPct, ut, pct, "model", u.Model) diff --git a/internal/storage/arc.go b/internal/storage/arc.go new file mode 100644 index 0000000..15df901 --- /dev/null +++ b/internal/storage/arc.go @@ -0,0 +1,292 @@ +package storage + +// The ZFS caching tiers, for the ZFS tab: what the ARC has done since boot and how it is +// tuned, and for each pool which cache devices it has — L2ARC, a separate log (SLOG), a +// special allocation class, a dedup table — or, as is common, none of them, which the tab +// should say plainly rather than leave a blank. +// +// The live numbers (size, the hit rate over the last interval) come from the sampler in +// internal/metrics, which also publishes them as zfs.arc.* sensors; this is the slower, +// fuller picture, read when the tab asks. + +import ( + "context" + "os" + "os/exec" + "strconv" + "strings" + "sync" + "time" +) + +// CacheDev is one member of a pool's cache, log, special, dedup or spare class. +type CacheDev struct { + Name string `json:"name"` + Size uint64 `json:"size,omitempty"` + Alloc uint64 `json:"alloc,omitempty"` + Health string `json:"health,omitempty"` +} + +// PoolCaching is what one pool has beyond its data vdevs, and the properties that steer +// what is cached. +type PoolCaching struct { + Pool string `json:"pool"` + Cache []CacheDev `json:"cache"` // L2ARC + Log []CacheDev `json:"log"` // SLOG + Special []CacheDev `json:"special"` // metadata / small blocks + Dedup []CacheDev `json:"dedup"` // dedup table vdevs + Spare []CacheDev `json:"spare"` + // dataset properties at the pool root + PrimaryCache string `json:"primarycache"` + SecondaryCache string `json:"secondarycache"` + Sync string `json:"sync"` + LogBias string `json:"logbias"` + DedupProp string `json:"dedup_prop"` + Compression string `json:"compression"` + RecordSize string `json:"recordsize"` + DedupRatio string `json:"dedup_ratio"` // zpool's, e.g. "1.00x" +} + +// ARCDetail is the full picture. +type ARCDetail struct { + Available bool `json:"available"` + MemTotal uint64 `json:"mem_total"` + Tunables map[string]string `json:"tunables"` // zfs module parameters that shape the cache; "" when unset + Lifetime struct { + Hits, Misses uint64 `json:",omitempty"` + DemandDataHits, DemandDataMisses uint64 + DemandMetaHits, DemandMetaMisses uint64 + PrefetchDataHits, PrefetchDataMisses uint64 + PrefetchMetaHits, PrefetchMetaMisses uint64 + L2Hits, L2Misses, L2ReadBytes, L2WriteBytes uint64 + L2Feeds, L2RebuildSize uint64 + EvictL2Eligible uint64 // bytes evicted from RAM that an L2ARC could have caught + Deleted, EvictSkip, MutexMiss, Prune, Throttle uint64 + } `json:"lifetime"` + Zfetch struct{ Hits, Misses, IOIssued uint64 } `json:"zfetch"` // the prefetcher + ZIL struct { + Commits uint64 `json:"commits"` + NormalBytes, NormalWrite uint64 // written to the pool's own vdevs + SlogBytes, SlogWrite uint64 // written to a separate log device + } `json:"zil"` + DBuf struct{ Size, Target, Hits, Misses uint64 } `json:"dbuf"` // the dbuf cache in front of the ARC + Pools []PoolCaching `json:"pools"` +} + +const kstatDir = "/proc/spl/kstat/zfs/" + +func readKstat(p string) map[string]uint64 { + b, err := os.ReadFile(p) + if err != nil { + return nil + } + out := map[string]uint64{} + for _, line := range strings.Split(string(b), "\n") { + f := strings.Fields(line) + if len(f) != 3 || f[1] != "4" { + continue + } + if v, err := strconv.ParseUint(f[2], 10, 64); err == nil { + out[f[0]] = v + } + } + return out +} + +var arcTunables = []string{"zfs_arc_max", "zfs_arc_min", "zfs_arc_sys_free", "zfs_arc_meta_balance", "zfs_prefetch_disable", "l2arc_write_max", "l2arc_write_boost", "l2arc_noprefetch", "l2arc_rebuild_enabled", "zfs_dirty_data_max"} + +const poolCachingEvery = 30 * time.Second // zpool list -v and zfs get are an exec each + +// ARC reads the kstats now and the pool layout from a short cache. +func (m *Manager) ARC(ctx context.Context) ARCDetail { + d := ARCDetail{Tunables: map[string]string{}, Pools: []PoolCaching{}} + k := readKstat(kstatDir + "arcstats") + if k == nil { + return d + } + d.Available = true + l := &d.Lifetime + l.Hits, l.Misses = k["hits"], k["misses"] + l.DemandDataHits, l.DemandDataMisses = k["demand_data_hits"], k["demand_data_misses"] + l.DemandMetaHits, l.DemandMetaMisses = k["demand_metadata_hits"], k["demand_metadata_misses"] + l.PrefetchDataHits, l.PrefetchDataMisses = k["prefetch_data_hits"], k["prefetch_data_misses"] + l.PrefetchMetaHits, l.PrefetchMetaMisses = k["prefetch_metadata_hits"], k["prefetch_metadata_misses"] + l.L2Hits, l.L2Misses, l.L2ReadBytes, l.L2WriteBytes = k["l2_hits"], k["l2_misses"], k["l2_read_bytes"], k["l2_write_bytes"] + l.L2Feeds, l.L2RebuildSize, l.EvictL2Eligible = k["l2_feeds"], k["l2_rebuild_size"], k["evict_l2_eligible"] + l.Deleted, l.EvictSkip, l.MutexMiss, l.Prune, l.Throttle = k["deleted"], k["evict_skip"], k["mutex_miss"], k["arc_prune"], k["memory_throttle_count"] + if z := readKstat(kstatDir + "zfetchstats"); z != nil { + d.Zfetch.Hits, d.Zfetch.Misses, d.Zfetch.IOIssued = z["hits"], z["misses"], z["io_issued"] + } + if z := readKstat(kstatDir + "zil"); z != nil { + d.ZIL.Commits = z["zil_commit_count"] + d.ZIL.NormalBytes, d.ZIL.NormalWrite = z["zil_itx_metaslab_normal_bytes"], z["zil_itx_metaslab_normal_write"] + d.ZIL.SlogBytes, d.ZIL.SlogWrite = z["zil_itx_metaslab_slog_bytes"], z["zil_itx_metaslab_slog_write"] + } + if z := readKstat(kstatDir + "dbufstats"); z != nil { + d.DBuf.Size, d.DBuf.Target, d.DBuf.Hits, d.DBuf.Misses = z["cache_size_bytes"], z["cache_target_bytes"], z["hash_hits"], z["hash_misses"] + } + for _, t := range arcTunables { + if b, err := os.ReadFile("/sys/module/zfs/parameters/" + t); err == nil { + d.Tunables[t] = strings.TrimSpace(string(b)) + } + } + if b, err := os.ReadFile("/proc/meminfo"); err == nil { + for _, line := range strings.Split(string(b), "\n") { + if strings.HasPrefix(line, "MemTotal:") { + f := strings.Fields(line) + if len(f) >= 2 { + d.MemTotal = u64(f[1]) << 10 + } + break + } + } + } + d.Pools = m.poolCaching(ctx) + return d +} + +var cachingMu sync.Mutex +var cachingAt time.Time +var cachingLast []PoolCaching + +func (m *Manager) poolCaching(ctx context.Context) []PoolCaching { + cachingMu.Lock() + defer cachingMu.Unlock() + if cachingLast != nil && time.Since(cachingAt) < poolCachingEvery { + return cachingLast + } + out := []PoolCaching{} + bin, ok := lookPath("zpool") + if !ok { + return out + } + // zpool list -v needs no root; the vdev tree under each pool, with the class headers + raw, code, err := m.runPlain(ctx, bin, "list", "-Hpv", "-o", "name,size,alloc,free,health") + if err != nil || code != 0 { + return out + } + out = parsePoolCaching(string(raw)) + if zfs, ok := lookPath("zfs"); ok { + for i := range out { + p := &out[i] + if props, code, err := m.runPlain(ctx, zfs, "get", "-Hp", "-o", "property,value", "primarycache,secondarycache,sync,logbias,dedup,compression,recordsize", p.Pool); err == nil && code == 0 { + for _, line := range strings.Split(strings.TrimSpace(string(props)), "\n") { + f := strings.Split(line, "\t") + if len(f) < 2 { + continue + } + switch f[0] { + case "primarycache": + p.PrimaryCache = f[1] + case "secondarycache": + p.SecondaryCache = f[1] + case "sync": + p.Sync = f[1] + case "logbias": + p.LogBias = f[1] + case "dedup": + p.DedupProp = f[1] + case "compression": + p.Compression = f[1] + case "recordsize": + p.RecordSize = humanSize(u64(f[1])) + } + } + } + } + } + for _, pool := range parsePools(func() []byte { + b, _, _ := m.runPlain(ctx, bin, "list", "-Hp", "-o", "name,size,alloc,free,frag,cap,health,dedup") + return b + }()) { + for i := range out { + if out[i].Pool == pool.Name { + out[i].DedupRatio = pool.Dedup + } + } + } + cachingLast, cachingAt = out, time.Now() + return out +} + +// runPlain runs a command as ourselves, never through sudo: these are read-only zpool/zfs +// queries that need no root on Linux, and a poll must not write to the auth log. +func (m *Manager) runPlain(ctx context.Context, bin string, args ...string) ([]byte, int, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, bin, args...).Output() + if ee, ok := err.(*exec.ExitError); ok { + return out, ee.ExitCode(), nil + } + return out, 0, err +} + +var cacheClasses = map[string]bool{"logs": true, "log": true, "cache": true, "special": true, "dedup": true, "spares": true, "spare": true} + +// parsePoolCaching reads `zpool list -Hpv -o name,size,alloc,free,health`: a pool line, then +// its vdevs indented, with a bare class name ("logs", "cache", "special", "dedup", "spares") +// introducing each extra class. Only those classes' members are kept; the data vdevs are the +// pool and are shown elsewhere. +func parsePoolCaching(text string) []PoolCaching { + var out []PoolCaching + var cur *PoolCaching + class := "" + for _, raw := range strings.Split(text, "\n") { + if strings.TrimSpace(raw) == "" { + continue + } + indented := strings.HasPrefix(raw, " ") || strings.HasPrefix(raw, "\t") + f := strings.Split(strings.TrimSpace(raw), "\t") + name := strings.TrimSpace(f[0]) + if !indented { + out = append(out, PoolCaching{Pool: name, Cache: []CacheDev{}, Log: []CacheDev{}, Special: []CacheDev{}, Dedup: []CacheDev{}, Spare: []CacheDev{}}) + cur, class = &out[len(out)-1], "" + continue + } + if cur == nil { + continue + } + if cacheClasses[name] && (len(f) < 2 || f[1] == "-" || f[1] == "") { + class = name + continue + } + if class == "" { + continue // a data vdev or one of its disks + } + // a mirror or raidz inside a class lists its members below it; keep them all, the + // grouping is visible in the vdev tree on the pool card + dev := CacheDev{Name: name} + if len(f) >= 5 { + dev.Size, dev.Alloc, dev.Health = u64(f[1]), u64(f[2]), strings.TrimSpace(f[4]) + if f[1] == "-" { + dev.Size = 0 + } + } + switch class { + case "logs", "log": + cur.Log = append(cur.Log, dev) + case "cache": + cur.Cache = append(cur.Cache, dev) + case "special": + cur.Special = append(cur.Special, dev) + case "dedup": + cur.Dedup = append(cur.Dedup, dev) + case "spares", "spare": + cur.Spare = append(cur.Spare, dev) + } + } + if out == nil { + out = []PoolCaching{} + } + return out +} + +func humanSize(b uint64) string { + switch { + case b >= 1<<20 && b%(1<<20) == 0: + return strconv.FormatUint(b>>20, 10) + "M" + case b >= 1<<10 && b%(1<<10) == 0: + return strconv.FormatUint(b>>10, 10) + "K" + } + return strconv.FormatUint(b, 10) +} diff --git a/internal/storage/arc_test.go b/internal/storage/arc_test.go new file mode 100644 index 0000000..e518571 --- /dev/null +++ b/internal/storage/arc_test.go @@ -0,0 +1,71 @@ +package storage + +import ( + "context" + "encoding/json" + "os" + "testing" +) + +// zpool list -Hpv on a pool with every extra class; the data raidz and its disks are skipped +const listV = "tank\t32000000000000\t23000000000000\t9000000000000\tONLINE\n" + + "\traidz2-0\t32000000000000\t23000000000000\t9000000000000\tONLINE\n" + + "\t\twwn-0x1\t4000000000000\t-\t-\tONLINE\n" + + "\t\twwn-0x2\t4000000000000\t-\t-\tONLINE\n" + + "\tspecial\t-\t-\t-\t-\n" + + "\t\tmirror-1\t500000000000\t120000000000\t380000000000\tONLINE\n" + + "\t\t\tnvme0n1p1\t500000000000\t-\t-\tONLINE\n" + + "\t\t\tnvme1n1p1\t500000000000\t-\t-\tONLINE\n" + + "\tlogs\t-\t-\t-\t-\n" + + "\t\tnvme2n1p1\t16000000000\t4000000\t15996000000\tONLINE\n" + + "\tcache\t-\t-\t-\t-\n" + + "\t\tnvme2n1p2\t400000000000\t210000000000\t190000000000\tONLINE\n" + + "\tspares\t-\t-\t-\t-\n" + + "\t\twwn-0x9\t4000000000000\t-\t-\tAVAIL\n" + + "scratch\t1000000000000\t1000000000\t999000000000\tONLINE\n" + + "\tsda\t1000000000000\t1000000000\t999000000000\tONLINE\n" + +func TestParsePoolCaching(t *testing.T) { + got := parsePoolCaching(listV) + if len(got) != 2 || got[0].Pool != "tank" || got[1].Pool != "scratch" { + t.Fatalf("pools: %+v", got) + } + tank := got[0] + if len(tank.Cache) != 1 || tank.Cache[0].Name != "nvme2n1p2" || tank.Cache[0].Alloc != 210000000000 { + t.Errorf("cache: %+v", tank.Cache) + } + if len(tank.Log) != 1 || tank.Log[0].Name != "nvme2n1p1" { + t.Errorf("log: %+v", tank.Log) + } + if len(tank.Special) != 3 || tank.Special[0].Name != "mirror-1" || tank.Special[0].Alloc != 120000000000 { + t.Errorf("special: %+v", tank.Special) + } + if len(tank.Spare) != 1 || tank.Spare[0].Health != "AVAIL" { + t.Errorf("spares: %+v", tank.Spare) + } + if len(tank.Dedup) != 0 { + t.Errorf("no dedup class: %+v", tank.Dedup) + } + s := got[1] + if len(s.Cache)+len(s.Log)+len(s.Special)+len(s.Spare) != 0 { + t.Errorf("a plain pool has no extra classes: %+v", s) + } +} + +func TestHumanSize(t *testing.T) { + for in, want := range map[uint64]string{131072: "128K", 1048576: "1M", 16384: "16K", 1000: "1000"} { + if got := humanSize(in); got != want { + t.Errorf("%d → %s, want %s", in, got, want) + } + } +} + +// TestLiveARC reads this machine's ZFS for real: HAMLAB_LIVE=1 go test -run TestLiveARC -v ./internal/storage +func TestLiveARC(t *testing.T) { + if os.Getenv("HAMLAB_LIVE") == "" { + t.Skip("set HAMLAB_LIVE=1 to read the real machine") + } + d := New(false).ARC(context.Background()) + b, _ := json.MarshalIndent(d, "", " ") + t.Logf("%s", b) +} diff --git a/web/src/ZFS.tsx b/web/src/ZFS.tsx index dc781f1..9578e24 100644 --- a/web/src/ZFS.tsx +++ b/web/src/ZFS.tsx @@ -8,10 +8,12 @@ // the retention in the name — rather than inventing a second convention for the two tools to // disagree over. An existing cron job is detected and shown, so the page tells the truth // about the machine whether Ham Lab or something else is doing the work. -import { FormEvent, useCallback, useEffect, useState } from 'react' -import { api, ZFSExternal, ZFSInfo, ZFSOverview, ZFSPolicy, ZFSRun, ZFSSnapshot } from './api' +import { FormEvent, Fragment, useCallback, useEffect, useState } from 'react' +import { api, ARCView, CacheDev, PoolCaching, ZFSExternal, ZFSInfo, ZFSOverview, ZFSPolicy, ZFSRun, ZFSSnapshot } from './api' import { fmtBytes } from './fmt' import { useRefreshMs } from './prefs' +import { fetchHistories } from './sensors' +import Sparkline from './Sparkline' import { askText, askYes, Btn, Loading, Meter, Spinner, toast, useEscape } from './ui' const ago = (t?: string | number) => { @@ -305,6 +307,145 @@ function adopt(x: ExternalJob, fallbackDataset: string): Partial { } } +// ---- the cache: the ARC in RAM, and what each pool has in front of its disks ---- +// +// Two kinds of number. The live ones (size, the hit rate over the last sampling interval) +// come from the sampler, which also keeps their history for the sparklines; the since-boot +// counters, the tunables and the pool layout come with the same request from the storage +// package. A tier a pool does not have is named and said to be absent, with what that means, +// because "nothing shown" and "none fitted" are different answers. + +const hitPct = (h: number, m: number) => (h + m > 0 ? (h / (h + m)) * 100 : NaN) +const fmtPct = (v: number) => (Number.isNaN(v) ? '—' : `${v.toFixed(1)}%`) +const fmtN = (n: number) => n.toLocaleString() +const cssVar = (n: string) => getComputedStyle(document.documentElement).getPropertyValue(n).trim() || '#3987e5' + +function Tier({ name, what, devs, none, note }: { name: string; what: string; devs: CacheDev[]; none: string; note?: string }) { + return ( +
+ {name} + + {devs.length === 0 ? {none} : devs.map((x) => ( + + {x.name}{x.size ? ` · ${fmtBytes(x.alloc ?? 0)} of ${fmtBytes(x.size)} used` : ''}{x.health && x.health !== 'ONLINE' ? ` · ${x.health}` : ''} + + ))} + {devs.length > 0 && note && {note}} + +
+ ) +} + +function PoolTiers({ p, v }: { p: PoolCaching; v: ARCView }) { + const L = v.detail.lifetime, z = v.detail.zil, a = v.live + return ( +
+
+ {p.pool} + caching tiers + + primarycache={p.primarycache || '?'} · secondarycache={p.secondarycache || '?'} +
+
+ + + + + {p.spare.length > 0 && } +
+
sync={p.sync || '?'} · logbias={p.logbias || '?'} · compression={p.compression || '?'} · recordsize={p.recordsize || '?'}
+
+ ) +} + +function ARCSection() { + const [v, setV] = useState(null) + const [hist, setHist] = useState<{ size: number[]; hit: number[] }>({ size: [], hit: [] }) + const [err, setErr] = useState('') + const rate = useRefreshMs() + useEffect(() => { + let stop = false + const poll = async () => { + try { const r = await api.zfsARC(); if (!stop) { setV(r); setErr('') } } catch (e) { if (!stop) setErr(e instanceof Error ? e.message : 'failed') } + try { + const hs = await fetchHistories('zfs.arc.size,zfs.arc.hit_pct') + if (!stop) setHist({ size: hs.find((h) => h.id === 'zfs.arc.size')?.v ?? [], hit: hs.find((h) => h.id === 'zfs.arc.hit_pct')?.v ?? [] }) + } catch { /* the sparklines stay empty; the numbers still show */ } + } + poll() + const t = setInterval(poll, Math.max(2000, rate)) + return () => { stop = true; clearInterval(t) } + }, [rate]) + + if (err && !v) return
{err}
+ if (!v) return
reading the cache…
+ const d = v.detail, a = v.live + if (!d.available || !a) return
The ZFS module reports no ARC statistics on this machine.
+ + const L = d.lifetime + const ratio = a.compressed > 0 ? a.uncompressed / a.compressed : 0 + const maxSet = d.tunables.zfs_arc_max && d.tunables.zfs_arc_max !== '0' + const atFloor = a.target <= a.min * 1.02 + const rows: [string, number, number][] = [ + ['Demand data', L.DemandDataHits, L.DemandDataMisses], ['Demand metadata', L.DemandMetaHits, L.DemandMetaMisses], + ['Prefetch data', L.PrefetchDataHits, L.PrefetchDataMisses], ['Prefetch metadata', L.PrefetchMetaHits, L.PrefetchMetaMisses], + ] + return ( + <> +
+
+ + ARC + in RAM + + {fmtBytes(a.size)} / {fmtBytes(a.max)} · {Math.round((a.size / a.max) * 100)}% +
+ + {a.no_grow && ( +
+ Held by memory pressure. ZFS is not letting the cache grow (arc_no_grow): the system is short of free memory + {atFloor ? <>, and the target sits at its floor of {fmtBytes(a.min)} against a ceiling of {fmtBytes(a.max)} : ''}. + {' '}Whatever no longer fits is read from the disks. +
+ )} +
+
+
Size · ceiling {fmtBytes(a.max)}
+ +
+
+
Hit rate · {a.hit_pct_known ? `${a.hit_pct.toFixed(1)}% now` : 'no reads yet'} · {fmtN(Math.round(a.reads_per_sec))} reads/s
+ `${x.toFixed(1)}%`} height={48} /> +
+
+
+
Target{fmtBytes(a.target)} · floor {fmtBytes(a.min)} · ceiling {fmtBytes(a.max)} {maxSet ? '(zfs_arc_max)' : '(default: half of RAM)'}
+
Hit rate{a.hit_pct_known ? `${a.hit_pct.toFixed(1)}% now` : 'idle'} · {a.lifetime_hit_pct.toFixed(1)}% since boot · {fmtN(a.hits)} hits, {fmtN(a.misses)} misses
+
HoldsMRU {fmtBytes(a.mru)} · MFU {fmtBytes(a.mfu)} · dirty {fmtBytes(a.anon)}
+
Of whichdata {fmtBytes(a.data)} · metadata {fmtBytes(a.metadata)} · headers {fmtBytes(a.headers)}
+
Compressed{ratio > 0 ? `${fmtBytes(a.uncompressed)} held in ${fmtBytes(a.compressed)} · ${ratio.toFixed(2)}x` : 'not compressed'}
+
GhostsMRU {fmtBytes(a.mru_ghost)} · MFU {fmtBytes(a.mfu_ghost)}
+
Prefetch{d.tunables.zfs_prefetch_disable === '1' ? 'disabled (zfs_prefetch_disable)' : `${fmtPct(hitPct(d.zfetch.Hits, d.zfetch.Misses))} of ${fmtN(d.zfetch.Hits + d.zfetch.Misses)} predictions since boot`}
+
dbuf cache{fmtBytes(d.dbuf.Size)} of {fmtBytes(d.dbuf.Target)} · {fmtPct(hitPct(d.dbuf.Hits, d.dbuf.Misses))} hits
+
Pressure{fmtN(a.throttles)} throttles · {fmtN(a.prunes)} prunes · {fmtN(a.evict_skip)} evict skips
+
+
+ Since boothit ratehitsmisses + {rows.map(([k, h, m]) => {k}{fmtPct(hitPct(h, m))}{fmtN(h)}{fmtN(m)})} + All reads{fmtPct(hitPct(L.Hits, L.Misses))}{fmtN(L.Hits)}{fmtN(L.Misses)} +
+
+ {d.pools.length > 0 &&
{d.pools.map((p) => )}
} + + ) +} + // ---- page ---- export default function ZFS() { @@ -333,6 +474,11 @@ export default function ZFS() { {zfs.pools.length === 0 &&
ZFS is present but no pools are imported.
}
{zfs.pools.map((p) => )}
+

Cache

+ the ARC in RAM, and what each pool has in front of its disks +
+ +

Snapshot schedules

setEditing({ ...BLANK, dataset: zfs.pools[0]?.name ?? '' })}>+ New schedule
diff --git a/web/src/api.ts b/web/src/api.ts index 3565c92..2d49623 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -226,6 +226,28 @@ export interface ZFSPolicy { id: string; name: string; dataset: string; recursiv export interface ZFSRun { at?: string; taken?: string; pruned?: string[]; error?: string } export interface ZFSExternal { tool: string; schedule: string; command: string; user: string; source: string; id: string; next?: number; last?: number } export interface ZFSOverview { zfs: ZFSInfo; policies: { policy: ZFSPolicy; cron: string; next?: string; last: ZFSRun }[]; external?: ZFSExternal | null } +// The ARC as the sampler sees it (sizes in bytes; hit_pct over the last sampling interval). +export interface ARCLive { + size: number; target: number; min: number; max: number + mru: number; mfu: number; anon: number; data: number; metadata: number; meta_used: number; headers: number + compressed: number; uncompressed: number; mru_ghost: number; mfu_ghost: number; l2_size: number; l2_asize: number + reads_per_sec: number; misses_per_sec: number; hit_pct: number; hit_pct_known: boolean; l2_hit_pct: number + hits: number; misses: number; lifetime_hit_pct: number; no_grow: boolean; throttles: number; prunes: number; evict_skip: number +} +export interface CacheDev { name: string; size?: number; alloc?: number; health?: string } +export interface PoolCaching { + pool: string; cache: CacheDev[]; log: CacheDev[]; special: CacheDev[]; dedup: CacheDev[]; spare: CacheDev[] + primarycache: string; secondarycache: string; sync: string; logbias: string; dedup_prop: string; compression: string; recordsize: string; dedup_ratio: string +} +export interface ARCDetail { + available: boolean; mem_total: number; tunables: Record + lifetime: { Hits: number; Misses: number; DemandDataHits: number; DemandDataMisses: number; DemandMetaHits: number; DemandMetaMisses: number; PrefetchDataHits: number; PrefetchDataMisses: number; PrefetchMetaHits: number; PrefetchMetaMisses: number; L2Hits: number; L2Misses: number; L2ReadBytes: number; L2WriteBytes: number; L2Feeds: number; L2RebuildSize: number; EvictL2Eligible: number; Deleted: number; EvictSkip: number; MutexMiss: number; Prune: number; Throttle: number } + zfetch: { Hits: number; Misses: number; IOIssued: number } + zil: { commits: number; NormalBytes: number; NormalWrite: number; SlogBytes: number; SlogWrite: number } + dbuf: { Size: number; Target: number; Hits: number; Misses: number } + pools: PoolCaching[] +} +export interface ARCView { detail: ARCDetail; live: ARCLive | null } export interface FileEntry { name: string; type: 'dir' | 'file' | 'link' | 'other'; size: number; mode: string; perm: string; mtime: number; user: string; group: string; target?: string; broken?: boolean } export interface FileInfo extends FileEntry { path: string; text: boolean } @@ -278,6 +300,7 @@ export const api = { addKVM: (name: string, url: string, machine: string) => req('POST', '/api/kvms', { name, url, machine }, true), removeKVM: (id: string) => req('DELETE', `/api/kvms/${encodeURIComponent(id)}`, undefined, true), zfsOverview: () => req('GET', '/api/zfs'), + zfsARC: () => req('GET', '/api/zfs/arc'), saveZfsSchedule: (p: Partial) => req('PUT', '/api/zfs/schedules', p), deleteZfsSchedule: (id: string) => req('DELETE', `/api/zfs/schedules/${encodeURIComponent(id)}`), runZfsSchedule: (id: string) => req<{ taken: string; pruned: string[] }>('POST', `/api/zfs/schedules/${encodeURIComponent(id)}/run`), diff --git a/web/src/styles.css b/web/src/styles.css index 1d9c93f..809215e 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -740,6 +740,16 @@ pre.diff .add { color: var(--ok); background: color-mix(in srgb, var(--ok) 12%, .top-proc { height: 70vh; min-height: 420px; } } +/* ---- ZFS: the ARC card and each pool's caching tiers ---- */ +.arc-sparks { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; margin-top: 8px; } .arc-sparks .small { margin-bottom: 2px; } +@media (max-width: 640px) { .arc-sparks { grid-template-columns: 1fr; } } +.arc-break { display: grid; grid-template-columns: minmax(140px, 1fr) 80px 110px 110px; gap: 2px 10px; font-size: 12px; margin-top: 10px; } +.arc-break .head { color: var(--muted); } .arc-break .num { text-align: right; } +.arc-tiers { display: flex; flex-direction: column; gap: 5px; font-size: 13px; margin-top: 4px; } +.arc-tier { display: flex; gap: 8px; min-width: 0; } .arc-tier .k { color: var(--muted); width: 80px; flex-shrink: 0; cursor: help; } +.arc-tier-v { display: flex; flex-direction: column; gap: 1px; min-width: 0; } .arc-tier-v > span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.arc-tier-v .none { color: var(--muted); white-space: normal; } + /* ---- Space: what is using a disk ---- */ .space { position: absolute; inset: 0; display: flex; flex-direction: column; background: var(--bg); min-height: 0; } .space-bar { display: flex; flex-direction: column; gap: 8px; padding: 10px 12px; border-bottom: 1px solid var(--border); background: var(--bg2); }