The ZFS tab shows the ARC, and names every caching tier a pool has or lacks
A Cache section between the pools and the schedules. One card for the ARC: size against its ceiling with the target and floor, sparklines of size and hit rate from the sampler's history, what it holds (MRU/MFU/dirty, data vs metadata, compression, the ghost lists), the prefetcher and dbuf cache, the memory-pressure counters, and the since-boot breakdown by demand/prefetch and data/metadata. Then a card per pool for L2ARC, SLOG, special vdev, dedup table and spares — the devices and their fill where they exist, and where they don't, plainly none and what that means (the ZIL writing to the pool's own vdevs, with how much it has written). Live numbers come from the sampler, which reads arcstats every tick and publishes zfs.arc.* sensors; the fuller picture is read on request, with zpool list -Hpv and zfs get run without sudo and cached for thirty seconds. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
db40220365
commit
b44faf3ac9
@@ -0,0 +1,292 @@
|
||||
package storage
|
||||
|
||||
// The ZFS caching tiers, for the ZFS tab: what the ARC has done since boot and how it is
|
||||
// tuned, and for each pool which cache devices it has — L2ARC, a separate log (SLOG), a
|
||||
// special allocation class, a dedup table — or, as is common, none of them, which the tab
|
||||
// should say plainly rather than leave a blank.
|
||||
//
|
||||
// The live numbers (size, the hit rate over the last interval) come from the sampler in
|
||||
// internal/metrics, which also publishes them as zfs.arc.* sensors; this is the slower,
|
||||
// fuller picture, read when the tab asks.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CacheDev is one member of a pool's cache, log, special, dedup or spare class.
|
||||
type CacheDev struct {
|
||||
Name string `json:"name"`
|
||||
Size uint64 `json:"size,omitempty"`
|
||||
Alloc uint64 `json:"alloc,omitempty"`
|
||||
Health string `json:"health,omitempty"`
|
||||
}
|
||||
|
||||
// PoolCaching is what one pool has beyond its data vdevs, and the properties that steer
|
||||
// what is cached.
|
||||
type PoolCaching struct {
|
||||
Pool string `json:"pool"`
|
||||
Cache []CacheDev `json:"cache"` // L2ARC
|
||||
Log []CacheDev `json:"log"` // SLOG
|
||||
Special []CacheDev `json:"special"` // metadata / small blocks
|
||||
Dedup []CacheDev `json:"dedup"` // dedup table vdevs
|
||||
Spare []CacheDev `json:"spare"`
|
||||
// dataset properties at the pool root
|
||||
PrimaryCache string `json:"primarycache"`
|
||||
SecondaryCache string `json:"secondarycache"`
|
||||
Sync string `json:"sync"`
|
||||
LogBias string `json:"logbias"`
|
||||
DedupProp string `json:"dedup_prop"`
|
||||
Compression string `json:"compression"`
|
||||
RecordSize string `json:"recordsize"`
|
||||
DedupRatio string `json:"dedup_ratio"` // zpool's, e.g. "1.00x"
|
||||
}
|
||||
|
||||
// ARCDetail is the full picture.
|
||||
type ARCDetail struct {
|
||||
Available bool `json:"available"`
|
||||
MemTotal uint64 `json:"mem_total"`
|
||||
Tunables map[string]string `json:"tunables"` // zfs module parameters that shape the cache; "" when unset
|
||||
Lifetime struct {
|
||||
Hits, Misses uint64 `json:",omitempty"`
|
||||
DemandDataHits, DemandDataMisses uint64
|
||||
DemandMetaHits, DemandMetaMisses uint64
|
||||
PrefetchDataHits, PrefetchDataMisses uint64
|
||||
PrefetchMetaHits, PrefetchMetaMisses uint64
|
||||
L2Hits, L2Misses, L2ReadBytes, L2WriteBytes uint64
|
||||
L2Feeds, L2RebuildSize uint64
|
||||
EvictL2Eligible uint64 // bytes evicted from RAM that an L2ARC could have caught
|
||||
Deleted, EvictSkip, MutexMiss, Prune, Throttle uint64
|
||||
} `json:"lifetime"`
|
||||
Zfetch struct{ Hits, Misses, IOIssued uint64 } `json:"zfetch"` // the prefetcher
|
||||
ZIL struct {
|
||||
Commits uint64 `json:"commits"`
|
||||
NormalBytes, NormalWrite uint64 // written to the pool's own vdevs
|
||||
SlogBytes, SlogWrite uint64 // written to a separate log device
|
||||
} `json:"zil"`
|
||||
DBuf struct{ Size, Target, Hits, Misses uint64 } `json:"dbuf"` // the dbuf cache in front of the ARC
|
||||
Pools []PoolCaching `json:"pools"`
|
||||
}
|
||||
|
||||
const kstatDir = "/proc/spl/kstat/zfs/"
|
||||
|
||||
func readKstat(p string) map[string]uint64 {
|
||||
b, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
out := map[string]uint64{}
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) != 3 || f[1] != "4" {
|
||||
continue
|
||||
}
|
||||
if v, err := strconv.ParseUint(f[2], 10, 64); err == nil {
|
||||
out[f[0]] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var arcTunables = []string{"zfs_arc_max", "zfs_arc_min", "zfs_arc_sys_free", "zfs_arc_meta_balance", "zfs_prefetch_disable", "l2arc_write_max", "l2arc_write_boost", "l2arc_noprefetch", "l2arc_rebuild_enabled", "zfs_dirty_data_max"}
|
||||
|
||||
const poolCachingEvery = 30 * time.Second // zpool list -v and zfs get are an exec each
|
||||
|
||||
// ARC reads the kstats now and the pool layout from a short cache.
|
||||
func (m *Manager) ARC(ctx context.Context) ARCDetail {
|
||||
d := ARCDetail{Tunables: map[string]string{}, Pools: []PoolCaching{}}
|
||||
k := readKstat(kstatDir + "arcstats")
|
||||
if k == nil {
|
||||
return d
|
||||
}
|
||||
d.Available = true
|
||||
l := &d.Lifetime
|
||||
l.Hits, l.Misses = k["hits"], k["misses"]
|
||||
l.DemandDataHits, l.DemandDataMisses = k["demand_data_hits"], k["demand_data_misses"]
|
||||
l.DemandMetaHits, l.DemandMetaMisses = k["demand_metadata_hits"], k["demand_metadata_misses"]
|
||||
l.PrefetchDataHits, l.PrefetchDataMisses = k["prefetch_data_hits"], k["prefetch_data_misses"]
|
||||
l.PrefetchMetaHits, l.PrefetchMetaMisses = k["prefetch_metadata_hits"], k["prefetch_metadata_misses"]
|
||||
l.L2Hits, l.L2Misses, l.L2ReadBytes, l.L2WriteBytes = k["l2_hits"], k["l2_misses"], k["l2_read_bytes"], k["l2_write_bytes"]
|
||||
l.L2Feeds, l.L2RebuildSize, l.EvictL2Eligible = k["l2_feeds"], k["l2_rebuild_size"], k["evict_l2_eligible"]
|
||||
l.Deleted, l.EvictSkip, l.MutexMiss, l.Prune, l.Throttle = k["deleted"], k["evict_skip"], k["mutex_miss"], k["arc_prune"], k["memory_throttle_count"]
|
||||
if z := readKstat(kstatDir + "zfetchstats"); z != nil {
|
||||
d.Zfetch.Hits, d.Zfetch.Misses, d.Zfetch.IOIssued = z["hits"], z["misses"], z["io_issued"]
|
||||
}
|
||||
if z := readKstat(kstatDir + "zil"); z != nil {
|
||||
d.ZIL.Commits = z["zil_commit_count"]
|
||||
d.ZIL.NormalBytes, d.ZIL.NormalWrite = z["zil_itx_metaslab_normal_bytes"], z["zil_itx_metaslab_normal_write"]
|
||||
d.ZIL.SlogBytes, d.ZIL.SlogWrite = z["zil_itx_metaslab_slog_bytes"], z["zil_itx_metaslab_slog_write"]
|
||||
}
|
||||
if z := readKstat(kstatDir + "dbufstats"); z != nil {
|
||||
d.DBuf.Size, d.DBuf.Target, d.DBuf.Hits, d.DBuf.Misses = z["cache_size_bytes"], z["cache_target_bytes"], z["hash_hits"], z["hash_misses"]
|
||||
}
|
||||
for _, t := range arcTunables {
|
||||
if b, err := os.ReadFile("/sys/module/zfs/parameters/" + t); err == nil {
|
||||
d.Tunables[t] = strings.TrimSpace(string(b))
|
||||
}
|
||||
}
|
||||
if b, err := os.ReadFile("/proc/meminfo"); err == nil {
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
if strings.HasPrefix(line, "MemTotal:") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) >= 2 {
|
||||
d.MemTotal = u64(f[1]) << 10
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
d.Pools = m.poolCaching(ctx)
|
||||
return d
|
||||
}
|
||||
|
||||
var cachingMu sync.Mutex
|
||||
var cachingAt time.Time
|
||||
var cachingLast []PoolCaching
|
||||
|
||||
func (m *Manager) poolCaching(ctx context.Context) []PoolCaching {
|
||||
cachingMu.Lock()
|
||||
defer cachingMu.Unlock()
|
||||
if cachingLast != nil && time.Since(cachingAt) < poolCachingEvery {
|
||||
return cachingLast
|
||||
}
|
||||
out := []PoolCaching{}
|
||||
bin, ok := lookPath("zpool")
|
||||
if !ok {
|
||||
return out
|
||||
}
|
||||
// zpool list -v needs no root; the vdev tree under each pool, with the class headers
|
||||
raw, code, err := m.runPlain(ctx, bin, "list", "-Hpv", "-o", "name,size,alloc,free,health")
|
||||
if err != nil || code != 0 {
|
||||
return out
|
||||
}
|
||||
out = parsePoolCaching(string(raw))
|
||||
if zfs, ok := lookPath("zfs"); ok {
|
||||
for i := range out {
|
||||
p := &out[i]
|
||||
if props, code, err := m.runPlain(ctx, zfs, "get", "-Hp", "-o", "property,value", "primarycache,secondarycache,sync,logbias,dedup,compression,recordsize", p.Pool); err == nil && code == 0 {
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(props)), "\n") {
|
||||
f := strings.Split(line, "\t")
|
||||
if len(f) < 2 {
|
||||
continue
|
||||
}
|
||||
switch f[0] {
|
||||
case "primarycache":
|
||||
p.PrimaryCache = f[1]
|
||||
case "secondarycache":
|
||||
p.SecondaryCache = f[1]
|
||||
case "sync":
|
||||
p.Sync = f[1]
|
||||
case "logbias":
|
||||
p.LogBias = f[1]
|
||||
case "dedup":
|
||||
p.DedupProp = f[1]
|
||||
case "compression":
|
||||
p.Compression = f[1]
|
||||
case "recordsize":
|
||||
p.RecordSize = humanSize(u64(f[1]))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, pool := range parsePools(func() []byte {
|
||||
b, _, _ := m.runPlain(ctx, bin, "list", "-Hp", "-o", "name,size,alloc,free,frag,cap,health,dedup")
|
||||
return b
|
||||
}()) {
|
||||
for i := range out {
|
||||
if out[i].Pool == pool.Name {
|
||||
out[i].DedupRatio = pool.Dedup
|
||||
}
|
||||
}
|
||||
}
|
||||
cachingLast, cachingAt = out, time.Now()
|
||||
return out
|
||||
}
|
||||
|
||||
// runPlain runs a command as ourselves, never through sudo: these are read-only zpool/zfs
|
||||
// queries that need no root on Linux, and a poll must not write to the auth log.
|
||||
func (m *Manager) runPlain(ctx context.Context, bin string, args ...string) ([]byte, int, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, bin, args...).Output()
|
||||
if ee, ok := err.(*exec.ExitError); ok {
|
||||
return out, ee.ExitCode(), nil
|
||||
}
|
||||
return out, 0, err
|
||||
}
|
||||
|
||||
var cacheClasses = map[string]bool{"logs": true, "log": true, "cache": true, "special": true, "dedup": true, "spares": true, "spare": true}
|
||||
|
||||
// parsePoolCaching reads `zpool list -Hpv -o name,size,alloc,free,health`: a pool line, then
|
||||
// its vdevs indented, with a bare class name ("logs", "cache", "special", "dedup", "spares")
|
||||
// introducing each extra class. Only those classes' members are kept; the data vdevs are the
|
||||
// pool and are shown elsewhere.
|
||||
func parsePoolCaching(text string) []PoolCaching {
|
||||
var out []PoolCaching
|
||||
var cur *PoolCaching
|
||||
class := ""
|
||||
for _, raw := range strings.Split(text, "\n") {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
continue
|
||||
}
|
||||
indented := strings.HasPrefix(raw, " ") || strings.HasPrefix(raw, "\t")
|
||||
f := strings.Split(strings.TrimSpace(raw), "\t")
|
||||
name := strings.TrimSpace(f[0])
|
||||
if !indented {
|
||||
out = append(out, PoolCaching{Pool: name, Cache: []CacheDev{}, Log: []CacheDev{}, Special: []CacheDev{}, Dedup: []CacheDev{}, Spare: []CacheDev{}})
|
||||
cur, class = &out[len(out)-1], ""
|
||||
continue
|
||||
}
|
||||
if cur == nil {
|
||||
continue
|
||||
}
|
||||
if cacheClasses[name] && (len(f) < 2 || f[1] == "-" || f[1] == "") {
|
||||
class = name
|
||||
continue
|
||||
}
|
||||
if class == "" {
|
||||
continue // a data vdev or one of its disks
|
||||
}
|
||||
// a mirror or raidz inside a class lists its members below it; keep them all, the
|
||||
// grouping is visible in the vdev tree on the pool card
|
||||
dev := CacheDev{Name: name}
|
||||
if len(f) >= 5 {
|
||||
dev.Size, dev.Alloc, dev.Health = u64(f[1]), u64(f[2]), strings.TrimSpace(f[4])
|
||||
if f[1] == "-" {
|
||||
dev.Size = 0
|
||||
}
|
||||
}
|
||||
switch class {
|
||||
case "logs", "log":
|
||||
cur.Log = append(cur.Log, dev)
|
||||
case "cache":
|
||||
cur.Cache = append(cur.Cache, dev)
|
||||
case "special":
|
||||
cur.Special = append(cur.Special, dev)
|
||||
case "dedup":
|
||||
cur.Dedup = append(cur.Dedup, dev)
|
||||
case "spares", "spare":
|
||||
cur.Spare = append(cur.Spare, dev)
|
||||
}
|
||||
}
|
||||
if out == nil {
|
||||
out = []PoolCaching{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func humanSize(b uint64) string {
|
||||
switch {
|
||||
case b >= 1<<20 && b%(1<<20) == 0:
|
||||
return strconv.FormatUint(b>>20, 10) + "M"
|
||||
case b >= 1<<10 && b%(1<<10) == 0:
|
||||
return strconv.FormatUint(b>>10, 10) + "K"
|
||||
}
|
||||
return strconv.FormatUint(b, 10)
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// zpool list -Hpv on a pool with every extra class; the data raidz and its disks are skipped
|
||||
const listV = "tank\t32000000000000\t23000000000000\t9000000000000\tONLINE\n" +
|
||||
"\traidz2-0\t32000000000000\t23000000000000\t9000000000000\tONLINE\n" +
|
||||
"\t\twwn-0x1\t4000000000000\t-\t-\tONLINE\n" +
|
||||
"\t\twwn-0x2\t4000000000000\t-\t-\tONLINE\n" +
|
||||
"\tspecial\t-\t-\t-\t-\n" +
|
||||
"\t\tmirror-1\t500000000000\t120000000000\t380000000000\tONLINE\n" +
|
||||
"\t\t\tnvme0n1p1\t500000000000\t-\t-\tONLINE\n" +
|
||||
"\t\t\tnvme1n1p1\t500000000000\t-\t-\tONLINE\n" +
|
||||
"\tlogs\t-\t-\t-\t-\n" +
|
||||
"\t\tnvme2n1p1\t16000000000\t4000000\t15996000000\tONLINE\n" +
|
||||
"\tcache\t-\t-\t-\t-\n" +
|
||||
"\t\tnvme2n1p2\t400000000000\t210000000000\t190000000000\tONLINE\n" +
|
||||
"\tspares\t-\t-\t-\t-\n" +
|
||||
"\t\twwn-0x9\t4000000000000\t-\t-\tAVAIL\n" +
|
||||
"scratch\t1000000000000\t1000000000\t999000000000\tONLINE\n" +
|
||||
"\tsda\t1000000000000\t1000000000\t999000000000\tONLINE\n"
|
||||
|
||||
func TestParsePoolCaching(t *testing.T) {
|
||||
got := parsePoolCaching(listV)
|
||||
if len(got) != 2 || got[0].Pool != "tank" || got[1].Pool != "scratch" {
|
||||
t.Fatalf("pools: %+v", got)
|
||||
}
|
||||
tank := got[0]
|
||||
if len(tank.Cache) != 1 || tank.Cache[0].Name != "nvme2n1p2" || tank.Cache[0].Alloc != 210000000000 {
|
||||
t.Errorf("cache: %+v", tank.Cache)
|
||||
}
|
||||
if len(tank.Log) != 1 || tank.Log[0].Name != "nvme2n1p1" {
|
||||
t.Errorf("log: %+v", tank.Log)
|
||||
}
|
||||
if len(tank.Special) != 3 || tank.Special[0].Name != "mirror-1" || tank.Special[0].Alloc != 120000000000 {
|
||||
t.Errorf("special: %+v", tank.Special)
|
||||
}
|
||||
if len(tank.Spare) != 1 || tank.Spare[0].Health != "AVAIL" {
|
||||
t.Errorf("spares: %+v", tank.Spare)
|
||||
}
|
||||
if len(tank.Dedup) != 0 {
|
||||
t.Errorf("no dedup class: %+v", tank.Dedup)
|
||||
}
|
||||
s := got[1]
|
||||
if len(s.Cache)+len(s.Log)+len(s.Special)+len(s.Spare) != 0 {
|
||||
t.Errorf("a plain pool has no extra classes: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanSize(t *testing.T) {
|
||||
for in, want := range map[uint64]string{131072: "128K", 1048576: "1M", 16384: "16K", 1000: "1000"} {
|
||||
if got := humanSize(in); got != want {
|
||||
t.Errorf("%d → %s, want %s", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLiveARC reads this machine's ZFS for real: HAMLAB_LIVE=1 go test -run TestLiveARC -v ./internal/storage
|
||||
func TestLiveARC(t *testing.T) {
|
||||
if os.Getenv("HAMLAB_LIVE") == "" {
|
||||
t.Skip("set HAMLAB_LIVE=1 to read the real machine")
|
||||
}
|
||||
d := New(false).ARC(context.Background())
|
||||
b, _ := json.MarshalIndent(d, "", " ")
|
||||
t.Logf("%s", b)
|
||||
}
|
||||
Reference in New Issue
Block a user