The ZFS tab shows the ARC, and names every caching tier a pool has or lacks
A Cache section between the pools and the schedules. One card for the ARC: size against its ceiling with the target and floor, sparklines of size and hit rate from the sampler's history, what it holds (MRU/MFU/dirty, data vs metadata, compression, the ghost lists), the prefetcher and dbuf cache, the memory-pressure counters, and the since-boot breakdown by demand/prefetch and data/metadata. Then a card per pool for L2ARC, SLOG, special vdev, dedup table and spares — the devices and their fill where they exist, and where they don't, plainly none and what that means (the ZIL writing to the pool's own vdevs, with how much it has written). Live numbers come from the sampler, which reads arcstats every tick and publishes zfs.arc.* sensors; the fuller picture is read on request, with zpool list -Hpv and zfs get run without sudo and cached for thirty seconds. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
db40220365
commit
b44faf3ac9
@@ -0,0 +1,113 @@
|
||||
package metrics
|
||||
|
||||
// The ZFS ARC — the read cache in RAM — read live from /proc/spl/kstat/zfs/arcstats on a
|
||||
// machine that has the module loaded, and left out of the sample everywhere else. The kstat
|
||||
// file is counters since boot; the hit rate here is over the sampling interval, which is the
|
||||
// number that says whether the cache is doing its job *now*. The lifetime figures and the
|
||||
// pool's caching tiers (L2ARC, SLOG, special vdevs) are the storage package's business.
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const arcstatsPath = "/proc/spl/kstat/zfs/arcstats"
|
||||
|
||||
// ARC is the read cache as one sample sees it. Sizes are bytes.
|
||||
type ARC struct {
|
||||
Size uint64 `json:"size"`
|
||||
Target uint64 `json:"target"` // c: what the ARC is currently allowed to hold
|
||||
Min uint64 `json:"min"` // c_min: the floor it will not shrink below
|
||||
Max uint64 `json:"max"` // c_max: the ceiling (zfs_arc_max, or half of RAM)
|
||||
// what is in it
|
||||
MRU uint64 `json:"mru"` // recently used once
|
||||
MFU uint64 `json:"mfu"` // used more than once
|
||||
Anon uint64 `json:"anon"` // in flight: dirty, not yet on disk
|
||||
Data uint64 `json:"data"` // file contents
|
||||
Metadata uint64 `json:"metadata"` // dnodes, indirect blocks, dbufs, bonus buffers
|
||||
MetaUsed uint64 `json:"meta_used"`
|
||||
Headers uint64 `json:"headers"`
|
||||
Compressed uint64 `json:"compressed"`
|
||||
Uncompressed uint64 `json:"uncompressed"`
|
||||
MRUGhost uint64 `json:"mru_ghost"` // what it recently evicted and still remembers
|
||||
MFUGhost uint64 `json:"mfu_ghost"`
|
||||
// L2ARC, the cache devices (0 when there are none)
|
||||
L2Size uint64 `json:"l2_size"`
|
||||
L2ASize uint64 `json:"l2_asize"` // as stored on the device, after compression
|
||||
// over this interval
|
||||
ReadsPerSec float64 `json:"reads_per_sec"`
|
||||
MissesPerSec float64 `json:"misses_per_sec"`
|
||||
HitPct float64 `json:"hit_pct"` // of the interval's reads; an idle interval keeps the last one
|
||||
HitPctKnown bool `json:"hit_pct_known"` // false until an interval has had any reads
|
||||
L2HitPct float64 `json:"l2_hit_pct"`
|
||||
// since boot
|
||||
Hits uint64 `json:"hits"`
|
||||
Misses uint64 `json:"misses"`
|
||||
LifetimeHitPct float64 `json:"lifetime_hit_pct"`
|
||||
// pressure
|
||||
NoGrow bool `json:"no_grow"` // arc_no_grow: memory pressure is stopping it growing
|
||||
Throttles uint64 `json:"throttles"` // memory_throttle_count since boot
|
||||
Prunes uint64 `json:"prunes"` // arc_prune: metadata evictions asked of the filesystems
|
||||
EvictSkip uint64 `json:"evict_skip"`
|
||||
}
|
||||
|
||||
type arcPrev struct {
|
||||
hits, misses, l2hits, l2misses uint64
|
||||
hitPct float64
|
||||
known bool
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// readKstat parses a kstat table ("name type data" rows after two header lines).
|
||||
func readKstat(p string) map[string]uint64 {
|
||||
s := readStr(p)
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
out := map[string]uint64{}
|
||||
for _, line := range strings.Split(s, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) != 3 || f[1] != "4" { // type 4 is KSTAT_DATA_UINT64
|
||||
continue
|
||||
}
|
||||
v, err := strconv.ParseUint(f[2], 10, 64)
|
||||
if err == nil {
|
||||
out[f[0]] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// arc reads the ARC; nil when this machine has no ZFS module loaded.
|
||||
func (c *Collector) arc(now time.Time) *ARC {
|
||||
k := readKstat(arcstatsPath)
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
a := &ARC{Size: k["size"], Target: k["c"], Min: k["c_min"], Max: k["c_max"],
|
||||
MRU: k["mru_size"], MFU: k["mfu_size"], Anon: k["anon_size"], Data: k["data_size"], Metadata: k["metadata_size"],
|
||||
MetaUsed: k["arc_meta_used"], Headers: k["hdr_size"], Compressed: k["compressed_size"], Uncompressed: k["uncompressed_size"],
|
||||
MRUGhost: k["mru_ghost_size"], MFUGhost: k["mfu_ghost_size"], L2Size: k["l2_size"], L2ASize: k["l2_asize"],
|
||||
Hits: k["hits"], Misses: k["misses"], NoGrow: k["arc_no_grow"] != 0, Throttles: k["memory_throttle_count"], Prunes: k["arc_prune"], EvictSkip: k["evict_skip"]}
|
||||
if n := a.Hits + a.Misses; n > 0 {
|
||||
a.LifetimeHitPct = round1(float64(a.Hits) / float64(n) * 100)
|
||||
}
|
||||
p := c.prevARC
|
||||
if !p.at.IsZero() && a.Hits >= p.hits && a.Misses >= p.misses {
|
||||
if dt := now.Sub(p.at).Seconds(); dt > 0 {
|
||||
hits, misses := a.Hits-p.hits, a.Misses-p.misses
|
||||
a.ReadsPerSec = round1(float64(hits+misses) / dt)
|
||||
a.MissesPerSec = round1(float64(misses) / dt)
|
||||
if hits+misses > 0 {
|
||||
p.hitPct, p.known = round1(float64(hits)/float64(hits+misses)*100), true
|
||||
}
|
||||
if l2 := (k["l2_hits"] - p.l2hits) + (k["l2_misses"] - p.l2misses); l2 > 0 && k["l2_hits"] >= p.l2hits {
|
||||
a.L2HitPct = round1(float64(k["l2_hits"]-p.l2hits) / float64(l2) * 100)
|
||||
}
|
||||
}
|
||||
}
|
||||
a.HitPct, a.HitPctKnown = p.hitPct, p.known
|
||||
c.prevARC = arcPrev{hits: a.Hits, misses: a.Misses, l2hits: k["l2_hits"], l2misses: k["l2_misses"], hitPct: p.hitPct, known: p.known, at: now}
|
||||
return a
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package metrics
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -134,3 +135,20 @@ func TestLiveGPU(t *testing.T) {
|
||||
t.Logf("%s = %v %s", x.ID, x.Value, x.Unit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLiveARC: HAMLAB_LIVE=1 go test -run TestLiveARC -v ./internal/metrics
|
||||
func TestLiveARC(t *testing.T) {
|
||||
if os.Getenv("HAMLAB_LIVE") == "" {
|
||||
t.Skip("set HAMLAB_LIVE=1 to read the real machine")
|
||||
}
|
||||
c := NewCollector()
|
||||
c.sample()
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
c.sample()
|
||||
s, _ := c.Latest()
|
||||
b, _ := json.MarshalIndent(s.ARC, "", " ")
|
||||
t.Logf("%s", b)
|
||||
for _, x := range flatten(s, func(id string) bool { return strings.HasPrefix(id, "zfs.arc.") }) {
|
||||
t.Logf("%s = %v %s", x.ID, x.Value, x.Unit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -147,6 +147,7 @@ type Sample struct {
|
||||
FS []FS `json:"fs"`
|
||||
UPS *UPS `json:"ups"`
|
||||
GPUs []GPU `json:"gpus"`
|
||||
ARC *ARC `json:"arc"` // the ZFS read cache; nil without the module
|
||||
Uptime float64 `json:"uptime"`
|
||||
|
||||
fsT, upsT int64 // when the slow sections were really read; sensors report these, not T
|
||||
@@ -176,6 +177,7 @@ type Collector struct {
|
||||
lastTable *ProcTable
|
||||
upsProbe func() *UPS // nil when no UPS daemon is installed
|
||||
gpu gpuState
|
||||
prevARC arcPrev
|
||||
|
||||
subMu sync.Mutex
|
||||
subs map[chan struct{}]struct{} // told after every sample; see Subscribe
|
||||
@@ -360,6 +362,7 @@ func (c *Collector) sample() {
|
||||
}
|
||||
s.UPS, s.upsT = c.lastUPS, c.upsAt.UnixMilli()
|
||||
s.GPUs = c.gpus(now, cpuTemp(s.Temps))
|
||||
s.ARC = c.arc(now)
|
||||
|
||||
if first {
|
||||
return // deltas are meaningless on the priming pass
|
||||
|
||||
@@ -173,6 +173,33 @@ func flatten(s Sample, want func(id string) bool) []sensors.Sensor {
|
||||
f.add(p+".clients", g.Name+" clients", "count", "", float64(len(g.Clients)), t, none, lbl...)
|
||||
}
|
||||
|
||||
if a := s.ARC; a != nil {
|
||||
p := "zfs.arc."
|
||||
cap := upTo(float64(a.Max))
|
||||
f.add(p+"size", "ARC size", "bytes", "B", float64(a.Size), t, cap)
|
||||
f.add(p+"target", "ARC target", "bytes", "B", float64(a.Target), t, cap)
|
||||
f.add(p+"min", "ARC floor", "bytes", "B", float64(a.Min), t, none)
|
||||
f.add(p+"max", "ARC ceiling", "bytes", "B", float64(a.Max), t, none)
|
||||
f.add(p+"mru", "ARC recently used", "bytes", "B", float64(a.MRU), t, cap)
|
||||
f.add(p+"mfu", "ARC frequently used", "bytes", "B", float64(a.MFU), t, cap)
|
||||
f.add(p+"metadata", "ARC metadata", "bytes", "B", float64(a.Metadata), t, cap)
|
||||
f.add(p+"data", "ARC data", "bytes", "B", float64(a.Data), t, cap)
|
||||
if a.HitPctKnown {
|
||||
f.add(p+"hit_pct", "ARC hit rate", "usage", "%", a.HitPct, t, pct)
|
||||
}
|
||||
f.add(p+"reads", "ARC reads", "rate", "/s", a.ReadsPerSec, t, none)
|
||||
f.add(p+"misses", "ARC misses", "rate", "/s", a.MissesPerSec, t, none)
|
||||
f.add(p+"lifetime_hit_pct", "ARC hit rate since boot", "usage", "%", a.LifetimeHitPct, t, pct)
|
||||
f.add(p+"no_grow", "ARC held by memory pressure", "state", "", a.NoGrow, t, none)
|
||||
if a.Compressed > 0 {
|
||||
f.add(p+"compress_ratio", "ARC compression", "ratio", "x", round1(float64(a.Uncompressed)/float64(a.Compressed)*10)/10, t, none)
|
||||
}
|
||||
if a.L2Size > 0 {
|
||||
f.add(p+"l2_size", "L2ARC size", "bytes", "B", float64(a.L2Size), t, none)
|
||||
f.add(p+"l2_hit_pct", "L2ARC hit rate", "usage", "%", a.L2HitPct, t, pct)
|
||||
}
|
||||
}
|
||||
|
||||
if u := s.UPS; u != nil {
|
||||
ut := s.upsT
|
||||
f.add("ups.battery", "UPS battery", "usage", "%", u.BatteryPct, ut, pct, "model", u.Model)
|
||||
|
||||
Reference in New Issue
Block a user