ZFS tab: Writes — the RAM write buffer, each pool's write rate, and a landing tier with its mover

Live from arcstats (dirty), dmu_tx (throttled writes), each pool's txgs (bytes synced to disk)
and objset kstats (bytes from programs). A landing tier — a mergerfs pool whose first branch is
fast storage and whose last is a ZFS dataset — is detected from mergerfs's branch xattr; a mover
status file in /run/landing-mover/ adds its queue, the file in flight and totals. Sensors
zfs.write.*, zfs.<pool>.write_bps|logical_bps and zfs.<pool>.landing.* carry the history.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

web/src/ZFS.tsx

@@ -9,8 +9,8 @@
 // disagree over. An existing cron job is detected and shown, so the page tells the truth
 // about the machine whether Ham Lab or something else is doing the work.
 import { FormEvent, Fragment, useCallback, useEffect, useState } from 'react'
-import { api, ARCView, CacheDev, PoolCaching, ZFSExternal, ZFSInfo, ZFSOverview, ZFSPolicy, ZFSRun, ZFSSnapshot } from './api'
-import { fmtBytes } from './fmt'
+import { api, ARCView, CacheDev, Landing, PoolCaching, PoolWrites, ZFSExternal, ZFSInfo, ZFSOverview, ZFSPolicy, ZFSRun, ZFSSnapshot, ZFSWrites } from './api'
+import { fmtBytes, fmtRate } from './fmt'
 import { useRefreshMs } from './prefs'
 import { fetchHistories } from './sensors'
 import Sparkline from './Sparkline'
@@ -367,18 +367,142 @@ function PoolTiers({ p, v }: { p: PoolCaching; v: ARCView }) {
   )
 }
 
+// ---- the write path: the dirty buffer in RAM, what each pool writes, and a landing tier ----
+//
+// A write reaches a pool in up to three steps. If a landing tier fronts the pool (a mergerfs
+// pool whose first branch is fast storage and whose last is the pool), new files land there
+// first and a mover drains them. Either way the data then waits in RAM as dirty buffers, and
+// each txg sync writes it to the disks. Each step's live number is here, with its history.
+
+const dur = (s: number) => (!Number.isFinite(s) || s <= 0 ? '—' : s < 90 ? `${Math.round(s)} s` : s < 5400 ? `${Math.round(s / 60)} min` : `${(s / 3600).toFixed(1)} h`)
+type WHist = Record<string, number[]>
+const wIds = (w: ZFSWrites) => ['zfs.write.dirty', ...w.pools.flatMap((p) => {
+  const b = `zfs.${p.pool}.`
+  return [b + 'write_bps', b + 'logical_bps', ...(p.landing ? [b + 'landing.pending_bytes', b + 'landing.in_bps', b + 'landing.drain_bps'] : [])]
+})]
+
+function MoverLine({ l }: { l: Landing }) {
+  const m = l.mover
+  if (!m) return <div className="notice" style={{ marginTop: 6 }}>No mover is reporting for this landing tier (nothing in <span className="mono">/run/landing-mover/</span>). Files that land here stay here until something moves them.</div>
+  const frac = m.current_size > 0 ? m.partial / m.current_size : 0
+  return (
+    <>
+      <div className="props" style={{ marginTop: 6 }}>
+        <div><span className="k">Mover</span><span>
+          <span className={'badge ' + (m.state === 'moving' ? 'on' : m.state === 'idle' ? '' : 'warn')}>{m.state}</span>
+          {' '}{m.state === 'moving' && m.batch_files > 1 ? `batch of ${m.batch_files} files, ${fmtBytes(m.batch_bytes)}` : m.state === 'holding' ? 'a mount it needs is missing — waiting' : m.state === 'stopped' ? `no word since ${ago(m.updated_ms)}` : `queue ${m.queue}`}
+          {m.state === 'moving' && m.queue > 0 ? ` · ${m.queue} more queued` : ''}
+        </span></div>
+        {m.state === 'moving' && m.current && <div><span className="k">Now</span><span className="mono small" title={m.current}>{m.current}{m.current_size ? ` · ${fmtBytes(m.partial)} of ${fmtBytes(m.current_size)}` : ''}</span></div>}
+        <div><span className="k">Since start</span><span>{fmtN(m.moved_files)} files · {fmtBytes(m.moved_bytes)} moved · running {ago(m.started_ms).replace(' ago', '')}</span></div>
+        {m.last_move && <div><span className="k">Last moved</span><span className="mono small" title={m.last_move}>{m.last_move} · {ago(m.last_move_ms)}</span></div>}
+        <div><span className="k">Rules</span><span className="muted">a file moves once it has been closed and left alone for {m.settle} s · limit {m.bwlimit && m.bwlimit !== '0' ? `${m.bwlimit}B/s` : 'none'} · oldest first</span></div>
+        {m.last_error && <div><span className="k">Last error</span><span className="warn-text" style={{ whiteSpace: 'normal' }}>{m.last_error} · {ago(m.last_error_ms)}</span></div>}
+      </div>
+      {m.state === 'moving' && m.current_size > 0 && <Meter frac={frac} cls="" />}
+    </>
+  )
+}
+
+function LandingBlockInner({ l, pending, inn, drain }: { l: Landing; pending: number[]; inn: number[]; drain: number[] }) {
+  const eta = l.drain_bps > 0 ? l.pending_bytes / l.drain_bps : NaN
+  const fill = l.size > 0 ? l.used / l.size : 0
+  return (
+    <div className="landing-tier">
+      <div className="arc-tier"><span className="k" title="a mergerfs pool puts new files on this branch first; a mover drains them onto the pool">Landing</span>
+        <span className="arc-tier-v"><span className="mono" title={`${l.branch} → ${l.pool_branch} via ${l.mount}`}>{l.branch}{l.device ? ` · ${l.device}` : ''}{l.fstype ? ` · ${l.fstype}` : ''} · in front of {l.pool_branch} as {l.mount}</span></span>
+      </div>
+      <div className="sub-head" style={{ marginTop: 6 }}>
+        <span className="muted small">{fmtBytes(l.used)} of {fmtBytes(l.size)} used</span><span className="spacer" />
+        <span className="num">{l.pending_files ? `${fmtN(l.pending_files)}${l.pending_partial ? '+' : ''} waiting · ${fmtBytes(l.pending_bytes)}` : 'nothing waiting'}</span>
+      </div>
+      <Meter frac={fill} />
+      <div className="props" style={{ marginTop: 6 }}>
+        <div><span className="k">Waiting</span><span>{l.pending_files ? `${fmtN(l.pending_files)}${l.pending_partial ? '+' : ''} files, ${fmtBytes(l.pending_bytes)} · oldest landed ${ago(l.oldest_ms)}` : 'nothing — everything that landed is on the pool'}<span className="muted small"> · counted {ago(l.counted_ms)}</span></span></div>
+        <div><span className="k">Arriving</span><span>{fmtRate(l.in_bps)}{l.device ? <span className="muted small"> · writes to {l.device}</span> : ''}</span></div>
+        <div><span className="k">To the pool</span><span>{fmtRate(l.drain_bps)}{l.pending_bytes > 0 ? ` · ${Number.isFinite(eta) ? `clear in about ${dur(eta)}` : 'not draining right now'}` : ''}</span></div>
+      </div>
+      <MoverLine l={l} />
+      <div className="arc-sparks">
+        <div>
+          <div className="muted small">Waiting on the landing tier · {fmtBytes(l.pending_bytes)}</div>
+          <Sparkline series={[{ name: 'waiting', values: pending, color: cssVar('--s1') }]} format={fmtBytes} height={40} />
+        </div>
+        <div>
+          <div className="muted small">Arriving {fmtRate(l.in_bps)} · to the pool {fmtRate(l.drain_bps)}</div>
+          <Sparkline series={[
+            { name: 'arriving', values: inn, color: cssVar('--s1') },
+            { name: 'to the pool', values: drain, color: cssVar('--ok') },
+          ]} format={fmtRate} height={40} />
+        </div>
+      </div>
+    </div>
+  )
+}
+
+function WritePath({ w, p, h }: { w: ZFSWrites; p: PoolWrites; h: WHist }) {
+  const b = `zfs.${p.pool}.`
+  const throttling = w.delayed_per_sec > 0 || w.over_max_per_sec > 0
+  const delayAt = (w.limit * w.delay_min_pct) / 100
+  const ratio = p.write_bps > 0 && p.logical_bps > 0 ? p.logical_bps / p.write_bps : 0
+  return (
+    <section className="card">
+      <div className="sub-head">
+        <span className={'dot ' + (w.over_max_per_sec > 0 ? 'bad' : throttling ? 'warn' : 'ok')} />
+        <span className="dname">{p.pool}</span>
+        <span className="badge">write path</span>
+        <span className="spacer" />
+        <span className="num">{fmtRate(p.write_bps)} to disk</span>
+      </div>
+      {p.landing
+        ? <LandingBlockWithHist l={p.landing} h={h} b={b} />
+        : <div className="arc-tiers"><div className="arc-tier"><span className="k" title="a fast tier new files land on before the pool">Landing</span><span className="arc-tier-v"><span className="none">none — writes go straight into the pool's RAM buffer</span></span></div></div>}
+      <div className="arc-sparks">
+        <div>
+          <div className="muted small">RAM write buffer · {fmtBytes(w.dirty)} of {fmtBytes(w.limit)}</div>
+          <Sparkline series={[{ name: 'waiting in RAM', values: h['zfs.write.dirty'] ?? [], color: cssVar('--s1') }]} max={w.limit || undefined} format={fmtBytes} height={44} />
+        </div>
+        <div>
+          <div className="muted small">Writes · {fmtRate(p.write_bps)} to disk · {fmtRate(p.logical_bps)} from programs</div>
+          <Sparkline series={[
+            { name: 'to disk', values: h[b + 'write_bps'] ?? [], color: cssVar('--s1') },
+            { name: 'from programs', values: h[b + 'logical_bps'] ?? [], color: cssVar('--ok') },
+          ]} format={fmtRate} height={44} />
+        </div>
+      </div>
+      <div className="props" style={{ marginTop: 8 }}>
+        <div><span className="k">In RAM</span><span title="dirty data: written by programs, not yet on the disks. For all pools together — ZFS keeps no live per-pool figure">{fmtBytes(w.dirty)} waiting · holds up to {fmtBytes(w.limit)} {w.limit < w.dirty_max ? `(half the ARC target; zfs_dirty_data_max is ${fmtBytes(w.dirty_max)})` : '(zfs_dirty_data_max)'}</span></div>
+        <div><span className="k">Throttle</span><span title="past zfs_delay_min_dirty_percent of the buffer, ZFS makes each write wait a little so the disks can catch up; at the limit, writes stop until a sync frees room">
+          {w.over_max_per_sec > 0 ? <span className="warn-text">{fmtN(w.over_max_per_sec)}/s writes stopped at the limit</span> : w.delayed_per_sec > 0 ? <span className="warn-text">slowing {fmtN(w.delayed_per_sec)} writes/s — the disks are the limit now</span> : 'not slowing anything'}
+          <span className="muted"> · starts at {w.delay_min_pct}% ({fmtBytes(delayAt)})</span></span></div>
+        <div><span className="k">To disk</span><span>{fmtRate(p.write_bps)} written by syncs{ratio > 0 ? ` · ${fmtRate(p.logical_bps)} from programs (${ratio >= 1 ? `${ratio.toFixed(2)}x compressed` : 'parity and metadata included'})` : ''}<span className="muted"> · a sync at least every {w.txg_timeout_sec} s, sooner as the buffer fills</span></span></div>
+      </div>
+    </section>
+  )
+}
+
+function LandingBlockWithHist({ l, h, b }: { l: Landing; h: WHist; b: string }) {
+  return <LandingBlockInner l={l} pending={h[b + 'landing.pending_bytes'] ?? []} inn={h[b + 'landing.in_bps'] ?? []} drain={h[b + 'landing.drain_bps'] ?? []} />
+}
+
 function ARCSection() {
   const [v, setV] = useState<ARCView | null>(null)
   const [hist, setHist] = useState<{ size: number[]; hit: number[] }>({ size: [], hit: [] })
+  const [whist, setWhist] = useState<WHist>({})
   const [err, setErr] = useState('')
   const rate = useRefreshMs()
   useEffect(() => {
     let stop = false
     const poll = async () => {
-      try { const r = await api.zfsARC(); if (!stop) { setV(r); setErr('') } } catch (e) { if (!stop) setErr(e instanceof Error ? e.message : 'failed') }
+      let r: ARCView | null = null
+      try { r = await api.zfsARC(); if (!stop) { setV(r); setErr('') } } catch (e) { if (!stop) setErr(e instanceof Error ? e.message : 'failed') }
       try {
-        const hs = await fetchHistories('zfs.arc.size,zfs.arc.hit_pct')
-        if (!stop) setHist({ size: hs.find((h) => h.id === 'zfs.arc.size')?.v ?? [], hit: hs.find((h) => h.id === 'zfs.arc.hit_pct')?.v ?? [] })
+        const extra = r?.writes ? wIds(r.writes) : []
+        const hs = await fetchHistories(['zfs.arc.size', 'zfs.arc.hit_pct', ...extra].join(','))
+        if (!stop) {
+          setHist({ size: hs.find((h) => h.id === 'zfs.arc.size')?.v ?? [], hit: hs.find((h) => h.id === 'zfs.arc.hit_pct')?.v ?? [] })
+          setWhist(Object.fromEntries(hs.filter((h) => extra.includes(h.id)).map((h) => [h.id, h.v])))
+        }
       } catch { /* the sparklines stay empty; the numbers still show */ }
     }
     poll()
@@ -445,6 +569,14 @@ function ARCSection() {
         </div>
       </section>
       {d.pools.length > 0 && <div className="cards" style={{ marginTop: 8 }}>{d.pools.map((p) => <PoolTiers key={p.pool} p={p} v={v} />)}</div>}
+      {v.writes && v.writes.pools.length > 0 && (
+        <>
+          <div className="dash-bar" style={{ marginTop: 12 }}><h2 className="section">Writes</h2><span className="spacer" />
+            <span className="muted small">what is waiting to reach each pool, and how fast it is getting there</span>
+          </div>
+          <div className="cards">{v.writes.pools.map((p) => <WritePath key={p.pool} w={v.writes!} p={p} h={whist} />)}</div>
+        </>
+      )}
     </>
   )
 }

web/src/api.ts

@@ -252,7 +252,23 @@ export interface ARCDetail {
   dbuf: { Size: number; Target: number; Hits: number; Misses: number }
   pools: PoolCaching[]
 }
-export interface ARCView { detail: ARCDetail; live: ARCLive | null }
+export interface LandingMover {
+  state: 'idle' | 'moving' | 'holding' | 'stopped' | string; src: string; dst: string; pid: number; started_ms: number; updated_ms: number; stale: boolean
+  queue: number; settle: number; bwlimit: string; batch_files: number; batch_bytes: number; batch_start_ms: number; batch_first: string
+  current: string; current_size: number; partial: number; moved_files: number; moved_bytes: number
+  last_move: string; last_move_ms: number; last_error: string; last_error_ms: number
+}
+export interface Landing {
+  mount: string; branch: string; pool_branch: string; device?: string; fstype?: string; size: number; used: number
+  pending_files: number; pending_bytes: number; pending_partial?: boolean; oldest_ms?: number; counted_ms: number
+  in_bps: number; drain_bps: number; mover?: LandingMover
+}
+export interface PoolWrites { pool: string; write_bps: number; logical_bps: number; landing?: Landing }
+export interface ZFSWrites {
+  dirty: number; dirty_max: number; delay_min_pct: number; limit: number; delayed_per_sec: number; over_max_per_sec: number; txg_timeout_sec: number
+  pools: PoolWrites[]
+}
+export interface ARCView { detail: ARCDetail; live: ARCLive | null; writes: ZFSWrites | null }
 
 export interface FileEntry { name: string; type: 'dir' | 'file' | 'link' | 'other'; size: number; mode: string; perm: string; mtime: number; user: string; group: string; target?: string; broken?: boolean; preview?: 'image' | 'video' | 'audio' }
 export interface PlayTrack { codec: string; profile?: string; pix_fmt?: string; width?: number; height?: number; fps?: number; channels?: number; spherical?: boolean }

web/src/styles.css

@@ -884,6 +884,8 @@ pre.diff .add { color: var(--ok); background: color-mix(in srgb, var(--ok) 12%,
 .arc-tier { display: flex; gap: 8px; min-width: 0; } .arc-tier .k { color: var(--muted); width: 80px; flex-shrink: 0; cursor: help; }
 .arc-tier-v { display: flex; flex-direction: column; gap: 1px; min-width: 0; } .arc-tier-v > span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
 .arc-tier-v .none { color: var(--muted); white-space: normal; }
+.landing-tier { border: 1px solid var(--border); border-radius: 8px; padding: 8px 10px; margin: 4px 0 6px; font-size: 13px; }
+.landing-tier .props .badge { margin-right: 2px; }
 
 /* ---- Space: what is using a disk ---- */
 .space { position: absolute; inset: 0; display: flex; flex-direction: column; background: var(--bg); min-height: 0; }