hamlab/internal/storage/storage.go

// Package storage is the Disks tab: block devices from lsblk, SMART health via
// smartctl (optional; the tab explains how to get it), and ZFS pools/datasets/snapshots
// when the zfs tools and module are present. Spinning disks are never woken just to
// read SMART (-n standby).
package storage

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

// ---- block devices ----

type Part struct {
	Name   string `json:"name"`
	Type   string `json:"type"` // part | crypt | lvm | md | …
	FSType string `json:"fstype,omitempty"`
	Mount  string `json:"mount,omitempty"`
	Label  string `json:"label,omitempty"`
	Size   uint64 `json:"size"`
	Used   uint64 `json:"used,omitempty"`
	FSSize uint64 `json:"fs_size,omitempty"`
}

type Device struct {
	Name       string   `json:"name"`
	Kind       string   `json:"kind"` // nvme | ssd | hdd | usb | other
	Model      string   `json:"model,omitempty"`
	Vendor     string   `json:"vendor,omitempty"`
	Serial     string   `json:"serial,omitempty"`
	Tran       string   `json:"tran,omitempty"`
	Size       uint64   `json:"size"`
	Rotational bool     `json:"rotational"`
	Removable  bool     `json:"removable"`
	Temp       float64  `json:"temp,omitempty"`
	Parts      []Part   `json:"parts"`
	Namespaces []string `json:"namespaces,omitempty"` // NVMe: the block devices behind this controller
}

type lsblkNode struct {
	Name       string      `json:"name"`
	Type       string      `json:"type"`
	Size       uint64      `json:"size"`
	Model      *string     `json:"model"`
	Serial     *string     `json:"serial"`
	Vendor     *string     `json:"vendor"`
	Rota       bool        `json:"rota"`
	Tran       *string     `json:"tran"`
	FSType     *string     `json:"fstype"`
	Mountpoint *string     `json:"mountpoint"`
	Label      *string     `json:"label"`
	RM         bool        `json:"rm"`
	Children   []lsblkNode `json:"children"`
}

func str(p *string) string {
	if p == nil {
		return ""
	}
	return strings.TrimSpace(*p)
}

func Devices(ctx context.Context) ([]Device, error) {
	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	defer cancel()
	out, err := exec.CommandContext(ctx, "lsblk", "-J", "-b", "-o", "NAME,TYPE,SIZE,MODEL,SERIAL,VENDOR,ROTA,TRAN,FSTYPE,MOUNTPOINT,LABEL,RM").Output()
	if err != nil {
		return nil, errors.New("lsblk: " + err.Error())
	}
	var root struct {
		Blockdevices []lsblkNode `json:"blockdevices"`
	}
	if err := json.Unmarshal(out, &root); err != nil {
		return nil, err
	}
	devs := []Device{}
	for _, n := range root.Blockdevices {
		if n.Type != "disk" || n.Size == 0 || strings.HasPrefix(n.Name, "zd") {
			continue
		}
		d := Device{Name: n.Name, Model: str(n.Model), Vendor: str(n.Vendor), Serial: str(n.Serial), Tran: str(n.Tran),
			Size: n.Size, Rotational: n.Rota, Removable: n.RM, Temp: blockTemp(n.Name), Parts: []Part{}}
		switch {
		case d.Tran == "usb":
			d.Kind = "usb"
		case d.Tran == "nvme" || strings.HasPrefix(n.Name, "nvme"):
			d.Kind = "nvme"
		case n.Rota:
			d.Kind = "hdd"
		case d.Tran == "sata" || d.Tran == "sas" || strings.HasPrefix(n.Name, "sd") || strings.HasPrefix(n.Name, "mmcblk"):
			d.Kind = "ssd"
		default:
			d.Kind = "other"
		}
		var walk func(nodes []lsblkNode)
		walk = func(nodes []lsblkNode) {
			for _, c := range nodes {
				p := Part{Name: c.Name, Type: c.Type, FSType: str(c.FSType), Mount: str(c.Mountpoint), Label: str(c.Label), Size: c.Size}
				if p.Mount != "" && p.Mount != "[SWAP]" {
					var st syscall.Statfs_t
					if syscall.Statfs(p.Mount, &st) == nil {
						bs := uint64(st.Bsize)
						p.FSSize, p.Used = st.Blocks*bs, (st.Blocks-st.Bfree)*bs
					}
				}
				d.Parts = append(d.Parts, p)
				walk(c.Children)
			}
		}
		walk(n.Children)
		devs = append(devs, d)
	}
	devs = foldNVMe(devs)
	sort.Slice(devs, func(i, j int) bool { return devs[i].Name < devs[j].Name })
	return devs, nil
}

var nsRe = regexp.MustCompile(`^(nvme\d+)n\d+$`)

// foldNVMe merges nvme0n1, nvme0n2, … into one physical device "nvme0": one SSD, one
// SMART record, however many namespaces the controller presents (Apple exposes three).
func foldNVMe(devs []Device) []Device {
	byCtrl := map[string]*Device{}
	var out []Device
	for _, d := range devs {
		m := nsRe.FindStringSubmatch(d.Name)
		if m == nil {
			out = append(out, d)
			continue
		}
		ctrl := m[1]
		c, ok := byCtrl[ctrl]
		if !ok {
			nd := d
			nd.Name = ctrl
			nd.Parts = []Part{}
			nd.Namespaces = nil
			if mdl := strings.TrimSpace(readStrFile("/sys/class/nvme/" + ctrl + "/model")); mdl != "" {
				nd.Model = mdl
			}
			if sn := strings.TrimSpace(readStrFile("/sys/class/nvme/" + ctrl + "/serial")); sn != "" {
				nd.Serial = sn
			}
			if t := ctrlTemp(ctrl); t > 0 {
				nd.Temp = t
			}
			nd.Size = 0
			c = &nd
			byCtrl[ctrl] = c
			out = append(out, Device{}) // placeholder keeps order; filled below
			c.Namespaces = []string{}
		}
		c.Size += d.Size
		c.Namespaces = append(c.Namespaces, d.Name)
		c.Parts = append(c.Parts, d.Parts...)
	}
	// replace placeholders with the folded controllers, in first-seen order
	res := out[:0]
	seen := map[string]bool{}
	for _, d := range out {
		if d.Name != "" {
			res = append(res, d)
			continue
		}
		for _, c := range byCtrl {
			if !seen[c.Name] {
				seen[c.Name] = true
				res = append(res, *c)
				break
			}
		}
	}
	return res
}

func readStrFile(p string) string {
	b, err := os.ReadFile(p)
	if err != nil {
		return ""
	}
	return string(b)
}

func ctrlTemp(ctrl string) float64 {
	dirs, _ := filepath.Glob("/sys/class/nvme/" + ctrl + "/hwmon*")
	for _, d := range dirs {
		if b, err := os.ReadFile(filepath.Join(d, "temp1_input")); err == nil {
			if v, err := strconv.Atoi(strings.TrimSpace(string(b))); err == nil {
				return float64(v) / 1000
			}
		}
	}
	return 0
}

func blockTemp(name string) float64 {
	for _, pat := range []string{"/sys/block/" + name + "/device/hwmon*", "/sys/block/" + name + "/device/hwmon/hwmon*"} {
		dirs, _ := filepath.Glob(pat)
		for _, d := range dirs {
			if b, err := os.ReadFile(filepath.Join(d, "temp1_input")); err == nil {
				if v, err := strconv.Atoi(strings.TrimSpace(string(b))); err == nil {
					return float64(v) / 1000
				}
			}
		}
	}
	return 0
}

var devRe = regexp.MustCompile(`^[a-z][a-z0-9]*$`)

func validDevice(name string) bool {
	if !devRe.MatchString(name) {
		return false
	}
	if _, err := os.Stat("/sys/block/" + name); err == nil {
		return true
	}
	_, err := os.Stat("/sys/class/nvme/" + name) // NVMe controller (nvme0), which is what smartctl wants anyway
	return err == nil
}

// ---- SMART ----

type Attr struct {
	ID     int    `json:"id"`
	Name   string `json:"name"`
	Value  int    `json:"value"`
	Worst  int    `json:"worst"`
	Thresh int    `json:"thresh"`
	Raw    string `json:"raw"`
	Failed string `json:"failed,omitempty"`
	Warn   bool   `json:"warn"`
}

type Smart struct {
	Device       string   `json:"device"`
	Available    bool     `json:"available"` // smartctl installed
	Supported    bool     `json:"supported"` // device answered
	Standby      bool     `json:"standby"`   // spun down; not read
	Passed       *bool    `json:"passed"`
	Temp         int      `json:"temp,omitempty"`
	PowerOnHours int      `json:"power_on_hours,omitempty"`
	PowerCycles  int      `json:"power_cycles,omitempty"`
	Model        string   `json:"model,omitempty"`
	Serial       string   `json:"serial,omitempty"`
	Firmware     string   `json:"firmware,omitempty"`
	Attrs        []Attr   `json:"attrs"`
	NVMe         *NVMeLog `json:"nvme,omitempty"`
	SelfTest     string   `json:"self_test,omitempty"`
	SelfTestPct  int      `json:"self_test_remaining,omitempty"`
	SelfTestOK   bool     `json:"self_test_supported"`
	SelfTestNote string   `json:"self_test_note,omitempty"`
	Warnings     []string `json:"warnings"`
	Error        string   `json:"error,omitempty"`
	FetchedAt    int64    `json:"fetched_at"`
}

type NVMeLog struct {
	CriticalWarning  int    `json:"critical_warning"`
	PercentageUsed   int    `json:"percentage_used"`
	AvailableSpare   int    `json:"available_spare"`
	SpareThreshold   int    `json:"spare_threshold"`
	MediaErrors      int64  `json:"media_errors"`
	ErrorLogEntries  int64  `json:"error_log_entries"`
	DataUnitsRead    uint64 `json:"data_read_bytes"`
	DataUnitsWritten uint64 `json:"data_written_bytes"`
	UnsafeShutdowns  int64  `json:"unsafe_shutdowns"`
}

type Manager struct {
	sudo     bool
	smartctl bool
	mu       sync.Mutex
	cache    map[string]Smart
}

// lookPath also checks the sbin directories, which a user's PATH often lacks.
func lookPath(bin string) (string, bool) {
	if p, err := exec.LookPath(bin); err == nil {
		return p, true
	}
	for _, d := range []string{"/usr/sbin", "/sbin", "/usr/local/sbin"} {
		if _, err := os.Stat(d + "/" + bin); err == nil {
			return d + "/" + bin, true
		}
	}
	return "", false
}

func New(sudo bool) *Manager {
	_, ok := lookPath("smartctl")
	return &Manager{sudo: sudo, smartctl: ok, cache: map[string]Smart{}}
}

// HasSmartctl re-probes when it was missing at startup, so an install from the Disks
// tab takes effect without a daemon restart.
func (m *Manager) HasSmartctl() bool {
	if !m.smartctl {
		_, m.smartctl = lookPath("smartctl")
	}
	return m.smartctl
}

func (m *Manager) run(ctx context.Context, args ...string) ([]byte, int, error) {
	if p, ok := lookPath(args[0]); ok {
		args[0] = p
	}
	if m.sudo {
		args = append([]string{"sudo", "-n"}, args...)
	}
	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()
	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
	var out, errb bytes.Buffer
	cmd.Stdout, cmd.Stderr = &out, &errb
	err := cmd.Run()
	code := 0
	var ee *exec.ExitError
	if errors.As(err, &ee) {
		code = ee.ExitCode()
		err = nil // smartctl and zpool use exit codes as status; the caller decides
		if len(out.Bytes()) == 0 && errb.Len() > 0 {
			err = errors.New(strings.TrimSpace(strings.Split(errb.String(), "\n")[0])) // nothing usable came back; say why
		}
	}
	if err != nil {
		msg := strings.TrimSpace(errb.String())
		if msg == "" {
			msg = err.Error()
		}
		return out.Bytes(), code, errors.New(msg)
	}
	return out.Bytes(), code, nil
}

func num(v any) int64 {
	switch x := v.(type) {
	case float64:
		return int64(x)
	case json.Number:
		n, _ := x.Int64()
		return n
	}
	return 0
}

// Smart reads SMART for one device. Cached for 10 minutes; spinning disks in standby are
// left asleep and reported as such. force re-reads (and will wake the disk).
func (m *Manager) Smart(ctx context.Context, dev string, force bool) Smart {
	s := Smart{Device: dev, Available: m.smartctl, Attrs: []Attr{}, Warnings: []string{}, FetchedAt: time.Now().UnixMilli()}
	if !validDevice(dev) {
		s.Error = "unknown device"
		return s
	}
	if !m.HasSmartctl() {
		return s
	}
	m.mu.Lock()
	if c, ok := m.cache[dev]; ok && !force && time.Since(time.UnixMilli(c.FetchedAt)) < 10*time.Minute {
		m.mu.Unlock()
		return c
	}
	m.mu.Unlock()

	args := []string{"smartctl", "-j", "-a"}
	if !force {
		args = append(args, "-n", "standby")
	}
	args = append(args, "/dev/"+dev)
	out, code, err := m.run(ctx, args...)
	if err != nil && len(out) == 0 {
		s.Error = err.Error()
		return s
	}
	var raw map[string]any
	if json.Unmarshal(out, &raw) != nil {
		s.Error = "smartctl: unreadable output"
		return s
	}
	if code&2 != 0 { // -n standby: device is asleep (also "open failed", but JSON will say)
		if msgs, ok := raw["smartctl"].(map[string]any); ok {
			for _, mm := range asList(msgs["messages"]) {
				if strings.Contains(strings.ToLower(asString(mm["string"])), "standby") {
					s.Standby = true
				}
			}
		}
		if s.Standby {
			m.mu.Lock()
			m.cache[dev] = s
			m.mu.Unlock()
			return s
		}
	}
	if st, ok := raw["smart_status"].(map[string]any); ok {
		p := st["passed"] == true
		s.Passed = &p
		s.Supported = true
	}
	s.Model = asString(raw["model_name"])
	s.Serial = asString(raw["serial_number"])
	s.Firmware = asString(raw["firmware_version"])
	if t, ok := raw["temperature"].(map[string]any); ok {
		s.Temp = int(num(t["current"]))
	}
	if t, ok := raw["power_on_time"].(map[string]any); ok {
		s.PowerOnHours = int(num(t["hours"]))
	}
	s.PowerCycles = int(num(raw["power_cycle_count"]))

	if a, ok := raw["ata_smart_attributes"].(map[string]any); ok {
		for _, row := range asList(a["table"]) {
			at := Attr{ID: int(num(row["id"])), Name: asString(row["name"]), Value: int(num(row["value"])), Worst: int(num(row["worst"])), Thresh: int(num(row["thresh"])), Failed: asString(row["when_failed"])}
			if rw, ok := row["raw"].(map[string]any); ok {
				at.Raw = asString(rw["string"])
				rv := num(rw["value"])
				switch at.ID {
				case 5, 187, 188, 196, 197, 198:
					at.Warn = rv > 0
				case 199:
					at.Warn = rv > 0
				case 10, 11:
					at.Warn = rv > 100
				}
				if at.Warn {
					s.Warnings = append(s.Warnings, at.Name+" = "+at.Raw)
				}
			}
			if at.Failed != "" {
				at.Warn = true
				s.Warnings = append(s.Warnings, at.Name+" failed ("+at.Failed+")")
			}
			s.Attrs = append(s.Attrs, at)
		}
	}
	if n, ok := raw["nvme_smart_health_information_log"].(map[string]any); ok {
		l := &NVMeLog{CriticalWarning: int(num(n["critical_warning"])), PercentageUsed: int(num(n["percentage_used"])),
			AvailableSpare: int(num(n["available_spare"])), SpareThreshold: int(num(n["available_spare_threshold"])),
			MediaErrors: num(n["media_errors"]), ErrorLogEntries: num(n["num_err_log_entries"]), UnsafeShutdowns: num(n["unsafe_shutdowns"]),
			DataUnitsRead: uint64(num(n["data_units_read"])) * 512000, DataUnitsWritten: uint64(num(n["data_units_written"])) * 512000}
		s.NVMe = l
		s.Supported = true
		if l.CriticalWarning != 0 {
			s.Warnings = append(s.Warnings, "critical warning flags set")
		}
		if l.MediaErrors > 0 {
			s.Warnings = append(s.Warnings, "media errors: "+strconv.FormatInt(l.MediaErrors, 10))
		}
		if l.SpareThreshold > 0 && l.AvailableSpare < l.SpareThreshold {
			s.Warnings = append(s.Warnings, "available spare below threshold")
		}
		if l.PercentageUsed >= 90 {
			s.Warnings = append(s.Warnings, "endurance "+strconv.Itoa(l.PercentageUsed)+"% used")
		}
	}
	// Can this drive run self-tests from here? ATA says so outright; NVMe drives prove it
	// by exposing the self-test log, and some (WD SN580 among them) advertise the command
	// but reject the log page, which makes the whole feature unusable.
	if d, ok := raw["ata_smart_data"].(map[string]any); ok {
		if caps, ok := d["capabilities"].(map[string]any); ok {
			s.SelfTestOK = caps["self_tests_supported"] == true
			if !s.SelfTestOK {
				s.SelfTestNote = "drive reports no self-test support"
			}
		}
	}
	if _, ok := raw["nvme_self_test_log"]; ok {
		s.SelfTestOK = true
	} else if s.NVMe != nil {
		s.SelfTestNote = "this drive's firmware doesn't expose the NVMe self-test log, so tests can't be run or read from here"
	}
	if msgs, ok := raw["smartctl"].(map[string]any); ok {
		for _, mm := range asList(msgs["messages"]) {
			if strings.Contains(asString(mm["string"]), "Self-test Log failed") {
				s.SelfTestOK = false
				s.SelfTestNote = "firmware rejects the self-test log page (" + asString(mm["string"]) + ")"
			}
		}
	}
	if d, ok := raw["ata_smart_data"].(map[string]any); ok {
		if st, ok := d["self_test"].(map[string]any); ok {
			if ss, ok := st["status"].(map[string]any); ok {
				s.SelfTest = asString(ss["string"])
				s.SelfTestPct = int(num(ss["remaining_percent"]))
			}
		}
	}
	if s.Passed != nil && !*s.Passed {
		s.Warnings = append([]string{"SMART overall health: FAILED"}, s.Warnings...)
	}
	if !s.Supported && s.Error == "" {
		s.Error = "device does not report SMART"
		if msgs, ok := raw["smartctl"].(map[string]any); ok {
			for _, mm := range asList(msgs["messages"]) {
				if asString(mm["severity"]) == "error" {
					s.Error = asString(mm["string"])
				}
			}
		}
	}
	m.mu.Lock()
	m.cache[dev] = s
	m.mu.Unlock()
	return s
}

func asList(v any) []map[string]any {
	var out []map[string]any
	if l, ok := v.([]any); ok {
		for _, x := range l {
			if mm, ok := x.(map[string]any); ok {
				out = append(out, mm)
			}
		}
	}
	return out
}

func asString(v any) string {
	if s, ok := v.(string); ok {
		return s
	}
	return ""
}

// SelfTest starts a short or long self-test.
func (m *Manager) SelfTest(ctx context.Context, dev, kind string) error {
	if !validDevice(dev) {
		return errors.New("unknown device")
	}
	if kind != "short" && kind != "long" {
		return errors.New("test must be short or long")
	}
	if !m.smartctl {
		return errors.New("smartctl is not installed")
	}
	if sm := m.Smart(ctx, dev, false); sm.Supported && !sm.SelfTestOK {
		return errors.New("self-tests aren't available on this drive: " + sm.SelfTestNote)
	}
	out, code, err := m.run(ctx, "smartctl", "-t", kind, "/dev/"+dev)
	if err != nil {
		return err
	}
	if code != 0 {
		return errors.New(strings.TrimSpace(lastLine(string(out))))
	}
	m.mu.Lock()
	delete(m.cache, dev)
	m.mu.Unlock()
	return nil
}

func lastLine(s string) string {
	lines := strings.Split(strings.TrimSpace(s), "\n")
	return lines[len(lines)-1]
}

// ---- ZFS ----

type Pool struct {
	Name   string `json:"name"`
	Size   uint64 `json:"size"`
	Alloc  uint64 `json:"alloc"`
	Free   uint64 `json:"free"`
	Frag   int    `json:"frag"`
	Cap    int    `json:"cap"`
	Health string `json:"health"`
	Dedup  string `json:"dedup"`
	Scan   string `json:"scan,omitempty"`
	Errors string `json:"errors,omitempty"`
	Config string `json:"config,omitempty"` // vdev tree, verbatim
}

type Dataset struct {
	Name       string `json:"name"`
	Type       string `json:"type"`
	Used       uint64 `json:"used"`
	Avail      uint64 `json:"avail"`
	Refer      uint64 `json:"refer"`
	UsedBySnap uint64 `json:"used_by_snapshots"`
	Mount      string `json:"mount,omitempty"`
	Compress   string `json:"compressratio"`
	Snapshots  int    `json:"snapshots"`
}

type Snapshot struct {
	Name    string `json:"name"`
	Used    uint64 `json:"used"`
	Created int64  `json:"created"` // unix ms
}

type ZFS struct {
	Available bool      `json:"available"`
	Message   string    `json:"message,omitempty"`
	Pools     []Pool    `json:"pools"`
	Datasets  []Dataset `json:"datasets"`
}

var poolRe = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_.-]*$`)
var datasetRe = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_.:/-]*$`)
var snapRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]*$`)

func u64(s string) uint64 { v, _ := strconv.ParseUint(s, 10, 64); return v }
func atoi(s string) int   { v, _ := strconv.Atoi(strings.TrimSuffix(s, "%")); return v }

func (m *Manager) ZFS(ctx context.Context) ZFS {
	z := ZFS{Pools: []Pool{}, Datasets: []Dataset{}}
	if _, ok := lookPath("zpool"); !ok {
		z.Message = "ZFS tools not installed"
		return z
	}
	out, code, err := m.run(ctx, "zpool", "list", "-Hp", "-o", "name,size,alloc,free,frag,cap,health,dedup")
	if err != nil || code != 0 {
		msg := "zpool list failed"
		if err != nil {
			msg = err.Error()
		}
		z.Message = msg
		return z
	}
	z.Available = true
	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
		f := strings.Split(line, "\t")
		if len(f) < 8 {
			continue
		}
		p := Pool{Name: f[0], Size: u64(f[1]), Alloc: u64(f[2]), Free: u64(f[3]), Frag: atoi(f[4]), Cap: atoi(f[5]), Health: f[6], Dedup: f[7]}
		if st, _, err := m.run(ctx, "zpool", "status", p.Name); err == nil {
			p.Scan, p.Errors, p.Config = parseStatus(string(st))
		}
		z.Pools = append(z.Pools, p)
	}
	snapCount := map[string]int{}
	if so, _, err := m.run(ctx, "zfs", "list", "-t", "snapshot", "-H", "-o", "name"); err == nil {
		for _, line := range strings.Split(strings.TrimSpace(string(so)), "\n") {
			if ds, _, ok := strings.Cut(line, "@"); ok {
				snapCount[ds]++
			}
		}
	}
	if do, _, err := m.run(ctx, "zfs", "list", "-Hp", "-t", "filesystem,volume", "-o", "name,type,used,avail,refer,usedbysnapshots,mountpoint,compressratio"); err == nil {
		for _, line := range strings.Split(strings.TrimSpace(string(do)), "\n") {
			f := strings.Split(line, "\t")
			if len(f) < 8 {
				continue
			}
			d := Dataset{Name: f[0], Type: f[1], Used: u64(f[2]), Avail: u64(f[3]), Refer: u64(f[4]), UsedBySnap: u64(f[5]), Compress: f[7], Snapshots: snapCount[f[0]]}
			if f[6] != "-" && f[6] != "none" && f[6] != "legacy" {
				d.Mount = f[6]
			}
			z.Datasets = append(z.Datasets, d)
		}
	}
	return z
}

// parseStatus pulls the scan line, the errors line and the vdev table out of zpool status.
func parseStatus(text string) (scan, errs, config string) {
	lines := strings.Split(text, "\n")
	inConfig := false
	var cfg []string
	for i := 0; i < len(lines); i++ {
		l := lines[i]
		t := strings.TrimSpace(l)
		switch {
		case strings.HasPrefix(t, "scan:"):
			scan = strings.TrimSpace(strings.TrimPrefix(t, "scan:"))
			for i+1 < len(lines) && strings.HasPrefix(lines[i+1], "\t") && !strings.Contains(lines[i+1], ":") {
				i++
				scan += " " + strings.TrimSpace(lines[i])
			}
		case strings.HasPrefix(t, "errors:"):
			errs = strings.TrimSpace(strings.TrimPrefix(t, "errors:"))
			inConfig = false
		case strings.HasPrefix(t, "config:"):
			inConfig = true
		case inConfig && t != "":
			cfg = append(cfg, strings.TrimPrefix(l, "\t"))
		}
	}
	return scan, errs, strings.Join(cfg, "\n")
}

func (m *Manager) Snapshots(ctx context.Context, dataset string) ([]Snapshot, error) {
	if !datasetRe.MatchString(dataset) {
		return nil, errors.New("bad dataset name")
	}
	out, code, err := m.run(ctx, "zfs", "list", "-t", "snapshot", "-Hp", "-d", "1", "-o", "name,used,creation", "-s", "creation", dataset)
	if err != nil {
		return nil, err
	}
	if code != 0 {
		return nil, errors.New("zfs list failed")
	}
	snaps := []Snapshot{}
	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
		f := strings.Split(line, "\t")
		if len(f) < 3 {
			continue
		}
		c, _ := strconv.ParseInt(f[2], 10, 64)
		snaps = append(snaps, Snapshot{Name: f[0], Used: u64(f[1]), Created: c * 1000})
	}
	return snaps, nil
}

// Scrub starts, pauses or stops a scrub. action: start | pause | stop.
func (m *Manager) Scrub(ctx context.Context, pool, action string) error {
	if !poolRe.MatchString(pool) {
		return errors.New("bad pool name")
	}
	args := []string{"zpool", "scrub"}
	switch action {
	case "start":
	case "pause":
		args = append(args, "-p")
	case "stop":
		args = append(args, "-s")
	default:
		return errors.New("unknown scrub action")
	}
	out, code, err := m.run(ctx, append(args, pool)...)
	if err != nil {
		return err
	}
	if code != 0 {
		return errors.New(strings.TrimSpace(string(out)))
	}
	return nil
}

// CreateSnapshot takes dataset@name.
func (m *Manager) CreateSnapshot(ctx context.Context, dataset, name string) error {
	if !datasetRe.MatchString(dataset) || !snapRe.MatchString(name) {
		return errors.New("bad snapshot name")
	}
	out, code, err := m.run(ctx, "zfs", "snapshot", dataset+"@"+name)
	if err != nil {
		return err
	}
	if code != 0 {
		return errors.New(strings.TrimSpace(string(out)))
	}
	return nil
}