Virtualization phase 2: ISOs, New VM from ISO, the VGA console in the browser, nightly exports, mounts, instance sensors

- Library: an ISO shelf in virt/isos: upload (the Files uploader), fetch by URL with an
  optional sha256, import a file from the machine by copying it in, rename, delete (refused
  while attached), attach to a VM as a CD-ROM live or stopped with boot-first, eject, and
  New VM from ISO: an empty VM with the ISO as first boot device, Secure Boot on or off.
- The VGA console: Ham Lab asks Incus for the VM's SPICE console over its unix socket and
  bridges the display websocket to the browser, where spice-html5 (vendored from upstream
  b372aab, LGPL, loaded on demand) draws it; Ctrl+Alt+Del, fit-to-window.
- Exports: every instance as a tarball in virt/exports/<name>/, kept to a count, as a cron
  line in /etc/cron.d/hamlab-virt calling hamlab virt-export, settings and the last run
  as files beside the data; Export now per instance or for all.
- Disks tab per instance: the root disk, attached ISOs with Eject, bind mounts in and out
  (shift for containers, virtiofs for VMs).
- instance.* sensors (running, down, state, cpu, mem) and the VM-or-container-down alert
  default, offered once to installs seeded before it existed.
- cloud-init: Use my SSH keys writes a cloud-config from the account's public keys.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
2026-10-09 22:21:57 -07:00
co-authored by Claude Fable 5.1
parent 66498a58c5
commit 43ebacdd92
46 changed files with 14493 additions and 25 deletions
+231
View File
@@ -0,0 +1,231 @@
package incus
// Nightly exports: every instance as a tarball in virt/exports/<name>/, kept to a count, as a
// cron line like the ZFS schedules — `hamlab virt-export` in /etc/cron.d/hamlab-virt — so the
// backup runs whether or not the console is up. The settings and the last run's record are
// files beside the data, not state.json, because the cron job is a separate process.
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"time"
)
const CronFile = "/etc/cron.d/hamlab-virt"
type ExportConfig struct {
Enabled bool `json:"enabled"`
Hour int `json:"hour"`
Minute int `json:"minute"`
Keep int `json:"keep"` // newest exports kept per instance
InstanceOnly bool `json:"instance_only"` // leave the snapshots out of the tarball
Instances []string `json:"instances,omitempty"` // empty: every instance
Updated time.Time `json:"updated,omitempty"`
}
func DefaultExportConfig() ExportConfig {
return ExportConfig{Hour: 2, Minute: 15, Keep: 3, InstanceOnly: true}
}
func (c ExportConfig) Validate() error {
if c.Hour < 0 || c.Hour > 23 || c.Minute < 0 || c.Minute > 59 {
return errors.New("time: hour 0–23, minute 0–59")
}
if c.Keep < 1 || c.Keep > 100 {
return errors.New("keep: between 1 and 100")
}
for _, n := range c.Instances {
if !ValidName(n) {
return errors.New("instance: " + n)
}
}
return nil
}
func (c ExportConfig) CronExpr() string { return fmt.Sprintf("%d %d * * *", c.Minute, c.Hour) }
// NextRun is the next time the line fires, for the page.
func (c ExportConfig) NextRun(now time.Time) time.Time {
n := time.Date(now.Year(), now.Month(), now.Day(), c.Hour, c.Minute, 0, 0, now.Location())
if !n.After(now) {
n = n.Add(24 * time.Hour)
}
return n
}
// CronText is the whole file; rewritten on every change, never edited.
func (c ExportConfig) CronText(user, exe, config string) string {
var b strings.Builder
b.WriteString("# Ham Lab instance exports — written by the Virtualization tab, rewritten whole on every change.\n")
b.WriteString("# Edit it there, not here.\n")
b.WriteString("SHELL=/bin/sh\nPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n")
fmt.Fprintf(&b, "# every instance to virt/exports, keep %d each\n", c.Keep)
fmt.Fprintf(&b, "%s %s %s -config %s virt-export\n", c.CronExpr(), user, exe, config)
return b.String()
}
func (m *Manager) exportConfigPath() string { return filepath.Join(m.dir, "exports.json") }
func (m *Manager) exportRunPath() string { return filepath.Join(m.dir, "export-runs.json") }
func (m *Manager) ExportConfig() ExportConfig {
c := DefaultExportConfig()
if b, err := os.ReadFile(m.exportConfigPath()); err == nil {
_ = json.Unmarshal(b, &c)
}
return c
}
func (m *Manager) SaveExportConfig(c ExportConfig) error {
if err := c.Validate(); err != nil {
return err
}
if err := m.EnsureDirs(); err != nil {
return err
}
c.Updated = time.Now()
b, _ := json.MarshalIndent(c, "", " ")
return os.WriteFile(m.exportConfigPath(), b, 0o644)
}
type ExportRun struct {
At time.Time `json:"at"`
Exported []string `json:"exported"`
Pruned []string `json:"pruned"`
Errors []string `json:"errors,omitempty"`
Seconds float64 `json:"seconds"`
}
func (m *Manager) LastExportRun() *ExportRun {
b, err := os.ReadFile(m.exportRunPath())
if err != nil {
return nil
}
var r ExportRun
if json.Unmarshal(b, &r) != nil {
return nil
}
return &r
}
// ExportFile is one tarball in the exports tree.
type ExportFile struct {
Instance string `json:"instance"`
Name string `json:"name"`
Size int64 `json:"size"`
Modified time.Time `json:"modified"`
}
func (m *Manager) ExportFiles() []ExportFile {
out := []ExportFile{}
dirs, _ := os.ReadDir(m.ExportsDir())
for _, d := range dirs {
if !d.IsDir() {
continue
}
files, _ := os.ReadDir(filepath.Join(m.ExportsDir(), d.Name()))
for _, f := range files {
if f.IsDir() || strings.HasSuffix(f.Name(), ".part") {
continue
}
if info, err := f.Info(); err == nil {
out = append(out, ExportFile{Instance: d.Name(), Name: f.Name(), Size: info.Size(), Modified: info.ModTime()})
}
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].Instance != out[j].Instance {
return out[i].Instance < out[j].Instance
}
return out[i].Modified.After(out[j].Modified)
})
return out
}
// RunExports exports the named instances (or every one when names is empty) and prunes each
// instance's folder to the configured count. Output goes to w line by line for a task to show.
func (m *Manager) RunExports(ctx context.Context, names []string, w func(string)) ExportRun {
start := time.Now()
c := m.ExportConfig()
run := ExportRun{At: start, Exported: []string{}, Pruned: []string{}}
if err := m.EnsureDirs(); err != nil {
run.Errors = append(run.Errors, err.Error())
return run
}
all, err := m.Instances(ctx)
if err != nil {
run.Errors = append(run.Errors, "listing instances: "+err.Error())
return run
}
want := map[string]bool{}
for _, n := range names {
want[n] = true
}
if len(names) == 0 && len(c.Instances) > 0 {
for _, n := range c.Instances {
want[n] = true
}
}
for _, in := range all {
if len(want) > 0 && !want[in.Name] {
continue
}
dir := filepath.Join(m.ExportsDir(), in.Name)
if err := os.MkdirAll(dir, 0o755); err != nil {
run.Errors = append(run.Errors, in.Name+": "+err.Error())
continue
}
stamp := start.Format("2006-01-02_15.04")
final := filepath.Join(dir, fmt.Sprintf("%s-%s.tar.gz", in.Name, stamp))
part := final + ".part"
args := []string{"export", in.Name, part}
if c.InstanceOnly {
args = append(args, "--instance-only")
}
w(fmt.Sprintf("exporting %s …", in.Name))
cctx, cancel := context.WithTimeout(ctx, 2*time.Hour)
out, err := exec.CommandContext(cctx, "incus", args...).CombinedOutput()
cancel()
if err != nil {
_ = os.Remove(part)
msg := strings.TrimSpace(string(out))
if msg == "" {
msg = err.Error()
}
run.Errors = append(run.Errors, in.Name+": "+msg)
w(" failed: " + msg)
continue
}
if err := os.Rename(part, final); err != nil {
run.Errors = append(run.Errors, in.Name+": "+err.Error())
continue
}
if st, err := os.Stat(final); err == nil {
w(fmt.Sprintf(" %s (%.1f MiB)", filepath.Base(final), float64(st.Size())/(1<<20)))
}
run.Exported = append(run.Exported, filepath.Base(final))
// prune: newest first, keep c.Keep
var files []ExportFile
for _, f := range m.ExportFiles() {
if f.Instance == in.Name {
files = append(files, f)
}
}
for i := c.Keep; i < len(files); i++ {
if err := os.Remove(filepath.Join(dir, files[i].Name)); err == nil {
run.Pruned = append(run.Pruned, files[i].Name)
w(" pruned " + files[i].Name)
}
}
}
run.Seconds = time.Since(start).Seconds()
b, _ := json.MarshalIndent(run, "", " ")
_ = os.WriteFile(m.exportRunPath(), b, 0o644)
return run
}
+90 -16
View File
@@ -57,7 +57,7 @@ type Manager struct {
version string
mu sync.Mutex
cpuPrev map[string]cpuMark
cpuPrev map[string]map[string]cpuMark // caller → instance → last reading, so two readers do not spoil each other's deltas
remote struct {
at time.Time
list []Image
@@ -75,7 +75,7 @@ type cpuMark struct {
}
func New(dataDir, user, version string) *Manager {
return &Manager{dir: filepath.Join(dataDir, "virt"), user: user, version: version, cpuPrev: map[string]cpuMark{}}
return &Manager{dir: filepath.Join(dataDir, "virt"), user: user, version: version, cpuPrev: map[string]map[string]cpuMark{}}
}
func (m *Manager) Dir() string { return m.dir }
@@ -512,6 +512,12 @@ func secretKey(k string) bool {
}
func (m *Manager) Instances(ctx context.Context) ([]Instance, error) {
return m.InstancesFor(ctx, "api")
}
// InstancesFor is Instances with its own CPU history per caller: the page and the sensors
// both read, at different rates, and a shared "previous reading" would give each the other's delta.
func (m *Manager) InstancesFor(ctx context.Context, caller string) ([]Instance, error) {
c, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
out, err := m.run(c, "list", "--format", "json")
@@ -525,6 +531,11 @@ func (m *Manager) Instances(ctx context.Context) ([]Instance, error) {
now := time.Now()
m.mu.Lock()
defer m.mu.Unlock()
prevs := m.cpuPrev[caller]
if prevs == nil {
prevs = map[string]cpuMark{}
m.cpuPrev[caller] = prevs
}
seen := map[string]bool{}
var list []Instance
for _, r := range raw {
@@ -577,18 +588,18 @@ func (m *Manager) Instances(ctx context.Context) ([]Instance, error) {
}
}
if r.Status == "Running" {
if prev, ok := m.cpuPrev[r.Name]; ok && now.After(prev.at) && r.State.CPU.Usage >= prev.ns {
if prev, ok := prevs[r.Name]; ok && now.After(prev.at) && r.State.CPU.Usage >= prev.ns {
in.CPUPct = float64(r.State.CPU.Usage-prev.ns) / float64(now.Sub(prev.at).Nanoseconds()) * 100
}
m.cpuPrev[r.Name] = cpuMark{ns: r.State.CPU.Usage, at: now}
prevs[r.Name] = cpuMark{ns: r.State.CPU.Usage, at: now}
seen[r.Name] = true
}
}
list = append(list, in)
}
for n := range m.cpuPrev {
for n := range prevs {
if !seen[n] {
delete(m.cpuPrev, n)
delete(prevs, n)
}
}
if list == nil {
@@ -797,11 +808,16 @@ func (m *Manager) Remote(ctx context.Context, refresh bool) ([]Image, error) {
// LibraryFile is one file in virt/images or virt/isos.
type LibraryFile struct {
Name string `json:"name"`
Size int64 `json:"size"`
Modified time.Time `json:"modified"`
Name string `json:"name"`
Size int64 `json:"size"`
Modified time.Time `json:"modified"`
AttachedTo []string `json:"attached_to,omitempty"`
}
// Installed reports whether the incus binary is on this machine at all; the cheap gate the
// sensors use before asking it anything.
func Installed() bool { return have("incus") }
func (m *Manager) Library(dir string) []LibraryFile {
ents, err := os.ReadDir(dir)
if err != nil {
@@ -822,16 +838,20 @@ func (m *Manager) Library(dir string) []LibraryFile {
// ---- validation ----
var (
nameRe = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9-]{0,62}$`)
imageRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,127}$`)
sizeRe = regexp.MustCompile(`^[0-9]+(\.[0-9]+)?(B|[KMGT]i?B)?$`)
cpuRe = regexp.MustCompile(`^[0-9]{1,3}(-[0-9]{1,3})?(,[0-9]{1,3}(-[0-9]{1,3})?)*$`)
snapRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`)
cronRe = regexp.MustCompile(`^(@(hourly|daily|weekly|monthly|yearly|annually|startup|never)|[0-9*/,-]+( [0-9*/,-]+){4})$`)
expRe = regexp.MustCompile(`^[0-9]+[mHdwMy]( [0-9]+[mHdwMy])*$|^$`)
nameRe = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9-]{0,62}$`)
imageRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,127}$`)
sizeRe = regexp.MustCompile(`^[0-9]+(\.[0-9]+)?(B|[KMGT]i?B)?$`)
cpuRe = regexp.MustCompile(`^[0-9]{1,3}(-[0-9]{1,3})?(,[0-9]{1,3}(-[0-9]{1,3})?)*$`)
snapRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`)
cronRe = regexp.MustCompile(`^(@(hourly|daily|weekly|monthly|yearly|annually|startup|never)|[0-9*/,-]+( [0-9*/,-]+){4})$`)
expRe = regexp.MustCompile(`^[0-9]+[mHdwMy]( [0-9]+[mHdwMy])*$|^$`)
deviceRe = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_-]{0,30}$`)
isoRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._ +-]{0,200}\.(?i:iso|img)$`)
)
func ValidName(s string) bool { return nameRe.MatchString(s) }
func ValidDevice(s string) bool { return deviceRe.MatchString(s) }
func ValidISOName(s string) bool { return isoRe.MatchString(s) && s == filepath.Base(s) }
func ValidImage(s string) bool { return imageRe.MatchString(s) }
func ValidSize(s string) bool { return s == "" || sizeRe.MatchString(s) }
func ValidCPU(s string) bool { return s == "" || cpuRe.MatchString(s) }
@@ -1075,3 +1095,57 @@ func (m *Manager) InstallScript(st Status, o InstallOpts) (string, Layout, error
w("if systemctl is-active --quiet %s; then systemd-run --quiet --on-active=3 --unit=hamlab-virt-restart systemctl restart %s; fi", HamlabUnit, HamlabUnit)
return b.String(), lay, nil
}
// FromISOSpec is the New VM from ISO form: an empty VM with the ISO as its first boot device.
type FromISOSpec struct {
Name string `json:"name"`
ISO string `json:"iso"` // a file name in virt/isos
CPU string `json:"cpu"`
Memory string `json:"memory"`
Disk string `json:"disk"`
Network string `json:"network"`
SecureBoot bool `json:"secureboot"`
Start bool `json:"start"`
}
// FromISOScript makes the VM in one shell script run through ShellCmd; the parameters carry
// the values so nothing is spliced into the text.
func FromISOScript(sp FromISOSpec, kvm bool) (script string, params []string, err error) {
switch {
case !kvm:
return "", nil, errors.New("this machine has no KVM")
case !ValidName(sp.Name):
return "", nil, errors.New("name: letters, digits and dashes, starting with a letter, up to 63")
case !ValidISOName(sp.ISO):
return "", nil, errors.New("iso: pick one from the library")
case !ValidCPU(sp.CPU) || !ValidSize(sp.Memory) || !ValidSize(sp.Disk) || !ValidNetwork(sp.Network):
return "", nil, errors.New("cpu, memory, disk: a count and two sizes, like 2, 2GiB, 20GiB")
}
cpu, mem, disk := sp.CPU, sp.Memory, sp.Disk
if cpu == "" {
cpu = "2"
}
if mem == "" {
mem = "2GiB"
}
if disk == "" {
disk = "20GiB"
}
sb := "true"
if !sp.SecureBoot {
sb = "false"
}
start := "0"
if sp.Start {
start = "1"
}
// $1 name $2 iso path $3 cpu $4 memory $5 disk $6 network $7 secureboot $8 start
script = `set -e
NET=""; [ -n "$6" ] && NET="-n $6"
incus init --empty --vm "$1" -c limits.cpu="$3" -c limits.memory="$4" -c security.secureboot="$7" -d root,size="$5" $NET
incus config device add "$1" iso0 disk source="$2" boot.priority=10
echo "made $1 with $2 as its first boot device"
[ "$8" = 1 ] && incus start "$1" && echo "started; open the VGA console to run the installer"
exit 0`
return script, []string{sp.Name, sp.ISO, cpu, mem, disk, sp.Network, sb, start}, nil
}
+116
View File
@@ -0,0 +1,116 @@
package incus
// The graphical console. Incus exposes a VM's SPICE display as a websocket: POST to the
// instance's console endpoint with type vga and it answers an operation with two secrets,
// one for the display stream and one for a control channel. Ham Lab dials both over Incus's
// unix socket and hands the display stream to the browser, where spice-html5 draws it.
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"os"
"time"
"github.com/coder/websocket"
)
// SocketPath is where Incus's API listens; the daemon's account reaches it by being in
// incus-admin, which Install arranges.
func SocketPath() string {
if p := os.Getenv("INCUS_SOCKET"); p != "" {
return p
}
for _, p := range []string{"/var/lib/incus/unix.socket", "/run/incus/unix.socket"} {
if _, err := os.Stat(p); err == nil {
return p
}
}
return "/var/lib/incus/unix.socket"
}
func (m *Manager) apiClient() *http.Client {
sock := SocketPath()
return &http.Client{Transport: &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "unix", sock)
},
ForceAttemptHTTP2: false,
}}
}
// VGA opens the display and control websockets for a running VM. The caller closes both.
func (m *Manager) VGA(ctx context.Context, name string) (display, control *websocket.Conn, err error) {
if !ValidName(name) {
return nil, nil, errors.New("bad name")
}
client := m.apiClient()
body, _ := json.Marshal(map[string]any{"type": "vga", "width": 0, "height": 0})
c, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
req, _ := http.NewRequestWithContext(c, "POST", "http://incus/1.0/instances/"+name+"/console", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
if os.IsPermission(errors.Unwrap(err)) || isPermission(err) {
return nil, nil, errors.New("Ham Lab's account can't reach Incus's socket; Finish setup on the Virtualization tab puts it in incus-admin")
}
return nil, nil, err
}
defer resp.Body.Close()
var op struct {
Error string `json:"error"`
Operation string `json:"operation"`
Metadata struct {
ID string `json:"id"`
Metadata struct {
FDs map[string]string `json:"fds"`
} `json:"metadata"`
} `json:"metadata"`
}
if err := json.NewDecoder(resp.Body).Decode(&op); err != nil {
return nil, nil, errors.New("unreadable answer from incus")
}
if op.Error != "" {
return nil, nil, errors.New(op.Error)
}
data, control0 := op.Metadata.Metadata.FDs["0"], op.Metadata.Metadata.FDs["control"]
if op.Metadata.ID == "" || data == "" {
return nil, nil, errors.New("incus gave no console; is the VM running?")
}
dial := func(secret string) (*websocket.Conn, error) {
u := fmt.Sprintf("ws://incus/1.0/operations/%s/websocket?secret=%s", op.Metadata.ID, secret)
ws, _, err := websocket.Dial(ctx, u, &websocket.DialOptions{HTTPClient: client, CompressionMode: websocket.CompressionDisabled})
return ws, err
}
if control0 != "" {
if control, err = dial(control0); err != nil {
return nil, nil, err
}
}
if display, err = dial(data); err != nil {
if control != nil {
control.Close(websocket.StatusInternalError, "")
}
return nil, nil, err
}
display.SetReadLimit(64 << 20) // a SPICE frame can carry a whole screen
return display, control, nil
}
func isPermission(err error) bool {
var pe *os.PathError
if errors.As(err, &pe) {
return os.IsPermission(pe)
}
var oe *net.OpError
if errors.As(err, &oe) {
return os.IsPermission(oe.Err) || isPermission(oe.Err)
}
return false
}