Virtualization phase 2: ISOs, New VM from ISO, the VGA console in the browser, nightly exports, mounts, instance sensors
- Library: an ISO shelf in virt/isos: upload (the Files uploader), fetch by URL with an optional sha256, import a file from the machine by copying it in, rename, delete (refused while attached), attach to a VM as a CD-ROM live or stopped with boot-first, eject, and New VM from ISO: an empty VM with the ISO as first boot device, Secure Boot on or off. - The VGA console: Ham Lab asks Incus for the VM's SPICE console over its unix socket and bridges the display websocket to the browser, where spice-html5 (vendored from upstream b372aab, LGPL, loaded on demand) draws it; Ctrl+Alt+Del, fit-to-window. - Exports: every instance as a tarball in virt/exports/<name>/, kept to a count, as a cron line in /etc/cron.d/hamlab-virt calling hamlab virt-export, settings and the last run as files beside the data; Export now per instance or for all. - Disks tab per instance: the root disk, attached ISOs with Eject, bind mounts in and out (shift for containers, virtiofs for VMs). - instance.* sensors (running, down, state, cpu, mem) and the VM-or-container-down alert default, offered once to installs seeded before it existed. - cloud-init: Use my SSH keys writes a cloud-config from the account's public keys. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mitchell-Johnson
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
66498a58c5
commit
43ebacdd92
@@ -71,6 +71,9 @@ type alertSaved struct {
|
||||
Channels []AlertChannel `json:"channels"`
|
||||
Events []AlertEvent `json:"events"`
|
||||
Seeded bool `json:"seeded"`
|
||||
// SawInstanceRule marks that the instance-down default has been offered once; set so that
|
||||
// removing it stays removed.
|
||||
SawInstanceRule bool `json:"saw_instance_rule,omitempty"`
|
||||
// Firing is what was firing when the hub last looked, so a restart (every deploy is one)
|
||||
// carries on where it was and doesn't announce the same alerts all over again.
|
||||
Firing map[string]alertKept `json:"firing,omitempty"`
|
||||
@@ -102,6 +105,7 @@ var defaultRules = []AlertRule{
|
||||
{ID: "machine-offline", Name: "Machine not answering", Sensor: "hamlab.reachable", Op: "==", Value: false, ForSec: 120, Severity: "crit", Enabled: true},
|
||||
{ID: "container-down", Name: "Container down", Sensor: "container.*.down", Op: "==", Value: true, ForSec: 120, Severity: "crit", Enabled: true},
|
||||
{ID: "container-unhealthy", Name: "Container unhealthy", Sensor: "container.*.health", Op: "==", Value: "unhealthy", ForSec: 180, Severity: "warn", Enabled: true},
|
||||
{ID: "instance-down", Name: "VM or container down", Sensor: "instance.*.down", Op: "==", Value: true, ForSec: 120, Severity: "crit", Enabled: true},
|
||||
{ID: "service-failed", Name: "A service has failed", Sensor: "services.failed", Op: ">", Value: 0.0, ForSec: 300, Severity: "warn", Enabled: true},
|
||||
{ID: "disk-full", Name: "Filesystem nearly full", Sensor: "fs.*.used_pct", Op: ">=", Value: 90.0, ForSec: 300, Severity: "warn", Enabled: true},
|
||||
{ID: "pool-unhealthy", Name: "ZFS pool not healthy", Sensor: "zfs.*.healthy", Op: "==", Value: false, ForSec: 60, Severity: "crit", Enabled: true},
|
||||
@@ -122,6 +126,23 @@ func newAlertEngine(s *Server) *alertEngine {
|
||||
for key, k := range e.saved.Firing {
|
||||
e.state[key] = &alertState{pending: time.UnixMilli(k.Since), firing: true, since: time.UnixMilli(k.Since), value: k.Value}
|
||||
}
|
||||
// a default added after an install was seeded joins its rules once; removing it afterwards sticks
|
||||
if e.saved.Seeded && !e.saved.SawInstanceRule {
|
||||
has := false
|
||||
for _, r := range e.saved.Rules {
|
||||
if r.ID == "instance-down" {
|
||||
has = true
|
||||
}
|
||||
}
|
||||
if !has {
|
||||
for _, d := range defaultRules {
|
||||
if d.ID == "instance-down" {
|
||||
e.saved.Rules = append(e.saved.Rules, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
e.saved.SawInstanceRule = true
|
||||
}
|
||||
for i, r := range e.saved.Rules { // the first day's default watched "not running", which a container stopped on purpose also is
|
||||
if r.ID == "container-down" && r.Sensor == "container.*.running" {
|
||||
e.saved.Rules[i].Name, e.saved.Rules[i].Sensor, e.saved.Rules[i].Value = "Container down", "container.*.down", true
|
||||
|
||||
@@ -255,7 +255,22 @@ func (s *Server) routes() *http.ServeMux {
|
||||
m.HandleFunc("PUT /api/virt/instances/{name}/config", s.auth(s.virtConfig))
|
||||
m.HandleFunc("POST /api/virt/instances/{name}/snapshots", s.auth(s.virtSnapshotCreate))
|
||||
m.HandleFunc("POST /api/virt/instances/{name}/snapshots/{snap}/{action}", s.auth(s.virtSnapshotAction))
|
||||
m.HandleFunc("POST /api/virt/instances/from-iso", s.auth(s.virtFromISO))
|
||||
m.HandleFunc("GET /api/virt/instances/{name}/vga", s.auth(s.virtVGA))
|
||||
m.HandleFunc("POST /api/virt/instances/{name}/iso", s.auth(s.virtISOAttach))
|
||||
m.HandleFunc("POST /api/virt/instances/{name}/iso/detach", s.auth(s.virtISODetach))
|
||||
m.HandleFunc("POST /api/virt/instances/{name}/mounts", s.auth(s.virtMountAdd))
|
||||
m.HandleFunc("POST /api/virt/instances/{name}/mounts/remove", s.auth(s.virtMountRemove))
|
||||
m.HandleFunc("POST /api/virt/instances/{name}/{action}", s.auth(s.virtAction))
|
||||
m.HandleFunc("POST /api/virt/isos/fetch", s.auth(s.virtISOFetch))
|
||||
m.HandleFunc("POST /api/virt/isos/import", s.auth(s.virtISOImport))
|
||||
m.HandleFunc("POST /api/virt/isos/delete", s.auth(s.virtISODelete))
|
||||
m.HandleFunc("POST /api/virt/isos/rename", s.auth(s.virtISORename))
|
||||
m.HandleFunc("GET /api/virt/exports", s.auth(s.virtExports))
|
||||
m.HandleFunc("PUT /api/virt/exports", s.auth(s.virtExportsPut))
|
||||
m.HandleFunc("POST /api/virt/exports/run", s.auth(s.virtExportsRun))
|
||||
m.HandleFunc("POST /api/virt/exports/delete", s.auth(s.virtExportDelete))
|
||||
m.HandleFunc("GET /api/virt/sshkeys", s.auth(s.virtSSHKeys))
|
||||
m.HandleFunc("GET /api/alerts", s.auth(s.alertsGet))
|
||||
m.HandleFunc("PUT /api/alerts", s.auth(s.alertsPut))
|
||||
m.HandleFunc("POST /api/alerts/test", s.auth(s.alertsTest))
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"hamlab/internal/incus"
|
||||
"hamlab/internal/priv"
|
||||
"hamlab/internal/sensors"
|
||||
"hamlab/internal/storage"
|
||||
@@ -54,6 +55,7 @@ func newStatusSensors(s *Server) *statusSensors {
|
||||
p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}, hist: map[string]*sensors.Series{}, acc: map[string]*bucket{}}
|
||||
go p.loop("services", servicesEvery, p.services)
|
||||
go p.loop("containers", containersEvery, p.containers)
|
||||
go p.loop("instances", containersEvery, p.instances)
|
||||
go p.loop("updates", updatesEvery, p.updates)
|
||||
go p.loop("zfs", zfsEvery, p.zfs)
|
||||
go p.loop("smart", smartEvery, p.smart)
|
||||
@@ -320,6 +322,43 @@ type cpuMark struct {
|
||||
t int64
|
||||
}
|
||||
|
||||
// instances are Incus's containers and VMs. "Down" means it should be up and is not: an
|
||||
// instance marked to start with the machine that is stopped, or one in Error — a stopped
|
||||
// instance with no such mark was stopped on purpose, as with Docker containers.
|
||||
func (p *statusSensors) instances(ctx context.Context, t int64) []sensors.Sensor {
|
||||
if !incus.Installed() {
|
||||
return nil
|
||||
}
|
||||
list, err := p.s.virt.InstancesFor(ctx, "sensors")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var res []sensors.Sensor
|
||||
var running, total float64
|
||||
for _, in := range list {
|
||||
total++
|
||||
id := "instance." + sensors.Seg(in.Name)
|
||||
l := map[string]string{"instance": in.Name, "type": in.Type, "image": in.Image}
|
||||
up := in.Status == "Running"
|
||||
if up {
|
||||
running++
|
||||
}
|
||||
down := in.Status == "Error" || (!up && in.Config["boot.autostart"] == "true")
|
||||
res = append(res,
|
||||
sensors.Sensor{ID: id + ".running", Name: in.Name, Kind: "state", Value: up, T: t, Labels: l},
|
||||
sensors.Sensor{ID: id + ".down", Name: in.Name + " down", Kind: "state", Value: down, T: t, Labels: l},
|
||||
sensors.Sensor{ID: id + ".state", Name: in.Name + " state", Kind: "state", Value: strings.ToLower(in.Status), T: t, Labels: l})
|
||||
if up {
|
||||
res = append(res,
|
||||
sensors.Sensor{ID: id + ".cpu", Name: in.Name + " CPU", Kind: "usage", Unit: "%", Value: float64(int(in.CPUPct*10+0.5)) / 10, T: t, Min: sensors.F(0), Labels: l},
|
||||
sensors.Sensor{ID: id + ".mem", Name: in.Name + " memory", Kind: "bytes", Unit: "B", Value: float64(in.MemUsage), T: t, Labels: l})
|
||||
}
|
||||
}
|
||||
return append(res,
|
||||
sensors.Sensor{ID: "instances.total", Name: "Instances", Kind: "count", Value: total, T: t},
|
||||
sensors.Sensor{ID: "instances.running", Name: "Running instances", Kind: "count", Value: running, T: t})
|
||||
}
|
||||
|
||||
func (p *statusSensors) updates(ctx context.Context, t int64) []sensors.Sensor {
|
||||
st := p.s.updates.Status(ctx)
|
||||
if st.Manager == "" {
|
||||
|
||||
+605
-1
@@ -13,12 +13,20 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
|
||||
"hamlab/internal/incus"
|
||||
"hamlab/internal/priv"
|
||||
"hamlab/internal/sensors"
|
||||
)
|
||||
|
||||
func readBody(r *http.Request, v any, max int64) error {
|
||||
@@ -42,7 +50,14 @@ func (s *Server) virtOverview(w http.ResponseWriter, r *http.Request) {
|
||||
out["pools"] = s.virt.Pools(ctx)
|
||||
out["networks"] = s.virt.Networks(ctx)
|
||||
}
|
||||
out["library"] = map[string]any{"images": s.virt.Library(s.virt.ImagesDir()), "isos": s.virt.Library(s.virt.ISOsDir())}
|
||||
isos := s.virt.Library(s.virt.ISOsDir())
|
||||
if inst, ok := out["instances"].([]incus.Instance); ok {
|
||||
users := s.isoUsers(inst)
|
||||
for i := range isos {
|
||||
isos[i].AttachedTo = users[isos[i].Name]
|
||||
}
|
||||
}
|
||||
out["library"] = map[string]any{"images": s.virt.Library(s.virt.ImagesDir()), "isos": isos}
|
||||
jsonOK(w, 200, out)
|
||||
}
|
||||
|
||||
@@ -500,3 +515,592 @@ func (s *Server) virtSnapshotAction(w http.ResponseWriter, r *http.Request) {
|
||||
log.Printf("virt: snapshot %s %s/%s (from %s)", action, name, snap, clientIP(r))
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
// ---- the graphical console ----
|
||||
|
||||
// virtVGA bridges the browser to the VM's SPICE display: the browser's websocket on one side,
|
||||
// Incus's display websocket (dialled over its unix socket) on the other, frames copied both
|
||||
// ways. spice-html5 in the page speaks SPICE; Ham Lab only carries it.
|
||||
func (s *Server) virtVGA(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("name")
|
||||
if !incus.ValidName(name) {
|
||||
jsonErr(w, 400, "bad name")
|
||||
return
|
||||
}
|
||||
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{CompressionMode: websocket.CompressionDisabled})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
conn.SetReadLimit(64 << 20)
|
||||
ctx, cancel := context.WithCancel(r.Context())
|
||||
defer cancel()
|
||||
display, control, err := s.virt.VGA(ctx, name)
|
||||
if err != nil {
|
||||
log.Printf("virt: vga %s refused: %v (from %s)", name, err, clientIP(r))
|
||||
conn.Close(websocket.StatusInternalError, truncate(err.Error(), 120))
|
||||
return
|
||||
}
|
||||
log.Printf("virt: vga console of %s opened (from %s)", name, clientIP(r))
|
||||
defer display.Close(websocket.StatusNormalClosure, "")
|
||||
if control != nil {
|
||||
defer control.Close(websocket.StatusNormalClosure, "")
|
||||
go func() { // the control channel carries nothing we send; keep it open and drain it
|
||||
for {
|
||||
if _, _, err := control.Read(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
defer cancel()
|
||||
for {
|
||||
_, data, err := display.Read(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := conn.Write(ctx, websocket.MessageBinary, data); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
for {
|
||||
_, data, err := conn.Read(ctx)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if err := display.Write(ctx, websocket.MessageBinary, data); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
conn.Close(websocket.StatusNormalClosure, "")
|
||||
log.Printf("virt: vga console of %s closed", name)
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) > n {
|
||||
return s[:n]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// ---- ISOs ----
|
||||
|
||||
func (s *Server) isoPath(name string) (string, bool) {
|
||||
if !incus.ValidISOName(name) {
|
||||
return "", false
|
||||
}
|
||||
return filepath.Join(s.virt.ISOsDir(), name), true
|
||||
}
|
||||
|
||||
// isoUsers says which instances have each ISO file attached, by its file name.
|
||||
func (s *Server) isoUsers(list []incus.Instance) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
dir := s.virt.ISOsDir() + "/"
|
||||
for _, in := range list {
|
||||
for _, d := range in.Devices {
|
||||
if d["type"] == "disk" && strings.HasPrefix(d["source"], dir) {
|
||||
n := strings.TrimPrefix(d["source"], dir)
|
||||
out[n] = append(out[n], in.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// userShell runs a small script as the daemon's own account — file work in Ham Lab's folder
|
||||
// needs no root — with positional parameters.
|
||||
func userShell(script string, params ...string) *exec.Cmd {
|
||||
return exec.Command("sh", append([]string{"-c", script, "hamlab"}, params...)...)
|
||||
}
|
||||
|
||||
var urlRe = regexp.MustCompile(`^https?://[^\s"'<>]+$`)
|
||||
|
||||
func (s *Server) virtISOFetch(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
URL string `json:"url"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = readBody(r, &body, 8192)
|
||||
u := strings.TrimSpace(body.URL)
|
||||
if !urlRe.MatchString(u) {
|
||||
jsonErr(w, 400, "url: http or https")
|
||||
return
|
||||
}
|
||||
name := strings.TrimSpace(body.Name)
|
||||
if name == "" {
|
||||
if pu, err := url.Parse(u); err == nil {
|
||||
name = path.Base(pu.Path)
|
||||
}
|
||||
}
|
||||
dest, ok := s.isoPath(name)
|
||||
if !ok {
|
||||
jsonErr(w, 400, "name: a file name ending in .iso")
|
||||
return
|
||||
}
|
||||
sum := strings.ToLower(strings.TrimSpace(body.SHA256))
|
||||
if sum != "" && !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(sum) {
|
||||
jsonErr(w, 400, "sha256: 64 hex digits")
|
||||
return
|
||||
}
|
||||
if _, err := os.Stat(dest); err == nil {
|
||||
jsonErr(w, 409, name+" is already in the library")
|
||||
return
|
||||
}
|
||||
if err := s.virt.EnsureDirs(); err != nil {
|
||||
jsonErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
script := `set -e; echo "fetching $1"; echo " to $2"; curl -fL --retry 3 -sS -o "$2.part" "$1"; if [ -n "$3" ]; then echo "$3 $2.part" | sha256sum -c - || { rm -f "$2.part"; exit 1; }; fi; mv "$2.part" "$2"; ls -l "$2" | awk '{print " " $5 " bytes"}'`
|
||||
t, err := s.tasks.Start("virt:iso", "fetch "+name, userShell(script, u, dest, sum))
|
||||
if err != nil {
|
||||
jsonErr(w, 409, err.Error())
|
||||
return
|
||||
}
|
||||
log.Printf("virt: iso fetch %s as task %s (from %s)", u, t.ID, clientIP(r))
|
||||
jsonOK(w, 202, t.Snapshot(false))
|
||||
}
|
||||
|
||||
func (s *Server) virtISOImport(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Path string `json:"path"`
|
||||
}
|
||||
_ = readBody(r, &body, 8192)
|
||||
p := filepath.Clean(body.Path)
|
||||
st, err := os.Stat(p)
|
||||
if !filepath.IsAbs(p) || err != nil || !st.Mode().IsRegular() {
|
||||
jsonErr(w, 400, "path: a file on this machine")
|
||||
return
|
||||
}
|
||||
dest, ok := s.isoPath(filepath.Base(p))
|
||||
if !ok {
|
||||
jsonErr(w, 400, "the file has to be a .iso or .img")
|
||||
return
|
||||
}
|
||||
if _, err := os.Stat(dest); err == nil {
|
||||
jsonErr(w, 409, filepath.Base(p)+" is already in the library")
|
||||
return
|
||||
}
|
||||
if err := s.virt.EnsureDirs(); err != nil {
|
||||
jsonErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
script := `set -e; echo "copying $1"; echo " to $2"; cp --reflink=auto "$1" "$2.part" && mv "$2.part" "$2"; ls -l "$2" | awk '{print " " $5 " bytes"}'`
|
||||
cmd := userShell(script, p, dest)
|
||||
if f, err := os.Open(p); err != nil && priv.Available() { // readable only by root: copy as root and hand it over
|
||||
argv := priv.Wrap("sh", "-c", script+`; chown "$3" "$2"`, "hamlab", p, dest, s.user)
|
||||
cmd = exec.Command(argv[0], argv[1:]...)
|
||||
} else if err == nil {
|
||||
f.Close()
|
||||
}
|
||||
t, err := s.tasks.Start("virt:iso", "import "+filepath.Base(p), cmd)
|
||||
if err != nil {
|
||||
jsonErr(w, 409, err.Error())
|
||||
return
|
||||
}
|
||||
log.Printf("virt: iso import %s as task %s (from %s)", p, t.ID, clientIP(r))
|
||||
jsonOK(w, 202, t.Snapshot(false))
|
||||
}
|
||||
|
||||
func (s *Server) virtISODelete(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = readBody(r, &body, 4096)
|
||||
p, ok := s.isoPath(body.Name)
|
||||
if !ok {
|
||||
jsonErr(w, 400, "bad name")
|
||||
return
|
||||
}
|
||||
if list, err := s.virt.Instances(r.Context()); err == nil {
|
||||
if users := s.isoUsers(list)[body.Name]; len(users) > 0 {
|
||||
jsonErr(w, 409, "attached to "+strings.Join(users, ", ")+" — detach it first")
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := os.Remove(p); err != nil {
|
||||
jsonErr(w, 404, "no such ISO")
|
||||
return
|
||||
}
|
||||
log.Printf("virt: iso %s deleted (from %s)", body.Name, clientIP(r))
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
func (s *Server) virtISORename(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Name string `json:"name"`
|
||||
To string `json:"to"`
|
||||
}
|
||||
_ = readBody(r, &body, 4096)
|
||||
from, ok1 := s.isoPath(body.Name)
|
||||
to, ok2 := s.isoPath(body.To)
|
||||
if !ok1 || !ok2 {
|
||||
jsonErr(w, 400, "bad name")
|
||||
return
|
||||
}
|
||||
if list, err := s.virt.Instances(r.Context()); err == nil {
|
||||
if users := s.isoUsers(list)[body.Name]; len(users) > 0 {
|
||||
jsonErr(w, 409, "attached to "+strings.Join(users, ", ")+" — detach it first")
|
||||
return
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(to); err == nil {
|
||||
jsonErr(w, 409, body.To+" already exists")
|
||||
return
|
||||
}
|
||||
if err := os.Rename(from, to); err != nil {
|
||||
jsonErr(w, 404, "no such ISO")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
func (s *Server) instanceByName(ctx context.Context, name string) (*incus.Instance, error) {
|
||||
list, err := s.virt.Instances(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range list {
|
||||
if list[i].Name == name {
|
||||
return &list[i], nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("no instance called %s", name)
|
||||
}
|
||||
|
||||
// virtISOAttach puts an ISO in a VM's drive — live or stopped, Incus takes either — as a ROM
|
||||
// device, first in the boot order when asked.
|
||||
func (s *Server) virtISOAttach(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("name")
|
||||
var body struct {
|
||||
ISO string `json:"iso"`
|
||||
BootFirst bool `json:"boot_first"`
|
||||
}
|
||||
_ = readBody(r, &body, 4096)
|
||||
p, ok := s.isoPath(body.ISO)
|
||||
if !incus.ValidName(name) || !ok {
|
||||
jsonErr(w, 400, "bad name")
|
||||
return
|
||||
}
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
jsonErr(w, 404, "no such ISO in the library")
|
||||
return
|
||||
}
|
||||
if !s.virtReady(w, r) {
|
||||
return
|
||||
}
|
||||
in, err := s.instanceByName(r.Context(), name)
|
||||
if err != nil {
|
||||
jsonErr(w, 404, err.Error())
|
||||
return
|
||||
}
|
||||
if in.Type != "virtual-machine" {
|
||||
jsonErr(w, 409, "only a virtual machine takes an ISO")
|
||||
return
|
||||
}
|
||||
dev := ""
|
||||
for i := 0; i < 10; i++ {
|
||||
if _, taken := in.Devices[fmt.Sprintf("iso%d", i)]; !taken {
|
||||
dev = fmt.Sprintf("iso%d", i)
|
||||
break
|
||||
}
|
||||
}
|
||||
if dev == "" {
|
||||
jsonErr(w, 409, "ten ISOs are already attached")
|
||||
return
|
||||
}
|
||||
args := []string{"config", "device", "add", name, dev, "disk", "source=" + p}
|
||||
if body.BootFirst {
|
||||
args = append(args, "boot.priority=10")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
|
||||
defer cancel()
|
||||
if out, err := s.virt.Cmd(ctx, args...).CombinedOutput(); err != nil {
|
||||
jsonErr(w, 409, strings.TrimPrefix(strings.TrimSpace(string(out)), "Error: "))
|
||||
return
|
||||
}
|
||||
log.Printf("virt: %s attached to %s as %s (from %s)", body.ISO, name, dev, clientIP(r))
|
||||
jsonOK(w, 200, map[string]string{"device": dev})
|
||||
}
|
||||
|
||||
func (s *Server) virtISODetach(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("name")
|
||||
var body struct {
|
||||
Device string `json:"device"`
|
||||
}
|
||||
_ = readBody(r, &body, 4096)
|
||||
if !incus.ValidName(name) || !incus.ValidDevice(body.Device) {
|
||||
jsonErr(w, 400, "bad name")
|
||||
return
|
||||
}
|
||||
if !s.virtReady(w, r) {
|
||||
return
|
||||
}
|
||||
in, err := s.instanceByName(r.Context(), name)
|
||||
if err != nil {
|
||||
jsonErr(w, 404, err.Error())
|
||||
return
|
||||
}
|
||||
d, ok := in.Devices[body.Device]
|
||||
if !ok || d["type"] != "disk" || !strings.HasPrefix(d["source"], s.virt.ISOsDir()+"/") {
|
||||
jsonErr(w, 409, body.Device+" isn't an ISO from the library")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
|
||||
defer cancel()
|
||||
if out, err := s.virt.Cmd(ctx, "config", "device", "remove", name, body.Device).CombinedOutput(); err != nil {
|
||||
jsonErr(w, 409, strings.TrimPrefix(strings.TrimSpace(string(out)), "Error: "))
|
||||
return
|
||||
}
|
||||
log.Printf("virt: %s detached from %s (from %s)", body.Device, name, clientIP(r))
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
func (s *Server) virtFromISO(w http.ResponseWriter, r *http.Request) {
|
||||
var sp incus.FromISOSpec
|
||||
if err := readBody(r, &sp, 16<<10); err != nil {
|
||||
jsonErr(w, 400, "bad request")
|
||||
return
|
||||
}
|
||||
st := s.virt.Detect(r.Context())
|
||||
if st.State != "ready" {
|
||||
jsonErr(w, 409, "Incus isn't ready on this machine")
|
||||
return
|
||||
}
|
||||
script, params, err := incus.FromISOScript(sp, st.KVM)
|
||||
if err != nil {
|
||||
jsonErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
p, _ := s.isoPath(sp.ISO)
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
jsonErr(w, 404, "no such ISO in the library")
|
||||
return
|
||||
}
|
||||
params[1] = p
|
||||
if _, err := s.instanceByName(r.Context(), sp.Name); err == nil {
|
||||
jsonErr(w, 409, "there is already an instance called "+sp.Name)
|
||||
return
|
||||
}
|
||||
t, err := s.tasks.Start("virt:create", "create VM "+sp.Name+" from "+sp.ISO, s.virt.ShellCmd(r.Context(), script, params...))
|
||||
if err != nil {
|
||||
jsonErr(w, 409, err.Error())
|
||||
return
|
||||
}
|
||||
log.Printf("virt: VM %s from %s as task %s (from %s)", sp.Name, sp.ISO, t.ID, clientIP(r))
|
||||
jsonOK(w, 202, t.Snapshot(false))
|
||||
}
|
||||
|
||||
// ---- mounts: disk devices other than root ----
|
||||
|
||||
func (s *Server) virtMountAdd(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("name")
|
||||
var body struct {
|
||||
Device string `json:"device"`
|
||||
Source string `json:"source"`
|
||||
Path string `json:"path"`
|
||||
ReadOnly bool `json:"readonly"`
|
||||
Shift bool `json:"shift"`
|
||||
}
|
||||
_ = readBody(r, &body, 8192)
|
||||
src, dst := filepath.Clean(body.Source), filepath.Clean(body.Path)
|
||||
dev := strings.TrimSpace(body.Device)
|
||||
if dev == "" {
|
||||
dev = "mnt-" + sensors.Seg(filepath.Base(src))
|
||||
}
|
||||
switch {
|
||||
case !incus.ValidName(name):
|
||||
jsonErr(w, 400, "bad name")
|
||||
return
|
||||
case !incus.ValidDevice(dev):
|
||||
jsonErr(w, 400, "device: letters, digits, dashes")
|
||||
return
|
||||
case !filepath.IsAbs(src) || !filepath.IsAbs(dst) || dst == "/":
|
||||
jsonErr(w, 400, "source and path: absolute paths; path not /")
|
||||
return
|
||||
}
|
||||
if _, err := os.Stat(src); err != nil && os.IsNotExist(err) {
|
||||
jsonErr(w, 404, src+" doesn't exist on this machine")
|
||||
return
|
||||
}
|
||||
if !s.virtReady(w, r) {
|
||||
return
|
||||
}
|
||||
in, err := s.instanceByName(r.Context(), name)
|
||||
if err != nil {
|
||||
jsonErr(w, 404, err.Error())
|
||||
return
|
||||
}
|
||||
if _, taken := in.Devices[dev]; taken {
|
||||
jsonErr(w, 409, "there is already a device called "+dev)
|
||||
return
|
||||
}
|
||||
args := []string{"config", "device", "add", name, dev, "disk", "source=" + src, "path=" + dst}
|
||||
if body.ReadOnly {
|
||||
args = append(args, "readonly=true")
|
||||
}
|
||||
if body.Shift && in.Type == "container" {
|
||||
args = append(args, "shift=true")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
|
||||
defer cancel()
|
||||
if out, err := s.virt.Cmd(ctx, args...).CombinedOutput(); err != nil {
|
||||
jsonErr(w, 409, strings.TrimPrefix(strings.TrimSpace(string(out)), "Error: "))
|
||||
return
|
||||
}
|
||||
log.Printf("virt: %s: mounted %s at %s as %s (from %s)", name, src, dst, dev, clientIP(r))
|
||||
jsonOK(w, 200, map[string]string{"device": dev})
|
||||
}
|
||||
|
||||
func (s *Server) virtMountRemove(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("name")
|
||||
var body struct {
|
||||
Device string `json:"device"`
|
||||
}
|
||||
_ = readBody(r, &body, 4096)
|
||||
if !incus.ValidName(name) || !incus.ValidDevice(body.Device) || body.Device == "root" {
|
||||
jsonErr(w, 400, "bad name")
|
||||
return
|
||||
}
|
||||
if !s.virtReady(w, r) {
|
||||
return
|
||||
}
|
||||
in, err := s.instanceByName(r.Context(), name)
|
||||
if err != nil {
|
||||
jsonErr(w, 404, err.Error())
|
||||
return
|
||||
}
|
||||
if d, ok := in.Devices[body.Device]; !ok || d["type"] != "disk" {
|
||||
jsonErr(w, 409, body.Device+" isn't a disk device of "+name)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
|
||||
defer cancel()
|
||||
if out, err := s.virt.Cmd(ctx, "config", "device", "remove", name, body.Device).CombinedOutput(); err != nil {
|
||||
jsonErr(w, 409, strings.TrimPrefix(strings.TrimSpace(string(out)), "Error: "))
|
||||
return
|
||||
}
|
||||
log.Printf("virt: %s: device %s removed (from %s)", name, body.Device, clientIP(r))
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
// ---- exports ----
|
||||
|
||||
func (s *Server) virtExports(w http.ResponseWriter, r *http.Request) {
|
||||
c := s.virt.ExportConfig()
|
||||
out := map[string]any{"config": c, "cron": c.CronExpr(), "last": s.virt.LastExportRun(), "files": s.virt.ExportFiles(), "dir": s.virt.ExportsDir(), "cron_file": incus.CronFile}
|
||||
if c.Enabled {
|
||||
out["next"] = c.NextRun(time.Now())
|
||||
}
|
||||
jsonOK(w, 200, out)
|
||||
}
|
||||
|
||||
func (s *Server) virtExportsPut(w http.ResponseWriter, r *http.Request) {
|
||||
var c incus.ExportConfig
|
||||
if err := readBody(r, &c, 16<<10); err != nil {
|
||||
jsonErr(w, 400, "bad request")
|
||||
return
|
||||
}
|
||||
if err := s.virt.SaveExportConfig(c); err != nil {
|
||||
jsonErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
exe, err := os.Executable()
|
||||
if err == nil {
|
||||
exe, _ = filepath.EvalSymlinks(exe)
|
||||
}
|
||||
if c.Enabled {
|
||||
err = s.storage.WriteRootFile(r.Context(), incus.CronFile, c.CronText(s.user, exe, s.cfg.Path()))
|
||||
} else {
|
||||
err = s.storage.RemoveRootFile(r.Context(), incus.CronFile)
|
||||
}
|
||||
if err != nil {
|
||||
jsonErr(w, 500, "schedule saved, but the cron file could not be written: "+err.Error())
|
||||
return
|
||||
}
|
||||
log.Printf("virt: exports %v at %02d:%02d keep %d (from %s)", c.Enabled, c.Hour, c.Minute, c.Keep, clientIP(r))
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
func (s *Server) virtExportsRun(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Instance string `json:"instance"`
|
||||
}
|
||||
_ = readBody(r, &body, 4096)
|
||||
if body.Instance != "" && !incus.ValidName(body.Instance) {
|
||||
jsonErr(w, 400, "bad name")
|
||||
return
|
||||
}
|
||||
if !s.virtReady(w, r) {
|
||||
return
|
||||
}
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
jsonErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
args := []string{"-config", s.cfg.Path(), "virt-export"}
|
||||
title := "export every instance"
|
||||
if body.Instance != "" {
|
||||
args = append(args, body.Instance)
|
||||
title = "export " + body.Instance
|
||||
}
|
||||
t, err := s.tasks.Start("virt:export", title, exec.Command(exe, args...))
|
||||
if err != nil {
|
||||
jsonErr(w, 409, err.Error())
|
||||
return
|
||||
}
|
||||
log.Printf("virt: %s as task %s (from %s)", title, t.ID, clientIP(r))
|
||||
jsonOK(w, 202, t.Snapshot(false))
|
||||
}
|
||||
|
||||
func (s *Server) virtExportDelete(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Instance string `json:"instance"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = readBody(r, &body, 4096)
|
||||
if !incus.ValidName(body.Instance) || body.Name != filepath.Base(body.Name) || !strings.HasSuffix(body.Name, ".tar.gz") {
|
||||
jsonErr(w, 400, "bad name")
|
||||
return
|
||||
}
|
||||
if err := os.Remove(filepath.Join(s.virt.ExportsDir(), body.Instance, body.Name)); err != nil {
|
||||
jsonErr(w, 404, "no such export")
|
||||
return
|
||||
}
|
||||
log.Printf("virt: export %s/%s deleted (from %s)", body.Instance, body.Name, clientIP(r))
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
// virtSSHKeys is the daemon account's public keys, for a cloud-init that lets him straight in.
|
||||
func (s *Server) virtSSHKeys(w http.ResponseWriter, _ *http.Request) {
|
||||
home, _ := os.UserHomeDir()
|
||||
seen := map[string]bool{}
|
||||
keys := []string{}
|
||||
add := func(line string) {
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 2 || !strings.HasPrefix(f[0], "ssh-") && !strings.HasPrefix(f[0], "ecdsa-") && !strings.HasPrefix(f[0], "sk-") {
|
||||
return
|
||||
}
|
||||
k := f[0] + " " + f[1]
|
||||
if seen[k] {
|
||||
return
|
||||
}
|
||||
seen[k] = true
|
||||
keys = append(keys, strings.TrimSpace(line))
|
||||
}
|
||||
if b, err := os.ReadFile(filepath.Join(home, ".ssh", "authorized_keys")); err == nil {
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
add(l)
|
||||
}
|
||||
}
|
||||
pubs, _ := filepath.Glob(filepath.Join(home, ".ssh", "*.pub"))
|
||||
for _, p := range pubs {
|
||||
if b, err := os.ReadFile(p); err == nil {
|
||||
add(string(b))
|
||||
}
|
||||
}
|
||||
jsonOK(w, 200, map[string]any{"keys": keys, "user": s.user})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
package incus
|
||||
|
||||
// Nightly exports: every instance as a tarball in virt/exports/<name>/, kept to a count, as a
|
||||
// cron line like the ZFS schedules — `hamlab virt-export` in /etc/cron.d/hamlab-virt — so the
|
||||
// backup runs whether or not the console is up. The settings and the last run's record are
|
||||
// files beside the data, not state.json, because the cron job is a separate process.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const CronFile = "/etc/cron.d/hamlab-virt"
|
||||
|
||||
type ExportConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Hour int `json:"hour"`
|
||||
Minute int `json:"minute"`
|
||||
Keep int `json:"keep"` // newest exports kept per instance
|
||||
InstanceOnly bool `json:"instance_only"` // leave the snapshots out of the tarball
|
||||
Instances []string `json:"instances,omitempty"` // empty: every instance
|
||||
Updated time.Time `json:"updated,omitempty"`
|
||||
}
|
||||
|
||||
func DefaultExportConfig() ExportConfig {
|
||||
return ExportConfig{Hour: 2, Minute: 15, Keep: 3, InstanceOnly: true}
|
||||
}
|
||||
|
||||
func (c ExportConfig) Validate() error {
|
||||
if c.Hour < 0 || c.Hour > 23 || c.Minute < 0 || c.Minute > 59 {
|
||||
return errors.New("time: hour 0–23, minute 0–59")
|
||||
}
|
||||
if c.Keep < 1 || c.Keep > 100 {
|
||||
return errors.New("keep: between 1 and 100")
|
||||
}
|
||||
for _, n := range c.Instances {
|
||||
if !ValidName(n) {
|
||||
return errors.New("instance: " + n)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c ExportConfig) CronExpr() string { return fmt.Sprintf("%d %d * * *", c.Minute, c.Hour) }
|
||||
|
||||
// NextRun is the next time the line fires, for the page.
|
||||
func (c ExportConfig) NextRun(now time.Time) time.Time {
|
||||
n := time.Date(now.Year(), now.Month(), now.Day(), c.Hour, c.Minute, 0, 0, now.Location())
|
||||
if !n.After(now) {
|
||||
n = n.Add(24 * time.Hour)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// CronText is the whole file; rewritten on every change, never edited.
|
||||
func (c ExportConfig) CronText(user, exe, config string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Ham Lab instance exports — written by the Virtualization tab, rewritten whole on every change.\n")
|
||||
b.WriteString("# Edit it there, not here.\n")
|
||||
b.WriteString("SHELL=/bin/sh\nPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n")
|
||||
fmt.Fprintf(&b, "# every instance to virt/exports, keep %d each\n", c.Keep)
|
||||
fmt.Fprintf(&b, "%s %s %s -config %s virt-export\n", c.CronExpr(), user, exe, config)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (m *Manager) exportConfigPath() string { return filepath.Join(m.dir, "exports.json") }
|
||||
func (m *Manager) exportRunPath() string { return filepath.Join(m.dir, "export-runs.json") }
|
||||
|
||||
func (m *Manager) ExportConfig() ExportConfig {
|
||||
c := DefaultExportConfig()
|
||||
if b, err := os.ReadFile(m.exportConfigPath()); err == nil {
|
||||
_ = json.Unmarshal(b, &c)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (m *Manager) SaveExportConfig(c ExportConfig) error {
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := m.EnsureDirs(); err != nil {
|
||||
return err
|
||||
}
|
||||
c.Updated = time.Now()
|
||||
b, _ := json.MarshalIndent(c, "", " ")
|
||||
return os.WriteFile(m.exportConfigPath(), b, 0o644)
|
||||
}
|
||||
|
||||
type ExportRun struct {
|
||||
At time.Time `json:"at"`
|
||||
Exported []string `json:"exported"`
|
||||
Pruned []string `json:"pruned"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
Seconds float64 `json:"seconds"`
|
||||
}
|
||||
|
||||
func (m *Manager) LastExportRun() *ExportRun {
|
||||
b, err := os.ReadFile(m.exportRunPath())
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var r ExportRun
|
||||
if json.Unmarshal(b, &r) != nil {
|
||||
return nil
|
||||
}
|
||||
return &r
|
||||
}
|
||||
|
||||
// ExportFile is one tarball in the exports tree.
|
||||
type ExportFile struct {
|
||||
Instance string `json:"instance"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
Modified time.Time `json:"modified"`
|
||||
}
|
||||
|
||||
func (m *Manager) ExportFiles() []ExportFile {
|
||||
out := []ExportFile{}
|
||||
dirs, _ := os.ReadDir(m.ExportsDir())
|
||||
for _, d := range dirs {
|
||||
if !d.IsDir() {
|
||||
continue
|
||||
}
|
||||
files, _ := os.ReadDir(filepath.Join(m.ExportsDir(), d.Name()))
|
||||
for _, f := range files {
|
||||
if f.IsDir() || strings.HasSuffix(f.Name(), ".part") {
|
||||
continue
|
||||
}
|
||||
if info, err := f.Info(); err == nil {
|
||||
out = append(out, ExportFile{Instance: d.Name(), Name: f.Name(), Size: info.Size(), Modified: info.ModTime()})
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Instance != out[j].Instance {
|
||||
return out[i].Instance < out[j].Instance
|
||||
}
|
||||
return out[i].Modified.After(out[j].Modified)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// RunExports exports the named instances (or every one when names is empty) and prunes each
|
||||
// instance's folder to the configured count. Output goes to w line by line for a task to show.
|
||||
func (m *Manager) RunExports(ctx context.Context, names []string, w func(string)) ExportRun {
|
||||
start := time.Now()
|
||||
c := m.ExportConfig()
|
||||
run := ExportRun{At: start, Exported: []string{}, Pruned: []string{}}
|
||||
if err := m.EnsureDirs(); err != nil {
|
||||
run.Errors = append(run.Errors, err.Error())
|
||||
return run
|
||||
}
|
||||
all, err := m.Instances(ctx)
|
||||
if err != nil {
|
||||
run.Errors = append(run.Errors, "listing instances: "+err.Error())
|
||||
return run
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, n := range names {
|
||||
want[n] = true
|
||||
}
|
||||
if len(names) == 0 && len(c.Instances) > 0 {
|
||||
for _, n := range c.Instances {
|
||||
want[n] = true
|
||||
}
|
||||
}
|
||||
for _, in := range all {
|
||||
if len(want) > 0 && !want[in.Name] {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Join(m.ExportsDir(), in.Name)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
run.Errors = append(run.Errors, in.Name+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
stamp := start.Format("2006-01-02_15.04")
|
||||
final := filepath.Join(dir, fmt.Sprintf("%s-%s.tar.gz", in.Name, stamp))
|
||||
part := final + ".part"
|
||||
args := []string{"export", in.Name, part}
|
||||
if c.InstanceOnly {
|
||||
args = append(args, "--instance-only")
|
||||
}
|
||||
w(fmt.Sprintf("exporting %s …", in.Name))
|
||||
cctx, cancel := context.WithTimeout(ctx, 2*time.Hour)
|
||||
out, err := exec.CommandContext(cctx, "incus", args...).CombinedOutput()
|
||||
cancel()
|
||||
if err != nil {
|
||||
_ = os.Remove(part)
|
||||
msg := strings.TrimSpace(string(out))
|
||||
if msg == "" {
|
||||
msg = err.Error()
|
||||
}
|
||||
run.Errors = append(run.Errors, in.Name+": "+msg)
|
||||
w(" failed: " + msg)
|
||||
continue
|
||||
}
|
||||
if err := os.Rename(part, final); err != nil {
|
||||
run.Errors = append(run.Errors, in.Name+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
if st, err := os.Stat(final); err == nil {
|
||||
w(fmt.Sprintf(" %s (%.1f MiB)", filepath.Base(final), float64(st.Size())/(1<<20)))
|
||||
}
|
||||
run.Exported = append(run.Exported, filepath.Base(final))
|
||||
// prune: newest first, keep c.Keep
|
||||
var files []ExportFile
|
||||
for _, f := range m.ExportFiles() {
|
||||
if f.Instance == in.Name {
|
||||
files = append(files, f)
|
||||
}
|
||||
}
|
||||
for i := c.Keep; i < len(files); i++ {
|
||||
if err := os.Remove(filepath.Join(dir, files[i].Name)); err == nil {
|
||||
run.Pruned = append(run.Pruned, files[i].Name)
|
||||
w(" pruned " + files[i].Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
run.Seconds = time.Since(start).Seconds()
|
||||
b, _ := json.MarshalIndent(run, "", " ")
|
||||
_ = os.WriteFile(m.exportRunPath(), b, 0o644)
|
||||
return run
|
||||
}
|
||||
+90
-16
@@ -57,7 +57,7 @@ type Manager struct {
|
||||
version string
|
||||
|
||||
mu sync.Mutex
|
||||
cpuPrev map[string]cpuMark
|
||||
cpuPrev map[string]map[string]cpuMark // caller → instance → last reading, so two readers do not spoil each other's deltas
|
||||
remote struct {
|
||||
at time.Time
|
||||
list []Image
|
||||
@@ -75,7 +75,7 @@ type cpuMark struct {
|
||||
}
|
||||
|
||||
func New(dataDir, user, version string) *Manager {
|
||||
return &Manager{dir: filepath.Join(dataDir, "virt"), user: user, version: version, cpuPrev: map[string]cpuMark{}}
|
||||
return &Manager{dir: filepath.Join(dataDir, "virt"), user: user, version: version, cpuPrev: map[string]map[string]cpuMark{}}
|
||||
}
|
||||
|
||||
func (m *Manager) Dir() string { return m.dir }
|
||||
@@ -512,6 +512,12 @@ func secretKey(k string) bool {
|
||||
}
|
||||
|
||||
func (m *Manager) Instances(ctx context.Context) ([]Instance, error) {
|
||||
return m.InstancesFor(ctx, "api")
|
||||
}
|
||||
|
||||
// InstancesFor is Instances with its own CPU history per caller: the page and the sensors
|
||||
// both read, at different rates, and a shared "previous reading" would give each the other's delta.
|
||||
func (m *Manager) InstancesFor(ctx context.Context, caller string) ([]Instance, error) {
|
||||
c, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
out, err := m.run(c, "list", "--format", "json")
|
||||
@@ -525,6 +531,11 @@ func (m *Manager) Instances(ctx context.Context) ([]Instance, error) {
|
||||
now := time.Now()
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
prevs := m.cpuPrev[caller]
|
||||
if prevs == nil {
|
||||
prevs = map[string]cpuMark{}
|
||||
m.cpuPrev[caller] = prevs
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var list []Instance
|
||||
for _, r := range raw {
|
||||
@@ -577,18 +588,18 @@ func (m *Manager) Instances(ctx context.Context) ([]Instance, error) {
|
||||
}
|
||||
}
|
||||
if r.Status == "Running" {
|
||||
if prev, ok := m.cpuPrev[r.Name]; ok && now.After(prev.at) && r.State.CPU.Usage >= prev.ns {
|
||||
if prev, ok := prevs[r.Name]; ok && now.After(prev.at) && r.State.CPU.Usage >= prev.ns {
|
||||
in.CPUPct = float64(r.State.CPU.Usage-prev.ns) / float64(now.Sub(prev.at).Nanoseconds()) * 100
|
||||
}
|
||||
m.cpuPrev[r.Name] = cpuMark{ns: r.State.CPU.Usage, at: now}
|
||||
prevs[r.Name] = cpuMark{ns: r.State.CPU.Usage, at: now}
|
||||
seen[r.Name] = true
|
||||
}
|
||||
}
|
||||
list = append(list, in)
|
||||
}
|
||||
for n := range m.cpuPrev {
|
||||
for n := range prevs {
|
||||
if !seen[n] {
|
||||
delete(m.cpuPrev, n)
|
||||
delete(prevs, n)
|
||||
}
|
||||
}
|
||||
if list == nil {
|
||||
@@ -797,11 +808,16 @@ func (m *Manager) Remote(ctx context.Context, refresh bool) ([]Image, error) {
|
||||
|
||||
// LibraryFile is one file in virt/images or virt/isos.
|
||||
type LibraryFile struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
Modified time.Time `json:"modified"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
Modified time.Time `json:"modified"`
|
||||
AttachedTo []string `json:"attached_to,omitempty"`
|
||||
}
|
||||
|
||||
// Installed reports whether the incus binary is on this machine at all; the cheap gate the
|
||||
// sensors use before asking it anything.
|
||||
func Installed() bool { return have("incus") }
|
||||
|
||||
func (m *Manager) Library(dir string) []LibraryFile {
|
||||
ents, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
@@ -822,16 +838,20 @@ func (m *Manager) Library(dir string) []LibraryFile {
|
||||
// ---- validation ----
|
||||
|
||||
var (
|
||||
nameRe = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9-]{0,62}$`)
|
||||
imageRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,127}$`)
|
||||
sizeRe = regexp.MustCompile(`^[0-9]+(\.[0-9]+)?(B|[KMGT]i?B)?$`)
|
||||
cpuRe = regexp.MustCompile(`^[0-9]{1,3}(-[0-9]{1,3})?(,[0-9]{1,3}(-[0-9]{1,3})?)*$`)
|
||||
snapRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`)
|
||||
cronRe = regexp.MustCompile(`^(@(hourly|daily|weekly|monthly|yearly|annually|startup|never)|[0-9*/,-]+( [0-9*/,-]+){4})$`)
|
||||
expRe = regexp.MustCompile(`^[0-9]+[mHdwMy]( [0-9]+[mHdwMy])*$|^$`)
|
||||
nameRe = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9-]{0,62}$`)
|
||||
imageRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,127}$`)
|
||||
sizeRe = regexp.MustCompile(`^[0-9]+(\.[0-9]+)?(B|[KMGT]i?B)?$`)
|
||||
cpuRe = regexp.MustCompile(`^[0-9]{1,3}(-[0-9]{1,3})?(,[0-9]{1,3}(-[0-9]{1,3})?)*$`)
|
||||
snapRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`)
|
||||
cronRe = regexp.MustCompile(`^(@(hourly|daily|weekly|monthly|yearly|annually|startup|never)|[0-9*/,-]+( [0-9*/,-]+){4})$`)
|
||||
expRe = regexp.MustCompile(`^[0-9]+[mHdwMy]( [0-9]+[mHdwMy])*$|^$`)
|
||||
deviceRe = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_-]{0,30}$`)
|
||||
isoRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._ +-]{0,200}\.(?i:iso|img)$`)
|
||||
)
|
||||
|
||||
func ValidName(s string) bool { return nameRe.MatchString(s) }
|
||||
func ValidDevice(s string) bool { return deviceRe.MatchString(s) }
|
||||
func ValidISOName(s string) bool { return isoRe.MatchString(s) && s == filepath.Base(s) }
|
||||
func ValidImage(s string) bool { return imageRe.MatchString(s) }
|
||||
func ValidSize(s string) bool { return s == "" || sizeRe.MatchString(s) }
|
||||
func ValidCPU(s string) bool { return s == "" || cpuRe.MatchString(s) }
|
||||
@@ -1075,3 +1095,57 @@ func (m *Manager) InstallScript(st Status, o InstallOpts) (string, Layout, error
|
||||
w("if systemctl is-active --quiet %s; then systemd-run --quiet --on-active=3 --unit=hamlab-virt-restart systemctl restart %s; fi", HamlabUnit, HamlabUnit)
|
||||
return b.String(), lay, nil
|
||||
}
|
||||
|
||||
// FromISOSpec is the New VM from ISO form: an empty VM with the ISO as its first boot device.
|
||||
type FromISOSpec struct {
|
||||
Name string `json:"name"`
|
||||
ISO string `json:"iso"` // a file name in virt/isos
|
||||
CPU string `json:"cpu"`
|
||||
Memory string `json:"memory"`
|
||||
Disk string `json:"disk"`
|
||||
Network string `json:"network"`
|
||||
SecureBoot bool `json:"secureboot"`
|
||||
Start bool `json:"start"`
|
||||
}
|
||||
|
||||
// FromISOScript makes the VM in one shell script run through ShellCmd; the parameters carry
|
||||
// the values so nothing is spliced into the text.
|
||||
func FromISOScript(sp FromISOSpec, kvm bool) (script string, params []string, err error) {
|
||||
switch {
|
||||
case !kvm:
|
||||
return "", nil, errors.New("this machine has no KVM")
|
||||
case !ValidName(sp.Name):
|
||||
return "", nil, errors.New("name: letters, digits and dashes, starting with a letter, up to 63")
|
||||
case !ValidISOName(sp.ISO):
|
||||
return "", nil, errors.New("iso: pick one from the library")
|
||||
case !ValidCPU(sp.CPU) || !ValidSize(sp.Memory) || !ValidSize(sp.Disk) || !ValidNetwork(sp.Network):
|
||||
return "", nil, errors.New("cpu, memory, disk: a count and two sizes, like 2, 2GiB, 20GiB")
|
||||
}
|
||||
cpu, mem, disk := sp.CPU, sp.Memory, sp.Disk
|
||||
if cpu == "" {
|
||||
cpu = "2"
|
||||
}
|
||||
if mem == "" {
|
||||
mem = "2GiB"
|
||||
}
|
||||
if disk == "" {
|
||||
disk = "20GiB"
|
||||
}
|
||||
sb := "true"
|
||||
if !sp.SecureBoot {
|
||||
sb = "false"
|
||||
}
|
||||
start := "0"
|
||||
if sp.Start {
|
||||
start = "1"
|
||||
}
|
||||
// $1 name $2 iso path $3 cpu $4 memory $5 disk $6 network $7 secureboot $8 start
|
||||
script = `set -e
|
||||
NET=""; [ -n "$6" ] && NET="-n $6"
|
||||
incus init --empty --vm "$1" -c limits.cpu="$3" -c limits.memory="$4" -c security.secureboot="$7" -d root,size="$5" $NET
|
||||
incus config device add "$1" iso0 disk source="$2" boot.priority=10
|
||||
echo "made $1 with $2 as its first boot device"
|
||||
[ "$8" = 1 ] && incus start "$1" && echo "started; open the VGA console to run the installer"
|
||||
exit 0`
|
||||
return script, []string{sp.Name, sp.ISO, cpu, mem, disk, sp.Network, sb, start}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package incus
|
||||
|
||||
// The graphical console. Incus exposes a VM's SPICE display as a websocket: POST to the
|
||||
// instance's console endpoint with type vga and it answers an operation with two secrets,
|
||||
// one for the display stream and one for a control channel. Ham Lab dials both over Incus's
|
||||
// unix socket and hands the display stream to the browser, where spice-html5 draws it.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
// SocketPath is where Incus's API listens; the daemon's account reaches it by being in
|
||||
// incus-admin, which Install arranges.
|
||||
func SocketPath() string {
|
||||
if p := os.Getenv("INCUS_SOCKET"); p != "" {
|
||||
return p
|
||||
}
|
||||
for _, p := range []string{"/var/lib/incus/unix.socket", "/run/incus/unix.socket"} {
|
||||
if _, err := os.Stat(p); err == nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return "/var/lib/incus/unix.socket"
|
||||
}
|
||||
|
||||
func (m *Manager) apiClient() *http.Client {
|
||||
sock := SocketPath()
|
||||
return &http.Client{Transport: &http.Transport{
|
||||
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, "unix", sock)
|
||||
},
|
||||
ForceAttemptHTTP2: false,
|
||||
}}
|
||||
}
|
||||
|
||||
// VGA opens the display and control websockets for a running VM. The caller closes both.
|
||||
func (m *Manager) VGA(ctx context.Context, name string) (display, control *websocket.Conn, err error) {
|
||||
if !ValidName(name) {
|
||||
return nil, nil, errors.New("bad name")
|
||||
}
|
||||
client := m.apiClient()
|
||||
body, _ := json.Marshal(map[string]any{"type": "vga", "width": 0, "height": 0})
|
||||
c, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
req, _ := http.NewRequestWithContext(c, "POST", "http://incus/1.0/instances/"+name+"/console", bytes.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
if os.IsPermission(errors.Unwrap(err)) || isPermission(err) {
|
||||
return nil, nil, errors.New("Ham Lab's account can't reach Incus's socket; Finish setup on the Virtualization tab puts it in incus-admin")
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var op struct {
|
||||
Error string `json:"error"`
|
||||
Operation string `json:"operation"`
|
||||
Metadata struct {
|
||||
ID string `json:"id"`
|
||||
Metadata struct {
|
||||
FDs map[string]string `json:"fds"`
|
||||
} `json:"metadata"`
|
||||
} `json:"metadata"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&op); err != nil {
|
||||
return nil, nil, errors.New("unreadable answer from incus")
|
||||
}
|
||||
if op.Error != "" {
|
||||
return nil, nil, errors.New(op.Error)
|
||||
}
|
||||
data, control0 := op.Metadata.Metadata.FDs["0"], op.Metadata.Metadata.FDs["control"]
|
||||
if op.Metadata.ID == "" || data == "" {
|
||||
return nil, nil, errors.New("incus gave no console; is the VM running?")
|
||||
}
|
||||
dial := func(secret string) (*websocket.Conn, error) {
|
||||
u := fmt.Sprintf("ws://incus/1.0/operations/%s/websocket?secret=%s", op.Metadata.ID, secret)
|
||||
ws, _, err := websocket.Dial(ctx, u, &websocket.DialOptions{HTTPClient: client, CompressionMode: websocket.CompressionDisabled})
|
||||
return ws, err
|
||||
}
|
||||
if control0 != "" {
|
||||
if control, err = dial(control0); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
if display, err = dial(data); err != nil {
|
||||
if control != nil {
|
||||
control.Close(websocket.StatusInternalError, "")
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
display.SetReadLimit(64 << 20) // a SPICE frame can carry a whole screen
|
||||
return display, control, nil
|
||||
}
|
||||
|
||||
func isPermission(err error) bool {
|
||||
var pe *os.PathError
|
||||
if errors.As(err, &pe) {
|
||||
return os.IsPermission(pe)
|
||||
}
|
||||
var oe *net.OpError
|
||||
if errors.As(err, &oe) {
|
||||
return os.IsPermission(oe.Err) || isPermission(oe.Err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user