diff --git a/cmd/hamlab/main.go b/cmd/hamlab/main.go index bb538e6..12c534e 100644 --- a/cmd/hamlab/main.go +++ b/cmd/hamlab/main.go @@ -30,6 +30,7 @@ import ( "hamlab/internal/config" "hamlab/internal/cron" "hamlab/internal/files" + "hamlab/internal/incus" "hamlab/internal/metrics" "hamlab/internal/nodes" "hamlab/internal/priv" @@ -62,6 +63,7 @@ func main() { passwd [user] set a user's password (makes the account if it is new; reads stdin when piped) users list the accounts zfs-snapshot run one ZFS snapshot schedule (what its cron line calls) + virt-export [name…] export Incus instances to the library (what its cron line calls) docker-update run one image-update schedule (what its cron line calls) user-rm remove an account and sign it out everywhere check verify the config and state load, then exit @@ -143,6 +145,10 @@ func main() { if err := zfsSnapshot(cfg, flag.Arg(1)); err != nil { log.Fatal(err) } + case "virt-export": // the nightly cron line from the Virtualization tab; optional instance names + if err := virtExport(cfg, flag.Args()[1:]); err != nil { + log.Fatal(err) + } case "users": migrateUsers(cfg) if len(cfg.Users) == 0 { @@ -293,6 +299,21 @@ func zfsSnapshot(cfg *config.Config, id string) error { return nil } +// virtExport runs the instance exports the Virtualization tab scheduled, as the account Ham +// Lab runs as (which is in incus-admin), and records the run beside the data. +func virtExport(cfg *config.Config, names []string) error { + who := os.Getenv("USER") + if u, err := user.Current(); err == nil && u.Username != "" { + who = u.Username + } + run := incus.New(cfg.DataDir(), who, version).RunExports(context.Background(), names, func(l string) { fmt.Println(l) }) + fmt.Printf("exported %d, pruned %d, %.0f s\n", len(run.Exported), len(run.Pruned), run.Seconds) + if len(run.Errors) > 0 { + return errors.New(strings.Join(run.Errors, "; ")) + } + return nil +} + func migrateUsers(cfg *config.Config) { name := os.Getenv("USER") if u, err := user.Current(); err == nil && u.Username != "" { diff --git a/internal/api/alerts.go b/internal/api/alerts.go index c9d04c6..0430185 100644 --- a/internal/api/alerts.go +++ b/internal/api/alerts.go @@ -71,6 +71,9 @@ type alertSaved struct { Channels []AlertChannel `json:"channels"` Events []AlertEvent `json:"events"` Seeded bool `json:"seeded"` + // SawInstanceRule marks that the instance-down default has been offered once; set so that + // removing it stays removed. + SawInstanceRule bool `json:"saw_instance_rule,omitempty"` // Firing is what was firing when the hub last looked, so a restart (every deploy is one) // carries on where it was and doesn't announce the same alerts all over again. Firing map[string]alertKept `json:"firing,omitempty"` @@ -102,6 +105,7 @@ var defaultRules = []AlertRule{ {ID: "machine-offline", Name: "Machine not answering", Sensor: "hamlab.reachable", Op: "==", Value: false, ForSec: 120, Severity: "crit", Enabled: true}, {ID: "container-down", Name: "Container down", Sensor: "container.*.down", Op: "==", Value: true, ForSec: 120, Severity: "crit", Enabled: true}, {ID: "container-unhealthy", Name: "Container unhealthy", Sensor: "container.*.health", Op: "==", Value: "unhealthy", ForSec: 180, Severity: "warn", Enabled: true}, + {ID: "instance-down", Name: "VM or container down", Sensor: "instance.*.down", Op: "==", Value: true, ForSec: 120, Severity: "crit", Enabled: true}, {ID: "service-failed", Name: "A service has failed", Sensor: "services.failed", Op: ">", Value: 0.0, ForSec: 300, Severity: "warn", Enabled: true}, {ID: "disk-full", Name: "Filesystem nearly full", Sensor: "fs.*.used_pct", Op: ">=", Value: 90.0, ForSec: 300, Severity: "warn", Enabled: true}, {ID: "pool-unhealthy", Name: "ZFS pool not healthy", Sensor: "zfs.*.healthy", Op: "==", Value: false, ForSec: 60, Severity: "crit", Enabled: true}, @@ -122,6 +126,23 @@ func newAlertEngine(s *Server) *alertEngine { for key, k := range e.saved.Firing { e.state[key] = &alertState{pending: time.UnixMilli(k.Since), firing: true, since: time.UnixMilli(k.Since), value: k.Value} } + // a default added after an install was seeded joins its rules once; removing it afterwards sticks + if e.saved.Seeded && !e.saved.SawInstanceRule { + has := false + for _, r := range e.saved.Rules { + if r.ID == "instance-down" { + has = true + } + } + if !has { + for _, d := range defaultRules { + if d.ID == "instance-down" { + e.saved.Rules = append(e.saved.Rules, d) + } + } + } + e.saved.SawInstanceRule = true + } for i, r := range e.saved.Rules { // the first day's default watched "not running", which a container stopped on purpose also is if r.ID == "container-down" && r.Sensor == "container.*.running" { e.saved.Rules[i].Name, e.saved.Rules[i].Sensor, e.saved.Rules[i].Value = "Container down", "container.*.down", true diff --git a/internal/api/server.go b/internal/api/server.go index 0622cde..99f7eb5 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -255,7 +255,22 @@ func (s *Server) routes() *http.ServeMux { m.HandleFunc("PUT /api/virt/instances/{name}/config", s.auth(s.virtConfig)) m.HandleFunc("POST /api/virt/instances/{name}/snapshots", s.auth(s.virtSnapshotCreate)) m.HandleFunc("POST /api/virt/instances/{name}/snapshots/{snap}/{action}", s.auth(s.virtSnapshotAction)) + m.HandleFunc("POST /api/virt/instances/from-iso", s.auth(s.virtFromISO)) + m.HandleFunc("GET /api/virt/instances/{name}/vga", s.auth(s.virtVGA)) + m.HandleFunc("POST /api/virt/instances/{name}/iso", s.auth(s.virtISOAttach)) + m.HandleFunc("POST /api/virt/instances/{name}/iso/detach", s.auth(s.virtISODetach)) + m.HandleFunc("POST /api/virt/instances/{name}/mounts", s.auth(s.virtMountAdd)) + m.HandleFunc("POST /api/virt/instances/{name}/mounts/remove", s.auth(s.virtMountRemove)) m.HandleFunc("POST /api/virt/instances/{name}/{action}", s.auth(s.virtAction)) + m.HandleFunc("POST /api/virt/isos/fetch", s.auth(s.virtISOFetch)) + m.HandleFunc("POST /api/virt/isos/import", s.auth(s.virtISOImport)) + m.HandleFunc("POST /api/virt/isos/delete", s.auth(s.virtISODelete)) + m.HandleFunc("POST /api/virt/isos/rename", s.auth(s.virtISORename)) + m.HandleFunc("GET /api/virt/exports", s.auth(s.virtExports)) + m.HandleFunc("PUT /api/virt/exports", s.auth(s.virtExportsPut)) + m.HandleFunc("POST /api/virt/exports/run", s.auth(s.virtExportsRun)) + m.HandleFunc("POST /api/virt/exports/delete", s.auth(s.virtExportDelete)) + m.HandleFunc("GET /api/virt/sshkeys", s.auth(s.virtSSHKeys)) m.HandleFunc("GET /api/alerts", s.auth(s.alertsGet)) m.HandleFunc("PUT /api/alerts", s.auth(s.alertsPut)) m.HandleFunc("POST /api/alerts/test", s.auth(s.alertsTest)) diff --git a/internal/api/status_sensors.go b/internal/api/status_sensors.go index d236539..b499701 100644 --- a/internal/api/status_sensors.go +++ b/internal/api/status_sensors.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "hamlab/internal/incus" "hamlab/internal/priv" "hamlab/internal/sensors" "hamlab/internal/storage" @@ -54,6 +55,7 @@ func newStatusSensors(s *Server) *statusSensors { p := &statusSensors{s: s, by: map[string][]sensors.Sensor{}, hist: map[string]*sensors.Series{}, acc: map[string]*bucket{}} go p.loop("services", servicesEvery, p.services) go p.loop("containers", containersEvery, p.containers) + go p.loop("instances", containersEvery, p.instances) go p.loop("updates", updatesEvery, p.updates) go p.loop("zfs", zfsEvery, p.zfs) go p.loop("smart", smartEvery, p.smart) @@ -320,6 +322,43 @@ type cpuMark struct { t int64 } +// instances are Incus's containers and VMs. "Down" means it should be up and is not: an +// instance marked to start with the machine that is stopped, or one in Error — a stopped +// instance with no such mark was stopped on purpose, as with Docker containers. +func (p *statusSensors) instances(ctx context.Context, t int64) []sensors.Sensor { + if !incus.Installed() { + return nil + } + list, err := p.s.virt.InstancesFor(ctx, "sensors") + if err != nil { + return nil + } + var res []sensors.Sensor + var running, total float64 + for _, in := range list { + total++ + id := "instance." + sensors.Seg(in.Name) + l := map[string]string{"instance": in.Name, "type": in.Type, "image": in.Image} + up := in.Status == "Running" + if up { + running++ + } + down := in.Status == "Error" || (!up && in.Config["boot.autostart"] == "true") + res = append(res, + sensors.Sensor{ID: id + ".running", Name: in.Name, Kind: "state", Value: up, T: t, Labels: l}, + sensors.Sensor{ID: id + ".down", Name: in.Name + " down", Kind: "state", Value: down, T: t, Labels: l}, + sensors.Sensor{ID: id + ".state", Name: in.Name + " state", Kind: "state", Value: strings.ToLower(in.Status), T: t, Labels: l}) + if up { + res = append(res, + sensors.Sensor{ID: id + ".cpu", Name: in.Name + " CPU", Kind: "usage", Unit: "%", Value: float64(int(in.CPUPct*10+0.5)) / 10, T: t, Min: sensors.F(0), Labels: l}, + sensors.Sensor{ID: id + ".mem", Name: in.Name + " memory", Kind: "bytes", Unit: "B", Value: float64(in.MemUsage), T: t, Labels: l}) + } + } + return append(res, + sensors.Sensor{ID: "instances.total", Name: "Instances", Kind: "count", Value: total, T: t}, + sensors.Sensor{ID: "instances.running", Name: "Running instances", Kind: "count", Value: running, T: t}) +} + func (p *statusSensors) updates(ctx context.Context, t int64) []sensors.Sensor { st := p.s.updates.Status(ctx) if st.Manager == "" { diff --git a/internal/api/virt.go b/internal/api/virt.go index a997c41..cc90c31 100644 --- a/internal/api/virt.go +++ b/internal/api/virt.go @@ -13,12 +13,20 @@ import ( "io" "log" "net/http" + "net/url" "os" + "os/exec" + "path" "path/filepath" + "regexp" "strings" "time" + "github.com/coder/websocket" + "hamlab/internal/incus" + "hamlab/internal/priv" + "hamlab/internal/sensors" ) func readBody(r *http.Request, v any, max int64) error { @@ -42,7 +50,14 @@ func (s *Server) virtOverview(w http.ResponseWriter, r *http.Request) { out["pools"] = s.virt.Pools(ctx) out["networks"] = s.virt.Networks(ctx) } - out["library"] = map[string]any{"images": s.virt.Library(s.virt.ImagesDir()), "isos": s.virt.Library(s.virt.ISOsDir())} + isos := s.virt.Library(s.virt.ISOsDir()) + if inst, ok := out["instances"].([]incus.Instance); ok { + users := s.isoUsers(inst) + for i := range isos { + isos[i].AttachedTo = users[isos[i].Name] + } + } + out["library"] = map[string]any{"images": s.virt.Library(s.virt.ImagesDir()), "isos": isos} jsonOK(w, 200, out) } @@ -500,3 +515,592 @@ func (s *Server) virtSnapshotAction(w http.ResponseWriter, r *http.Request) { log.Printf("virt: snapshot %s %s/%s (from %s)", action, name, snap, clientIP(r)) w.WriteHeader(204) } + +// ---- the graphical console ---- + +// virtVGA bridges the browser to the VM's SPICE display: the browser's websocket on one side, +// Incus's display websocket (dialled over its unix socket) on the other, frames copied both +// ways. spice-html5 in the page speaks SPICE; Ham Lab only carries it. +func (s *Server) virtVGA(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if !incus.ValidName(name) { + jsonErr(w, 400, "bad name") + return + } + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{CompressionMode: websocket.CompressionDisabled}) + if err != nil { + return + } + conn.SetReadLimit(64 << 20) + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + display, control, err := s.virt.VGA(ctx, name) + if err != nil { + log.Printf("virt: vga %s refused: %v (from %s)", name, err, clientIP(r)) + conn.Close(websocket.StatusInternalError, truncate(err.Error(), 120)) + return + } + log.Printf("virt: vga console of %s opened (from %s)", name, clientIP(r)) + defer display.Close(websocket.StatusNormalClosure, "") + if control != nil { + defer control.Close(websocket.StatusNormalClosure, "") + go func() { // the control channel carries nothing we send; keep it open and drain it + for { + if _, _, err := control.Read(ctx); err != nil { + return + } + } + }() + } + go func() { + defer cancel() + for { + _, data, err := display.Read(ctx) + if err != nil { + return + } + if err := conn.Write(ctx, websocket.MessageBinary, data); err != nil { + return + } + } + }() + for { + _, data, err := conn.Read(ctx) + if err != nil { + break + } + if err := display.Write(ctx, websocket.MessageBinary, data); err != nil { + break + } + } + cancel() + conn.Close(websocket.StatusNormalClosure, "") + log.Printf("virt: vga console of %s closed", name) +} + +func truncate(s string, n int) string { + if len(s) > n { + return s[:n] + } + return s +} + +// ---- ISOs ---- + +func (s *Server) isoPath(name string) (string, bool) { + if !incus.ValidISOName(name) { + return "", false + } + return filepath.Join(s.virt.ISOsDir(), name), true +} + +// isoUsers says which instances have each ISO file attached, by its file name. +func (s *Server) isoUsers(list []incus.Instance) map[string][]string { + out := map[string][]string{} + dir := s.virt.ISOsDir() + "/" + for _, in := range list { + for _, d := range in.Devices { + if d["type"] == "disk" && strings.HasPrefix(d["source"], dir) { + n := strings.TrimPrefix(d["source"], dir) + out[n] = append(out[n], in.Name) + } + } + } + return out +} + +// userShell runs a small script as the daemon's own account — file work in Ham Lab's folder +// needs no root — with positional parameters. +func userShell(script string, params ...string) *exec.Cmd { + return exec.Command("sh", append([]string{"-c", script, "hamlab"}, params...)...) +} + +var urlRe = regexp.MustCompile(`^https?://[^\s"'<>]+$`) + +func (s *Server) virtISOFetch(w http.ResponseWriter, r *http.Request) { + var body struct { + URL string `json:"url"` + SHA256 string `json:"sha256"` + Name string `json:"name"` + } + _ = readBody(r, &body, 8192) + u := strings.TrimSpace(body.URL) + if !urlRe.MatchString(u) { + jsonErr(w, 400, "url: http or https") + return + } + name := strings.TrimSpace(body.Name) + if name == "" { + if pu, err := url.Parse(u); err == nil { + name = path.Base(pu.Path) + } + } + dest, ok := s.isoPath(name) + if !ok { + jsonErr(w, 400, "name: a file name ending in .iso") + return + } + sum := strings.ToLower(strings.TrimSpace(body.SHA256)) + if sum != "" && !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(sum) { + jsonErr(w, 400, "sha256: 64 hex digits") + return + } + if _, err := os.Stat(dest); err == nil { + jsonErr(w, 409, name+" is already in the library") + return + } + if err := s.virt.EnsureDirs(); err != nil { + jsonErr(w, 500, err.Error()) + return + } + script := `set -e; echo "fetching $1"; echo " to $2"; curl -fL --retry 3 -sS -o "$2.part" "$1"; if [ -n "$3" ]; then echo "$3 $2.part" | sha256sum -c - || { rm -f "$2.part"; exit 1; }; fi; mv "$2.part" "$2"; ls -l "$2" | awk '{print " " $5 " bytes"}'` + t, err := s.tasks.Start("virt:iso", "fetch "+name, userShell(script, u, dest, sum)) + if err != nil { + jsonErr(w, 409, err.Error()) + return + } + log.Printf("virt: iso fetch %s as task %s (from %s)", u, t.ID, clientIP(r)) + jsonOK(w, 202, t.Snapshot(false)) +} + +func (s *Server) virtISOImport(w http.ResponseWriter, r *http.Request) { + var body struct { + Path string `json:"path"` + } + _ = readBody(r, &body, 8192) + p := filepath.Clean(body.Path) + st, err := os.Stat(p) + if !filepath.IsAbs(p) || err != nil || !st.Mode().IsRegular() { + jsonErr(w, 400, "path: a file on this machine") + return + } + dest, ok := s.isoPath(filepath.Base(p)) + if !ok { + jsonErr(w, 400, "the file has to be a .iso or .img") + return + } + if _, err := os.Stat(dest); err == nil { + jsonErr(w, 409, filepath.Base(p)+" is already in the library") + return + } + if err := s.virt.EnsureDirs(); err != nil { + jsonErr(w, 500, err.Error()) + return + } + script := `set -e; echo "copying $1"; echo " to $2"; cp --reflink=auto "$1" "$2.part" && mv "$2.part" "$2"; ls -l "$2" | awk '{print " " $5 " bytes"}'` + cmd := userShell(script, p, dest) + if f, err := os.Open(p); err != nil && priv.Available() { // readable only by root: copy as root and hand it over + argv := priv.Wrap("sh", "-c", script+`; chown "$3" "$2"`, "hamlab", p, dest, s.user) + cmd = exec.Command(argv[0], argv[1:]...) + } else if err == nil { + f.Close() + } + t, err := s.tasks.Start("virt:iso", "import "+filepath.Base(p), cmd) + if err != nil { + jsonErr(w, 409, err.Error()) + return + } + log.Printf("virt: iso import %s as task %s (from %s)", p, t.ID, clientIP(r)) + jsonOK(w, 202, t.Snapshot(false)) +} + +func (s *Server) virtISODelete(w http.ResponseWriter, r *http.Request) { + var body struct { + Name string `json:"name"` + } + _ = readBody(r, &body, 4096) + p, ok := s.isoPath(body.Name) + if !ok { + jsonErr(w, 400, "bad name") + return + } + if list, err := s.virt.Instances(r.Context()); err == nil { + if users := s.isoUsers(list)[body.Name]; len(users) > 0 { + jsonErr(w, 409, "attached to "+strings.Join(users, ", ")+" — detach it first") + return + } + } + if err := os.Remove(p); err != nil { + jsonErr(w, 404, "no such ISO") + return + } + log.Printf("virt: iso %s deleted (from %s)", body.Name, clientIP(r)) + w.WriteHeader(204) +} + +func (s *Server) virtISORename(w http.ResponseWriter, r *http.Request) { + var body struct { + Name string `json:"name"` + To string `json:"to"` + } + _ = readBody(r, &body, 4096) + from, ok1 := s.isoPath(body.Name) + to, ok2 := s.isoPath(body.To) + if !ok1 || !ok2 { + jsonErr(w, 400, "bad name") + return + } + if list, err := s.virt.Instances(r.Context()); err == nil { + if users := s.isoUsers(list)[body.Name]; len(users) > 0 { + jsonErr(w, 409, "attached to "+strings.Join(users, ", ")+" — detach it first") + return + } + } + if _, err := os.Stat(to); err == nil { + jsonErr(w, 409, body.To+" already exists") + return + } + if err := os.Rename(from, to); err != nil { + jsonErr(w, 404, "no such ISO") + return + } + w.WriteHeader(204) +} + +func (s *Server) instanceByName(ctx context.Context, name string) (*incus.Instance, error) { + list, err := s.virt.Instances(ctx) + if err != nil { + return nil, err + } + for i := range list { + if list[i].Name == name { + return &list[i], nil + } + } + return nil, fmt.Errorf("no instance called %s", name) +} + +// virtISOAttach puts an ISO in a VM's drive — live or stopped, Incus takes either — as a ROM +// device, first in the boot order when asked. +func (s *Server) virtISOAttach(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + var body struct { + ISO string `json:"iso"` + BootFirst bool `json:"boot_first"` + } + _ = readBody(r, &body, 4096) + p, ok := s.isoPath(body.ISO) + if !incus.ValidName(name) || !ok { + jsonErr(w, 400, "bad name") + return + } + if _, err := os.Stat(p); err != nil { + jsonErr(w, 404, "no such ISO in the library") + return + } + if !s.virtReady(w, r) { + return + } + in, err := s.instanceByName(r.Context(), name) + if err != nil { + jsonErr(w, 404, err.Error()) + return + } + if in.Type != "virtual-machine" { + jsonErr(w, 409, "only a virtual machine takes an ISO") + return + } + dev := "" + for i := 0; i < 10; i++ { + if _, taken := in.Devices[fmt.Sprintf("iso%d", i)]; !taken { + dev = fmt.Sprintf("iso%d", i) + break + } + } + if dev == "" { + jsonErr(w, 409, "ten ISOs are already attached") + return + } + args := []string{"config", "device", "add", name, dev, "disk", "source=" + p} + if body.BootFirst { + args = append(args, "boot.priority=10") + } + ctx, cancel := context.WithTimeout(r.Context(), time.Minute) + defer cancel() + if out, err := s.virt.Cmd(ctx, args...).CombinedOutput(); err != nil { + jsonErr(w, 409, strings.TrimPrefix(strings.TrimSpace(string(out)), "Error: ")) + return + } + log.Printf("virt: %s attached to %s as %s (from %s)", body.ISO, name, dev, clientIP(r)) + jsonOK(w, 200, map[string]string{"device": dev}) +} + +func (s *Server) virtISODetach(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + var body struct { + Device string `json:"device"` + } + _ = readBody(r, &body, 4096) + if !incus.ValidName(name) || !incus.ValidDevice(body.Device) { + jsonErr(w, 400, "bad name") + return + } + if !s.virtReady(w, r) { + return + } + in, err := s.instanceByName(r.Context(), name) + if err != nil { + jsonErr(w, 404, err.Error()) + return + } + d, ok := in.Devices[body.Device] + if !ok || d["type"] != "disk" || !strings.HasPrefix(d["source"], s.virt.ISOsDir()+"/") { + jsonErr(w, 409, body.Device+" isn't an ISO from the library") + return + } + ctx, cancel := context.WithTimeout(r.Context(), time.Minute) + defer cancel() + if out, err := s.virt.Cmd(ctx, "config", "device", "remove", name, body.Device).CombinedOutput(); err != nil { + jsonErr(w, 409, strings.TrimPrefix(strings.TrimSpace(string(out)), "Error: ")) + return + } + log.Printf("virt: %s detached from %s (from %s)", body.Device, name, clientIP(r)) + w.WriteHeader(204) +} + +func (s *Server) virtFromISO(w http.ResponseWriter, r *http.Request) { + var sp incus.FromISOSpec + if err := readBody(r, &sp, 16<<10); err != nil { + jsonErr(w, 400, "bad request") + return + } + st := s.virt.Detect(r.Context()) + if st.State != "ready" { + jsonErr(w, 409, "Incus isn't ready on this machine") + return + } + script, params, err := incus.FromISOScript(sp, st.KVM) + if err != nil { + jsonErr(w, 400, err.Error()) + return + } + p, _ := s.isoPath(sp.ISO) + if _, err := os.Stat(p); err != nil { + jsonErr(w, 404, "no such ISO in the library") + return + } + params[1] = p + if _, err := s.instanceByName(r.Context(), sp.Name); err == nil { + jsonErr(w, 409, "there is already an instance called "+sp.Name) + return + } + t, err := s.tasks.Start("virt:create", "create VM "+sp.Name+" from "+sp.ISO, s.virt.ShellCmd(r.Context(), script, params...)) + if err != nil { + jsonErr(w, 409, err.Error()) + return + } + log.Printf("virt: VM %s from %s as task %s (from %s)", sp.Name, sp.ISO, t.ID, clientIP(r)) + jsonOK(w, 202, t.Snapshot(false)) +} + +// ---- mounts: disk devices other than root ---- + +func (s *Server) virtMountAdd(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + var body struct { + Device string `json:"device"` + Source string `json:"source"` + Path string `json:"path"` + ReadOnly bool `json:"readonly"` + Shift bool `json:"shift"` + } + _ = readBody(r, &body, 8192) + src, dst := filepath.Clean(body.Source), filepath.Clean(body.Path) + dev := strings.TrimSpace(body.Device) + if dev == "" { + dev = "mnt-" + sensors.Seg(filepath.Base(src)) + } + switch { + case !incus.ValidName(name): + jsonErr(w, 400, "bad name") + return + case !incus.ValidDevice(dev): + jsonErr(w, 400, "device: letters, digits, dashes") + return + case !filepath.IsAbs(src) || !filepath.IsAbs(dst) || dst == "/": + jsonErr(w, 400, "source and path: absolute paths; path not /") + return + } + if _, err := os.Stat(src); err != nil && os.IsNotExist(err) { + jsonErr(w, 404, src+" doesn't exist on this machine") + return + } + if !s.virtReady(w, r) { + return + } + in, err := s.instanceByName(r.Context(), name) + if err != nil { + jsonErr(w, 404, err.Error()) + return + } + if _, taken := in.Devices[dev]; taken { + jsonErr(w, 409, "there is already a device called "+dev) + return + } + args := []string{"config", "device", "add", name, dev, "disk", "source=" + src, "path=" + dst} + if body.ReadOnly { + args = append(args, "readonly=true") + } + if body.Shift && in.Type == "container" { + args = append(args, "shift=true") + } + ctx, cancel := context.WithTimeout(r.Context(), time.Minute) + defer cancel() + if out, err := s.virt.Cmd(ctx, args...).CombinedOutput(); err != nil { + jsonErr(w, 409, strings.TrimPrefix(strings.TrimSpace(string(out)), "Error: ")) + return + } + log.Printf("virt: %s: mounted %s at %s as %s (from %s)", name, src, dst, dev, clientIP(r)) + jsonOK(w, 200, map[string]string{"device": dev}) +} + +func (s *Server) virtMountRemove(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + var body struct { + Device string `json:"device"` + } + _ = readBody(r, &body, 4096) + if !incus.ValidName(name) || !incus.ValidDevice(body.Device) || body.Device == "root" { + jsonErr(w, 400, "bad name") + return + } + if !s.virtReady(w, r) { + return + } + in, err := s.instanceByName(r.Context(), name) + if err != nil { + jsonErr(w, 404, err.Error()) + return + } + if d, ok := in.Devices[body.Device]; !ok || d["type"] != "disk" { + jsonErr(w, 409, body.Device+" isn't a disk device of "+name) + return + } + ctx, cancel := context.WithTimeout(r.Context(), time.Minute) + defer cancel() + if out, err := s.virt.Cmd(ctx, "config", "device", "remove", name, body.Device).CombinedOutput(); err != nil { + jsonErr(w, 409, strings.TrimPrefix(strings.TrimSpace(string(out)), "Error: ")) + return + } + log.Printf("virt: %s: device %s removed (from %s)", name, body.Device, clientIP(r)) + w.WriteHeader(204) +} + +// ---- exports ---- + +func (s *Server) virtExports(w http.ResponseWriter, r *http.Request) { + c := s.virt.ExportConfig() + out := map[string]any{"config": c, "cron": c.CronExpr(), "last": s.virt.LastExportRun(), "files": s.virt.ExportFiles(), "dir": s.virt.ExportsDir(), "cron_file": incus.CronFile} + if c.Enabled { + out["next"] = c.NextRun(time.Now()) + } + jsonOK(w, 200, out) +} + +func (s *Server) virtExportsPut(w http.ResponseWriter, r *http.Request) { + var c incus.ExportConfig + if err := readBody(r, &c, 16<<10); err != nil { + jsonErr(w, 400, "bad request") + return + } + if err := s.virt.SaveExportConfig(c); err != nil { + jsonErr(w, 400, err.Error()) + return + } + exe, err := os.Executable() + if err == nil { + exe, _ = filepath.EvalSymlinks(exe) + } + if c.Enabled { + err = s.storage.WriteRootFile(r.Context(), incus.CronFile, c.CronText(s.user, exe, s.cfg.Path())) + } else { + err = s.storage.RemoveRootFile(r.Context(), incus.CronFile) + } + if err != nil { + jsonErr(w, 500, "schedule saved, but the cron file could not be written: "+err.Error()) + return + } + log.Printf("virt: exports %v at %02d:%02d keep %d (from %s)", c.Enabled, c.Hour, c.Minute, c.Keep, clientIP(r)) + w.WriteHeader(204) +} + +func (s *Server) virtExportsRun(w http.ResponseWriter, r *http.Request) { + var body struct { + Instance string `json:"instance"` + } + _ = readBody(r, &body, 4096) + if body.Instance != "" && !incus.ValidName(body.Instance) { + jsonErr(w, 400, "bad name") + return + } + if !s.virtReady(w, r) { + return + } + exe, err := os.Executable() + if err != nil { + jsonErr(w, 500, err.Error()) + return + } + args := []string{"-config", s.cfg.Path(), "virt-export"} + title := "export every instance" + if body.Instance != "" { + args = append(args, body.Instance) + title = "export " + body.Instance + } + t, err := s.tasks.Start("virt:export", title, exec.Command(exe, args...)) + if err != nil { + jsonErr(w, 409, err.Error()) + return + } + log.Printf("virt: %s as task %s (from %s)", title, t.ID, clientIP(r)) + jsonOK(w, 202, t.Snapshot(false)) +} + +func (s *Server) virtExportDelete(w http.ResponseWriter, r *http.Request) { + var body struct { + Instance string `json:"instance"` + Name string `json:"name"` + } + _ = readBody(r, &body, 4096) + if !incus.ValidName(body.Instance) || body.Name != filepath.Base(body.Name) || !strings.HasSuffix(body.Name, ".tar.gz") { + jsonErr(w, 400, "bad name") + return + } + if err := os.Remove(filepath.Join(s.virt.ExportsDir(), body.Instance, body.Name)); err != nil { + jsonErr(w, 404, "no such export") + return + } + log.Printf("virt: export %s/%s deleted (from %s)", body.Instance, body.Name, clientIP(r)) + w.WriteHeader(204) +} + +// virtSSHKeys is the daemon account's public keys, for a cloud-init that lets him straight in. +func (s *Server) virtSSHKeys(w http.ResponseWriter, _ *http.Request) { + home, _ := os.UserHomeDir() + seen := map[string]bool{} + keys := []string{} + add := func(line string) { + f := strings.Fields(line) + if len(f) < 2 || !strings.HasPrefix(f[0], "ssh-") && !strings.HasPrefix(f[0], "ecdsa-") && !strings.HasPrefix(f[0], "sk-") { + return + } + k := f[0] + " " + f[1] + if seen[k] { + return + } + seen[k] = true + keys = append(keys, strings.TrimSpace(line)) + } + if b, err := os.ReadFile(filepath.Join(home, ".ssh", "authorized_keys")); err == nil { + for _, l := range strings.Split(string(b), "\n") { + add(l) + } + } + pubs, _ := filepath.Glob(filepath.Join(home, ".ssh", "*.pub")) + for _, p := range pubs { + if b, err := os.ReadFile(p); err == nil { + add(string(b)) + } + } + jsonOK(w, 200, map[string]any{"keys": keys, "user": s.user}) +} diff --git a/internal/incus/export.go b/internal/incus/export.go new file mode 100644 index 0000000..c29a7db --- /dev/null +++ b/internal/incus/export.go @@ -0,0 +1,231 @@ +package incus + +// Nightly exports: every instance as a tarball in virt/exports//, kept to a count, as a +// cron line like the ZFS schedules — `hamlab virt-export` in /etc/cron.d/hamlab-virt — so the +// backup runs whether or not the console is up. The settings and the last run's record are +// files beside the data, not state.json, because the cron job is a separate process. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" +) + +const CronFile = "/etc/cron.d/hamlab-virt" + +type ExportConfig struct { + Enabled bool `json:"enabled"` + Hour int `json:"hour"` + Minute int `json:"minute"` + Keep int `json:"keep"` // newest exports kept per instance + InstanceOnly bool `json:"instance_only"` // leave the snapshots out of the tarball + Instances []string `json:"instances,omitempty"` // empty: every instance + Updated time.Time `json:"updated,omitempty"` +} + +func DefaultExportConfig() ExportConfig { + return ExportConfig{Hour: 2, Minute: 15, Keep: 3, InstanceOnly: true} +} + +func (c ExportConfig) Validate() error { + if c.Hour < 0 || c.Hour > 23 || c.Minute < 0 || c.Minute > 59 { + return errors.New("time: hour 0–23, minute 0–59") + } + if c.Keep < 1 || c.Keep > 100 { + return errors.New("keep: between 1 and 100") + } + for _, n := range c.Instances { + if !ValidName(n) { + return errors.New("instance: " + n) + } + } + return nil +} + +func (c ExportConfig) CronExpr() string { return fmt.Sprintf("%d %d * * *", c.Minute, c.Hour) } + +// NextRun is the next time the line fires, for the page. +func (c ExportConfig) NextRun(now time.Time) time.Time { + n := time.Date(now.Year(), now.Month(), now.Day(), c.Hour, c.Minute, 0, 0, now.Location()) + if !n.After(now) { + n = n.Add(24 * time.Hour) + } + return n +} + +// CronText is the whole file; rewritten on every change, never edited. +func (c ExportConfig) CronText(user, exe, config string) string { + var b strings.Builder + b.WriteString("# Ham Lab instance exports — written by the Virtualization tab, rewritten whole on every change.\n") + b.WriteString("# Edit it there, not here.\n") + b.WriteString("SHELL=/bin/sh\nPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n\n") + fmt.Fprintf(&b, "# every instance to virt/exports, keep %d each\n", c.Keep) + fmt.Fprintf(&b, "%s %s %s -config %s virt-export\n", c.CronExpr(), user, exe, config) + return b.String() +} + +func (m *Manager) exportConfigPath() string { return filepath.Join(m.dir, "exports.json") } +func (m *Manager) exportRunPath() string { return filepath.Join(m.dir, "export-runs.json") } + +func (m *Manager) ExportConfig() ExportConfig { + c := DefaultExportConfig() + if b, err := os.ReadFile(m.exportConfigPath()); err == nil { + _ = json.Unmarshal(b, &c) + } + return c +} + +func (m *Manager) SaveExportConfig(c ExportConfig) error { + if err := c.Validate(); err != nil { + return err + } + if err := m.EnsureDirs(); err != nil { + return err + } + c.Updated = time.Now() + b, _ := json.MarshalIndent(c, "", " ") + return os.WriteFile(m.exportConfigPath(), b, 0o644) +} + +type ExportRun struct { + At time.Time `json:"at"` + Exported []string `json:"exported"` + Pruned []string `json:"pruned"` + Errors []string `json:"errors,omitempty"` + Seconds float64 `json:"seconds"` +} + +func (m *Manager) LastExportRun() *ExportRun { + b, err := os.ReadFile(m.exportRunPath()) + if err != nil { + return nil + } + var r ExportRun + if json.Unmarshal(b, &r) != nil { + return nil + } + return &r +} + +// ExportFile is one tarball in the exports tree. +type ExportFile struct { + Instance string `json:"instance"` + Name string `json:"name"` + Size int64 `json:"size"` + Modified time.Time `json:"modified"` +} + +func (m *Manager) ExportFiles() []ExportFile { + out := []ExportFile{} + dirs, _ := os.ReadDir(m.ExportsDir()) + for _, d := range dirs { + if !d.IsDir() { + continue + } + files, _ := os.ReadDir(filepath.Join(m.ExportsDir(), d.Name())) + for _, f := range files { + if f.IsDir() || strings.HasSuffix(f.Name(), ".part") { + continue + } + if info, err := f.Info(); err == nil { + out = append(out, ExportFile{Instance: d.Name(), Name: f.Name(), Size: info.Size(), Modified: info.ModTime()}) + } + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Instance != out[j].Instance { + return out[i].Instance < out[j].Instance + } + return out[i].Modified.After(out[j].Modified) + }) + return out +} + +// RunExports exports the named instances (or every one when names is empty) and prunes each +// instance's folder to the configured count. Output goes to w line by line for a task to show. +func (m *Manager) RunExports(ctx context.Context, names []string, w func(string)) ExportRun { + start := time.Now() + c := m.ExportConfig() + run := ExportRun{At: start, Exported: []string{}, Pruned: []string{}} + if err := m.EnsureDirs(); err != nil { + run.Errors = append(run.Errors, err.Error()) + return run + } + all, err := m.Instances(ctx) + if err != nil { + run.Errors = append(run.Errors, "listing instances: "+err.Error()) + return run + } + want := map[string]bool{} + for _, n := range names { + want[n] = true + } + if len(names) == 0 && len(c.Instances) > 0 { + for _, n := range c.Instances { + want[n] = true + } + } + for _, in := range all { + if len(want) > 0 && !want[in.Name] { + continue + } + dir := filepath.Join(m.ExportsDir(), in.Name) + if err := os.MkdirAll(dir, 0o755); err != nil { + run.Errors = append(run.Errors, in.Name+": "+err.Error()) + continue + } + stamp := start.Format("2006-01-02_15.04") + final := filepath.Join(dir, fmt.Sprintf("%s-%s.tar.gz", in.Name, stamp)) + part := final + ".part" + args := []string{"export", in.Name, part} + if c.InstanceOnly { + args = append(args, "--instance-only") + } + w(fmt.Sprintf("exporting %s …", in.Name)) + cctx, cancel := context.WithTimeout(ctx, 2*time.Hour) + out, err := exec.CommandContext(cctx, "incus", args...).CombinedOutput() + cancel() + if err != nil { + _ = os.Remove(part) + msg := strings.TrimSpace(string(out)) + if msg == "" { + msg = err.Error() + } + run.Errors = append(run.Errors, in.Name+": "+msg) + w(" failed: " + msg) + continue + } + if err := os.Rename(part, final); err != nil { + run.Errors = append(run.Errors, in.Name+": "+err.Error()) + continue + } + if st, err := os.Stat(final); err == nil { + w(fmt.Sprintf(" %s (%.1f MiB)", filepath.Base(final), float64(st.Size())/(1<<20))) + } + run.Exported = append(run.Exported, filepath.Base(final)) + // prune: newest first, keep c.Keep + var files []ExportFile + for _, f := range m.ExportFiles() { + if f.Instance == in.Name { + files = append(files, f) + } + } + for i := c.Keep; i < len(files); i++ { + if err := os.Remove(filepath.Join(dir, files[i].Name)); err == nil { + run.Pruned = append(run.Pruned, files[i].Name) + w(" pruned " + files[i].Name) + } + } + } + run.Seconds = time.Since(start).Seconds() + b, _ := json.MarshalIndent(run, "", " ") + _ = os.WriteFile(m.exportRunPath(), b, 0o644) + return run +} diff --git a/internal/incus/incus.go b/internal/incus/incus.go index 25439ac..25d23e0 100644 --- a/internal/incus/incus.go +++ b/internal/incus/incus.go @@ -57,7 +57,7 @@ type Manager struct { version string mu sync.Mutex - cpuPrev map[string]cpuMark + cpuPrev map[string]map[string]cpuMark // caller → instance → last reading, so two readers do not spoil each other's deltas remote struct { at time.Time list []Image @@ -75,7 +75,7 @@ type cpuMark struct { } func New(dataDir, user, version string) *Manager { - return &Manager{dir: filepath.Join(dataDir, "virt"), user: user, version: version, cpuPrev: map[string]cpuMark{}} + return &Manager{dir: filepath.Join(dataDir, "virt"), user: user, version: version, cpuPrev: map[string]map[string]cpuMark{}} } func (m *Manager) Dir() string { return m.dir } @@ -512,6 +512,12 @@ func secretKey(k string) bool { } func (m *Manager) Instances(ctx context.Context) ([]Instance, error) { + return m.InstancesFor(ctx, "api") +} + +// InstancesFor is Instances with its own CPU history per caller: the page and the sensors +// both read, at different rates, and a shared "previous reading" would give each the other's delta. +func (m *Manager) InstancesFor(ctx context.Context, caller string) ([]Instance, error) { c, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() out, err := m.run(c, "list", "--format", "json") @@ -525,6 +531,11 @@ func (m *Manager) Instances(ctx context.Context) ([]Instance, error) { now := time.Now() m.mu.Lock() defer m.mu.Unlock() + prevs := m.cpuPrev[caller] + if prevs == nil { + prevs = map[string]cpuMark{} + m.cpuPrev[caller] = prevs + } seen := map[string]bool{} var list []Instance for _, r := range raw { @@ -577,18 +588,18 @@ func (m *Manager) Instances(ctx context.Context) ([]Instance, error) { } } if r.Status == "Running" { - if prev, ok := m.cpuPrev[r.Name]; ok && now.After(prev.at) && r.State.CPU.Usage >= prev.ns { + if prev, ok := prevs[r.Name]; ok && now.After(prev.at) && r.State.CPU.Usage >= prev.ns { in.CPUPct = float64(r.State.CPU.Usage-prev.ns) / float64(now.Sub(prev.at).Nanoseconds()) * 100 } - m.cpuPrev[r.Name] = cpuMark{ns: r.State.CPU.Usage, at: now} + prevs[r.Name] = cpuMark{ns: r.State.CPU.Usage, at: now} seen[r.Name] = true } } list = append(list, in) } - for n := range m.cpuPrev { + for n := range prevs { if !seen[n] { - delete(m.cpuPrev, n) + delete(prevs, n) } } if list == nil { @@ -797,11 +808,16 @@ func (m *Manager) Remote(ctx context.Context, refresh bool) ([]Image, error) { // LibraryFile is one file in virt/images or virt/isos. type LibraryFile struct { - Name string `json:"name"` - Size int64 `json:"size"` - Modified time.Time `json:"modified"` + Name string `json:"name"` + Size int64 `json:"size"` + Modified time.Time `json:"modified"` + AttachedTo []string `json:"attached_to,omitempty"` } +// Installed reports whether the incus binary is on this machine at all; the cheap gate the +// sensors use before asking it anything. +func Installed() bool { return have("incus") } + func (m *Manager) Library(dir string) []LibraryFile { ents, err := os.ReadDir(dir) if err != nil { @@ -822,16 +838,20 @@ func (m *Manager) Library(dir string) []LibraryFile { // ---- validation ---- var ( - nameRe = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9-]{0,62}$`) - imageRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,127}$`) - sizeRe = regexp.MustCompile(`^[0-9]+(\.[0-9]+)?(B|[KMGT]i?B)?$`) - cpuRe = regexp.MustCompile(`^[0-9]{1,3}(-[0-9]{1,3})?(,[0-9]{1,3}(-[0-9]{1,3})?)*$`) - snapRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`) - cronRe = regexp.MustCompile(`^(@(hourly|daily|weekly|monthly|yearly|annually|startup|never)|[0-9*/,-]+( [0-9*/,-]+){4})$`) - expRe = regexp.MustCompile(`^[0-9]+[mHdwMy]( [0-9]+[mHdwMy])*$|^$`) + nameRe = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9-]{0,62}$`) + imageRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,127}$`) + sizeRe = regexp.MustCompile(`^[0-9]+(\.[0-9]+)?(B|[KMGT]i?B)?$`) + cpuRe = regexp.MustCompile(`^[0-9]{1,3}(-[0-9]{1,3})?(,[0-9]{1,3}(-[0-9]{1,3})?)*$`) + snapRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`) + cronRe = regexp.MustCompile(`^(@(hourly|daily|weekly|monthly|yearly|annually|startup|never)|[0-9*/,-]+( [0-9*/,-]+){4})$`) + expRe = regexp.MustCompile(`^[0-9]+[mHdwMy]( [0-9]+[mHdwMy])*$|^$`) + deviceRe = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_-]{0,30}$`) + isoRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._ +-]{0,200}\.(?i:iso|img)$`) ) func ValidName(s string) bool { return nameRe.MatchString(s) } +func ValidDevice(s string) bool { return deviceRe.MatchString(s) } +func ValidISOName(s string) bool { return isoRe.MatchString(s) && s == filepath.Base(s) } func ValidImage(s string) bool { return imageRe.MatchString(s) } func ValidSize(s string) bool { return s == "" || sizeRe.MatchString(s) } func ValidCPU(s string) bool { return s == "" || cpuRe.MatchString(s) } @@ -1075,3 +1095,57 @@ func (m *Manager) InstallScript(st Status, o InstallOpts) (string, Layout, error w("if systemctl is-active --quiet %s; then systemd-run --quiet --on-active=3 --unit=hamlab-virt-restart systemctl restart %s; fi", HamlabUnit, HamlabUnit) return b.String(), lay, nil } + +// FromISOSpec is the New VM from ISO form: an empty VM with the ISO as its first boot device. +type FromISOSpec struct { + Name string `json:"name"` + ISO string `json:"iso"` // a file name in virt/isos + CPU string `json:"cpu"` + Memory string `json:"memory"` + Disk string `json:"disk"` + Network string `json:"network"` + SecureBoot bool `json:"secureboot"` + Start bool `json:"start"` +} + +// FromISOScript makes the VM in one shell script run through ShellCmd; the parameters carry +// the values so nothing is spliced into the text. +func FromISOScript(sp FromISOSpec, kvm bool) (script string, params []string, err error) { + switch { + case !kvm: + return "", nil, errors.New("this machine has no KVM") + case !ValidName(sp.Name): + return "", nil, errors.New("name: letters, digits and dashes, starting with a letter, up to 63") + case !ValidISOName(sp.ISO): + return "", nil, errors.New("iso: pick one from the library") + case !ValidCPU(sp.CPU) || !ValidSize(sp.Memory) || !ValidSize(sp.Disk) || !ValidNetwork(sp.Network): + return "", nil, errors.New("cpu, memory, disk: a count and two sizes, like 2, 2GiB, 20GiB") + } + cpu, mem, disk := sp.CPU, sp.Memory, sp.Disk + if cpu == "" { + cpu = "2" + } + if mem == "" { + mem = "2GiB" + } + if disk == "" { + disk = "20GiB" + } + sb := "true" + if !sp.SecureBoot { + sb = "false" + } + start := "0" + if sp.Start { + start = "1" + } + // $1 name $2 iso path $3 cpu $4 memory $5 disk $6 network $7 secureboot $8 start + script = `set -e +NET=""; [ -n "$6" ] && NET="-n $6" +incus init --empty --vm "$1" -c limits.cpu="$3" -c limits.memory="$4" -c security.secureboot="$7" -d root,size="$5" $NET +incus config device add "$1" iso0 disk source="$2" boot.priority=10 +echo "made $1 with $2 as its first boot device" +[ "$8" = 1 ] && incus start "$1" && echo "started; open the VGA console to run the installer" +exit 0` + return script, []string{sp.Name, sp.ISO, cpu, mem, disk, sp.Network, sb, start}, nil +} diff --git a/internal/incus/vga.go b/internal/incus/vga.go new file mode 100644 index 0000000..c8f6854 --- /dev/null +++ b/internal/incus/vga.go @@ -0,0 +1,116 @@ +package incus + +// The graphical console. Incus exposes a VM's SPICE display as a websocket: POST to the +// instance's console endpoint with type vga and it answers an operation with two secrets, +// one for the display stream and one for a control channel. Ham Lab dials both over Incus's +// unix socket and hands the display stream to the browser, where spice-html5 draws it. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "os" + "time" + + "github.com/coder/websocket" +) + +// SocketPath is where Incus's API listens; the daemon's account reaches it by being in +// incus-admin, which Install arranges. +func SocketPath() string { + if p := os.Getenv("INCUS_SOCKET"); p != "" { + return p + } + for _, p := range []string{"/var/lib/incus/unix.socket", "/run/incus/unix.socket"} { + if _, err := os.Stat(p); err == nil { + return p + } + } + return "/var/lib/incus/unix.socket" +} + +func (m *Manager) apiClient() *http.Client { + sock := SocketPath() + return &http.Client{Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", sock) + }, + ForceAttemptHTTP2: false, + }} +} + +// VGA opens the display and control websockets for a running VM. The caller closes both. +func (m *Manager) VGA(ctx context.Context, name string) (display, control *websocket.Conn, err error) { + if !ValidName(name) { + return nil, nil, errors.New("bad name") + } + client := m.apiClient() + body, _ := json.Marshal(map[string]any{"type": "vga", "width": 0, "height": 0}) + c, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + req, _ := http.NewRequestWithContext(c, "POST", "http://incus/1.0/instances/"+name+"/console", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + if os.IsPermission(errors.Unwrap(err)) || isPermission(err) { + return nil, nil, errors.New("Ham Lab's account can't reach Incus's socket; Finish setup on the Virtualization tab puts it in incus-admin") + } + return nil, nil, err + } + defer resp.Body.Close() + var op struct { + Error string `json:"error"` + Operation string `json:"operation"` + Metadata struct { + ID string `json:"id"` + Metadata struct { + FDs map[string]string `json:"fds"` + } `json:"metadata"` + } `json:"metadata"` + } + if err := json.NewDecoder(resp.Body).Decode(&op); err != nil { + return nil, nil, errors.New("unreadable answer from incus") + } + if op.Error != "" { + return nil, nil, errors.New(op.Error) + } + data, control0 := op.Metadata.Metadata.FDs["0"], op.Metadata.Metadata.FDs["control"] + if op.Metadata.ID == "" || data == "" { + return nil, nil, errors.New("incus gave no console; is the VM running?") + } + dial := func(secret string) (*websocket.Conn, error) { + u := fmt.Sprintf("ws://incus/1.0/operations/%s/websocket?secret=%s", op.Metadata.ID, secret) + ws, _, err := websocket.Dial(ctx, u, &websocket.DialOptions{HTTPClient: client, CompressionMode: websocket.CompressionDisabled}) + return ws, err + } + if control0 != "" { + if control, err = dial(control0); err != nil { + return nil, nil, err + } + } + if display, err = dial(data); err != nil { + if control != nil { + control.Close(websocket.StatusInternalError, "") + } + return nil, nil, err + } + display.SetReadLimit(64 << 20) // a SPICE frame can carry a whole screen + return display, control, nil +} + +func isPermission(err error) bool { + var pe *os.PathError + if errors.As(err, &pe) { + return os.IsPermission(pe) + } + var oe *net.OpError + if errors.As(err, &oe) { + return os.IsPermission(oe.Err) || isPermission(oe.Err) + } + return false +} diff --git a/web/src/VgaConsole.tsx b/web/src/VgaConsole.tsx new file mode 100644 index 0000000..5fea277 --- /dev/null +++ b/web/src/VgaConsole.tsx @@ -0,0 +1,84 @@ +// The graphical console of a VM: spice-html5 (vendored, loaded on demand) drawing the SPICE +// stream that Ham Lab bridges from Incus into a canvas. Used for installers from an ISO, and +// anything else with a screen rather than a serial line. +import { useEffect, useRef, useState } from 'react' +import { wsURLPath } from './api' +import { Btn, useEscape } from './ui' +import './vendor/spice-html5/spice.css' + +type Spice = typeof import('./vendor/spice-html5/src/main.js') + +export default function VgaConsole({ name, onClose }: { name: string; onClose: () => void }) { + const [state, setState] = useState<'loading' | 'connecting' | 'connected' | 'error'>('loading') + const [err, setErr] = useState('') + const [fit, setFit] = useState(true) + const [size, setSize] = useState('') + const conn = useRef | null>(null) + const mod = useRef(null) + const area = useRef(null) + const screen = useRef(null) + useEscape(onClose) + + useEffect(() => { + let dead = false + import('./vendor/spice-html5/src/main.js').then((m) => { + if (dead) return + mod.current = m + setState('connecting') + try { + conn.current = new m.SpiceMainConn({ + uri: wsURLPath(`/api/virt/instances/${encodeURIComponent(name)}/vga`), + screen_id: 'spice-screen', message_id: 'spice-message', + onerror: (e) => { setState('error'); setErr(e instanceof Error ? e.message : (e as Event & { reason?: string })?.reason || 'the console closed') }, + onsuccess: () => setState('connected'), + onagent: () => { try { m.handle_resize() } catch { /* no agent in the guest yet */ } }, + }) + } catch (e) { setState('error'); setErr(String(e)) } + }).catch((e) => { setState('error'); setErr('could not load the SPICE viewer: ' + String(e)) }) + return () => { dead = true; try { conn.current?.stop() } catch { /* already gone */ } conn.current = null } + }, [name]) + + // The guest draws at its own resolution; scale the canvas to the pane unless asked not to. + useEffect(() => { + const a = area.current, s = screen.current + if (!a || !s) return + const apply = () => { + const c = s.querySelector('canvas') + if (!c) return + setSize(`${c.width}×${c.height}`) + c.tabIndex = 1 + if (!fit) { s.style.transform = ''; s.style.width = ''; s.style.height = ''; return } + const k = Math.min(1, (a.clientWidth - 8) / c.width, (a.clientHeight - 8) / c.height) + s.style.transformOrigin = 'top left' + s.style.transform = `scale(${k})` + s.style.width = `${c.width}px`; s.style.height = `${c.height}px` + } + apply() + const ro = new ResizeObserver(apply); ro.observe(a) + const mo = new MutationObserver(apply); mo.observe(s, { childList: true, attributes: true, subtree: true, attributeFilter: ['width', 'height'] }) + return () => { ro.disconnect(); mo.disconnect() } + }, [fit, state]) + + const cad = () => { if (conn.current && mod.current) mod.current.sendCtrlAltDel(conn.current) } + const focus = () => screen.current?.querySelector('canvas')?.focus() + return ( +
+
e.stopPropagation()}> +
+ + {name} + {state === 'connected' ? `VGA · ${size || 'waiting for the first frame'}` : state === 'error' ? err : state === 'loading' ? 'loading the viewer…' : 'connecting…'} + + Ctrl+Alt+Del + + +
+
+
+
+
+
Click the screen to type into it. Keys go to the guest while it has focus; Escape with focus elsewhere closes this.
+
+
+ ) +} diff --git a/web/src/Virtualization.tsx b/web/src/Virtualization.tsx index a6eab1f..234dcdb 100644 --- a/web/src/Virtualization.tsx +++ b/web/src/Virtualization.tsx @@ -3,7 +3,8 @@ // selected machine can do, never by a rule about what should run where. Everything the // tab makes lives under Ham Lab's own folder on that machine. import { FormEvent, Fragment, useCallback, useEffect, useMemo, useState } from 'react' -import { api, LibraryFile, TaskInfo, VirtImage, VirtInstance, VirtOverview, VirtStatus } from './api' +import { api, ExportsView, LibraryFile, TaskInfo, VirtImage, VirtInstance, VirtOverview, VirtStatus } from './api' +import VgaConsole from './VgaConsole' import TaskOutput from './TaskOutput' import { useHashSel } from './hashsel' import { useRefreshMs } from './prefs' @@ -24,6 +25,7 @@ export default function Virtualization() { const [tab, setTab] = useState('instances') const [sel, setSel] = useHashSel('virt') const [making, setMaking] = useState(null) + const [fromIso, setFromIso] = useState(null) const [task, setTask] = useState<{ id: string; title: string } | null>(null) const [filter, setFilter] = useState('') const rate = useRefreshMs() @@ -77,10 +79,11 @@ export default function Virtualization() {
)} {tab === 'images' && { setTab('instances'); setMaking({ image, vm }) }} onChanged={load} />} - {tab === 'library' && } + {tab === 'library' && setFromIso(iso)} />}
{tab === 'instances' && (cur ? setSel('')} onChanged={load} onTask={startTask} /> : )} {making && setMaking(null)} onStarted={(t) => { setMaking(null); startTask(t) }} />} + {fromIso !== null && setFromIso(null)} onStarted={(t) => { setFromIso(null); startTask(t); setTab('instances') }} />} ) } @@ -209,8 +212,10 @@ function Summary({ ov }: { ov: VirtOverview }) { // ---- one instance ---- function Detail({ i, st, onBack, onChanged, onTask }: { i: VirtInstance; st: VirtStatus; onBack: () => void; onChanged: () => void; onTask: (t: TaskInfo) => void }) { - const [tab, setTab] = useState<'overview' | 'snapshots' | 'config'>('overview') + const [tab, setTab] = useState<'overview' | 'snapshots' | 'mounts' | 'config'>('overview') + const [vga, setVga] = useState(false) const running = i.status === 'Running' + const isVM = i.type === 'virtual-machine' const stopped = i.status === 'Stopped' const act = (a: 'start' | 'stop' | 'restart' | 'kill') => async () => { try { await api.virtAction(i.name, a); toast(`${i.name}: ${a} ok`); onChanged() } catch (e) { toast(`${i.name}: ${errMsg(e)}`, 'err') } } const term = (k: 'console' | 'exec') => async () => { @@ -251,10 +256,12 @@ function Detail({ i, st, onBack, onChanged, onTask }: { i: VirtInstance; st: Vir {!running && Start} {running && Shell} Console + {isVM && running && setVga(true)} title="the VM's screen, keyboard and mouse — what an installer needs">VGA} {running && Restart} {running && Stop} {running && Force stop} Snapshot + { try { onTask(await api.virtExportsRun(i.name)) } catch (e) { toast(errMsg(e), 'err') } }} title="a tarball of it in the library's exports folder">Export {stopped && Rename} {stopped && Publish} Delete… @@ -262,8 +269,11 @@ function Detail({ i, st, onBack, onChanged, onTask }: { i: VirtInstance; st: Vir
+
+ {tab === 'mounts' && } + {vga && setVga(false)} />} {tab === 'overview' && } {tab === 'snapshots' && } {tab === 'config' && ( @@ -415,6 +425,7 @@ function NewInstance({ st, ov, preset, onClose, onStarted }: { st: VirtStatus; o + {showCloud &&
{ try { const k = await api.virtSSHKeys(); if (!k.keys.length) { toast('no public keys found for ' + k.user, 'info'); return } setCloud(`#cloud-config\nusers:\n - name: ${k.user}\n groups: [sudo]\n shell: /bin/bash\n sudo: ALL=(ALL) NOPASSWD:ALL\n ssh_authorized_keys:\n${k.keys.map((x) => ' - ' + x).join('\n')}\npackage_update: true\npackages: [curl, vim]\n`) } catch (e) { toast(errMsg(e), 'err') } }} title="a cloud-config that makes your account inside, with the public keys this machine's Ham Lab account holds">Use my SSH keys
} {showCloud &&