From 04269fa222bdeea3cd5ee9ce27de817e27432a23 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Wed, 14 Oct 2026 02:19:13 +0300 Subject: [PATCH 01/33] fix(gateway): refuse to unlink a non-socket at the socket path clearStaleUnixSocket treats ECONNREFUSED as "a socket nobody is listening on" and removes the path before binding. On macOS a connect to a regular file fails with ENOTSOCK, which falls through to the refusal, so the existing TestClaimUnixSocketRefusesANonSocketPath passes there. Linux answers the same connect with ECONNREFUSED, so on a Linux host the gateway deleted whatever regular file sat at the path it was told to bind. The test fails on Linux without this change. Lstat the path on ECONNREFUSED and refuse unless it is a socket. Signed-off-by: Panagiotis Moustafellos --- cmd/hull/gateway.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/hull/gateway.go b/cmd/hull/gateway.go index 5a47d5dc..4173cdde 100644 --- a/cmd/hull/gateway.go +++ b/cmd/hull/gateway.go @@ -569,7 +569,15 @@ func clearStaleUnixSocket(what, path, remedy string) error { return fmt.Errorf("socket %s is in use by a running gateway", path) case errors.Is(err, syscall.ECONNREFUSED): // Nobody is listening: the file the last gateway left behind is - // ours to clear. + // ours to clear -- if it is a socket. macOS answers a connect to a + // regular file with ENOTSOCK, which lands in the refusal below, but + // Linux answers it with ECONNREFUSED, the same errno as a dead + // socket, and this removed whatever file the operator had at the + // path. Look at what is there before unlinking it. + if info, stErr := os.Lstat(path); stErr == nil && info.Mode()&fs.ModeSocket == 0 { + return fmt.Errorf("%s exists and is not a socket; refusing to remove it to "+ + "bind the %s socket there", path, what) + } if rmErr := os.Remove(path); rmErr != nil && !errors.Is(rmErr, fs.ErrNotExist) { return fmt.Errorf("failed to remove stale socket %s: %w", path, rmErr) } From a0df36016a72471137dbac6d8370cfd7836a317b Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Tue, 20 Oct 2026 16:44:35 +0100 Subject: [PATCH 02/33] fix(gateway): close the netstack when the gateway stops runGateway made a netgw.Network and never closed its gvisor stack, so the stack's TCP workers outlived the gateway. A gateway runs in its own process, where that cost nothing, but one cmd/hull test runs two of them inside the test binary. Under -race their 30 idle workers spin on about eight cores for the rest of the package, which made every later test ten times slower: the streamed move tests broke their wall-clock bounds and the race step hit its 10 minute package timeout. netgw.Network now has Close, which closes the stack, and runGateway defers it. Close does not wait for the endpoints, since a member still attached at shutdown could open a connection that nothing aborts, and the wait would then hold the gateway's exit. Both packages' tests check that no TCP worker is left behind. Signed-off-by: Panagiotis Moustafellos --- cmd/hull/gateway.go | 3 +++ cmd/hull/gateway_egress_test.go | 27 +++++++++++++++++++++++++ internal/netgw/network.go | 11 ++++++++++ internal/netgw/network_test.go | 36 +++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+) diff --git a/cmd/hull/gateway.go b/cmd/hull/gateway.go index 4173cdde..2888557a 100644 --- a/cmd/hull/gateway.go +++ b/cmd/hull/gateway.go @@ -206,6 +206,9 @@ func runGateway(ctx context.Context, sockPath, apiPath, qemuSockPath, subnet, ga if err != nil { return fmt.Errorf("failed to create virtual network: %w", err) } + // Deferred first, so it runs last: the stack goes once the sockets that + // feed it are closed. + defer vn.Close() // Keep the named hosts in the rules resolved for as long as the gateway // runs, so the policy follows a name whose addresses rotate. diff --git a/cmd/hull/gateway_egress_test.go b/cmd/hull/gateway_egress_test.go index 9d9796d1..71ff8b94 100644 --- a/cmd/hull/gateway_egress_test.go +++ b/cmd/hull/gateway_egress_test.go @@ -18,6 +18,7 @@ import ( "context" "net" "path/filepath" + "runtime" "strings" "testing" "time" @@ -113,6 +114,7 @@ func TestGatewayListensOnTheDerivedQemuSocketByDefault(t *testing.T) { args = append(args, append(tc.extra, want)...) } + workers := tcpWorkers() ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { @@ -122,6 +124,16 @@ func TestGatewayListensOnTheDerivedQemuSocketByDefault(t *testing.T) { t.Cleanup(func() { cancel() <-done + // The gateway's stack goes with it: its idle TCP workers + // would otherwise burn CPU under -race for the rest of the + // package. + deadline := time.Now().Add(5 * time.Second) + for tcpWorkers() > workers && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if left := tcpWorkers(); left > workers { + t.Errorf("the stopped gateway left %d TCP workers running", left-workers) + } }) deadline := time.Now().Add(5 * time.Second) @@ -144,3 +156,18 @@ func TestGatewayListensOnTheDerivedQemuSocketByDefault(t *testing.T) { }) } } + +// tcpWorkers counts the goroutines of every netstack's TCP workers in the +// process. +func tcpWorkers() int { + buf := make([]byte, 1<<20) + for { + n := runtime.Stack(buf, true) + if n < len(buf) { + // Each worker's trace names its creator, whether or not it + // has run yet. + return strings.Count(string(buf[:n]), "created by gvisor.dev/gvisor/pkg/tcpip/transport/tcp.(*dispatcher).init") + } + buf = make([]byte, 2*len(buf)) + } +} diff --git a/internal/netgw/network.go b/internal/netgw/network.go index c5ee4185..f475dbe8 100644 --- a/internal/netgw/network.go +++ b/internal/netgw/network.go @@ -147,6 +147,17 @@ func (n *Network) Leases() map[string]string { return n.ipPool.Leases() } +// Close stops the network's stack: it aborts every endpoint, the DHCP and +// DNS servers' among them, and tells the TCP workers to stop, which they do +// shortly after. It does not wait for the endpoints to close: a member still +// attached may open a connection after the abort, and waiting on it would +// hold the gateway's exit. The stack does not stop with the ctx given to +// Start, so a gateway that returns without Close leaves its TCP workers +// running for the rest of the process. +func (n *Network) Close() { + n.stack.Close() +} + // Start runs the network's background work until ctx is done. It is separate // from New so the caller owns the lifetime, and returns having started // nothing when the policy has no literal host rule to keep current. diff --git a/internal/netgw/network_test.go b/internal/netgw/network_test.go index 4fd6f888..f7c3835b 100644 --- a/internal/netgw/network_test.go +++ b/internal/netgw/network_test.go @@ -17,6 +17,8 @@ package netgw import ( "context" "net" + "runtime" + "strings" "testing" "time" @@ -179,6 +181,40 @@ func testNetwork(t *testing.T) *Network { return n } +// tcpWorkers counts the goroutines of every stack's TCP workers in the +// process. +func tcpWorkers() int { + buf := make([]byte, 1<<20) + for { + n := runtime.Stack(buf, true) + if n < len(buf) { + // Each worker's trace names its creator, whether or not it + // has run yet. + return strings.Count(string(buf[:n]), "created by gvisor.dev/gvisor/pkg/tcpip/transport/tcp.(*dispatcher).init") + } + buf = make([]byte, 2*len(buf)) + } +} + +// A closed network leaves no TCP workers behind. The stack does not stop +// with the ctx given to Start, and its idle workers burn CPU under -race, +// which slowed every later test in a package that ran a gateway in process. +func TestCloseStopsTheStacksWorkers(t *testing.T) { + before := tcpWorkers() + n := testNetwork(t) + if tcpWorkers() == before { + t.Fatal("found no TCP workers to stop") + } + n.Close() + deadline := time.Now().Add(5 * time.Second) + for tcpWorkers() > before { + if time.Now().After(deadline) { + t.Fatalf("%d TCP workers left after Close, %d before New", tcpWorkers(), before) + } + time.Sleep(10 * time.Millisecond) + } +} + func arpRequest(t *testing.T, m *member, target net.IP) []byte { t.Helper() frame := make([]byte, header.EthernetMinimumSize+header.ARPSize) From cafea4a8949fc69361509e29e043a7067ec31251 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Fri, 16 Oct 2026 00:46:23 +0300 Subject: [PATCH 03/33] style: gofmt internal/bootassets/verify.go A doubled blank line there since the first commit made `gofmt -l` list the file, so the gofmt gate could not pass on the whole tree. Signed-off-by: Panagiotis Moustafellos --- internal/bootassets/verify.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/bootassets/verify.go b/internal/bootassets/verify.go index 04d6efd0..c2375dcc 100644 --- a/internal/bootassets/verify.go +++ b/internal/bootassets/verify.go @@ -322,7 +322,6 @@ func repoOf(ref string) string { return parsed.Context().Name() } - // verifyLookPath and verifyRun are variables so the decision table can be // driven in tests on a host that has no cosign, or one that has a working one. var verifyLookPath = exec.LookPath From e79f06c58734bcf89f7f31ebf1f919a62af572f7 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Wed, 14 Oct 2026 02:20:41 +0300 Subject: [PATCH 04/33] refactor(cmd): move the macOS-only helpers into _darwin files cmd/hull is darwin-only because a handful of helpers in otherwise portable files call macOS interfaces: the termios ioctl numbers, the hdiutil store mount, the ps-based process identity check, the kern.procargs2 and kern.osproductversion sysctls, Stat_t.Ctimespec, Statfs_t.Fstypename, the Homebrew e2fsprogs paths, and urunc's darwin-only VzVmm and VzDarwin symbols. Moving each behind a small _darwin.go file is what lets the shared files build for Linux in the next change, with a Linux counterpart beside each one. Nothing changes on macOS. The moved code is the same code, with three small differences. processIsAVMM and reapOrphanVMM now read process facts through inspectProcess and listProcesses: the ps parsing they did inline lives in proc_darwin.go, and the identity rules (base name, recorded argv, start-time slack) stay in stop.go, where both platforms share them. e2fsck is named by a constant like mke2fs and debugfs instead of an inline Homebrew path, and buildBlockRootfs checks that all three exist before mke2fs writes a 15 GiB image. A missing debugfs used to surface only after the image was written, as an error about the wrong step. The command list and usage line come from commands_darwin.go. Signed-off-by: Panagiotis Moustafellos --- cmd/hull/apfs_darwin.go | 49 ++++ cmd/hull/blockrootfs.go | 15 +- cmd/hull/commands_darwin.go | 44 ++++ cmd/hull/compose.go | 40 --- cmd/hull/e2fs_darwin.go | 24 ++ cmd/hull/main.go | 252 +----------------- cmd/hull/orphan_instance_test.go | 5 +- cmd/hull/proc_darwin.go | 126 +++++++++ cmd/hull/prune.go | 6 +- cmd/hull/run.go | 55 +--- cmd/hull/run_network_test.go | 2 +- cmd/hull/stop.go | 75 ++---- cmd/hull/storemount_darwin.go | 260 +++++++++++++++++++ cmd/hull/{terminal_darwin.go => terminal.go} | 8 +- cmd/hull/terminal_ioctl_darwin.go | 25 ++ cmd/hull/terminal_review_test.go | 4 +- cmd/hull/vmm_darwin.go | 41 +++ 17 files changed, 630 insertions(+), 401 deletions(-) create mode 100644 cmd/hull/apfs_darwin.go create mode 100644 cmd/hull/commands_darwin.go create mode 100644 cmd/hull/e2fs_darwin.go create mode 100644 cmd/hull/proc_darwin.go create mode 100644 cmd/hull/storemount_darwin.go rename cmd/hull/{terminal_darwin.go => terminal.go} (93%) create mode 100644 cmd/hull/terminal_ioctl_darwin.go create mode 100644 cmd/hull/vmm_darwin.go diff --git a/cmd/hull/apfs_darwin.go b/cmd/hull/apfs_darwin.go new file mode 100644 index 00000000..971a2350 --- /dev/null +++ b/cmd/hull/apfs_darwin.go @@ -0,0 +1,49 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +// requireSameAPFSVolume refuses an hvi writable rootfs unless the image store +// and the instance directory are one APFS volume, which is what makes the +// per-instance clone in cloneRootfsAPFS a clone rather than a full copy. +func requireSameAPFSVolume(source, destinationParent string) error { + var sourceFS, destinationFS unix.Statfs_t + if err := unix.Statfs(source, &sourceFS); err != nil { + return fmt.Errorf("inspect source filesystem for APFS clone: %w", err) + } + if err := unix.Statfs(destinationParent, &destinationFS); err != nil { + return fmt.Errorf("inspect destination filesystem for APFS clone: %w", err) + } + fsName := func(raw [16]byte) string { + for i, value := range raw { + if value == 0 { + return string(raw[:i]) + } + } + return string(raw[:]) + } + if fsName(sourceFS.Fstypename) != "apfs" || fsName(destinationFS.Fstypename) != "apfs" { + return fmt.Errorf("HVI writable rootfs requires an APFS image store and instance directory") + } + if sourceFS.Fsid.Val != destinationFS.Fsid.Val { + return fmt.Errorf("HVI writable rootfs requires the image store and instance directory on the same APFS volume") + } + return nil +} diff --git a/cmd/hull/blockrootfs.go b/cmd/hull/blockrootfs.go index a784fa99..f182fb67 100644 --- a/cmd/hull/blockrootfs.go +++ b/cmd/hull/blockrootfs.go @@ -31,11 +31,6 @@ import ( "github.com/brig-sh/hull/pkg/ociclient" ) -const ( - mke2fsBin = "/opt/homebrew/opt/e2fsprogs/sbin/mke2fs" - debugfsBin = "/opt/homebrew/opt/e2fsprogs/sbin/debugfs" -) - // blockInject is a per-instance file placed into the image after it is built. // // These used to be written into a staged copy of the rootfs before imaging, @@ -64,6 +59,14 @@ type blockInject struct { // (see ociclient.GuestAttr) and is restored here, straight into the finished // filesystem, where ext4's inodes can hold what APFS could not. func buildBlockRootfs(ctx context.Context, diskPath, rootfsDir string, injects []blockInject, sizeMB int) error { + // All three are needed, and the last two run only after mke2fs has written + // a full-size image; finding out that debugfs is missing at that point + // leaves a 15 GiB sparse file and an error that names the wrong step. + for _, bin := range []string{mke2fsBin, debugfsBin, e2fsckBin} { + if _, err := os.Stat(bin); err != nil { + return fmt.Errorf("the block rootfs needs e2fsprogs (mke2fs, debugfs, e2fsck): %w", err) + } + } mkCmd := exec.CommandContext(ctx, mke2fsBin, "-t", "ext4", "-d", rootfsDir, "-L", "rootfs", "-m", "0", "-q", diskPath, fmt.Sprintf("%dM", sizeMB)) if out, err := mkCmd.CombinedOutput(); err != nil { @@ -167,7 +170,7 @@ func repairBlockImage(ctx context.Context, diskPath string) error { // -f forces a full check even though the filesystem looks clean, which it // does: nothing has mounted it. -y answers the questions, all of which are // "recount this". - cmd := exec.CommandContext(ctx, "/opt/homebrew/opt/e2fsprogs/sbin/e2fsck", "-f", "-y", diskPath) + cmd := exec.CommandContext(ctx, e2fsckBin, "-f", "-y", diskPath) out, err := cmd.CombinedOutput() if err := ctx.Err(); err != nil { return err diff --git a/cmd/hull/commands_darwin.go b/cmd/hull/commands_darwin.go new file mode 100644 index 00000000..ff8a14ce --- /dev/null +++ b/cmd/hull/commands_darwin.go @@ -0,0 +1,44 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "github.com/urfave/cli/v3" + +const appUsage = "native macOS container CLI for unikernels" + +// platformCommands is every verb hull has on macOS. +func platformCommands() []*cli.Command { + return []*cli.Command{ + pullCommand(), + runCommand(), + execCommand(), + psCommand(), + stopCommand(), + checkpointCommand(), + restoreCommand(), + rmCommand(), + logsCommand(), + inspectCommand(), + imagesCommand(), + rmiCommand(), + pruneCommand(), + assetsCommand(), + storeCommand(), + composeCommand(), + networkGatewayCommand(), + telemetryCommand(), + versionCommand(), + } +} diff --git a/cmd/hull/compose.go b/cmd/hull/compose.go index 547079c6..21bc6fd7 100644 --- a/cmd/hull/compose.go +++ b/cmd/hull/compose.go @@ -17,7 +17,6 @@ package main import ( "bytes" "context" - "encoding/binary" "encoding/json" "errors" "fmt" @@ -41,7 +40,6 @@ import ( "github.com/brig-sh/hull/internal/netgw" "github.com/urfave/cli/v3" - "golang.org/x/sys/unix" "gopkg.in/yaml.v3" "github.com/brig-sh/hull/internal/compose" @@ -1905,44 +1903,6 @@ func gatewayProcessMatches(pid int, sockPath string) bool { return subcommand && socket } -// processArgv returns the argv of pid as the kernel holds it, via the -// kern.procargs2 sysctl. The buffer is argc (a native int32), then the exec -// path, NUL padding, and argc NUL-terminated argv strings, then the -// environment, which is not read. -func processArgv(pid int) ([]string, error) { - buf, err := unix.SysctlRaw("kern.procargs2", pid) - if err != nil { - return nil, err - } - if len(buf) < 4 { - return nil, fmt.Errorf("procargs2 for pid %d: short buffer", pid) - } - argc := int(int32(binary.NativeEndian.Uint32(buf))) - if argc < 0 { - return nil, fmt.Errorf("procargs2 for pid %d: argc %d", pid, argc) - } - rest := buf[4:] - // Skip the exec path, then the NUL padding after it. - i := bytes.IndexByte(rest, 0) - if i < 0 { - return nil, fmt.Errorf("procargs2 for pid %d: no exec path", pid) - } - rest = rest[i:] - for len(rest) > 0 && rest[0] == 0 { - rest = rest[1:] - } - argv := make([]string, 0, argc) - for range argc { - j := bytes.IndexByte(rest, 0) - if j < 0 { - return nil, fmt.Errorf("procargs2 for pid %d: truncated argv", pid) - } - argv = append(argv, string(rest[:j])) - rest = rest[j+1:] - } - return argv, nil -} - // waitHealthy polls the gateway probe API until the TCP address answers. func waitHealthy(apiSock, addr string, interval, timeout time.Duration) error { client := &http.Client{ diff --git a/cmd/hull/e2fs_darwin.go b/cmd/hull/e2fs_darwin.go new file mode 100644 index 00000000..6f55e8f6 --- /dev/null +++ b/cmd/hull/e2fs_darwin.go @@ -0,0 +1,24 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// e2fsprogs from Homebrew. macOS ships none of it, and Homebrew keeps the +// formula keg-only, so the binaries are not on PATH and are named by their +// keg path. +const ( + mke2fsBin = "/opt/homebrew/opt/e2fsprogs/sbin/mke2fs" + debugfsBin = "/opt/homebrew/opt/e2fsprogs/sbin/debugfs" + e2fsckBin = "/opt/homebrew/opt/e2fsprogs/sbin/e2fsck" +) diff --git a/cmd/hull/main.go b/cmd/hull/main.go index c213b2c2..da3d0686 100644 --- a/cmd/hull/main.go +++ b/cmd/hull/main.go @@ -20,11 +20,9 @@ import ( "context" "fmt" "os" - "os/exec" "path/filepath" "runtime/debug" "strings" - "syscall" "github.com/brig-sh/hull/internal/buildinfo" "github.com/brig-sh/hull/pkg/store" @@ -60,7 +58,7 @@ func main() { app := &cli.Command{ Name: "hull", - Usage: "native macOS container CLI for unikernels", + Usage: appUsage, Version: build.String(), Flags: []cli.Flag{ &cli.BoolFlag{ @@ -88,27 +86,7 @@ func main() { initTelemetry(cmd) return nil, nil }, - Commands: []*cli.Command{ - pullCommand(), - runCommand(), - execCommand(), - psCommand(), - stopCommand(), - checkpointCommand(), - restoreCommand(), - rmCommand(), - logsCommand(), - inspectCommand(), - imagesCommand(), - rmiCommand(), - pruneCommand(), - assetsCommand(), - storeCommand(), - composeCommand(), - networkGatewayCommand(), - telemetryCommand(), - versionCommand(), - }, + Commands: platformCommands(), } if err := app.Run(context.Background(), os.Args); err != nil { @@ -117,183 +95,6 @@ func main() { sendCommandEvent("ok", "") } -const defaultStoreImageSize = "200g" - -// storeMarkerName marks a directory as a store hull mounted itself. -const storeMarkerName = ".case-sensitive-apfs" - -// storeImagePath is where the backing sparse image for a store lives. -// -// The default store keeps the name it has always had, so an existing install -// finds its images after an upgrade. Any other store dir gets an image named -// after itself rather than after its parent: two stores under one parent -- -// `_work/store-a` and `_work/store-b` -- would otherwise both resolve to -// `_work/hull-store.sparseimage`, and the second attach fails with "Resource -// busy" because the first already has it. -func storeImagePath(storeDir, defaultStoreDir string) string { - if storeDir == defaultStoreDir { - return filepath.Join(filepath.Dir(storeDir), "hull-store.sparseimage") - } - return storeDir + ".sparseimage" -} - -// ensureStore guarantees that storeDir is case-sensitive, mounting a -// case-sensitive APFS sparse image there if it is not already. -// -// This runs for every store, not just the default one. It used to run only -// when --store-dir was left alone, which meant anyone passing an explicit path -// -- CI, most of all -- silently got a case-insensitive store on the boot -// volume: exactly the thing the default store exists to avoid. Nothing -// complained, because store.New only calls MkdirAll. The failure surfaced much -// later, as a guest that unpacks a Linux rootfs with names that differ only by -// case and then does not boot. -func ensureStore(storeDir, defaultStoreDir string) error { - marker := filepath.Join(storeDir, storeMarkerName) - if _, err := os.Stat(marker); err == nil { - // Already ours and already mounted -- but possibly mounted the way - // hull used to mount it, with ownership ignored. - return ensureStoreOwnership(storeDir, storeImagePath(storeDir, defaultStoreDir)) - } else if !os.IsNotExist(err) { - return fmt.Errorf("check store mount: %w", err) - } - if err := os.MkdirAll(storeDir, 0700); err != nil { - return fmt.Errorf("create store mountpoint: %w", err) - } - - // A store that is already case-sensitive needs nothing: the user pointed - // --store-dir at a case-sensitive volume of their own, and nesting a 200G - // image inside it would be waste and surprise. - sensitive, err := dirIsCaseSensitive(storeDir) - if err != nil { - return fmt.Errorf("check whether %s is case-sensitive: %w", storeDir, err) - } - if sensitive { - return nil - } - - // Mounting hides whatever is already in the directory -- the files stay on - // disk, invisible and unused, and the store looks empty. Someone with an - // existing plain store dir would appear to lose every image and quietly - // re-pull the lot. Refuse instead, and say what to do. - empty, err := dirIsEmpty(storeDir) - if err != nil { - return fmt.Errorf("check whether %s is empty: %w", storeDir, err) - } - if !empty { - return fmt.Errorf("%s is on a case-insensitive filesystem and is not empty, so hull "+ - "cannot mount a case-sensitive store over it without hiding what is there. "+ - "Move or remove its contents, or point --store-dir at an empty directory or a "+ - "case-sensitive volume", storeDir) - } - - image := storeImagePath(storeDir, defaultStoreDir) - if _, err := os.Stat(image); os.IsNotExist(err) { - if err := runHdiutil("create", "-type", "SPARSE", "-fs", "Case-sensitive APFS", "-size", defaultStoreImageSize, "-volname", "hull-store", image); err != nil { - return fmt.Errorf("create case-sensitive APFS store: %w", err) - } - } else if err != nil { - return fmt.Errorf("check store image: %w", err) - } - // -owners on, because macOS mounts disk images with ownership ignored by - // default (mount(8) prints it as "noowners"). On such a volume the kernel - // does not enforce uid or mode at all, so every 0600 hull writes inside - // the store -- state.json, urunit.conf, and the container-boot initrd that - // carries the environment secrets handed to the guest -- is readable by - // any other local account on the machine. The mode bits are the only - // access control the store has; without this flag they are decoration. - if err := runHdiutil(storeAttachArgs(image, storeDir)...); err != nil { - return fmt.Errorf("mount case-sensitive APFS store at %s: %w", storeDir, err) - } - if err := ensureStoreOwnership(storeDir, image); err != nil { - return err - } - - // Belt and braces: if the mount somehow produced a case-insensitive store - // anyway, fail here with the reason rather than letting a guest boot on a - // rootfs whose files have collided. - if sensitive, err := dirIsCaseSensitive(storeDir); err != nil { - return fmt.Errorf("verify the mounted store: %w", err) - } else if !sensitive { - return fmt.Errorf("%s is still case-insensitive after mounting %s", storeDir, image) - } - if err := os.WriteFile(marker, []byte("case-sensitive APFS\n"), 0600); err != nil { - return fmt.Errorf("write store marker: %w", err) - } - return nil -} - -// storeAttachArgs is the hdiutil command line that mounts a store image. -// Kept in one place so the ownership flag cannot be dropped from one call -// site and kept on another, and so a test can assert it is there. -func storeAttachArgs(image, storeDir string) []string { - return []string{"attach", image, "-mountpoint", storeDir, "-nobrowse", "-owners", "on"} -} - -// mntIgnoreOwnership is MNT_IGNORE_OWNERSHIP from : the volume is -// mounted "noowners" and the kernel ignores the uid, gid and mode of every -// file on it. Not exported by the syscall package. -const mntIgnoreOwnership = 0x00200000 - -// storeIgnoresOwnership reports whether the volume backing dir was mounted -// with ownership disabled, i.e. whether the 0600 on the instance state, the -// urunit config and the container-boot initrd is actually being enforced. -func storeIgnoresOwnership(dir string) (bool, error) { - var st syscall.Statfs_t - if err := syscall.Statfs(dir, &st); err != nil { - return false, fmt.Errorf("statfs %s: %w", dir, err) - } - return st.Flags&mntIgnoreOwnership != 0, nil -} - -// ensureStoreOwnership repairs a store that an older hull mounted without -// ownership enforcement. -// -// Every store created before the -owners flag existed is still mounted -// "noowners" on every machine that has one, and the fix on the create path -// does nothing for them: the marker file short-circuits ensureStore. There is -// no unprivileged way to flip the flag on a live mount, so the repair is a -// detach and a re-attach. It is attempted rather than required, because the -// detach fails while another hull process (a running instance's `run`, a -// concurrent `pull`) has the volume busy -- and a store whose permissions are -// weaker than advertised is a reason to warn loudly, not a reason to refuse to -// run at all. -func ensureStoreOwnership(storeDir, image string) error { - ignored, err := storeIgnoresOwnership(storeDir) - if err != nil { - return err - } - if !ignored { - return nil - } - - if _, err := os.Stat(image); err != nil { - // Not a store hull mounted from an image it can name: nothing to - // re-attach, so all that is left is to say what is wrong. - log.Warnf("%s is mounted with ownership ignored (noowners), so the 0600 on the "+ - "instance state and the container-boot initrd inside it does not keep another "+ - "local account out. Remount that volume with owners enabled.", storeDir) - return nil - } - - log.Infof("%s was mounted without ownership enforcement; remounting it with -owners on", storeDir) - if err := runHdiutil("detach", storeDir); err != nil { - log.Warnf("could not remount %s with ownership enforced (%v). The store stays usable, "+ - "but the 0600 on the instance state and the container-boot initrd inside it does "+ - "not keep another local account out. Close other hull processes and re-run, or "+ - "remount by hand: hdiutil detach %s && hdiutil attach %s -mountpoint %s -nobrowse -owners on", - storeDir, err, storeDir, image, storeDir) - return nil - } - if err := runHdiutil(storeAttachArgs(image, storeDir)...); err != nil { - return fmt.Errorf("re-mount case-sensitive APFS store at %s with ownership enforced: %w", storeDir, err) - } - if ignored, err := storeIgnoresOwnership(storeDir); err == nil && ignored { - log.Warnf("%s still ignores ownership after remounting; the file modes inside the "+ - "store are not enforced against other local accounts", storeDir) - } - return nil -} - // dirIsCaseSensitive answers the only question that matters about a store: can // it hold xt_CONNMARK.h and xt_connmark.h at the same time. // @@ -319,55 +120,6 @@ func dirIsCaseSensitive(dir string) (bool, error) { return true, nil } -// storeOwnFiles are files hull itself drops in the store directory before the -// store is opened, and which mounting over is therefore harmless: they are -// bookkeeping hull rewrites, not anything a user put there. -// -// telemetry is the reason this list exists. It initialises from the root -// command, so by the time a command opens the store its two files are already -// sitting in the directory -- and without this the emptiness check below would -// refuse to mount over hull's own footprints on a genuinely fresh store. -// -// The crash queue and the outbox are here for the same reason: under a -// --store-dir other than the default one, the state lives in the store, and -// a crash before the first mount queues its report there. The default store -// keeps no telemetry state; see telemetry.DefaultStateDir. -var storeOwnFiles = map[string]bool{ - "telemetry.json": true, - "telemetry.lock": true, - "crashes": true, - "outbox": true, - storeMarkerName: true, - ".DS_Store": true, -} - -// dirIsEmpty reports whether mounting over dir would hide anything worth -// keeping. Hull's own bookkeeping does not count; anything else does. That -// includes a temporary state file the telemetry client left behind when it -// was killed between writing and renaming it. -func dirIsEmpty(dir string) (bool, error) { - entries, err := os.ReadDir(dir) - if err != nil { - return false, err - } - for _, e := range entries { - name := e.Name() - if storeOwnFiles[name] || strings.HasPrefix(name, ".telemetry-") && strings.HasSuffix(name, ".json") { - continue - } - return false, nil - } - return true, nil -} - -func runHdiutil(args ...string) error { - command := exec.Command("hdiutil", args...) - if output, err := command.CombinedOutput(); err != nil { - return fmt.Errorf("hdiutil %v: %w: %s", args, err, output) - } - return nil -} - // globalStore returns the store instance from the CLI context. // // The case-sensitive volume is ensured here rather than in the root Before diff --git a/cmd/hull/orphan_instance_test.go b/cmd/hull/orphan_instance_test.go index 1f5f06d2..3069753a 100644 --- a/cmd/hull/orphan_instance_test.go +++ b/cmd/hull/orphan_instance_test.go @@ -47,7 +47,6 @@ import ( "time" "github.com/urfave/cli/v3" - "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" "github.com/brig-sh/hull/pkg/store" ) @@ -97,7 +96,7 @@ func TestNoVMMIsSpawnedWithoutARecordOfIt(t *testing.T) { marker := filepath.Join(t.TempDir(), "vmm-started") state := &store.InstanceState{ID: "vm1", LogFile: s.InstanceLogFile("vm1")} started, err := launchVMM(context.Background(), &cli.Command{}, s, state, standInVMM(t, marker), - nil, hypervisors.VzVmm, true, "none", "") + nil, vzVmm, true, "none", "") if err == nil { t.Errorf("launchVMM reported success with a store it could not write to; "+ @@ -228,7 +227,7 @@ func TestFailedSpawnLeavesNoStartingRecord(t *testing.T) { started, err := launchVMM(context.Background(), &cli.Command{}, s, state, []string{filepath.Join(t.TempDir(), "no-such-vmm")}, nil, - hypervisors.VzVmm, true, "none", "") + vzVmm, true, "none", "") if err == nil { t.Fatal("launchVMM succeeded with a VMM binary that does not exist") } diff --git a/cmd/hull/proc_darwin.go b/cmd/hull/proc_darwin.go new file mode 100644 index 00000000..4e176739 --- /dev/null +++ b/cmd/hull/proc_darwin.go @@ -0,0 +1,126 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/binary" + "fmt" + "os/exec" + "strconv" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +// inspectProcess reads what processIsAVMM needs to know about pid from ps. +// +// ps prints lstart as a fixed 24-character date, then the command. The command +// is argv joined with spaces, which is also how the recorded argv is compared. +func inspectProcess(pid int) (procInfo, bool) { + out, err := exec.Command("/bin/ps", "-p", strconv.Itoa(pid), "-o", "lstart=,command=").Output() + if err != nil { + return procInfo{}, false + } + line := strings.TrimSpace(string(out)) + const lstartLen = 24 + if len(line) <= lstartLen { + return procInfo{}, false + } + started, command := strings.TrimSpace(line[:lstartLen]), strings.TrimSpace(line[lstartLen:]) + fields := strings.Fields(command) + if len(fields) == 0 { + return procInfo{}, false + } + info := procInfo{argv0: fields[0], command: command} + // ParseInLocation, not Parse: `ps -o lstart=` prints LOCAL wall-clock time + // with no zone, and parsing it as UTC makes every process look + // (UTC-offset) hours out. On a UTC+3 machine that made every live VMM + // appear to have started three hours in the future, so the reuse check + // rejected it and stop refused to signal a VM it should have stopped. + when, perr := time.ParseInLocation("Mon Jan 2 15:04:05 2006", started, time.Local) + if perr != nil { + // Unparseable: the caller falls back to the name match rather than + // refusing to act at all; say so in the log rather than silently. + log.Debugf("could not parse the start time %q for pid %d: %v", started, pid, perr) + return info, true + } + info.started, info.startKnown = when, true + return info, true +} + +// listProcesses returns every process's pid and space-joined argv. +// +// -ww so long argv is not truncated to the terminal width; the paths that +// make a recorded argv unique are at the end of the line. +func listProcesses() ([]procEntry, error) { + out, err := exec.Command("/bin/ps", "-axww", "-o", "pid=,command=").Output() + if err != nil { + return nil, err + } + var procs []procEntry + for _, line := range strings.Split(string(out), "\n") { + line = strings.TrimSpace(line) + pidStr, cmd, ok := strings.Cut(line, " ") + if !ok { + continue + } + pid, perr := strconv.Atoi(pidStr) + if perr != nil || pid <= 0 { + continue + } + procs = append(procs, procEntry{pid: pid, command: strings.TrimSpace(cmd)}) + } + return procs, nil +} + +// processArgv returns the argv of pid as the kernel holds it, via the +// kern.procargs2 sysctl. The buffer is argc (a native int32), then the exec +// path, NUL padding, and argc NUL-terminated argv strings, then the +// environment, which is not read. +func processArgv(pid int) ([]string, error) { + buf, err := unix.SysctlRaw("kern.procargs2", pid) + if err != nil { + return nil, err + } + if len(buf) < 4 { + return nil, fmt.Errorf("procargs2 for pid %d: short buffer", pid) + } + argc := int(int32(binary.NativeEndian.Uint32(buf))) + if argc < 0 { + return nil, fmt.Errorf("procargs2 for pid %d: argc %d", pid, argc) + } + rest := buf[4:] + // Skip the exec path, then the NUL padding after it. + i := bytes.IndexByte(rest, 0) + if i < 0 { + return nil, fmt.Errorf("procargs2 for pid %d: no exec path", pid) + } + rest = rest[i:] + for len(rest) > 0 && rest[0] == 0 { + rest = rest[1:] + } + argv := make([]string, 0, argc) + for range argc { + j := bytes.IndexByte(rest, 0) + if j < 0 { + return nil, fmt.Errorf("procargs2 for pid %d: truncated argv", pid) + } + argv = append(argv, string(rest[:j])) + rest = rest[j+1:] + } + return argv, nil +} diff --git a/cmd/hull/prune.go b/cmd/hull/prune.go index 76f23fd8..966302c5 100644 --- a/cmd/hull/prune.go +++ b/cmd/hull/prune.go @@ -65,11 +65,7 @@ func pruneStore(cmd *cli.Command) error { if err != nil { return err } - storeDir := cmd.String("store-dir") - hint := "" - if _, statErr := os.Stat(storeImagePath(storeDir, defaultStoreDir())); statErr == nil { - hint = "Run `hull store compact` to return the reclaimed space to the host." - } + hint := storeCompactHint(cmd.String("store-dir")) return pruneStoreIn(os.Stdout, s, cmd.Bool("all"), cmd.Bool("dry-run"), hint) } diff --git a/cmd/hull/run.go b/cmd/hull/run.go index 110d94fd..5b4fc875 100644 --- a/cmd/hull/run.go +++ b/cmd/hull/run.go @@ -525,7 +525,7 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { telemetryBackend = vmmName // Graphics is only supported on the Vz backend; the QEMU darwin builder // never emits window flags. - if gui && vmmType != hypervisors.VzVmm { + if gui && vmmType != vzVmm { return fmt.Errorf("--gui is only supported with the Vz hypervisor (got %q)", vmmName) } @@ -533,7 +533,7 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { // from Virtualization.framework, which the QEMU backend cannot provide. // The annotation lets an image opt in; the flag covers everything else. rosetta := cmd.Bool("rosetta") || ociSpec.Annotations["com.urunc.darwin.rosetta"] == "true" - if rosetta && vmmType != hypervisors.VzVmm { + if rosetta && vmmType != vzVmm { return fmt.Errorf("--rosetta is only supported with the Vz hypervisor (got %q)", vmmName) } @@ -552,7 +552,7 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { if !cmd.Bool("no-boot-assets") && annotationOverrides["com.urunc.unikernel.bootKernel"] == "" && annotationOverrides["com.urunc.unikernel.bootInitrd"] == "" && - (vmmType == hypervisors.VzVmm || vmmType == hypervisors.HviVmm) && + (vmmType == vzVmm || vmmType == hypervisors.HviVmm) && !imageCarriesKernel(ociSpec.Annotations) { kernel, initrdFile, assetErr := bootassets.Ensure(ctx, cmd.String("store-dir")) if assetErr != nil { @@ -577,7 +577,7 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { unikernelType, unikernelTypeDeclared := unikernelFamily(ociSpec.Annotations, containerBoot) // Create VMM instance - vmm, err := hypervisors.NewVMM(vmmType, uruncCfg.Monitors) + vmm, err := newVMM(vmmType, uruncCfg.Monitors) if err != nil { return fmt.Errorf("failed to create VMM: %w", err) } @@ -593,9 +593,7 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { if qemuD, ok := vmm.(*hypervisors.Qemu); ok { qemuD.SetQMPSocket(qmpSocket) } - if vzD, ok := vmm.(*hypervisors.VzDarwin); ok { - vzD.SetQMPSocket(qmpSocket) - } + setVzQMPSocket(vmm, qmpSocket) // Create unikernel instance unikernel, err := unikernels.New(unikernelType) @@ -680,7 +678,7 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { if err := verifyContainerBootAssets(bootKernel, bootInitrd); err != nil { return err } - if containerBoot && vmmType != hypervisors.VzVmm && vmmType != hypervisors.HviVmm { + if containerBoot && vmmType != vzVmm && vmmType != hypervisors.HviVmm { return fmt.Errorf("generic virtiofs container boot requires the vz or hvi backend (got %q)", vmmName) } if bootKernel != "" { @@ -712,7 +710,7 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { if err != nil { return fmt.Errorf("boot kernel from image annotation: %w", err) } - } else if vmmType == hypervisors.VzVmm || vmmType == hypervisors.HviVmm { + } else if vmmType == vzVmm || vmmType == hypervisors.HviVmm { // For native arm64 backends, look for common kernel locations in rootfs. // // The names are hull's, but the files are the image's, and an image @@ -800,7 +798,7 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { // Determine rootfs sharing mode rootfsType := rootfsTypeOverride if rootfsType == "" { - if vmmType == hypervisors.VzVmm || vmmType == hypervisors.HviVmm { + if vmmType == vzVmm || vmmType == hypervisors.HviVmm { rootfsType = "virtiofs" } else { rootfsType = "9pfs" @@ -982,7 +980,7 @@ exec %s "$@" shareMounts := "" var containerMounts strings.Builder for _, sh := range shares { - if vmmType == hypervisors.VzVmm || vmmType == hypervisors.HviVmm { + if vmmType == vzVmm || vmmType == hypervisors.HviVmm { mountOpts := "" if sh.readOnly { mountOpts = "-o ro " @@ -1057,7 +1055,7 @@ exec %s "$@" // Console device depends on VMM backend consoleDev := "ttyS0" if runtime.GOARCH == "arm64" { - if vmmType == hypervisors.VzVmm { + if vmmType == vzVmm { consoleDev = "hvc0" } else { consoleDev = "ttyAMA0" @@ -1090,7 +1088,7 @@ exec %s "$@" initrdEntry = "/init" } kernelCmdline = fmt.Sprintf("rdinit=%s console=%s", initrdEntry, consoleDev) + ipArg - } else if rootfsDir != "" && vmmType == hypervisors.VzVmm { + } else if rootfsDir != "" && vmmType == vzVmm { // Vz: virtiofs rootfs boot (no initramfs) kernelCmdline = fmt.Sprintf("root=rootfs rootfstype=virtiofs rw console=%s", consoleDev) + ipArg @@ -1443,7 +1441,7 @@ exec %s "$@" // Join the user-mode gateway: the datagram end backs the guest's single // NIC (fd 3 in the child); the control connection (fd 4) stays open for // the VMM's lifetime and its closure removes the gateway member. - if gatewaySock != "" && vmmType == hypervisors.VzVmm { + if gatewaySock != "" && vmmType == vzVmm { dataF, ctlConn, err := joinGateway(gatewaySock) if err != nil { return err @@ -1458,7 +1456,7 @@ exec %s "$@" gatewayFiles = []*os.File{dataF, ctlF} cmdArgs = append(cmdArgs, "--net-fd", "3") } - if vmmType == hypervisors.VzVmm { + if vmmType == vzVmm { cmdArgs = append(cmdArgs, vzNetArgs(netMode)...) cmdArgs = append(cmdArgs, "--stop-grace", strconv.Itoa(int(cmd.Int("stop-grace")))) // Every Vz instance is checkpoint-ready: the state dir persists the @@ -1779,31 +1777,6 @@ func cloneRootfsAPFS(ctx context.Context, source, rootfsLink string) error { return nil } -func requireSameAPFSVolume(source, destinationParent string) error { - var sourceFS, destinationFS unix.Statfs_t - if err := unix.Statfs(source, &sourceFS); err != nil { - return fmt.Errorf("inspect source filesystem for APFS clone: %w", err) - } - if err := unix.Statfs(destinationParent, &destinationFS); err != nil { - return fmt.Errorf("inspect destination filesystem for APFS clone: %w", err) - } - fsName := func(raw [16]byte) string { - for i, value := range raw { - if value == 0 { - return string(raw[:i]) - } - } - return string(raw[:]) - } - if fsName(sourceFS.Fstypename) != "apfs" || fsName(destinationFS.Fstypename) != "apfs" { - return fmt.Errorf("HVI writable rootfs requires an APFS image store and instance directory") - } - if sourceFS.Fsid.Val != destinationFS.Fsid.Val { - return fmt.Errorf("HVI writable rootfs requires the image store and instance directory on the same APFS volume") - } - return nil -} - func restoreClonedHardlinks(source, destination string) error { type inodeKey struct { device uint64 @@ -2461,7 +2434,7 @@ func launchVMM(ctx context.Context, cmd *cli.Command, s *store.Store, state *sto origTermios, _ = getTermios(int(os.Stdin.Fd())) defer restoreTerminal(origTermios) defer claimForeground() - if vmmType == hypervisors.VzVmm && origTermios != nil { + if vmmType == vzVmm && origTermios != nil { raw := *origTermios raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN raw.Iflag &^= syscall.ICRNL diff --git a/cmd/hull/run_network_test.go b/cmd/hull/run_network_test.go index 137feba0..7b5cd62a 100644 --- a/cmd/hull/run_network_test.go +++ b/cmd/hull/run_network_test.go @@ -111,7 +111,7 @@ func TestCheckHviNetworking(t *testing.T) { // treated as "networking on" everywhere else in the run path. It has // to be refused here too, or the check is bypassed by a typo. {name: "hvi unrecognised net mode is refused", vmm: hypervisors.HviVmm, netMode: "shred", wantErr: true}, - {name: "vz shared is untouched", vmm: hypervisors.VzVmm, netMode: "shared"}, + {name: "vz shared is untouched", vmm: vzVmm, netMode: "shared"}, {name: "qemu shared is untouched", vmm: hypervisors.QemuVmm, netMode: "shared"}, } for _, tc := range tests { diff --git a/cmd/hull/stop.go b/cmd/hull/stop.go index 97457dee..d7209797 100644 --- a/cmd/hull/stop.go +++ b/cmd/hull/stop.go @@ -21,9 +21,7 @@ import ( "errors" "fmt" "os" - "os/exec" "path/filepath" - "strconv" "strings" "syscall" "time" @@ -285,26 +283,16 @@ func reapOrphanVMM(s *store.Store, state *store.InstanceState) bool { return false } want := strings.Join(state.CmdLine, " ") - // -ww so long argv is not truncated to the terminal width; the paths that - // make this unique are at the end of the line. - out, err := exec.Command("/bin/ps", "-axww", "-o", "pid=,command=").Output() + procs, err := listProcesses() if err != nil { return false } killed := false - for _, line := range strings.Split(string(out), "\n") { - line = strings.TrimSpace(line) - pidStr, cmd, ok := strings.Cut(line, " ") - if !ok { - continue - } - if strings.TrimSpace(cmd) != want { - continue - } - pid, perr := strconv.Atoi(pidStr) - if perr != nil || pid <= 0 { + for _, p := range procs { + if p.command != want { continue } + pid := p.pid // Re-check immediately before signalling. // // The decision to reap comes from a record read before the `ps` above, @@ -372,26 +360,11 @@ func processIsAVMM(pid int, state *store.InstanceState) bool { if pid <= 0 { return false } - out, err := exec.Command("/bin/ps", "-p", strconv.Itoa(pid), "-o", "lstart=,command=").Output() - if err != nil { - return false - } - line := strings.TrimSpace(string(out)) - if line == "" { - return false - } - // ps prints lstart as a fixed 24-character date, then the command. - const lstartLen = 24 - if len(line) <= lstartLen { + info, ok := inspectProcess(pid) + if !ok { return false } - started, cmdline := strings.TrimSpace(line[:lstartLen]), strings.TrimSpace(line[lstartLen:]) - - fields := strings.Fields(cmdline) - if len(fields) == 0 { - return false - } - base := filepath.Base(fields[0]) + base := filepath.Base(info.argv0) isVMM := vmmExecutables[base] if !isVMM { for _, prefix := range vmmExecutablePrefixes { @@ -409,23 +382,27 @@ func processIsAVMM(pid int, state *store.InstanceState) bool { } // If the record carries the argv we spawned, that is the strongest signal // available and it costs nothing to use. - if len(state.CmdLine) > 0 && strings.Join(state.CmdLine, " ") == cmdline { - return true - } - if state.StartTime.IsZero() { + if len(state.CmdLine) > 0 && strings.Join(state.CmdLine, " ") == info.command { return true } - // ParseInLocation, not Parse: `ps -o lstart=` prints LOCAL wall-clock time - // with no zone, and parsing it as UTC makes every process look - // (UTC-offset) hours out. On a UTC+3 machine that made every live VMM - // appear to have started three hours in the future, so the reuse check - // rejected it and stop refused to signal a VM it should have stopped. - when, perr := time.ParseInLocation("Mon Jan 2 15:04:05 2006", started, time.Local) - if perr != nil { - // Unparseable: fall back to the name match rather than refusing to act - // at all, and say so in the log rather than silently. - log.Debugf("could not parse the start time %q for pid %d: %v", started, pid, perr) + if state.StartTime.IsZero() || !info.startKnown { return true } - return !when.After(state.StartTime.Add(5 * time.Second)) + return !info.started.After(state.StartTime.Add(5 * time.Second)) +} + +// procInfo is what processIsAVMM reads about a live process: the program it +// runs, its argv joined with spaces the way a record's CmdLine is compared, +// and when it started, if the platform could say. +type procInfo struct { + argv0 string + command string + started time.Time + startKnown bool +} + +// procEntry is one line of a process listing: a pid and its joined argv. +type procEntry struct { + pid int + command string } diff --git a/cmd/hull/storemount_darwin.go b/cmd/hull/storemount_darwin.go new file mode 100644 index 00000000..776b84fc --- /dev/null +++ b/cmd/hull/storemount_darwin.go @@ -0,0 +1,260 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" +) + +const defaultStoreImageSize = "200g" + +// storeMarkerName marks a directory as a store hull mounted itself. +const storeMarkerName = ".case-sensitive-apfs" + +// storeImagePath is where the backing sparse image for a store lives. +// +// The default store keeps the name it has always had, so an existing install +// finds its images after an upgrade. Any other store dir gets an image named +// after itself rather than after its parent: two stores under one parent -- +// `_work/store-a` and `_work/store-b` -- would otherwise both resolve to +// `_work/hull-store.sparseimage`, and the second attach fails with "Resource +// busy" because the first already has it. +func storeImagePath(storeDir, defaultStoreDir string) string { + if storeDir == defaultStoreDir { + return filepath.Join(filepath.Dir(storeDir), "hull-store.sparseimage") + } + return storeDir + ".sparseimage" +} + +// ensureStore guarantees that storeDir is case-sensitive, mounting a +// case-sensitive APFS sparse image there if it is not already. +// +// This runs for every store, not just the default one. It used to run only +// when --store-dir was left alone, which meant anyone passing an explicit path +// -- CI, most of all -- silently got a case-insensitive store on the boot +// volume: exactly the thing the default store exists to avoid. Nothing +// complained, because store.New only calls MkdirAll. The failure surfaced much +// later, as a guest that unpacks a Linux rootfs with names that differ only by +// case and then does not boot. +func ensureStore(storeDir, defaultStoreDir string) error { + marker := filepath.Join(storeDir, storeMarkerName) + if _, err := os.Stat(marker); err == nil { + // Already ours and already mounted -- but possibly mounted the way + // hull used to mount it, with ownership ignored. + return ensureStoreOwnership(storeDir, storeImagePath(storeDir, defaultStoreDir)) + } else if !os.IsNotExist(err) { + return fmt.Errorf("check store mount: %w", err) + } + if err := os.MkdirAll(storeDir, 0700); err != nil { + return fmt.Errorf("create store mountpoint: %w", err) + } + + // A store that is already case-sensitive needs nothing: the user pointed + // --store-dir at a case-sensitive volume of their own, and nesting a 200G + // image inside it would be waste and surprise. + sensitive, err := dirIsCaseSensitive(storeDir) + if err != nil { + return fmt.Errorf("check whether %s is case-sensitive: %w", storeDir, err) + } + if sensitive { + return nil + } + + // Mounting hides whatever is already in the directory -- the files stay on + // disk, invisible and unused, and the store looks empty. Someone with an + // existing plain store dir would appear to lose every image and quietly + // re-pull the lot. Refuse instead, and say what to do. + empty, err := dirIsEmpty(storeDir) + if err != nil { + return fmt.Errorf("check whether %s is empty: %w", storeDir, err) + } + if !empty { + return fmt.Errorf("%s is on a case-insensitive filesystem and is not empty, so hull "+ + "cannot mount a case-sensitive store over it without hiding what is there. "+ + "Move or remove its contents, or point --store-dir at an empty directory or a "+ + "case-sensitive volume", storeDir) + } + + image := storeImagePath(storeDir, defaultStoreDir) + if _, err := os.Stat(image); os.IsNotExist(err) { + if err := runHdiutil("create", "-type", "SPARSE", "-fs", "Case-sensitive APFS", "-size", defaultStoreImageSize, "-volname", "hull-store", image); err != nil { + return fmt.Errorf("create case-sensitive APFS store: %w", err) + } + } else if err != nil { + return fmt.Errorf("check store image: %w", err) + } + // -owners on, because macOS mounts disk images with ownership ignored by + // default (mount(8) prints it as "noowners"). On such a volume the kernel + // does not enforce uid or mode at all, so every 0600 hull writes inside + // the store -- state.json, urunit.conf, and the container-boot initrd that + // carries the environment secrets handed to the guest -- is readable by + // any other local account on the machine. The mode bits are the only + // access control the store has; without this flag they are decoration. + if err := runHdiutil(storeAttachArgs(image, storeDir)...); err != nil { + return fmt.Errorf("mount case-sensitive APFS store at %s: %w", storeDir, err) + } + if err := ensureStoreOwnership(storeDir, image); err != nil { + return err + } + + // Belt and braces: if the mount somehow produced a case-insensitive store + // anyway, fail here with the reason rather than letting a guest boot on a + // rootfs whose files have collided. + if sensitive, err := dirIsCaseSensitive(storeDir); err != nil { + return fmt.Errorf("verify the mounted store: %w", err) + } else if !sensitive { + return fmt.Errorf("%s is still case-insensitive after mounting %s", storeDir, image) + } + if err := os.WriteFile(marker, []byte("case-sensitive APFS\n"), 0600); err != nil { + return fmt.Errorf("write store marker: %w", err) + } + return nil +} + +// storeAttachArgs is the hdiutil command line that mounts a store image. +// Kept in one place so the ownership flag cannot be dropped from one call +// site and kept on another, and so a test can assert it is there. +func storeAttachArgs(image, storeDir string) []string { + return []string{"attach", image, "-mountpoint", storeDir, "-nobrowse", "-owners", "on"} +} + +// mntIgnoreOwnership is MNT_IGNORE_OWNERSHIP from : the volume is +// mounted "noowners" and the kernel ignores the uid, gid and mode of every +// file on it. Not exported by the syscall package. +const mntIgnoreOwnership = 0x00200000 + +// storeIgnoresOwnership reports whether the volume backing dir was mounted +// with ownership disabled, i.e. whether the 0600 on the instance state, the +// urunit config and the container-boot initrd is actually being enforced. +func storeIgnoresOwnership(dir string) (bool, error) { + var st syscall.Statfs_t + if err := syscall.Statfs(dir, &st); err != nil { + return false, fmt.Errorf("statfs %s: %w", dir, err) + } + return st.Flags&mntIgnoreOwnership != 0, nil +} + +// ensureStoreOwnership repairs a store that an older hull mounted without +// ownership enforcement. +// +// Every store created before the -owners flag existed is still mounted +// "noowners" on every machine that has one, and the fix on the create path +// does nothing for them: the marker file short-circuits ensureStore. There is +// no unprivileged way to flip the flag on a live mount, so the repair is a +// detach and a re-attach. It is attempted rather than required, because the +// detach fails while another hull process (a running instance's `run`, a +// concurrent `pull`) has the volume busy -- and a store whose permissions are +// weaker than advertised is a reason to warn loudly, not a reason to refuse to +// run at all. +func ensureStoreOwnership(storeDir, image string) error { + ignored, err := storeIgnoresOwnership(storeDir) + if err != nil { + return err + } + if !ignored { + return nil + } + + if _, err := os.Stat(image); err != nil { + // Not a store hull mounted from an image it can name: nothing to + // re-attach, so all that is left is to say what is wrong. + log.Warnf("%s is mounted with ownership ignored (noowners), so the 0600 on the "+ + "instance state and the container-boot initrd inside it does not keep another "+ + "local account out. Remount that volume with owners enabled.", storeDir) + return nil + } + + log.Infof("%s was mounted without ownership enforcement; remounting it with -owners on", storeDir) + if err := runHdiutil("detach", storeDir); err != nil { + log.Warnf("could not remount %s with ownership enforced (%v). The store stays usable, "+ + "but the 0600 on the instance state and the container-boot initrd inside it does "+ + "not keep another local account out. Close other hull processes and re-run, or "+ + "remount by hand: hdiutil detach %s && hdiutil attach %s -mountpoint %s -nobrowse -owners on", + storeDir, err, storeDir, image, storeDir) + return nil + } + if err := runHdiutil(storeAttachArgs(image, storeDir)...); err != nil { + return fmt.Errorf("re-mount case-sensitive APFS store at %s with ownership enforced: %w", storeDir, err) + } + if ignored, err := storeIgnoresOwnership(storeDir); err == nil && ignored { + log.Warnf("%s still ignores ownership after remounting; the file modes inside the "+ + "store are not enforced against other local accounts", storeDir) + } + return nil +} + +// storeOwnFiles are files hull itself drops in the store directory before the +// store is opened, and which mounting over is therefore harmless: they are +// bookkeeping hull rewrites, not anything a user put there. +// +// telemetry is the reason this list exists. It initialises from the root +// command, so by the time a command opens the store its two files are already +// sitting in the directory -- and without this the emptiness check below would +// refuse to mount over hull's own footprints on a genuinely fresh store. +// +// The crash queue and the outbox are here for the same reason: under a +// --store-dir other than the default one, the state lives in the store, and +// a crash before the first mount queues its report there. The default store +// keeps no telemetry state; see telemetry.DefaultStateDir. +var storeOwnFiles = map[string]bool{ + "telemetry.json": true, + "telemetry.lock": true, + "crashes": true, + "outbox": true, + storeMarkerName: true, + ".DS_Store": true, +} + +// dirIsEmpty reports whether mounting over dir would hide anything worth +// keeping. Hull's own bookkeeping does not count; anything else does. That +// includes a temporary state file the telemetry client left behind when it +// was killed between writing and renaming it. +func dirIsEmpty(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return false, err + } + for _, e := range entries { + name := e.Name() + if storeOwnFiles[name] || strings.HasPrefix(name, ".telemetry-") && strings.HasSuffix(name, ".json") { + continue + } + return false, nil + } + return true, nil +} + +func runHdiutil(args ...string) error { + command := exec.Command("hdiutil", args...) + if output, err := command.CombinedOutput(); err != nil { + return fmt.Errorf("hdiutil %v: %w: %s", args, err, output) + } + return nil +} + +// storeCompactHint is what prune adds after freeing space: a sparse image +// keeps the blocks it once held until it is compacted, so on a store hull +// mounted from one, the space is not back on the host yet. +func storeCompactHint(storeDir string) string { + if _, err := os.Stat(storeImagePath(storeDir, defaultStoreDir())); err == nil { + return "Run `hull store compact` to return the reclaimed space to the host." + } + return "" +} diff --git a/cmd/hull/terminal_darwin.go b/cmd/hull/terminal.go similarity index 93% rename from cmd/hull/terminal_darwin.go rename to cmd/hull/terminal.go index af6ac474..73a427eb 100644 --- a/cmd/hull/terminal_darwin.go +++ b/cmd/hull/terminal.go @@ -10,7 +10,7 @@ import ( func getTermios(fd int) (*syscall.Termios, error) { var t syscall.Termios _, _, errno := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), - uintptr(syscall.TIOCGETA), uintptr(unsafe.Pointer(&t)), 0, 0, 0) + uintptr(ioctlGetTermios), uintptr(unsafe.Pointer(&t)), 0, 0, 0) if errno != 0 { return nil, errno } @@ -22,7 +22,7 @@ func restoreTerminal(t *syscall.Termios) { return } _, _, _ = syscall.Syscall6(syscall.SYS_IOCTL, uintptr(syscall.Stdin), - uintptr(syscall.TIOCSETA), uintptr(unsafe.Pointer(t)), 0, 0, 0) + uintptr(ioctlSetTermios), uintptr(unsafe.Pointer(t)), 0, 0, 0) } // setTermios applies t to stdin. @@ -31,7 +31,7 @@ func setTermios(t *syscall.Termios) { return } _, _, _ = syscall.Syscall6(syscall.SYS_IOCTL, uintptr(syscall.Stdin), - uintptr(syscall.TIOCSETA), uintptr(unsafe.Pointer(t)), 0, 0, 0) + uintptr(ioctlSetTermios), uintptr(unsafe.Pointer(t)), 0, 0, 0) } // makeRawTerminal puts fd into raw mode (cfmakeraw semantics) and returns @@ -51,7 +51,7 @@ func makeRawTerminal(fd int) (*syscall.Termios, error) { raw.Cc[syscall.VMIN] = 1 raw.Cc[syscall.VTIME] = 0 _, _, errno := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), - uintptr(syscall.TIOCSETA), uintptr(unsafe.Pointer(&raw)), 0, 0, 0) + uintptr(ioctlSetTermios), uintptr(unsafe.Pointer(&raw)), 0, 0, 0) if errno != 0 { return nil, errno } diff --git a/cmd/hull/terminal_ioctl_darwin.go b/cmd/hull/terminal_ioctl_darwin.go new file mode 100644 index 00000000..1bd20d40 --- /dev/null +++ b/cmd/hull/terminal_ioctl_darwin.go @@ -0,0 +1,25 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "syscall" + +// The termios get and set requests. BSD names them TIOCGETA and TIOCSETA; +// Linux calls the same pair TCGETS and TCSETS. Everything else terminal.go +// does (TIOCGWINSZ, TIOCSPGRP) has one name on both. +const ( + ioctlGetTermios = syscall.TIOCGETA + ioctlSetTermios = syscall.TIOCSETA +) diff --git a/cmd/hull/terminal_review_test.go b/cmd/hull/terminal_review_test.go index 5040fc9a..c45fc392 100644 --- a/cmd/hull/terminal_review_test.go +++ b/cmd/hull/terminal_review_test.go @@ -86,9 +86,9 @@ func TestSizingPolicyDoesNotOpenDevTTY(t *testing.T) { fset := token.NewFileSet() // Parsed rather than grepped: the doc comment explains why /dev/tty is not // consulted, and mentioning it there is the point. - file, err := parser.ParseFile(fset, "terminal_darwin.go", nil, 0) + file, err := parser.ParseFile(fset, "terminal.go", nil, 0) if err != nil { - t.Fatalf("parse terminal_darwin.go: %v", err) + t.Fatalf("parse terminal.go: %v", err) } ast.Inspect(file, func(n ast.Node) bool { lit, ok := n.(*ast.BasicLit) diff --git a/cmd/hull/vmm_darwin.go b/cmd/hull/vmm_darwin.go new file mode 100644 index 00000000..7519900e --- /dev/null +++ b/cmd/hull/vmm_darwin.go @@ -0,0 +1,41 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// vzVmm names the Virtualization.framework backend. urunc declares it only in +// its darwin files, so code shared with the Linux build spells it through this +// constant instead; on macOS it is urunc's own value, which keeps a typo in +// the Linux copy from compiling here unnoticed (see vmm_linux.go). +const vzVmm = hypervisors.VzVmm + +// newVMM is urunc's monitor factory. On macOS it already resolves vz-runner, +// hvi and qemu next to the hull executable and builds their argv, so hull has +// nothing to add. +func newVMM(vmmType hypervisors.VmmType, monitors map[string]types.MonitorConfig) (types.VMM, error) { + return hypervisors.NewVMM(vmmType, monitors) +} + +// setVzQMPSocket hands vz-runner the per-instance control socket path. The +// type it asserts on exists only in urunc's darwin build. +func setVzQMPSocket(vmm types.VMM, sock string) { + if vzD, ok := vmm.(*hypervisors.VzDarwin); ok { + vzD.SetQMPSocket(sock) + } +} From 4695a3ba804876ef249f7074e8cb4a4180274c88 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Wed, 14 Oct 2026 02:19:22 +0300 Subject: [PATCH 05/33] feat(linux): build cmd/hull for Linux and run hvi with a block rootfs Brig's teleport plan needs a destination runtime on Linux/KVM that drives hvi the way hull does on a Mac. cmd/hull did not compile for Linux: the untagged files called darwin-only helpers. The shared files drop their darwin tag and each helper moved into a _darwin.go file gets a Linux counterpart: termios through TCGETS and TCSETS, process identity from /proc, and a plain store directory in place of the hdiutil mount. hull builds the Linux hvi argv itself instead of using urunc's Linux monitor, which is written for the containerd shim and passes no --agent-sock, --net-gateway or --net-mac. hviLinuxArgv is the darwin argv minus the virtio-fs shares, and both builds take the vz name from one constant. A darwin test fails if urunc's darwin argv grows, drops or reorders a flag for the same share-less arguments. hull on Linux offers the one configuration hvi on KVM can run: hvi with a block rootfs. It refuses other backends, virtiofs and 9pfs roots, host shares and --net shared without a gateway by name, before the store is opened or the image pulled, and it refuses an image annotated for another hypervisor instead of booting it under hvi. make linux cross-builds the CLI, CI builds, vets, lints and unit-tests hull for GOOS=linux, and backends.md and build.md describe what Linux hull does, what differs from macOS and what was tested on a Raspberry Pi 5. Signed-off-by: Panagiotis Moustafellos --- .github/workflows/ci.yml | 38 ++++++- Makefile | 10 +- README.md | 4 + cmd/hull/apfs_linux.go | 28 +++++ cmd/hull/assets.go | 2 - cmd/hull/blockrootfs.go | 2 - cmd/hull/commands_linux.go | 78 ++++++++++++++ cmd/hull/compose_down_test.go | 2 + cmd/hull/e2fs_linux.go | 50 +++++++++ cmd/hull/exec.go | 2 - cmd/hull/exec_compose.go | 2 - cmd/hull/hviargv.go | 104 ++++++++++++++++++ cmd/hull/hviargv_darwin_test.go | 106 ++++++++++++++++++ cmd/hull/hviargv_test.go | 86 +++++++++++++++ cmd/hull/images.go | 2 - cmd/hull/inspect.go | 2 - cmd/hull/jsonout.go | 2 - cmd/hull/linuxrun.go | 116 ++++++++++++++++++++ cmd/hull/linuxrun_test.go | 117 ++++++++++++++++++++ cmd/hull/logcapture_test.go | 38 +++++++ cmd/hull/logs.go | 2 - cmd/hull/main.go | 5 +- cmd/hull/oneshot.go | 2 - cmd/hull/proc_linux.go | 131 +++++++++++++++++++++++ cmd/hull/proc_linux_test.go | 95 ++++++++++++++++ cmd/hull/procfs.go | 93 ++++++++++++++++ cmd/hull/procfs_test.go | 82 ++++++++++++++ cmd/hull/prune.go | 2 - cmd/hull/ps.go | 2 - cmd/hull/pull.go | 2 - cmd/hull/rm.go | 2 - cmd/hull/rm_test.go | 2 - cmd/hull/rmi.go | 2 - cmd/hull/rootfs_containment_test.go | 2 - cmd/hull/run.go | 44 ++++---- cmd/hull/runpreflight_darwin.go | 27 +++++ cmd/hull/runpreflight_linux.go | 24 +++++ cmd/hull/setuidwarn_test.go | 21 +--- cmd/hull/shortflags_test.go | 2 + cmd/hull/stop.go | 2 - cmd/hull/storemount_linux.go | 52 +++++++++ cmd/hull/supervisor.go | 2 - cmd/hull/telemetry.go | 2 - cmd/hull/terminal.go | 2 - cmd/hull/terminal_ioctl_linux.go | 25 +++++ cmd/hull/version.go | 2 - cmd/hull/vmm_linux.go | 110 +++++++++++++++++++ docs/backends.md | 92 ++++++++++++---- docs/build.md | 54 +++++++++- go.mod | 9 +- go.sum | 16 +-- pkg/ociclient/annotationdefaults_test.go | 18 ++++ pkg/ociclient/bundle.go | 10 +- 53 files changed, 1611 insertions(+), 118 deletions(-) create mode 100644 cmd/hull/apfs_linux.go create mode 100644 cmd/hull/commands_linux.go create mode 100644 cmd/hull/e2fs_linux.go create mode 100644 cmd/hull/hviargv.go create mode 100644 cmd/hull/hviargv_darwin_test.go create mode 100644 cmd/hull/hviargv_test.go create mode 100644 cmd/hull/linuxrun.go create mode 100644 cmd/hull/linuxrun_test.go create mode 100644 cmd/hull/logcapture_test.go create mode 100644 cmd/hull/proc_linux.go create mode 100644 cmd/hull/proc_linux_test.go create mode 100644 cmd/hull/procfs.go create mode 100644 cmd/hull/procfs_test.go create mode 100644 cmd/hull/runpreflight_darwin.go create mode 100644 cmd/hull/runpreflight_linux.go create mode 100644 cmd/hull/storemount_linux.go create mode 100644 cmd/hull/terminal_ioctl_linux.go create mode 100644 cmd/hull/vmm_linux.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be3db34a..011417e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,13 +147,43 @@ jobs: - name: Check the cask renderer run: python3 scripts/render-cask-test.py + linux: + name: linux (build, vet, lint, unit) + # hull builds for arm64 Linux as the hvi runtime on KVM (docs/backends.md, + # "hvi on Linux"). Nothing else in this file compiles it for Linux: lint + # above type-checks GOOS=darwin and every test lane is a Mac. A darwin-only + # symbol reaching a shared file would break the Linux binary with every + # gate green -- which is how cmd/hull stopped compiling for Linux in the + # first place. The unit tests need no /dev/kvm; no guest boots here. + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + + - name: Build + run: go build -o /dev/null ./cmd/hull + + - name: Vet + run: go vet ./... + + - uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8 + with: + version: v2.12.2 + + # -p 2 for the same reason as everywhere else in this file. -count=1 so + # a cached result cannot stand in for a run. + - name: Unit tests + run: go test -count=1 -p 2 -timeout=10m ./... + telemetry: name: telemetry module (linux) # pkg/telemetry is a module of its own, so `go test ./...` and the lint - # above, both run from the root module, never reach it. It is also the - # one part of this repository that builds for Linux, because brig links - # it there: this job is where that is checked, natively rather than with - # GOOS set. + # above, both run from the root module, never reach it. brig links it on + # Linux, so this job checks it there as its own module, natively rather + # than with GOOS set. runs-on: ubuntu-latest defaults: run: diff --git a/Makefile b/Makefile index d8530ad1..93472ef3 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,8 @@ BUILD_DIR ?= ${CURDIR}/dist #? PREFIX Directory to install binaries (default: /usr/local/bin) PREFIX ?= /usr/local/bin -# Architecture (Apple Silicon only for now) +# Architecture. arm64 is the only one built or tested: Apple Silicon for the +# macOS targets, and arm64 Linux/KVM for `make linux`. ARCH ?= arm64 # Binary variables @@ -101,6 +102,13 @@ urunc_macos: GOARCH=$(ARCH) $(GO) build -ldflags "$(LDFLAGS)" \ -o $(HULL_BIN)_$(ARCH) $(CURDIR)/cmd/hull +## linux Build the hull CLI for Linux (GOOS=linux, GOARCH=$(ARCH)); build hvi on the Linux host itself. +.PHONY: linux +linux: + mkdir -p $(BUILD_DIR) + GOOS=linux GOARCH=$(ARCH) $(GO) build -ldflags "$(LDFLAGS)" \ + -o $(HULL_BIN)_linux_$(ARCH) $(CURDIR)/cmd/hull + ## vz_runner Build the Swift Virtualization.framework runner. .PHONY: vz_runner vz_runner: diff --git a/README.md b/README.md index b812b068..d829bb0a 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,10 @@ Kubernetes integration, and it implements a subset of Compose. See `vz-runner` declares a macOS 14 floor in `vz-runner/Package.swift`, so earlier versions can work. Nothing blocks them, and nothing tests them. - About 2 GB of free disk space for the store, plus each image you pull. +- On Linux, hull builds from source for arm64 with KVM and runs only the + `hvi` backend with a block rootfs, as the destination for a sandbox that + started on a Mac. See [docs/backends.md](docs/backends.md#hvi-on-linux) + and [docs/build.md](docs/build.md#linux). ## Install diff --git a/cmd/hull/apfs_linux.go b/cmd/hull/apfs_linux.go new file mode 100644 index 00000000..28b3b1b5 --- /dev/null +++ b/cmd/hull/apfs_linux.go @@ -0,0 +1,28 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "errors" + +// requireSameAPFSVolume always refuses on Linux. The hvi writable-directory +// rootfs is an APFS clone served over virtio-fs, and Linux has neither: no +// APFS, and no virtio-fs server in hvi's KVM backend. checkLinuxRun refuses +// that mode before anything reaches here; this is the backstop, so that a +// path which skipped the preflight fails instead of falling back to a full +// copy of the image. +func requireSameAPFSVolume(_, _ string) error { + return errors.New("the hvi writable-directory rootfs needs APFS and virtio-fs; " + + "on Linux use --rootfs-type block") +} diff --git a/cmd/hull/assets.go b/cmd/hull/assets.go index d083eb5f..d6d14b79 100644 --- a/cmd/hull/assets.go +++ b/cmd/hull/assets.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/blockrootfs.go b/cmd/hull/blockrootfs.go index f182fb67..5563f599 100644 --- a/cmd/hull/blockrootfs.go +++ b/cmd/hull/blockrootfs.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/commands_linux.go b/cmd/hull/commands_linux.go new file mode 100644 index 00000000..5dace824 --- /dev/null +++ b/cmd/hull/commands_linux.go @@ -0,0 +1,78 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + + "github.com/urfave/cli/v3" +) + +const appUsage = "container CLI for hvi sandboxes on Linux/KVM" + +// platformCommands is the Linux verb set: what it takes to run an hvi sandbox +// that may have started on a Mac, and what brig calls to drive one (run, exec, +// ps, inspect, logs, stop, rm, assets dir, network-gateway). +// +// Left out, because they drive something Linux does not have: +// - checkpoint and restore: vz-runner's saved-state files, which are +// Virtualization.framework's and bound to the Mac that wrote them. +// - store: detaches the APFS sparse image the macOS store is mounted from. +// +// compose is listed but refuses; see linuxCompose. +func platformCommands() []*cli.Command { + return []*cli.Command{ + pullCommand(), + runCommand(), + execCommand(), + psCommand(), + stopCommand(), + rmCommand(), + logsCommand(), + inspectCommand(), + imagesCommand(), + rmiCommand(), + pruneCommand(), + assetsCommand(), + linuxCompose(), + networkGatewayCommand(), + telemetryCommand(), + versionCommand(), + } +} + +// errComposeMacOnly is what every compose subcommand answers on Linux. +var errComposeMacOnly = errors.New("hull compose runs on macOS only: its services default to vz " + + "or qemu, and its health checks, restart policy and shared gateway have only been " + + "exercised there. On Linux, run each sandbox with `hull run --hypervisor hvi`") + +// linuxCompose is the compose command with a Before hook that refuses it. +// +// Refused rather than left out, for two reasons. Left out, `hull compose up` +// answers "No help topic for 'compose'", which reads as a broken install +// rather than a platform limit. And the compose code stays compiled on Linux +// either way -- the gateway hosts compose's service supervisor, so the two +// cannot be separated without splitting network-gateway -- and keeping its +// entry point reachable keeps it type-checked and linted like any other code +// on this platform, instead of rotting as a pile of unused functions. +func linuxCompose() *cli.Command { + c := composeCommand() + c.Hidden = true + c.Before = func(ctx context.Context, _ *cli.Command) (context.Context, error) { + return ctx, errComposeMacOnly + } + return c +} diff --git a/cmd/hull/compose_down_test.go b/cmd/hull/compose_down_test.go index a9086745..adf40fc9 100644 --- a/cmd/hull/compose_down_test.go +++ b/cmd/hull/compose_down_test.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build darwin + package main import ( diff --git a/cmd/hull/e2fs_linux.go b/cmd/hull/e2fs_linux.go new file mode 100644 index 00000000..9321122d --- /dev/null +++ b/cmd/hull/e2fs_linux.go @@ -0,0 +1,50 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "os/exec" + "path/filepath" +) + +// e2fsprogs on Linux, found where distributions install it. +// +// /usr/sbin and /sbin are searched before PATH because they are usually not +// ON an ordinary user's PATH: Debian and Raspberry Pi OS put mke2fs in /sbin +// and give a non-root login a PATH without it, so a LookPath-only search fails +// on exactly the hosts that have it installed. +var ( + mke2fsBin = findE2fsTool("mke2fs") + debugfsBin = findE2fsTool("debugfs") + e2fsckBin = findE2fsTool("e2fsck") +) + +// findE2fsTool returns the first executable named name in /usr/sbin, /sbin or +// PATH. When there is none it still returns a path, /usr/sbin/, so the +// error buildBlockRootfs reports names a concrete file rather than a bare +// program name. +func findE2fsTool(name string) string { + for _, dir := range []string{"/usr/sbin", "/sbin"} { + p := filepath.Join(dir, name) + if info, err := os.Stat(p); err == nil && !info.IsDir() && info.Mode()&0o111 != 0 { + return p + } + } + if p, err := exec.LookPath(name); err == nil { + return p + } + return filepath.Join("/usr/sbin", name) +} diff --git a/cmd/hull/exec.go b/cmd/hull/exec.go index 8faeabd6..d683fda9 100644 --- a/cmd/hull/exec.go +++ b/cmd/hull/exec.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/exec_compose.go b/cmd/hull/exec_compose.go index 21ea6fa8..8cf2498c 100644 --- a/cmd/hull/exec_compose.go +++ b/cmd/hull/exec_compose.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/hviargv.go b/cmd/hull/hviargv.go new file mode 100644 index 00000000..6780dcd4 --- /dev/null +++ b/cmd/hull/hviargv.go @@ -0,0 +1,104 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "strconv" + + "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// vzVmmName is urunc's name for the Virtualization.framework backend, spelled +// out for the Linux build, where urunc does not declare it. +// hviargv_darwin_test.go pins it to urunc's own value. +const vzVmmName = "vz" + +// hviLinuxArgv builds the hvi command line hull runs on Linux. +// +// It is hull's own copy of urunc's HviDarwin.BuildExecCmd with the virtio-fs +// shares taken out, rather than a call into urunc's Linux Hvi monitor. That +// one is written for the containerd shim, not for hull: it passes no +// --agent-sock, so hull exec has nothing to dial; no --net-gateway, so the +// guest gets hvi's stub NIC instead of the gateway's network and policy; and +// no --net-mac, so every guest presents the same default MAC and the gateway, +// which keys its leases by it, hands two sandboxes the same address. A Linux +// destination for a sandbox started on a Mac has to boot it with the same +// devices it had there, so the argv here must match the darwin one flag for +// flag. Patching urunc would fork a dependency hull pins at a branch tip for +// a contract that belongs to hull; mirroring the dozen lines keeps the +// contract here, and hviargv_darwin_test.go fails if the two drift. +// +// Shares are refused rather than dropped: hvi's KVM backend has no virtio-fs +// server (it parses --share-* and configures nothing), so a guest that asked +// for a host directory would boot without it and fail somewhere far from +// here. --fs-uid and --fs-gid only describe shares, so they go too. +func hviLinuxArgv(binary string, args types.ExecArgs) ([]string, error) { + kernel := args.KernelPath + if kernel == "" { + kernel = args.UnikernelPath + } + if kernel == "" { + return nil, errors.New("hvi: no kernel image to boot") + } + if args.Sharedfs.Path != "" || len(args.SharedDirs) > 0 { + return nil, errors.New("hvi on Linux has no virtio-fs server, so it cannot share a host " + + "directory with the guest; use --rootfs-type block and no --shared-dir") + } + // A bare --net on hvi is its stub NIC, which forwards nothing. On macOS + // that path exists for vmnet; on Linux there is no vmnet and no reason to + // boot a guest whose network silently goes nowhere. + if args.Net.TapDev != "" && args.Net.UnixSocket == "" { + return nil, errors.New("hvi on Linux reaches the network only through a gateway: " + + "pass --gateway-sock and --gateway-cidr, or --net none") + } + mem := hypervisors.DefaultMemory + if args.MemSizeB != 0 { + mem = args.MemSizeB / (1024 * 1024) + } + vcpus := args.VCPUs + if vcpus == 0 { + vcpus = 1 + } + cmd := []string{ + binary, "boot", + "--kernel", kernel, + "--mem-mib", strconv.FormatUint(mem, 10), + "--cpus", strconv.FormatUint(uint64(vcpus), 10), + } + if args.InitrdPath != "" { + cmd = append(cmd, "--initramfs", args.InitrdPath) + } + if args.Command != "" { + cmd = append(cmd, "--cmdline", args.Command) + } + if args.BlockDevPath != "" { + cmd = append(cmd, "--disk", args.BlockDevPath) + } + if args.Net.UnixSocket != "" { + cmd = append(cmd, "--net-gateway", args.Net.UnixSocket) + } + if args.Net.MAC != "" { + cmd = append(cmd, "--net-mac", args.Net.MAC) + } + if args.AgentSockPath != "" { + cmd = append(cmd, "--agent-sock", args.AgentSockPath) + } + if args.ContainerID != "" { + cmd = append(cmd, "--sandbox-id", args.ContainerID) + } + return cmd, nil +} diff --git a/cmd/hull/hviargv_darwin_test.go b/cmd/hull/hviargv_darwin_test.go new file mode 100644 index 00000000..2ba62e49 --- /dev/null +++ b/cmd/hull/hviargv_darwin_test.go @@ -0,0 +1,106 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "reflect" + "testing" + + "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// hviLinuxArgv is a copy of urunc's darwin builder minus shares, kept so hvi +// has one command-line contract on every host hull runs it on. This compares +// the two on the argument sets below, so it catches a flag urunc's darwin argv +// adds, drops or reorders for them; a flag added under some other condition +// needs a case here before it is caught. +func TestHviLinuxArgvMatchesTheDarwinBuilder(t *testing.T) { + for _, args := range []types.ExecArgs{ + {KernelPath: "/k"}, + { + ContainerID: "t1", + KernelPath: "/s/i/t1/kernel", + InitrdPath: "/s/i/t1/initrd", + BlockDevPath: "/s/i/t1/bundle/rootfs.ext4", + AgentSockPath: "/s/i/t1/agent.sock", + Command: "rdinit=/init console=ttyAMA0", + MemSizeB: 768 << 20, + VCPUs: 3, + Net: types.NetDevParams{UnixSocket: "/g/gw.sock.qemu", MAC: "52:54:00:aa:bb:cc"}, + }, + {KernelPath: "/k", Net: types.NetDevParams{MAC: "52:54:00:00:00:01"}, ContainerID: "none"}, + } { + darwin, err := hypervisors.NewHviDarwin("/opt/hull/hvi").BuildExecCmd(args, nil) + if err != nil { + t.Fatal(err) + } + linux, err := hviLinuxArgv("/opt/hull/hvi", args) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(linux, darwin) { + t.Errorf("argv drifted from urunc's darwin builder:\n linux %q\n darwin %q", linux, darwin) + } + } +} + +// vmm_linux.go spells the vz backend through vzVmmName, because urunc +// declares it only here. Pin that constant to urunc's value. +func TestLinuxVzNameMatchesUrunc(t *testing.T) { + if hypervisors.VzVmm != vzVmmName { + t.Fatalf("urunc's VzVmm is %q but vzVmmName is %q; update vzVmmName in hviargv.go", + hypervisors.VzVmm, vzVmmName) + } +} + +// --fs-uid and --fs-gid describe virtio-fs shares to hvi's file server, so the +// Linux argv, which has no shares, drops them where urunc's darwin builder +// would add them. Everything else must still match, in order. +func TestHviLinuxArgvDropsOnlyTheShareOwnerFlags(t *testing.T) { + args := types.ExecArgs{ + ContainerID: "t1", + KernelPath: "/k", + BlockDevPath: "/s/rootfs.ext4", + GuestUID: 1000, + GuestGID: 1000, + Net: types.NetDevParams{UnixSocket: "/g/gw.sock", MAC: "52:54:00:aa:bb:cc"}, + } + darwin, err := hypervisors.NewHviDarwin("/opt/hull/hvi").BuildExecCmd(args, nil) + if err != nil { + t.Fatal(err) + } + var want []string + dropped := 0 + for i := 0; i < len(darwin); i++ { + if darwin[i] == "--fs-uid" || darwin[i] == "--fs-gid" { + i++ // and its value + dropped++ + continue + } + want = append(want, darwin[i]) + } + if dropped != 2 { + t.Fatalf("urunc's darwin argv no longer carries --fs-uid and --fs-gid for a "+ + "non-root guest; re-check what hviLinuxArgv should drop: %q", darwin) + } + linux, err := hviLinuxArgv("/opt/hull/hvi", args) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(linux, want) { + t.Errorf("argv drifted from urunc's darwin builder:\n linux %q\n darwin %q (minus the share owner)", linux, want) + } +} diff --git a/cmd/hull/hviargv_test.go b/cmd/hull/hviargv_test.go new file mode 100644 index 00000000..0490eb4e --- /dev/null +++ b/cmd/hull/hviargv_test.go @@ -0,0 +1,86 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "reflect" + "strings" + "testing" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// The flags urunc's Linux hvi monitor leaves out are the ones this builder +// exists for: without --agent-sock there is no hull exec, without +// --net-gateway the guest has no network or policy, and without --net-mac two +// guests share a lease. +func TestHviLinuxArgvCarriesAgentGatewayAndMAC(t *testing.T) { + got, err := hviLinuxArgv("/opt/hull/hvi", types.ExecArgs{ + ContainerID: "t1", + KernelPath: "/s/i/t1/kernel", + InitrdPath: "/s/i/t1/initrd", + BlockDevPath: "/s/i/t1/bundle/rootfs.ext4", + AgentSockPath: "/s/i/t1/agent.sock", + Command: "rdinit=/init console=ttyAMA0", + MemSizeB: 1024 << 20, + VCPUs: 2, + Net: types.NetDevParams{UnixSocket: "/g/gw.sock.qemu", MAC: "52:54:00:01:02:03"}, + }) + if err != nil { + t.Fatal(err) + } + want := []string{ + "/opt/hull/hvi", "boot", + "--kernel", "/s/i/t1/kernel", + "--mem-mib", "1024", + "--cpus", "2", + "--initramfs", "/s/i/t1/initrd", + "--cmdline", "rdinit=/init console=ttyAMA0", + "--disk", "/s/i/t1/bundle/rootfs.ext4", + "--net-gateway", "/g/gw.sock.qemu", + "--net-mac", "52:54:00:01:02:03", + "--agent-sock", "/s/i/t1/agent.sock", + "--sandbox-id", "t1", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("argv:\n got %q\nwant %q", got, want) + } +} + +func TestHviLinuxArgvRefusesWhatKVMHviCannotServe(t *testing.T) { + base := types.ExecArgs{KernelPath: "/k"} + + withShare := base + withShare.SharedDirs = []types.SharedDirParams{{Path: "/home/u/src", Tag: "share0"}} + if _, err := hviLinuxArgv("hvi", withShare); err == nil || !strings.Contains(err.Error(), "virtio-fs") { + t.Errorf("a shared directory was not refused: %v", err) + } + + withRootShare := base + withRootShare.Sharedfs = types.SharedfsParams{Path: "/s/rootfs", Tag: "rootfs"} + if _, err := hviLinuxArgv("hvi", withRootShare); err == nil { + t.Error("a virtio-fs rootfs was not refused") + } + + stubNet := base + stubNet.Net = types.NetDevParams{TapDev: "en0"} + if _, err := hviLinuxArgv("hvi", stubNet); err == nil || !strings.Contains(err.Error(), "gateway") { + t.Errorf("a NIC with no gateway was not refused: %v", err) + } + + if _, err := hviLinuxArgv("hvi", types.ExecArgs{}); err == nil { + t.Error("an argv with no kernel was built") + } +} diff --git a/cmd/hull/images.go b/cmd/hull/images.go index b53692c4..c12ccbab 100644 --- a/cmd/hull/images.go +++ b/cmd/hull/images.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/inspect.go b/cmd/hull/inspect.go index 78b6c508..6906f4c4 100644 --- a/cmd/hull/inspect.go +++ b/cmd/hull/inspect.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/jsonout.go b/cmd/hull/jsonout.go index 183bdbc3..69e10393 100644 --- a/cmd/hull/jsonout.go +++ b/cmd/hull/jsonout.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/linuxrun.go b/cmd/hull/linuxrun.go new file mode 100644 index 00000000..3fc298a6 --- /dev/null +++ b/cmd/hull/linuxrun.go @@ -0,0 +1,116 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "fmt" +) + +// runRequest is the part of a `hull run` command line that decides whether +// this host can run it at all, read before anything is pulled or created. +type runRequest struct { + hypervisor string + rootfsType string + netMode string + gatewaySock string + sharedDirs int + sharedDirFds int + gui bool + rosetta bool +} + +// hypervisorAnnotation is the image's own choice of backend. +const hypervisorAnnotation = "com.urunc.unikernel.hypervisor" + +// selectHypervisor picks a run's backend and says where the choice came from, +// for the telemetry and for the refusal that names it: the --hypervisor flag, +// then the image's annotation, then the platform's default (qemu on macOS, +// for backward compatibility; hvi on Linux, the only backend there). +func selectHypervisor(flag string, annotations map[string]string, platformDefault string) (name, source string) { + if flag != "" { + return flag, "flag" + } + if annotation, ok := annotations[hypervisorAnnotation]; ok { + return annotation, "annotation" + } + return platformDefault, "default" +} + +// checkLinuxHypervisor refuses a backend hull on Linux does not have, naming +// where it was asked for. The flag is refused earlier, by checkLinuxRun; the +// annotation can only be read once the image is pulled. Before this existed +// checkLinuxRun filled in hvi whenever the flag was unset, which left the +// annotation unread: an image built for qemu or firecracker, a unikernel most +// of all, booted on hvi without a word. +func checkLinuxHypervisor(name, source string) error { + if name == "hvi" { + return nil + } + if source == "annotation" { + return fmt.Errorf("the image's %s annotation asks for %q, and hull on Linux runs only "+ + "hvi; pass --hypervisor hvi to run it on hvi regardless", hypervisorAnnotation, name) + } + return fmt.Errorf("hull on Linux runs only --hypervisor hvi (got %q)", name) +} + +// checkLinuxRun narrows a run to what hull on Linux can deliver, and fills in +// the Linux default rootfs, block. The backend is left as given: an unset +// flag leaves the choice to the image's annotation and then to the Linux +// default, which selectHypervisor and checkLinuxHypervisor apply once the +// image is pulled. +// +// Linux hull exists to be the other end of a moved hvi sandbox, so it offers +// the one configuration that can be moved and refuses the rest by name before +// anything is pulled -- a refusal after a multi-gigabyte pull and a 15 GiB +// image build is the same answer, late. What it refuses, and why: +// +// - any backend but hvi: vz and the macOS qemu builder do not exist here. +// - --rootfs-type virtiofs and 9pfs: hvi's KVM backend has no file server, +// so a directory rootfs would boot with no root at all. +// - --shared-dir and --shared-dir-fd: the same missing file server. +// - --net shared without --gateway-sock: on macOS that is vmnet, and here it +// would be hvi's stub NIC, which forwards nothing, plus a wait on macOS +// DHCP leases that never come. The gateway is the network on Linux. +// - --gui and --rosetta: Virtualization.framework features. +func checkLinuxRun(r runRequest) (runRequest, error) { + if r.hypervisor != "" { + if err := checkLinuxHypervisor(r.hypervisor, "flag"); err != nil { + return r, err + } + } + switch r.rootfsType { + case "", "block": + r.rootfsType = "block" + default: + return r, fmt.Errorf("hull on Linux runs only --rootfs-type block (got %q): hvi's KVM "+ + "backend has no virtio-fs server to serve a directory rootfs", r.rootfsType) + } + if r.sharedDirs > 0 || r.sharedDirFds > 0 { + return r, errors.New("hull on Linux cannot share host directories (--shared-dir, " + + "--shared-dir-fd): hvi's KVM backend has no virtio-fs server") + } + if r.gui { + return r, errors.New("--gui needs Virtualization.framework and is macOS only") + } + if r.rosetta { + return r, errors.New("--rosetta needs Virtualization.framework and is macOS only") + } + if r.netMode != "none" && r.gatewaySock == "" { + return r, fmt.Errorf("hull on Linux reaches the network only through a gateway: "+ + "pass --gateway-sock and --gateway-cidr, or --net none (got --net %s)", r.netMode) + } + return r, nil +} diff --git a/cmd/hull/linuxrun_test.go b/cmd/hull/linuxrun_test.go new file mode 100644 index 00000000..02b49736 --- /dev/null +++ b/cmd/hull/linuxrun_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "strings" + "testing" +) + +// What brig sends, in both network modes, must pass and come out on a block +// rootfs. An unset backend stays unset, so the image's annotation still gets +// its say once the image is pulled. +func TestCheckLinuxRunAcceptsWhatBrigSends(t *testing.T) { + for _, r := range []runRequest{ + {hypervisor: "hvi", rootfsType: "block", netMode: "none"}, + {hypervisor: "hvi", rootfsType: "block", netMode: "shared", gatewaySock: "/tmp/gw.sock"}, + {netMode: "none"}, + } { + got, err := checkLinuxRun(r) + if err != nil { + t.Errorf("%+v refused: %v", r, err) + continue + } + if got.hypervisor != r.hypervisor || got.rootfsType != "block" { + t.Errorf("%+v came out as %q/%q, want %q/block", r, got.hypervisor, got.rootfsType, r.hypervisor) + } + } +} + +// The flag wins over the image's annotation, which wins over the platform's +// default, on both platforms. +func TestSelectHypervisorPrecedence(t *testing.T) { + qemuImage := map[string]string{hypervisorAnnotation: "qemu"} + for _, tc := range []struct { + flag string + annotations map[string]string + def string + want, src string + }{ + {"vz", qemuImage, "qemu", "vz", "flag"}, + {"", qemuImage, "qemu", "qemu", "annotation"}, + {"", nil, "qemu", "qemu", "default"}, + {"hvi", qemuImage, "hvi", "hvi", "flag"}, + {"", qemuImage, "hvi", "qemu", "annotation"}, + {"", map[string]string{"other": "x"}, "hvi", "hvi", "default"}, + } { + name, src := selectHypervisor(tc.flag, tc.annotations, tc.def) + if name != tc.want || src != tc.src { + t.Errorf("selectHypervisor(%q, %v, %q) = %q from %s, want %q from %s", + tc.flag, tc.annotations, tc.def, name, src, tc.want, tc.src) + } + } +} + +// An image annotated for another backend is refused on Linux, naming the +// annotation, rather than booted on hvi without a word. The flag still +// overrides it, and the Linux default passes. +func TestLinuxRefusesAnImageAnnotatedForAnotherBackend(t *testing.T) { + name, src := selectHypervisor("", map[string]string{hypervisorAnnotation: "qemu"}, "hvi") + err := checkLinuxHypervisor(name, src) + if err == nil { + t.Fatal("an image annotated for qemu was accepted on Linux") + } + if !strings.Contains(err.Error(), hypervisorAnnotation) || !strings.Contains(err.Error(), "--hypervisor hvi") { + t.Errorf("refusal %q names neither the annotation nor the way out", err) + } + for _, flag := range []string{"hvi", ""} { + name, src := selectHypervisor(flag, nil, "hvi") + if err := checkLinuxHypervisor(name, src); err != nil { + t.Errorf("selectHypervisor(%q) on Linux was refused: %v", flag, err) + } + } + name, src = selectHypervisor("hvi", map[string]string{hypervisorAnnotation: "firecracker"}, "hvi") + if err := checkLinuxHypervisor(name, src); err != nil { + t.Errorf("--hypervisor hvi did not override the annotation: %v", err) + } +} + +// Each refusal has to name the thing that was refused, so the operator knows +// which flag to drop. +func TestCheckLinuxRunRefusesWhatLinuxCannotRun(t *testing.T) { + for _, tc := range []struct { + r runRequest + want string + }{ + {runRequest{hypervisor: "vz", netMode: "none"}, "--hypervisor hvi"}, + {runRequest{hypervisor: "qemu", netMode: "none"}, "--hypervisor hvi"}, + {runRequest{rootfsType: "virtiofs", netMode: "none"}, "--rootfs-type block"}, + {runRequest{rootfsType: "9pfs", netMode: "none"}, "--rootfs-type block"}, + {runRequest{netMode: "none", sharedDirs: 1}, "--shared-dir"}, + {runRequest{netMode: "none", sharedDirFds: 1}, "--shared-dir-fd"}, + {runRequest{netMode: "none", gui: true}, "--gui"}, + {runRequest{netMode: "none", rosetta: true}, "--rosetta"}, + {runRequest{netMode: "shared"}, "--gateway-sock"}, + } { + _, err := checkLinuxRun(tc.r) + if err == nil { + t.Errorf("%+v was accepted", tc.r) + continue + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("%+v: error %q does not mention %q", tc.r, err, tc.want) + } + } +} diff --git a/cmd/hull/logcapture_test.go b/cmd/hull/logcapture_test.go new file mode 100644 index 00000000..ae7c04c1 --- /dev/null +++ b/cmd/hull/logcapture_test.go @@ -0,0 +1,38 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "testing" + + "github.com/sirupsen/logrus" +) + +// captureWarnings collects everything logged while fn runs. +func captureWarnings(t *testing.T, fn func()) string { + t.Helper() + var buf bytes.Buffer + prev := logrus.StandardLogger().Out + prevLevel := logrus.GetLevel() + logrus.SetOutput(&buf) + logrus.SetLevel(logrus.DebugLevel) + t.Cleanup(func() { + logrus.SetOutput(prev) + logrus.SetLevel(prevLevel) + }) + fn() + return buf.String() +} diff --git a/cmd/hull/logs.go b/cmd/hull/logs.go index 039d35d9..c5d90231 100644 --- a/cmd/hull/logs.go +++ b/cmd/hull/logs.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/main.go b/cmd/hull/main.go index da3d0686..0926e147 100644 --- a/cmd/hull/main.go +++ b/cmd/hull/main.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( @@ -53,7 +51,8 @@ func main() { // Keep the image/instance store on the case-sensitive APFS volume mounted // at ~/.hull/store. Linux package trees contain names that differ only by // case (for example xt_CONNMARK.h and xt_connmark.h), which cannot coexist - // on the default case-insensitive home volume. + // on the default case-insensitive home volume. On Linux the same path is a + // plain directory; see ensureStore in storemount_linux.go. storeDirDefault := filepath.Join(homeDir, ".hull", "store") app := &cli.Command{ diff --git a/cmd/hull/oneshot.go b/cmd/hull/oneshot.go index c4fdcedf..3b32bff0 100644 --- a/cmd/hull/oneshot.go +++ b/cmd/hull/oneshot.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - // One-shot compose services. // // A one-shot service is a job: it runs a command to completion and its exit diff --git a/cmd/hull/proc_linux.go b/cmd/hull/proc_linux.go new file mode 100644 index 00000000..9bb01588 --- /dev/null +++ b/cmd/hull/proc_linux.go @@ -0,0 +1,131 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +// inspectProcess reads what processIsAVMM needs to know about pid from /proc. +// +// procps' ps takes the same "-p PID -o lstart=,command=" the macOS build runs, +// but its lstart has one-second resolution and is printed in local time to be +// parsed back, and its command column has already joined argv with spaces. +// /proc has the argv exactly and the start time in clock ticks, so it is read +// directly. argv0 comes from cmdline, not from comm: comm is cut to fifteen +// characters and is whatever the process last set with prctl, while argv[0] is +// what hull itself put there when it spawned the VMM. +func inspectProcess(pid int) (procInfo, bool) { + dir := "/proc/" + strconv.Itoa(pid) + argv, ok := readProcArgv(dir) + if !ok { + return procInfo{}, false + } + info := procInfo{argv0: argv[0], command: strings.Join(argv, " ")} + started, err := processStartTime(dir) + if err != nil { + log.Debugf("could not read the start time of pid %d: %v", pid, err) + return info, true + } + info.started, info.startKnown = started, true + return info, true +} + +// readProcArgv reads a process's argv, waiting briefly when it reads empty. +// +// Empty is what a kernel thread and a zombie read as, and neither is a VMM +// hull can stop. But it is also what a process reads as for a moment in the +// middle of execve: the kernel releases a vfork parent -- Go's os/exec spawns +// with CLONE_VFORK, so that is every child hull starts -- as soon as the new +// address space is installed, and lays out the new argv in it only after. A +// check that runs straight after spawning a VMM, as reapOrphanVMM's re-check +// can, would otherwise see "not a VMM" and walk past one. 50ms is far longer +// than that window and is paid only on an empty read. +func readProcArgv(dir string) ([]string, bool) { + for attempt := 0; ; attempt++ { + raw, err := os.ReadFile(dir + "/cmdline") + if err != nil { + return nil, false + } + if argv := parseProcCmdline(raw); len(argv) > 0 { + return argv, true + } + if attempt == 10 { + return nil, false + } + time.Sleep(5 * time.Millisecond) + } +} + +func processStartTime(dir string) (t time.Time, err error) { + stat, err := os.ReadFile(dir + "/stat") + if err != nil { + return t, err + } + ticks, err := parseProcStatStartTicks(stat) + if err != nil { + return t, err + } + sys, err := os.ReadFile("/proc/stat") + if err != nil { + return t, err + } + boot, err := parseProcBootTime(sys) + if err != nil { + return t, err + } + return procStartTime(boot, ticks), nil +} + +// listProcesses returns every process's pid and space-joined argv, from the +// numeric directories under /proc. +func listProcesses() ([]procEntry, error) { + entries, err := os.ReadDir("/proc") + if err != nil { + return nil, err + } + var procs []procEntry + for _, e := range entries { + pid, perr := strconv.Atoi(e.Name()) + if perr != nil || pid <= 0 { + continue + } + // A process can exit between the listing and the read; that is not + // an error, it is one fewer process. + raw, rerr := os.ReadFile("/proc/" + e.Name() + "/cmdline") + if rerr != nil { + continue + } + argv := parseProcCmdline(raw) + if len(argv) == 0 { + continue + } + procs = append(procs, procEntry{pid: pid, command: strings.Join(argv, " ")}) + } + return procs, nil +} + +// processArgv returns the argv of pid as the kernel holds it. +func processArgv(pid int) ([]string, error) { + argv, ok := readProcArgv("/proc/" + strconv.Itoa(pid)) + if !ok { + return nil, fmt.Errorf("pid %d has no argv", pid) + } + return argv, nil +} diff --git a/cmd/hull/proc_linux_test.go b/cmd/hull/proc_linux_test.go new file mode 100644 index 00000000..fcdf728e --- /dev/null +++ b/cmd/hull/proc_linux_test.go @@ -0,0 +1,95 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// The Linux identity check reads /proc, and has to agree with the recorded +// argv exactly -- spaces inside an argument included -- and report a start +// time that the pid-reuse check can trust. +func TestInspectProcessReadsArgvAndStartTimeFromProc(t *testing.T) { + before := time.Now() + cmd := exec.Command("/bin/sh", "-c", "while :; do sleep 1; done # an argument with spaces") + cmd.Args[0] = "hvi" + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + + info, ok := inspectProcess(cmd.Process.Pid) + if !ok { + t.Fatal("a live child was not found in /proc") + } + if want := strings.Join(cmd.Args, " "); info.command != want { + t.Errorf("command = %q, want %q", info.command, want) + } + if info.argv0 != "hvi" { + t.Errorf("argv0 = %q, want hvi", info.argv0) + } + if !info.startKnown { + t.Fatal("no start time") + } + // Clock ticks are 10ms and btime is whole seconds, so allow a second of + // slack either side. + if info.started.Before(before.Add(-2*time.Second)) || info.started.After(time.Now().Add(2*time.Second)) { + t.Errorf("start time %v is not around %v", info.started, before) + } + + procs, err := listProcesses() + if err != nil { + t.Fatal(err) + } + found := false + for _, p := range procs { + if p.pid == cmd.Process.Pid && p.command == info.command { + found = true + } + } + if !found { + t.Error("listProcesses did not list the child with its argv") + } +} + +// e2fsprogs lives in /sbin on Debian and Raspberry Pi OS, which an ordinary +// login's PATH does not include. The lookup must still find it there. +// +// findE2fsTool falls back to /usr/sbin/ when it finds nothing, so a +// prefix check alone would pass on a host with no e2fsprogs at all. The +// result has to exist. +func TestE2fsToolsAreFoundOutsidePath(t *testing.T) { + installed := false + for _, p := range []string{"/usr/sbin/mke2fs", "/sbin/mke2fs"} { + if _, err := os.Stat(p); err == nil { + installed = true + } + } + if !installed { + t.Skip("e2fsprogs is not installed in /usr/sbin or /sbin") + } + t.Setenv("PATH", "/nonexistent") + got := findE2fsTool("mke2fs") + if !strings.HasPrefix(got, "/usr/sbin/") && !strings.HasPrefix(got, "/sbin/") { + t.Fatalf("findE2fsTool(mke2fs) = %q, want a path under /usr/sbin or /sbin", got) + } + if _, err := os.Stat(got); err != nil { + t.Fatalf("findE2fsTool(mke2fs) = %q, which does not exist: %v", got, err) + } +} diff --git a/cmd/hull/procfs.go b/cmd/hull/procfs.go new file mode 100644 index 00000000..98c3f329 --- /dev/null +++ b/cmd/hull/procfs.go @@ -0,0 +1,93 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +// The parsers for Linux's /proc files that process identity rests on. They +// take bytes rather than a pid so they can be tested against fixtures on any +// host: proc_linux.go reads the files and hands them over. + +// procClockTicks is USER_HZ, the unit of /proc/PID/stat's starttime. The kernel +// reports times there in USER_HZ whatever its internal HZ is, and USER_HZ is +// 100 on every architecture Linux ships for, which is what sysconf(_SC_CLK_TCK) +// returns; reading it through cgo would buy nothing but a cgo dependency. +const procClockTicks = 100 + +// parseProcCmdline splits /proc/PID/cmdline into argv. +// +// The file is the argv strings each followed by a NUL. Splitting on NUL keeps +// an argument that contains spaces whole, which a ps listing cannot: there the +// boundaries are already gone. +func parseProcCmdline(data []byte) []string { + data = bytes.TrimSuffix(data, []byte{0}) + if len(data) == 0 { + return nil + } + parts := bytes.Split(data, []byte{0}) + argv := make([]string, len(parts)) + for i, p := range parts { + argv[i] = string(p) + } + return argv +} + +// parseProcStatStartTicks returns field 22 of /proc/PID/stat, the process's +// start time in clock ticks after boot. +// +// Fields are counted from after the LAST closing parenthesis. Field 2 is the +// command name in parentheses, and that name is whatever the process chose: +// it can contain spaces and parentheses of its own, so splitting the whole +// line on spaces lets a process called "a) 1 2 3" shift every field after it +// and pass off any number it likes as its start time. +func parseProcStatStartTicks(data []byte) (uint64, error) { + i := bytes.LastIndexByte(data, ')') + if i < 0 { + return 0, errors.New("stat: no command name") + } + // After ") " the next field is 3 (state); starttime is field 22, so it is + // the twentieth field from here. + fields := strings.Fields(string(data[i+1:])) + const startField = 22 - 3 + if len(fields) <= startField { + return 0, fmt.Errorf("stat: %d fields after the command name, want more than %d", len(fields), startField) + } + return strconv.ParseUint(fields[startField], 10, 64) +} + +// parseProcBootTime returns btime from /proc/stat: the boot time in seconds +// since the epoch, which is what starttime counts from. +func parseProcBootTime(data []byte) (int64, error) { + for _, line := range strings.Split(string(data), "\n") { + if v, ok := strings.CutPrefix(line, "btime "); ok { + return strconv.ParseInt(strings.TrimSpace(v), 10, 64) + } + } + return 0, errors.New("/proc/stat has no btime line") +} + +// procStartTime converts a starttime in clock ticks after boot to wall time. +func procStartTime(bootTime int64, ticks uint64) time.Time { + sec := int64(ticks / procClockTicks) + nsec := int64(ticks%procClockTicks) * int64(time.Second/procClockTicks) + return time.Unix(bootTime+sec, nsec) +} diff --git a/cmd/hull/procfs_test.go b/cmd/hull/procfs_test.go new file mode 100644 index 00000000..96736b5b --- /dev/null +++ b/cmd/hull/procfs_test.go @@ -0,0 +1,82 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "reflect" + "testing" + "time" +) + +func TestParseProcCmdlineKeepsArgumentsWhole(t *testing.T) { + raw := []byte("/home/u/hull-bin/hvi\x00boot\x00--cmdline\x00rdinit=/init console=ttyAMA0\x00") + want := []string{"/home/u/hull-bin/hvi", "boot", "--cmdline", "rdinit=/init console=ttyAMA0"} + if got := parseProcCmdline(raw); !reflect.DeepEqual(got, want) { + t.Fatalf("parseProcCmdline = %q, want %q", got, want) + } + // A kernel thread or a zombie has an empty cmdline, and is no process + // hull could have spawned. + if got := parseProcCmdline(nil); got != nil { + t.Fatalf("empty cmdline parsed as %q", got) + } + // An empty argument in the middle is still an argument. + if got := parseProcCmdline([]byte("a\x00\x00b\x00")); !reflect.DeepEqual(got, []string{"a", "", "b"}) { + t.Fatalf("empty middle argument lost: %q", got) + } +} + +// The command name in /proc/PID/stat is the process's own choice and may hold +// spaces and parentheses. Counting fields from the first ")" would let a +// process named to imitate the rest of the line pick its own start time, and +// with it get past the pid-reuse check in processIsAVMM. +func TestParseProcStatStartTicksSkipsAHostileCommandName(t *testing.T) { + // Fields 3..21 are filler; field 22 (starttime) is 4242. + tail := "S 1 1 1 0 -1 4194560 100 0 0 0 1 2 0 0 20 0 1 0 4242 1000 10" + for _, comm := range []string{"hvi", "a b", "x) 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 99 (y", "))"} { + line := []byte("1234 (" + comm + ") " + tail + "\n") + got, err := parseProcStatStartTicks(line) + if err != nil { + t.Fatalf("comm %q: %v", comm, err) + } + if got != 4242 { + t.Errorf("comm %q: starttime = %d, want 4242", comm, got) + } + } + if _, err := parseProcStatStartTicks([]byte("1234 (hvi) S 1 2")); err == nil { + t.Error("a truncated stat line was accepted") + } + if _, err := parseProcStatStartTicks([]byte("garbage")); err == nil { + t.Error("a stat line with no command name was accepted") + } +} + +func TestParseProcBootTime(t *testing.T) { + raw := []byte("cpu 1 2 3 4\nintr 5\nctxt 6\nbtime 1757000000\nprocesses 7\n") + got, err := parseProcBootTime(raw) + if err != nil || got != 1757000000 { + t.Fatalf("parseProcBootTime = %d, %v", got, err) + } + if _, err := parseProcBootTime([]byte("cpu 1\n")); err == nil { + t.Error("a /proc/stat with no btime was accepted") + } +} + +func TestProcStartTimeCountsClockTicksFromBoot(t *testing.T) { + got := procStartTime(1000, 12345) // 123.45 s after boot + want := time.Unix(1123, 450_000_000) + if !got.Equal(want) { + t.Fatalf("procStartTime = %v, want %v", got, want) + } +} diff --git a/cmd/hull/prune.go b/cmd/hull/prune.go index 966302c5..2adc9cdc 100644 --- a/cmd/hull/prune.go +++ b/cmd/hull/prune.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/ps.go b/cmd/hull/ps.go index 19ac21a8..132837e9 100644 --- a/cmd/hull/ps.go +++ b/cmd/hull/ps.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/pull.go b/cmd/hull/pull.go index 9ed12839..0e72d719 100644 --- a/cmd/hull/pull.go +++ b/cmd/hull/pull.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/rm.go b/cmd/hull/rm.go index 00d9cfac..4cf60040 100644 --- a/cmd/hull/rm.go +++ b/cmd/hull/rm.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/rm_test.go b/cmd/hull/rm_test.go index 6e0c6b4c..83849a71 100644 --- a/cmd/hull/rm_test.go +++ b/cmd/hull/rm_test.go @@ -1,5 +1,3 @@ -//go:build darwin - // Copyright (c) 2026, NOFire AI // SPDX-License-Identifier: Apache-2.0 diff --git a/cmd/hull/rmi.go b/cmd/hull/rmi.go index 3081f953..ef5fa260 100644 --- a/cmd/hull/rmi.go +++ b/cmd/hull/rmi.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/rootfs_containment_test.go b/cmd/hull/rootfs_containment_test.go index 70d7994c..fb361aa7 100644 --- a/cmd/hull/rootfs_containment_test.go +++ b/cmd/hull/rootfs_containment_test.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main // --- review finding: the pull escape ---------------------------------------- diff --git a/cmd/hull/run.go b/cmd/hull/run.go index 5b4fc875..9ba4eeb8 100644 --- a/cmd/hull/run.go +++ b/cmd/hull/run.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( @@ -248,6 +246,22 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { if guiTitle != "" && !gui { return errors.New("--gui-title requires --gui") } + // Before the store is opened and the image pulled: a host that cannot run + // this combination should say so before it has downloaded anything. + preflight, err := platformRunPreflight(runRequest{ + hypervisor: hypervisorOverride, + rootfsType: rootfsTypeOverride, + netMode: netMode, + gatewaySock: gatewaySock, + sharedDirs: len(sharedDirs), + sharedDirFds: len(sharedDirFds), + gui: gui, + rosetta: cmd.Bool("rosetta"), + }) + if err != nil { + return err + } + hypervisorOverride, rootfsTypeOverride = preflight.hypervisor, preflight.rootfsType // Generate instance ID if not provided if instanceName == "" { @@ -500,27 +514,17 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { } // Determine VMM and unikernel types from annotations or CLI override - // Default to QEMU for backward compatibility - var vmmType hypervisors.VmmType - vmmName := "" + vmmName, vmmSource := selectHypervisor(hypervisorOverride, ociSpec.Annotations, defaultHypervisor) + log.Debugf("Using hypervisor %s (from %s)", vmmName, vmmSource) - // CLI override takes precedence - vmmSource := "default" - if hypervisorOverride != "" { - vmmName = hypervisorOverride - vmmSource = "flag" - log.Debugf("Using hypervisor from --hypervisor flag: %s", vmmName) - } else if annotation, ok := ociSpec.Annotations["com.urunc.unikernel.hypervisor"]; ok { - vmmName = annotation - vmmSource = "annotation" - log.Debugf("Using hypervisor from annotation: %s", vmmName) - } else { - vmmName = "qemu" - log.Debugf("No hypervisor override or annotation, using default: qemu") + vmmName = normalizeVmmName(vmmName) + // The flag was checked before the pull; an annotation can only be checked + // now that the image is here. + if err := platformCheckHypervisor(vmmName, vmmSource); err != nil { + return err } - vmmName = normalizeVmmName(vmmName) - vmmType = hypervisors.VmmType(vmmName) + vmmType := hypervisors.VmmType(vmmName) telemetryBackendSource = vmmSource telemetryBackend = vmmName // Graphics is only supported on the Vz backend; the QEMU darwin builder diff --git a/cmd/hull/runpreflight_darwin.go b/cmd/hull/runpreflight_darwin.go new file mode 100644 index 00000000..f8ac12f9 --- /dev/null +++ b/cmd/hull/runpreflight_darwin.go @@ -0,0 +1,27 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// platformRunPreflight has nothing to narrow on macOS: every backend and +// rootfs mode run.go knows is available here. +func platformRunPreflight(r runRequest) (runRequest, error) { return r, nil } + +// defaultHypervisor is qemu, for backward compatibility: it was the only +// backend when the flag and the annotation were both optional. +const defaultHypervisor = "qemu" + +// platformCheckHypervisor has nothing to refuse on macOS; an unknown name +// fails in newVMM. +func platformCheckHypervisor(string, string) error { return nil } diff --git a/cmd/hull/runpreflight_linux.go b/cmd/hull/runpreflight_linux.go new file mode 100644 index 00000000..a5c01da3 --- /dev/null +++ b/cmd/hull/runpreflight_linux.go @@ -0,0 +1,24 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// platformRunPreflight applies checkLinuxRun. +func platformRunPreflight(r runRequest) (runRequest, error) { return checkLinuxRun(r) } + +// defaultHypervisor is hvi: it is the only backend hull on Linux has. +const defaultHypervisor = "hvi" + +// platformCheckHypervisor applies checkLinuxHypervisor. +func platformCheckHypervisor(name, source string) error { return checkLinuxHypervisor(name, source) } diff --git a/cmd/hull/setuidwarn_test.go b/cmd/hull/setuidwarn_test.go index 8b5643d6..733fbb01 100644 --- a/cmd/hull/setuidwarn_test.go +++ b/cmd/hull/setuidwarn_test.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build darwin + package main import ( - "bytes" "go/ast" "go/parser" "go/token" @@ -23,26 +24,8 @@ import ( "path/filepath" "strings" "testing" - - "github.com/sirupsen/logrus" ) -// captureWarnings collects everything logged while fn runs. -func captureWarnings(t *testing.T, fn func()) string { - t.Helper() - var buf bytes.Buffer - prev := logrus.StandardLogger().Out - prevLevel := logrus.GetLevel() - logrus.SetOutput(&buf) - logrus.SetLevel(logrus.DebugLevel) - t.Cleanup(func() { - logrus.SetOutput(prev) - logrus.SetLevel(prevLevel) - }) - fn() - return buf.String() -} - // writeRootfs lays out a rootfs containing rel at the given mode. func writeRootfs(t *testing.T, rel string, mode os.FileMode) string { t.Helper() diff --git a/cmd/hull/shortflags_test.go b/cmd/hull/shortflags_test.go index 48d890dd..3028c113 100644 --- a/cmd/hull/shortflags_test.go +++ b/cmd/hull/shortflags_test.go @@ -1,6 +1,8 @@ // Copyright (c) 2026, NOFire AI // SPDX-License-Identifier: Apache-2.0 +//go:build darwin + package main import ( diff --git a/cmd/hull/stop.go b/cmd/hull/stop.go index d7209797..bf622a9c 100644 --- a/cmd/hull/stop.go +++ b/cmd/hull/stop.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/storemount_linux.go b/cmd/hull/storemount_linux.go new file mode 100644 index 00000000..26414e32 --- /dev/null +++ b/cmd/hull/storemount_linux.go @@ -0,0 +1,52 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" +) + +// ensureStore makes storeDir ready to hold a store on Linux: a plain directory. +// +// The macOS build mounts a case-sensitive APFS sparse image here (see +// storemount_darwin.go), because the default macOS volume folds case and a +// Linux rootfs holds names that differ only by it. A Linux filesystem is case +// sensitive already, and there is no hdiutil, so the directory itself is the +// store. It is still probed rather than assumed, because a store pointed at a +// vfat or exFAT mount, or a case-folding ext4 directory, would unpack an image +// whose colliding names silently overwrite each other and boot a guest that +// is not the image. +// +// 0700 because the mode bits are the store's only access control: the +// instance state, the urunit config with the guest's environment, and the +// block image all live under it. +func ensureStore(storeDir, _ string) error { + if err := os.MkdirAll(storeDir, 0o700); err != nil { + return fmt.Errorf("create store directory: %w", err) + } + sensitive, err := dirIsCaseSensitive(storeDir) + if err != nil { + return fmt.Errorf("check whether %s is case-sensitive: %w", storeDir, err) + } + if !sensitive { + return fmt.Errorf("%s is on a case-insensitive filesystem; a Linux rootfs holds names "+ + "that differ only by case, so point --store-dir at a case-sensitive one", storeDir) + } + return nil +} + +// storeCompactHint is empty: a plain directory gives freed space back at once. +func storeCompactHint(string) string { return "" } diff --git a/cmd/hull/supervisor.go b/cmd/hull/supervisor.go index 37f56dbb..e440d300 100644 --- a/cmd/hull/supervisor.go +++ b/cmd/hull/supervisor.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - // Restart policies and the project supervisor. // // The per-project gateway daemon is already the one host process with the diff --git a/cmd/hull/telemetry.go b/cmd/hull/telemetry.go index 207ef559..2a92dd9b 100644 --- a/cmd/hull/telemetry.go +++ b/cmd/hull/telemetry.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/terminal.go b/cmd/hull/terminal.go index 73a427eb..62287b11 100644 --- a/cmd/hull/terminal.go +++ b/cmd/hull/terminal.go @@ -1,5 +1,3 @@ -//go:build darwin - package main import ( diff --git a/cmd/hull/terminal_ioctl_linux.go b/cmd/hull/terminal_ioctl_linux.go new file mode 100644 index 00000000..cf40012e --- /dev/null +++ b/cmd/hull/terminal_ioctl_linux.go @@ -0,0 +1,25 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "syscall" + +// See terminal_ioctl_darwin.go. TCSETS applies the change at once, which is +// what TIOCSETA does on macOS; TCSETSW would wait for output to drain and +// change when a raw-mode switch lands relative to the guest's first bytes. +const ( + ioctlGetTermios = syscall.TCGETS + ioctlSetTermios = syscall.TCSETS +) diff --git a/cmd/hull/version.go b/cmd/hull/version.go index 93377191..6aa57610 100644 --- a/cmd/hull/version.go +++ b/cmd/hull/version.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build darwin - package main import ( diff --git a/cmd/hull/vmm_linux.go b/cmd/hull/vmm_linux.go new file mode 100644 index 00000000..fe8cc3ce --- /dev/null +++ b/cmd/hull/vmm_linux.go @@ -0,0 +1,110 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" + "golang.org/x/sys/unix" +) + +// vzVmm is the value urunc gives Virtualization.framework on macOS. urunc does +// not declare it on Linux, but run.go compares against it on both, so the +// Linux build carries the same string, from vzVmmName. +const vzVmm hypervisors.VmmType = vzVmmName + +// newVMM builds the monitor for a Linux run. Only hvi is offered: it is the +// backend hull's Linux support is built around, and the one a sandbox moved +// from another KVM host was running (see checkLinuxRun). +// +// urunc's Linux factory is not used for it. It finds hvi only on PATH, so the +// hvi installed beside hull -- the layout make and the release archive +// produce, and the one macOS resolves first -- is not found; and its argv is +// the containerd shim's, not hull's. See hviLinuxArgv. +func newVMM(vmmType hypervisors.VmmType, monitors map[string]types.MonitorConfig) (types.VMM, error) { + if vmmType != hypervisors.HviVmm { + return nil, fmt.Errorf("hull on Linux runs only the hvi backend (got %q)", vmmType) + } + path, err := linuxHviPath(monitors) + if err != nil { + return nil, err + } + return &hviLinux{binaryPath: path}, nil +} + +// setVzQMPSocket has nothing to do on Linux: there is no vz-runner. +func setVzQMPSocket(types.VMM, string) {} + +// linuxHviPath resolves hvi the way urunc's darwin factory does: an explicit +// monitor config first, then the binary installed beside hull, then PATH. +// Beside hull comes before PATH so that a hull and hvi built together are run +// together, rather than this hull driving whatever older hvi is on PATH. +func linuxHviPath(monitors map[string]types.MonitorConfig) (string, error) { + if p := monitors[string(hypervisors.HviVmm)].BinaryPath; p != "" { + return p, nil + } + if exe, err := os.Executable(); err == nil { + candidate := filepath.Join(filepath.Dir(exe), hypervisors.HviBinary) + if info, statErr := os.Stat(candidate); statErr == nil && !info.IsDir() && info.Mode()&0o111 != 0 { + return candidate, nil + } + } + if p, err := exec.LookPath(hypervisors.HviBinary); err == nil { + return p, nil + } + return "", errors.New("hvi not found next to hull or in PATH") +} + +// hviLinux is hull's hvi monitor on Linux. Only BuildExecCmd carries any +// logic; the rest answers the urunc VMM interface the way urunc's own hvi +// monitors do. +type hviLinux struct { + binaryPath string +} + +func (h *hviLinux) BuildExecCmd(args types.ExecArgs, _ types.Unikernel) ([]string, error) { + return hviLinuxArgv(h.binaryPath, args) +} + +func (h *hviLinux) PreExec(types.ExecArgs) error { return nil } + +func (h *hviLinux) Stop(pid int) error { + if err := unix.Kill(pid, unix.SIGKILL); err != nil && !errors.Is(err, unix.ESRCH) { + return err + } + return nil +} + +func (h *hviLinux) Signal(pid int, sig unix.Signal) error { return unix.Kill(pid, sig) } + +func (h *hviLinux) Path() string { return h.binaryPath } + +func (h *hviLinux) UsesKVM() bool { return true } + +func (h *hviLinux) SupportsSharedfs(string) bool { return false } + +func (h *hviLinux) Ok() error { + info, err := os.Stat(h.binaryPath) + if err != nil || info.IsDir() || info.Mode()&0o111 == 0 { + return fmt.Errorf("hvi not found or not executable at %s", h.binaryPath) + } + return nil +} diff --git a/docs/backends.md b/docs/backends.md index 03d2e621..5d553eb4 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -10,7 +10,7 @@ hull resolves the backend in this order: 1. the `--hypervisor` flag 2. the image's `com.urunc.unikernel.hypervisor` annotation -3. `qemu` +3. `qemu` on macOS, `hvi` on Linux That last step is worth knowing. If you pass no flag and the image carries no annotation, hull runs `qemu`, which hull does not install and which cannot boot @@ -36,23 +36,26 @@ Start with `vz`. Move to another backend only for a reason on this list. ## Capability comparison -| | `vz` | `hvi` | `qemu` | -|---|---|---|---| -| Interface | Virtualization.framework | Hypervisor.framework, called directly | Hypervisor.framework, via QEMU's `-accel hvf` | -| Entitlement the runner needs | `com.apple.security.virtualization` | `com.apple.security.hypervisor` | `com.apple.vm.networking`, and only for `--net shared` | -| Ships with hull | yes, as `vz-runner` | yes, as `hvi` | no, `brew install qemu` | -| Guest architecture | arm64 | arm64 | arm64 | -| Device transport | Virtualization.framework's | virtio-mmio, no PCI | QEMU's | -| Rootfs: shared directory | virtiofs | virtiofs | 9pfs | -| Rootfs: block | ext4 image | ext4 image | ext4 image | -| Boots a plain image carrying no kernel | yes | yes | **no** | -| Read-only `--shared-dir` | yes | yes | **refused** | -| `--net shared` | Apple NAT, no extra privilege | **refused**, see below | vmnet, needs root or a signed QEMU | -| `--gateway-sock` | yes | yes | yes | -| `exec` | yes | yes | yes | -| Checkpoint and restore | yes, with a block rootfs | **no** | **no** | -| `--gui`, `--gui-title` | yes | no | no | -| `--rosetta` | yes | no | no | +| | `vz` | `hvi` | `qemu` | Linux `hvi` | +|---|---|---|---|---| +| Interface | Virtualization.framework | Hypervisor.framework, called directly | Hypervisor.framework, via QEMU's `-accel hvf` | KVM | +| Entitlement the runner needs | `com.apple.security.virtualization` | `com.apple.security.hypervisor` | `com.apple.vm.networking`, and only for `--net shared` | none; read-write `/dev/kvm` | +| Ships with hull | yes, as `vz-runner` | yes, as `hvi` | no, `brew install qemu` | no release yet, see [build.md](build.md#linux) | +| Guest architecture | arm64 | arm64 | arm64 | arm64 | +| Device transport | Virtualization.framework's | virtio-mmio, no PCI | QEMU's | virtio-mmio, no PCI | +| Rootfs: shared directory | virtiofs | virtiofs | 9pfs | **refused** | +| Rootfs: block | ext4 image | ext4 image | ext4 image | ext4 image, the only mode | +| Boots a plain image carrying no kernel | yes | yes | **no** | yes | +| Read-only `--shared-dir` | yes | yes | **refused** | **refused** | +| `--net shared` | Apple NAT, no extra privilege | **refused**, see below | vmnet, needs root or a signed QEMU | **refused** without `--gateway-sock` | +| `--gateway-sock` | yes | yes | yes | yes | +| `exec` | yes | yes | yes | yes | +| Checkpoint and restore | yes, with a block rootfs | **no** | **no** | **no** | +| `--gui`, `--gui-title` | yes | no | no | no | +| `--rosetta` | yes | no | no | no | + +The last column is hull built for Linux, which runs only `hvi`; see +[`hvi` on Linux](#hvi-on-linux). Read the gaps as gaps. Three backends existing does not make them interchangeable, and hull does not emulate a missing feature on a backend that @@ -220,6 +223,59 @@ or against CPU side channels. guest clock once at boot by writing the host epoch into the boot initrd. A long-running `hvi` guest has no host time source to correct drift against. +## `hvi` on Linux + +hull also builds for arm64 Linux, where it runs `hvi` on KVM and nothing else. +It is there to be the other end of a sandbox that started on a Mac, so it +offers the one configuration a sandbox can carry between the two: `hvi`, a +block rootfs, and either no network or a gateway. brig drives it with the same +verbs it uses on macOS: `run`, `exec`, `ps`, `inspect`, `logs`, `stop`, `rm`, +`assets dir` and `network-gateway`. `pull`, `images`, `rmi`, `prune`, +`telemetry` and `version` work too. + +What is different from macOS: + +- **The backend is always `hvi`.** With no `--hypervisor` flag, hull on Linux + picks `hvi`, not `qemu`, because nothing else is offered. Any other backend is + refused before the image is pulled. +- **The rootfs is always a block image.** `--rootfs-type` defaults to `block`; + `virtiofs` and `9pfs` are refused, and so are `--shared-dir` and + `--shared-dir-fd`. `hvi`'s KVM backend has no virtio-fs server: it accepts + the share flags and serves nothing, so a guest would boot without them. +- **The network is the gateway or nothing.** `--net none`, or `--net shared` + with `--gateway-sock` and `--gateway-cidr`. A bare `--net shared` is refused: + on macOS it means vmnet, and on Linux it would be `hvi`'s stub NIC, which + forwards nothing. `hull network-gateway` is the same gvisor gateway as on + macOS, with the same egress rules. +- **The `hvi` command line is hull's own.** urunc's Linux `hvi` monitor is + written for the containerd shim and passes no `--agent-sock`, + `--net-gateway` or `--net-mac`, so `exec`, the gateway and per-guest DHCP + leases would not work. hull builds the argv itself, the same as urunc's + darwin builder minus the shares, and a test fails if the two drift. +- **`hvi` is found beside hull first**, then on `PATH`, as on macOS. +- **The store is a plain directory**, created 0700. There is no sparse image + and no `hull store`. hull refuses a store directory on a case-insensitive + filesystem. +- **e2fsprogs comes from the distribution.** hull looks for `mke2fs`, + `debugfs` and `e2fsck` in `/usr/sbin`, then `/sbin`, then `PATH`. Debian and + Raspberry Pi OS keep them in `/sbin`, which an ordinary login's `PATH` does + not include. +- **Image ownership is recorded on unpack.** Run as an ordinary user, hull + cannot chown the files it unpacks, so it records each file's uid, gid and + mode in a `user.com.nofire.hvi.linux-attr` xattr, as macOS does in its own + namespace, and writes them into the block image. Run as root, the files + carry the real owners and nothing is recorded. Symlinks cannot hold a + `user.` xattr on Linux, so an unprivileged unpack leaves them owned by the + user who ran hull. +- **Not on Linux:** `compose`, `checkpoint`, `restore`, `store`, `--gui` and + `--rosetta`. + +The user running hull needs read-write access to `/dev/kvm`, usually through +the `kvm` group. A group added to a logged-in user reaches only new sessions. + +Tested on a Raspberry Pi 5 (Cortex-A76, KVM with a GICv2) running Raspberry Pi +OS with kernel 6.12, using the `linux-arm64` boot assets. + ## `qemu` `hull` builds a `qemu-system-aarch64` command line using `-accel hvf`. QEMU is diff --git a/docs/build.md b/docs/build.md index 980e7128..c3b8cbe6 100644 --- a/docs/build.md +++ b/docs/build.md @@ -14,7 +14,8 @@ two runners must be signed before they can start a VM. ## Prerequisites -- An Apple Silicon Mac. There is no other build target. +- An Apple Silicon Mac for the macOS build. The Linux build (`make linux`, + arm64 only) is described under [Linux](#linux). - **Go** 1.26 or later. `go.mod` declares 1.26.4. - **Xcode command line tools**, which supply Swift. `vz-runner` declares `swift-tools-version:5.9` and a macOS 14 deployment target. @@ -113,6 +114,57 @@ entitlement. Putting only `hull` on your PATH is the most common local-build mistake. The `vz` and `hvi` backends then fail with a message about not finding the runner. +## Linux + +hull also builds for arm64 Linux, as the runtime for `hvi` sandboxes on KVM. +What it runs there, and what it refuses, is in +[backends.md](backends.md#hvi-on-linux). + +There is no release archive for Linux yet. Build both halves yourself: + +```bash +make linux # dist/hull_linux_arm64, cross-built from any host +``` + +Build `hvi` on the Linux host itself, from the same submodule commit. That is +the tested path; hvi-vmm's own CI builds its Linux targets on Linux runners +too. + +```bash +cd hvi-vmm && cargo build --release # target/release/hvi +``` + +Install the two side by side. hull finds `hvi` next to its own executable +before it looks on `PATH`, exactly as on macOS: + +```bash +mkdir -p ~/hull-bin +cp dist/hull_linux_arm64 ~/hull-bin/hull +cp hvi-vmm/target/release/hvi ~/hull-bin/hvi +``` + +The host needs: + +- **KVM**, with `/dev/kvm` readable and writable by the user running hull. + On Debian-family systems: `sudo usermod -aG kvm "$USER"`, then log in again. +- **e2fsprogs** (`mke2fs`, `debugfs`, `e2fsck`) for the block rootfs. hull + looks in `/usr/sbin` and `/sbin` before `PATH`. +- A filesystem with user xattrs for the store (ext4, xfs, btrfs; tmpfs from + Linux 6.6), when hull runs as an ordinary user. See "Image ownership" in + [backends.md](backends.md#hvi-on-linux). + +Then fetch the boot assets and run: + +```bash +~/hull-bin/hull assets pull # ghcr.io/nofireai/hull-assets:linux-arm64 +~/hull-bin/hull run --detach --name t1 alpine:latest +~/hull-bin/hull exec t1 -- uname -a +``` + +`go test ./...` runs on Linux too. Tests that need macOS (hdiutil, APFS, +xattrs in the macOS namespace, the vz and compose paths) are tagged +`//go:build darwin` and skipped there. + ## Signing A freshly built runner cannot start a VM until it is signed with the right diff --git a/go.mod b/go.mod index 43fba3f8..ccd3554a 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/apparentlymart/go-cidr v1.1.1 // indirect github.com/cavaliergopher/cpio v1.0.1 // indirect github.com/checkpoint-restore/go-criu/v6 v6.3.0 // indirect - github.com/cilium/ebpf v0.22.0 // indirect + github.com/cilium/ebpf v0.18.0 // indirect github.com/containerd/cgroups/v3 v3.1.0 // indirect github.com/containerd/console v1.0.5 // indirect github.com/containerd/continuity v0.5.0 // indirect @@ -111,3 +111,10 @@ require ( // hull builds it from this tree: a change to the client and its use here land // in one commit, and the version above is a placeholder the replace resolves. replace github.com/brig-sh/hull/pkg/telemetry => ./pkg/telemetry + +// github.com/cilium/ebpf is held at v0.18.0. It reaches hull only in the +// native Linux build, through urunc's cgo libcontainer code and +// github.com/opencontainers/cgroups v0.0.4, which reads ProgramInfo.RunCount +// and ProgramInfo.Runtime. ebpf v0.19.0 removed both, so a newer ebpf breaks +// that build. cgroups v0.1.0 no longer reads them but does not build with +// runc v1.3.6, so ebpf moves when cgroups and runc move. diff --git a/go.sum b/go.sum index b0186b98..c4e5fd7d 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/checkpoint-restore/go-criu/v6 v6.3.0 h1:mIdrSO2cPNWQY1truPg6uHLXyKHk3Z5Odx4wjKOASzA= github.com/checkpoint-restore/go-criu/v6 v6.3.0/go.mod h1:rrRTN/uSwY2X+BPRl/gkulo9gsKOSAeVp9/K2tv7xZI= -github.com/cilium/ebpf v0.22.0 h1:v2ktp0roffpMOj2MMf3idtCQZOsAoC4BJbAJN+ke2bY= -github.com/cilium/ebpf v0.22.0/go.mod h1:CDzZbe2hC5JjlDC+CY3KFCzlYwN4gbxppYM+Z10bQt4= +github.com/cilium/ebpf v0.18.0 h1:OsSwqS4y+gQHxaKgg2U/+Fev834kdnsQbtzRnbVC6Gs= +github.com/cilium/ebpf v0.18.0/go.mod h1:vmsAT73y4lW2b4peE+qcOqw6MxvWQdC+LiU5gd/xyo4= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/compose-spec/compose-go/v2 v2.14.0 h1:uaJeo5B3+OVlu+Rx2qLBcAdXPEUUzm5nQrRiGJafRAQ= @@ -140,8 +140,8 @@ github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/ github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY= github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4= -github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= -github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= +github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= +github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -180,8 +180,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -310,8 +310,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= -gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= gvisor.dev/gvisor v0.0.0-20240916094835-a174eb65023f h1:O2w2DymsOlM/nv2pLNWCMCYOldgBBMkD7H0/prN5W2k= gvisor.dev/gvisor v0.0.0-20240916094835-a174eb65023f/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/ociclient/annotationdefaults_test.go b/pkg/ociclient/annotationdefaults_test.go index 7b202aee..48cbcf5a 100644 --- a/pkg/ociclient/annotationdefaults_test.go +++ b/pkg/ociclient/annotationdefaults_test.go @@ -15,6 +15,7 @@ package ociclient import ( + "runtime" "testing" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -68,3 +69,20 @@ func TestNoUnikernelTypeIsInjectedBesideOtherUruncLabels(t *testing.T) { }) } } + +// hull on Linux treats the hypervisor annotation as the image's own choice and +// refuses anything but hvi, so the defaults written for a plain image must not +// name one there. macOS keeps qemu-hvf, its historical default. +func TestPlainImageDefaultsNameAHypervisorOnlyOnMacOS(t *testing.T) { + cfg := &v1.ConfigFile{} + cfg.Config.Labels = map[string]string{"org.opencontainers.image.title": "x"} + spec := createOCISpec(cfg, "rootfs") + got, ok := spec.Annotations[HypervisorAnnotation] + if runtime.GOOS == "darwin" { + if got != "qemu-hvf" { + t.Errorf("macOS default hypervisor annotation = %q, want qemu-hvf", got) + } + } else if ok { + t.Errorf("a plain image was given hypervisor annotation %q on %s", got, runtime.GOOS) + } +} diff --git a/pkg/ociclient/bundle.go b/pkg/ociclient/bundle.go index c0667439..ddf8e7d5 100644 --- a/pkg/ociclient/bundle.go +++ b/pkg/ociclient/bundle.go @@ -19,6 +19,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -182,7 +183,14 @@ func createOCISpec(imgConfig *v1.ConfigFile, rootfsDir string) *specs.Spec { if !hasUruncAnnotations { spec.Annotations[UnikernelTypeAnnotation] = "linux" - spec.Annotations[HypervisorAnnotation] = "qemu-hvf" + // macOS only. hull on Linux reads this annotation as the image's own + // choice of backend and refuses anything but hvi, so a qemu-hvf + // written here on the image's behalf would refuse every plain + // container image. Left out, the backend falls to the platform + // default, which on Linux is hvi. + if runtime.GOOS == "darwin" { + spec.Annotations[HypervisorAnnotation] = "qemu-hvf" + } spec.Annotations[UnikernelBinaryAnnotation] = "unikernel" spec.Annotations["com.urunc.unikernel.kernel"] = "unikernel" } From 71d3da2ad3202b5e08112bc6d2ee15428811affb Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Wed, 14 Oct 2026 02:22:31 +0300 Subject: [PATCH 06/33] feat(ociclient): keep image ownership on unprivileged Linux unpacks Run as an ordinary user on Linux, an unpack cannot chown, so every file came out owned by the user who ran the pull and mke2fs copied that into the guest's block image. On a Raspberry Pi 5 alpine's /bin/busybox and /etc/shadow were 1000:1000 in the guest, where sudo refuses to run and a setuid binary drops its caller. The unpack now records the tar header's mode, uid and gid in the user.com.nofire.hvi.linux-attr attribute, the only namespace an unprivileged process may write, and the block image build applies the recorded owners and modes. Files keep owner read and directories owner rwx on the host, so images that ship /etc/shadow at 0000 (Fedora, RHEL and their relatives) can be read by mke2fs, and the builder writes the image's own mode into the image with debugfs. Symlinks, which take no user.* attributes, get their owners from a side record, and the ownership record itself is stripped from the image so the guest never sees it. A Linux test drives the unprivileged unpack through to the block image, and the docs say what Linux hull is for and what was checked on it. Signed-off-by: Panagiotis Moustafellos --- README.md | 6 +- cmd/hull/blockrootfs.go | 120 ++++++++++++---- cmd/hull/blockrootfs_linux_test.go | 193 ++++++++++++++++++++++++++ cmd/hull/commands_linux.go | 4 +- cmd/hull/commands_linux_test.go | 111 +++++++++++++++ cmd/hull/hviargv.go | 12 +- cmd/hull/linuxrun.go | 9 +- cmd/hull/storemount_linux.go | 9 +- docs/backends.md | 47 +++++-- docs/build.md | 10 +- pkg/ociclient/guestattr_darwin.go | 19 +++ pkg/ociclient/guestattr_linux.go | 142 +++++++++++++++++++ pkg/ociclient/guestattr_linux_test.go | 161 +++++++++++++++++++++ pkg/ociclient/guestattr_other.go | 16 ++- pkg/ociclient/linkowners.go | 87 ++++++++++++ pkg/ociclient/unpack.go | 27 +++- pkg/ociclient/unpack_test.go | 10 +- 17 files changed, 916 insertions(+), 67 deletions(-) create mode 100644 cmd/hull/blockrootfs_linux_test.go create mode 100644 cmd/hull/commands_linux_test.go create mode 100644 pkg/ociclient/guestattr_linux.go create mode 100644 pkg/ociclient/guestattr_linux_test.go create mode 100644 pkg/ociclient/linkowners.go diff --git a/README.md b/README.md index d829bb0a..d0582c5a 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,9 @@ Kubernetes integration, and it implements a subset of Compose. See earlier versions can work. Nothing blocks them, and nothing tests them. - About 2 GB of free disk space for the store, plus each image you pull. - On Linux, hull builds from source for arm64 with KVM and runs only the - `hvi` backend with a block rootfs, as the destination for a sandbox that - started on a Mac. See [docs/backends.md](docs/backends.md#hvi-on-linux) - and [docs/build.md](docs/build.md#linux). + `hvi` backend with a block rootfs. A sandbox cannot be moved there from a + Mac. See [docs/backends.md](docs/backends.md#hvi-on-linux) and + [docs/build.md](docs/build.md#linux). ## Install diff --git a/cmd/hull/blockrootfs.go b/cmd/hull/blockrootfs.go index 5563f599..eb1ce2dd 100644 --- a/cmd/hull/blockrootfs.go +++ b/cmd/hull/blockrootfs.go @@ -97,7 +97,23 @@ func buildBlockRootfs(ctx context.Context, diskPath, rootfsDir string, injects [ } defer func() { _ = os.RemoveAll(stage) }() + // The image's own ownership and modes go first, so that the injects below + // have the last word on the paths they replace. The other way round, an + // image that ships its own /urunit.conf at 0644 would have that mode put + // back over the injected 0600 file that carries the guest's credentials. var script bytes.Buffer + skipped, err := appendOwnershipScript(&script, rootfsDir) + if err != nil { + return err + } + if skipped > 0 { + // Not fatal, but not silent either: those files keep the host's + // ownership inside the guest, and a caller staring at one of them + // deserves to know why. + log.Warnf("%d path(s) could not be given their recorded ownership in the "+ + "block image because the name is not expressible as a debugfs argument", skipped) + } + for i, inj := range injects { hostPath := filepath.Join(stage, fmt.Sprintf("f%d", i)) if err := os.WriteFile(hostPath, inj.content, 0600); err != nil { @@ -116,17 +132,6 @@ func buildBlockRootfs(ctx context.Context, diskPath, rootfsDir string, injects [ fmt.Fprintf(&script, "sif %s gid 0\n", debugfsPath(inj.guestPath)) } - skipped, err := appendOwnershipScript(&script, rootfsDir) - if err != nil { - return err - } - if skipped > 0 { - // Not fatal, but not silent either: those files keep the host's - // ownership inside the guest, and a caller staring at one of them - // deserves to know why. - log.Warnf("%d path(s) could not be given their recorded ownership in the "+ - "block image because the name is not expressible as a debugfs argument", skipped) - } if script.Len() == 0 { return nil } @@ -187,7 +192,8 @@ func repairBlockImage(ctx context.Context, diskPath string) error { return fmt.Errorf("failed to check the block image: %s: %w", string(out), err) } -// appendOwnershipScript emits the ownership every path in the image declares. +// appendOwnershipScript emits the ownership every path in the image declares, +// and the mode wherever the host file does not already carry it. // // Only paths that carry a record are touched. Anything else keeps what mke2fs // gave it, which is the right answer for a file the image did not bring. @@ -198,6 +204,14 @@ func appendOwnershipScript(w *bytes.Buffer, rootfsDir string) (skipped int, err } defer func() { _ = root.Close() }() + // Symlink ownership from an unprivileged Linux unpack, which could not + // put it on the links themselves. A tree without it keeps the host's + // ownership on its links, as before. + links, linkErr := ociclient.LinkOwners(rootfsDir) + if linkErr != nil { + log.Warnf("symlinks keep the host's ownership in the block image: %v", linkErr) + } + buf := bufio.NewWriter(w) // The walk runs over root.FS() rather than over rootfsDir with // filepath.WalkDir, for the reason spelled out on errImagePlantedPath in @@ -218,6 +232,11 @@ func appendOwnershipScript(w *bytes.Buffer, rootfsDir string) (skipped int, err } symlink := d.Type()&fs.ModeSymlink != 0 mode, uid, gid, ok := ociclient.GuestAttr(rootfsEntry(rootfsDir, name), symlink) + if !ok && symlink { + if o, found := links[name]; found { + uid, gid, ok = o.UID, o.GID, true + } + } if !ok { return nil } @@ -232,18 +251,16 @@ func appendOwnershipScript(w *bytes.Buffer, rootfsDir string) (skipped int, err // a failure here is reported once rather than at every line. _, _ = fmt.Fprintf(buf, "sif %s uid %d\n", q, uid) _, _ = fmt.Fprintf(buf, "sif %s gid %d\n", q, gid) - // Ownership only. The mode is already right -- the unpack put the - // image's mode on the host file, setuid included, and mke2fs copies it - // faithfully. - // - // Setting it here would also be actively destructive: debugfs writes - // the whole i_mode field, file-type bits and all, so a permission-only - // value silently clears S_IFDIR and turns every directory it touches - // into a regular file. That corrupts the filesystem so thoroughly that - // debugfs cannot reopen it, every later command fails with - // "Filesystem not open" -- and debugfs still exits 0, so nothing - // reports it until the guest kernel refuses the mount. - _ = mode + if line := modeFixup(q, d, mode); line != "" { + _, _ = buf.WriteString(line) + } + // The record has done its job once the lines above exist. Where mke2fs + // copied it into the image, take it out: it is host bookkeeping, and + // the guest has no use for it. Removing an attribute the inode does + // not have is silent in debugfs, so this needs no check of its own. + if ociclient.CopiedRecordXattr != "" && !symlink { + _, _ = fmt.Fprintf(buf, "ea_rm %s %s\n", q, ociclient.CopiedRecordXattr) + } return nil }) if flushErr := buf.Flush(); flushErr != nil { @@ -255,6 +272,61 @@ func appendOwnershipScript(w *bytes.Buffer, rootfsDir string) (skipped int, err return skipped, nil } +// modeFixup returns the debugfs line that gives a regular file or directory +// its recorded mode, or "" when mke2fs already copied that mode from the host. +// +// Usually it did: the unpack puts the image's mode on the host file, setuid +// included, and mke2fs copies it faithfully. The exception is an unprivileged +// Linux unpack, which keeps owner read on files and owner rwx on directories +// so mke2fs can read them at all (see ociclient's hostFileMode). A 0000 +// /etc/shadow is 0400 on the host, and this puts the 0000 back. +// +// The file type comes from the entry itself, never from the record. debugfs +// writes the whole i_mode field, file-type bits and all, and a tar header may +// or may not carry them, so a permission-only value would silently clear +// S_IFDIR and turn every directory it touches into a regular file. That +// corrupts the filesystem so thoroughly that debugfs cannot reopen it, every +// later command fails with "Filesystem not open" -- and debugfs still exits 0. +// Worse, e2fsck then "repairs" what is left, so the build succeeds and the +// guest finds its directories gone. Symlinks and everything else are left +// alone: their mode carries nothing a guest consults. +func modeFixup(q string, d fs.DirEntry, recorded uint32) string { + var fileType uint32 + switch { + case d.IsDir(): + fileType = 0o040000 + case d.Type().IsRegular(): + fileType = 0o100000 + default: + return "" + } + info, err := d.Info() + if err != nil { + return "" + } + want := recorded & 0o7777 + if posixPermBits(info.Mode()) == want { + return "" + } + return fmt.Sprintf("sif %s mode 0%o\n", q, fileType|want) +} + +// posixPermBits is guestFileMode in reverse: the permission, setuid, setgid +// and sticky bits of an fs.FileMode at their POSIX positions. +func posixPermBits(m fs.FileMode) uint32 { + bits := uint32(m.Perm()) + if m&fs.ModeSetuid != 0 { + bits |= 0o4000 + } + if m&fs.ModeSetgid != 0 { + bits |= 0o2000 + } + if m&fs.ModeSticky != 0 { + bits |= 0o1000 + } + return bits +} + // rootfsEntry returns the host path of an entry a rooted rootfs walk produced. // // The join comes last and only because it has already been validated, which is diff --git a/cmd/hull/blockrootfs_linux_test.go b/cmd/hull/blockrootfs_linux_test.go new file mode 100644 index 00000000..b40406b8 --- /dev/null +++ b/cmd/hull/blockrootfs_linux_test.go @@ -0,0 +1,193 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "archive/tar" + "bytes" + "context" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/brig-sh/hull/pkg/ociclient" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// tarLayer is an uncompressed image layer held in memory, which is all +// UnpackLayers reads from one. +type tarLayer struct{ data []byte } + +func (l tarLayer) Uncompressed() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(l.data)), nil +} +func (l tarLayer) Compressed() (io.ReadCloser, error) { return l.Uncompressed() } +func (l tarLayer) Size() (int64, error) { return int64(len(l.data)), nil } +func (l tarLayer) Digest() (v1.Hash, error) { return v1.Hash{}, nil } +func (l tarLayer) DiffID() (v1.Hash, error) { return v1.Hash{}, nil } +func (l tarLayer) MediaType() (types.MediaType, error) { + return types.DockerUncompressedLayer, nil +} + +type layerEntry struct { + hdr tar.Header + body string +} + +func buildLayer(t *testing.T, entries ...layerEntry) tarLayer { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, e := range entries { + h := e.hdr + h.Size = int64(len(e.body)) + if err := tw.WriteHeader(&h); err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(tw, e.body); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + return tarLayer{data: buf.Bytes()} +} + +var ( + debugfsTypeMode = regexp.MustCompile(`Type: (\S+)\s+Mode:\s+(\d+)`) + debugfsOwner = regexp.MustCompile(`User:\s+(\d+)\s+Group:\s+(\d+)`) +) + +// imageStat reads one inode back out of the finished image, as type, octal +// mode and "uid:gid". +func imageStat(t *testing.T, disk, path string) (fileType, mode, owner string) { + t.Helper() + out, err := exec.Command(debugfsBin, "-R", "stat "+path, disk).CombinedOutput() + if err != nil { + t.Fatalf("debugfs stat %s: %s: %v", path, out, err) + } + tm := debugfsTypeMode.FindSubmatch(out) + ow := debugfsOwner.FindSubmatch(out) + if tm == nil || ow == nil { + t.Fatalf("debugfs stat %s: unexpected output:\n%s", path, out) + } + return string(tm[1]), string(tm[2]), string(ow[1]) + ":" + string(ow[2]) +} + +// The whole unprivileged Linux path, the one an ordinary user's `hull run` +// takes: a layer unpacked through ociclient, where archive.Apply stops on its +// first chown and the manual extractor records what the host cannot hold, +// then the block image built from that tree. The image has to come out as the +// layer describes it -- owners, modes, file types, symlink owners -- with +// none of hull's host-side records left in it. +// +// The layer carries the cases that broke: Fedora's 0000 shadow files and +// 04111 sudo, which an owner cannot read, so mke2fs could not either; a 0555 +// directory, which the owner cannot delete from; symlinks, which cannot hold +// a user.* record on Linux; and an image-supplied /urunit.conf, whose mode and +// owner must not leak onto the injected 0600 one. +func TestUnprivilegedUnpackBuildsTheImageTheLayerDescribes(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("a root unpack chowns for real and records nothing") + } + for _, bin := range []string{mke2fsBin, debugfsBin, e2fsckBin} { + if _, err := os.Stat(bin); err != nil { + t.Skipf("e2fsprogs is not installed: %v", err) + } + } + base := t.TempDir() + rootfs := filepath.Join(base, "rootfs") + dir := func(name string, mode int64, uid, gid int) layerEntry { + return layerEntry{hdr: tar.Header{Name: name, Typeflag: tar.TypeDir, Mode: mode, Uid: uid, Gid: gid}} + } + reg := func(name string, mode int64, uid, gid int, body string) layerEntry { + return layerEntry{hdr: tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: mode, Uid: uid, Gid: gid}, body: body} + } + link := func(name, target string, uid, gid int) layerEntry { + return layerEntry{hdr: tar.Header{Name: name, Typeflag: tar.TypeSymlink, Linkname: target, Uid: uid, Gid: gid}} + } + layer := buildLayer(t, + dir("bin/", 0o755, 0, 0), + reg("bin/busybox", 0o755, 0, 0, "#!/bin/sh\n"), + link("bin/sh", "busybox", 0, 0), + dir("etc/", 0o755, 0, 0), + reg("etc/shadow", 0o640, 0, 42, "root:!::0:::::\n"), + reg("etc/gshadow", 0o000, 0, 0, "root:::\n"), + dir("usr/", 0o755, 0, 0), + dir("usr/bin/", 0o755, 0, 0), + reg("usr/bin/sudo", 0o4111, 0, 0, "#!/bin/sh\n"), + dir("usr/lib/", 0o755, 0, 0), + dir("usr/lib/sealed/", 0o555, 0, 0), + reg("usr/lib/sealed/f", 0o444, 0, 0, "x"), + dir("home/", 0o755, 0, 0), + dir("home/app/", 0o700, 1000, 1000), + link("home/app/.profile", "/etc/profile", 1000, 1000), + // 0000, so the host copy is widened and the builder emits a mode line + // for it -- the line that must not land on the injected file. + reg("urunit.conf", 0o000, 1000, 1000, "IMAGE=1\n"), + ) + if err := ociclient.UnpackLayers([]v1.Layer{layer}, rootfs, nil); err != nil { + t.Fatalf("UnpackLayers: %v", err) + } + if _, _, _, ok := ociclient.GuestAttr(filepath.Join(rootfs, "etc/shadow"), false); !ok { + t.Skipf("%s has no user xattr support, so nothing could be recorded", base) + } + + disk := filepath.Join(base, "rootfs.ext4") + injects := []blockInject{{guestPath: "/urunit.conf", content: []byte("SECRET=1\n"), mode: 0o600}} + if err := buildBlockRootfs(context.Background(), disk, rootfs, injects, 32); err != nil { + t.Fatalf("buildBlockRootfs: %v", err) + } + + for _, tc := range []struct{ path, fileType, mode, owner string }{ + {"/bin/busybox", "regular", "0755", "0:0"}, + {"/bin/sh", "symlink", "0777", "0:0"}, + {"/etc/shadow", "regular", "0640", "0:42"}, + {"/etc/gshadow", "regular", "0000", "0:0"}, + {"/usr/bin/sudo", "regular", "04111", "0:0"}, + {"/usr/lib/sealed", "directory", "0555", "0:0"}, + {"/usr/lib/sealed/f", "regular", "0444", "0:0"}, + {"/home/app", "directory", "0700", "1000:1000"}, + {"/home/app/.profile", "symlink", "0777", "1000:1000"}, + {"/urunit.conf", "regular", "0600", "0:0"}, + } { + fileType, mode, owner := imageStat(t, disk, tc.path) + if fileType != tc.fileType || mode != tc.mode || owner != tc.owner { + t.Errorf("%s is a %s %s owned %s in the image, want a %s %s owned %s", + tc.path, fileType, mode, owner, tc.fileType, tc.mode, tc.owner) + } + } + + for _, p := range []string{"/etc/shadow", "/bin/busybox", "/usr/lib/sealed"} { + out, err := exec.Command(debugfsBin, "-R", "ea_list "+p, disk).CombinedOutput() + if err != nil { + t.Fatalf("debugfs ea_list %s: %s: %v", p, out, err) + } + if strings.Contains(string(out), "linux-attr") { + t.Errorf("%s carries hull's host-side record into the image:\n%s", p, out) + } + } + + // And the owner can still delete what was unpacked. + if err := os.RemoveAll(rootfs); err != nil { + t.Errorf("the unpacked tree cannot be removed by its owner: %v", err) + } +} diff --git a/cmd/hull/commands_linux.go b/cmd/hull/commands_linux.go index 5dace824..87b7c6b8 100644 --- a/cmd/hull/commands_linux.go +++ b/cmd/hull/commands_linux.go @@ -24,8 +24,8 @@ import ( const appUsage = "container CLI for hvi sandboxes on Linux/KVM" // platformCommands is the Linux verb set: what it takes to run an hvi sandbox -// that may have started on a Mac, and what brig calls to drive one (run, exec, -// ps, inspect, logs, stop, rm, assets dir, network-gateway). +// on KVM, and what brig calls to drive one (run, exec, ps, inspect, logs, +// stop, rm, assets dir, network-gateway). // // Left out, because they drive something Linux does not have: // - checkpoint and restore: vz-runner's saved-state files, which are diff --git a/cmd/hull/commands_linux_test.go b/cmd/hull/commands_linux_test.go new file mode 100644 index 00000000..58d11b2b --- /dev/null +++ b/cmd/hull/commands_linux_test.go @@ -0,0 +1,111 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/urfave/cli/v3" +) + +// A fresh store is a 0700 directory: its mode is the only thing keeping other +// accounts out of the guest's environment and the block image. +func TestEnsureStoreCreatesAPrivateDirectory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "a", "store") + if err := ensureStore(dir, ""); err != nil { + t.Fatalf("ensureStore on a case-sensitive filesystem: %v", err) + } + fi, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() || fi.Mode().Perm() != 0o700 { + t.Errorf("store is %v, want a 0700 directory", fi.Mode()) + } +} + +// A store on a case-folding filesystem would unpack colliding names over each +// other, so it is refused by name; a probe that cannot answer is an error, not +// a pass. +func TestEnsureStoreRefusesACaseInsensitiveDirectory(t *testing.T) { + dir := t.TempDir() + err := ensurePlainStore(dir, func(string) (bool, error) { return false, nil }) + if err == nil || !strings.Contains(err.Error(), "case-insensitive") { + t.Errorf("a case-insensitive store was not refused: %v", err) + } + probeErr := errors.New("probe failed") + if err := ensurePlainStore(dir, func(string) (bool, error) { return false, probeErr }); !errors.Is(err, probeErr) { + t.Errorf("a failed probe was not reported: %v", err) + } +} + +// compose is registered on Linux only to refuse with the reason, rather than +// answering "No help topic", and it stays out of the help listing. +func TestLinuxComposeRefusesWithAReason(t *testing.T) { + c := linuxCompose() + if !c.Hidden { + t.Error("compose is listed in the Linux help") + } + if c.Before == nil { + t.Fatal("compose has no refusing Before hook") + } + if _, err := c.Before(context.Background(), c); !errors.Is(err, errComposeMacOnly) { + t.Errorf("compose Before = %v, want errComposeMacOnly", err) + } + found := false + for _, pc := range platformCommands() { + if pc.Name == "compose" { + found = pc.Before != nil + } + } + if !found { + t.Error("the Linux command list does not carry the refusing compose") + } +} + +// A run Linux cannot deliver is refused before the store is opened, so before +// anything is pulled: the image reference here does not resolve, and the +// store directory must not even be created. +func TestLinuxRunRefusesBeforeTheStoreIsOpened(t *testing.T) { + for _, tc := range []struct { + args []string + want string + }{ + {[]string{"--hypervisor", "qemu", "--net", "none"}, "--hypervisor hvi"}, + {[]string{"--rootfs-type", "virtiofs", "--net", "none"}, "--rootfs-type block"}, + {[]string{"--net", "shared"}, "--gateway-sock"}, + } { + storeDir := filepath.Join(t.TempDir(), "store") + app := &cli.Command{ + Name: "hull", + Flags: []cli.Flag{&cli.StringFlag{Name: "store-dir", Value: storeDir}}, + Commands: []*cli.Command{runCommand()}, + } + argv := append([]string{"hull", "run"}, tc.args...) + argv = append(argv, "registry.invalid/never-pulled:1") + err := app.Run(context.Background(), argv) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%v: err = %v, want a refusal naming %s", tc.args, err, tc.want) + } + if _, statErr := os.Stat(storeDir); !os.IsNotExist(statErr) { + t.Errorf("%v: the store was touched before the refusal: %v", tc.args, statErr) + } + } +} diff --git a/cmd/hull/hviargv.go b/cmd/hull/hviargv.go index 6780dcd4..bce0e1e2 100644 --- a/cmd/hull/hviargv.go +++ b/cmd/hull/hviargv.go @@ -35,12 +35,12 @@ const vzVmmName = "vz" // --agent-sock, so hull exec has nothing to dial; no --net-gateway, so the // guest gets hvi's stub NIC instead of the gateway's network and policy; and // no --net-mac, so every guest presents the same default MAC and the gateway, -// which keys its leases by it, hands two sandboxes the same address. A Linux -// destination for a sandbox started on a Mac has to boot it with the same -// devices it had there, so the argv here must match the darwin one flag for -// flag. Patching urunc would fork a dependency hull pins at a branch tip for -// a contract that belongs to hull; mirroring the dozen lines keeps the -// contract here, and hviargv_darwin_test.go fails if the two drift. +// which keys its leases by it, hands two sandboxes the same address. hull's +// guests should get one hvi command line on every host, so the argv here +// matches the darwin one flag for flag, shares aside. Patching urunc would +// fork a dependency hull pins at a branch tip for a contract that belongs to +// hull; mirroring the dozen lines keeps the contract here, and +// hviargv_darwin_test.go compares the two. // // Shares are refused rather than dropped: hvi's KVM backend has no virtio-fs // server (it parses --share-* and configures nothing), so a guest that asked diff --git a/cmd/hull/linuxrun.go b/cmd/hull/linuxrun.go index 3fc298a6..77accea8 100644 --- a/cmd/hull/linuxrun.go +++ b/cmd/hull/linuxrun.go @@ -72,10 +72,11 @@ func checkLinuxHypervisor(name, source string) error { // default, which selectHypervisor and checkLinuxHypervisor apply once the // image is pulled. // -// Linux hull exists to be the other end of a moved hvi sandbox, so it offers -// the one configuration that can be moved and refuses the rest by name before -// anything is pulled -- a refusal after a multi-gigabyte pull and a 15 GiB -// image build is the same answer, late. What it refuses, and why: +// Linux hull offers the one configuration hvi on KVM can run, which is also +// the one a sandbox moved between two KVM hosts will need, and refuses the +// rest by name before anything is pulled -- a refusal after a multi-gigabyte +// pull and a 15 GiB image build is the same answer, late. What it refuses, +// and why: // // - any backend but hvi: vz and the macOS qemu builder do not exist here. // - --rootfs-type virtiofs and 9pfs: hvi's KVM backend has no file server, diff --git a/cmd/hull/storemount_linux.go b/cmd/hull/storemount_linux.go index 26414e32..07a15dca 100644 --- a/cmd/hull/storemount_linux.go +++ b/cmd/hull/storemount_linux.go @@ -34,10 +34,17 @@ import ( // instance state, the urunit config with the guest's environment, and the // block image all live under it. func ensureStore(storeDir, _ string) error { + return ensurePlainStore(storeDir, dirIsCaseSensitive) +} + +// ensurePlainStore is ensureStore with the case probe passed in, so the +// refusal can be tested on a host that has no case-folding filesystem to +// point it at. +func ensurePlainStore(storeDir string, caseSensitive func(string) (bool, error)) error { if err := os.MkdirAll(storeDir, 0o700); err != nil { return fmt.Errorf("create store directory: %w", err) } - sensitive, err := dirIsCaseSensitive(storeDir) + sensitive, err := caseSensitive(storeDir) if err != nil { return fmt.Errorf("check whether %s is case-sensitive: %w", storeDir, err) } diff --git a/docs/backends.md b/docs/backends.md index 5d553eb4..07e99886 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -225,19 +225,29 @@ long-running `hvi` guest has no host time source to correct drift against. ## `hvi` on Linux -hull also builds for arm64 Linux, where it runs `hvi` on KVM and nothing else. -It is there to be the other end of a sandbox that started on a Mac, so it -offers the one configuration a sandbox can carry between the two: `hvi`, a -block rootfs, and either no network or a gateway. brig drives it with the same -verbs it uses on macOS: `run`, `exec`, `ps`, `inspect`, `logs`, `stop`, `rm`, -`assets dir` and `network-gateway`. `pull`, `images`, `rmi`, `prune`, -`telemetry` and `version` work too. +hull also builds for arm64 Linux, where it runs `hvi` on KVM and nothing else: +`hvi`, a block rootfs, and either no network or a gateway. That is the +configuration a sandbox needs to be moved between two KVM hosts, which is +planned and not built yet; no checkpoint, restore or migration exists on +Linux today. brig drives it with the same verbs it uses on macOS: `run`, +`exec`, `ps`, `inspect`, `logs`, `stop`, `rm`, `assets dir` and +`network-gateway`. `pull`, `images`, `rmi`, `prune`, `telemetry` and `version` +work too. + +A sandbox started on a Mac cannot be moved to Linux, and none of the planned +work changes that. A guest reads its counter frequency once, at boot, and +KVM cannot change the one a host gives it: 24 MHz on Apple silicon, 54 MHz on +a Raspberry Pi 5. The interrupt controllers differ too (GICv3 under +Hypervisor.framework, a GICv2 on the Pi), and so do the CPU ID registers. What is different from macOS: -- **The backend is always `hvi`.** With no `--hypervisor` flag, hull on Linux - picks `hvi`, not `qemu`, because nothing else is offered. Any other backend is - refused before the image is pulled. +- **The backend is always `hvi`.** With no `--hypervisor` flag and no + annotation, hull on Linux picks `hvi`, not `qemu`, because nothing else is + offered. Any other `--hypervisor` is refused before the image is pulled. An + image whose `com.urunc.unikernel.hypervisor` annotation names another + backend is refused once it is pulled; `--hypervisor hvi` runs it on `hvi` + regardless. - **The rootfs is always a block image.** `--rootfs-type` defaults to `block`; `virtiofs` and `9pfs` are refused, and so are `--shared-dir` and `--shared-dir-fd`. `hvi`'s KVM backend has no virtio-fs server: it accepts @@ -263,10 +273,19 @@ What is different from macOS: - **Image ownership is recorded on unpack.** Run as an ordinary user, hull cannot chown the files it unpacks, so it records each file's uid, gid and mode in a `user.com.nofire.hvi.linux-attr` xattr, as macOS does in its own - namespace, and writes them into the block image. Run as root, the files - carry the real owners and nothing is recorded. Symlinks cannot hold a - `user.` xattr on Linux, so an unprivileged unpack leaves them owned by the - user who ran hull. + namespace, and writes them into the block image, then removes the xattr + from the image. Symlinks cannot hold a `user.` xattr on Linux, so their + owners go in `rootfs.link-owners.json` beside the image's rootfs. On the + host, unpacked files keep owner read and directories owner rwx even where + the image asks for less (Fedora's `/etc/shadow` is 0000), so mke2fs can + read them and `hull rmi` can delete them; the image's own mode is what the + guest sees. Run as root, the files carry the real owners and modes and + nothing is recorded. +- **No pull inside a single-id user namespace.** Under `unshare -r`, hull is + uid 0, so it chowns as root would and the chown fails with `invalid + argument` on the first file owned by an id the namespace does not map + (`/etc/shadow`, gid 42, in alpine). The pull stops there. Run hull as an + ordinary user or as real root instead. - **Not on Linux:** `compose`, `checkpoint`, `restore`, `store`, `--gui` and `--rosetta`. diff --git a/docs/build.md b/docs/build.md index c3b8cbe6..e9167c97 100644 --- a/docs/build.md +++ b/docs/build.md @@ -126,9 +126,13 @@ There is no release archive for Linux yet. Build both halves yourself: make linux # dist/hull_linux_arm64, cross-built from any host ``` -Build `hvi` on the Linux host itself, from the same submodule commit. That is -the tested path; hvi-vmm's own CI builds its Linux targets on Linux runners -too. +Build `hvi` on the Linux host itself, from the `hvi-vmm` submodule commit +hull pins. That is the tested path: the Raspberry Pi 5 runs that reviewed +and then checked hull on Linux, gateway mode included, used `hvi` built on the +Pi from the pin, `890f9e0`. hvi-vmm's own CI does not build or test +aarch64 Linux at that commit. It builds and tests x86_64 Linux, boots it on an +x86 KVM runner, and only lints the `aarch64-unknown-linux-gnu` target, from an +x86 runner. ```bash cd hvi-vmm && cargo build --release # target/release/hvi diff --git a/pkg/ociclient/guestattr_darwin.go b/pkg/ociclient/guestattr_darwin.go index ba9e33fe..d3d6180e 100644 --- a/pkg/ociclient/guestattr_darwin.go +++ b/pkg/ociclient/guestattr_darwin.go @@ -38,6 +38,11 @@ import ( // change with it. const guestAttrXattr = "com.nofire.hvi.linux-attr" +// CopiedRecordXattr is empty on macOS: the record's name is in no namespace +// e2fsprogs maps to ext4, so mke2fs -d leaves it out of the block image and +// there is nothing for the builder to remove. +const CopiedRecordXattr = "" + // recordsOwnership says the manual extractor has to run on every layer, not // only on the ones archive.Apply could not finish. // @@ -47,6 +52,20 @@ const guestAttrXattr = "com.nofire.hvi.linux-attr" // which is not yet pinned down, so this adds a pass rather than replacing one. const recordsOwnership = true +// recordsLinksAside is false on macOS: XATTR_NOFOLLOW puts a symlink's +// record on the link itself. +const recordsLinksAside = false + +// hostFileMode and hostDirMode return the image's mode unchanged on macOS. +// +// The Linux build widens what the owner may do with an unpacked file, because +// there the host file only feeds mke2fs. Here it is also what vz and qemu +// serve to the guest over virtiofs and 9pfs, which report the host mode +// straight through, so a widened /etc/shadow would reach those guests readable. +func hostFileMode(mode int64) int64 { return mode } + +func hostDirMode(mode int64) int64 { return mode } + // recordGuestAttr stores the ownership and mode an entry has inside the image. // // symlink is set for entries whose own attributes matter rather than their diff --git a/pkg/ociclient/guestattr_linux.go b/pkg/ociclient/guestattr_linux.go new file mode 100644 index 00000000..6645678a --- /dev/null +++ b/pkg/ociclient/guestattr_linux.go @@ -0,0 +1,142 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package ociclient + +import ( + "encoding/binary" + "os" + + "golang.org/x/sys/unix" +) + +// guestAttrXattr is the darwin record's name in the user namespace, which is +// the only one an unprivileged process may write on Linux. +// +// Linux has the same gap macOS does whenever hull is not root: archive.Apply +// fails its first chown, the manual extractor skips chown altogether, and +// every file in the unpacked rootfs ends up owned by whoever ran the pull -- +// /etc/shadow, setuid-root binaries and all. mke2fs copies that ownership into +// the block image verbatim, so without a record the guest boots with a root +// filesystem owned by uid 1000, where sudo refuses to run and a setuid binary +// drops its caller to an ordinary user. The record carries what the tar header +// said, and buildBlockRootfs writes it into the finished image through debugfs, +// exactly as on macOS. +// +// The layout is the darwin one, twelve bytes, mode then uid then gid. hvi's +// Linux backend has no virtio-fs server, so nothing outside hull reads it here +// and the name only has to agree with GuestAttr below. +// +// Symlinks get no record here: Linux refuses user.* attributes on a symlink +// (EPERM). Their ownership is set aside in a file beside the rootfs instead; +// see recordsLinksAside and LinkOwners. +const guestAttrXattr = "user.com.nofire.hvi.linux-attr" + +// CopiedRecordXattr names the record as it arrives in a block image built +// from the unpacked tree, so the builder can take it out again. +// +// mke2fs -d copies every user.* attribute it finds, so without that every +// file in a Linux-built guest image carries hull's host-side bookkeeping, +// visible to the guest through getfattr, `tar --xattrs` and `rsync -X`, and +// the image differs from the one a Mac builds from the same layers. +const CopiedRecordXattr = guestAttrXattr + +// recordsOwnership is true when the unpack cannot chown, i.e. when hull is not +// root. A root unpack keeps the image's ownership in the filesystem itself, +// mke2fs then copies the real one, and there is nothing to record. +// +// It is decided by the effective uid, not by whether archive.Apply happened to +// fail: an image whose files all belong to the unpacking user unpacks without +// a chown error, and its ownership still has to be recorded. +// +// A single-id user namespace (`unshare -r`) is not handled. There the euid is +// 0, so nothing is recorded, and containerd chowns as root would: the first +// file owned by an unmapped id fails with EINVAL, which is not the expected +// permission error, so the pull stops. Treating EINVAL as expected would also +// swallow failures that are not about ownership at all. +var recordsOwnership = os.Geteuid() != 0 + +// recordsLinksAside says symlink ownership goes in the file LinkOwners reads, +// because a symlink cannot hold the record. Without it every link in a +// Linux-built image is owned by the user who ran the pull, /bin/sh -> busybox +// included, where the same image built on a Mac has 0:0 links. +var recordsLinksAside = recordsOwnership + +// hostFileMode is the mode an unpacked regular file is given on the host: +// the image's own, with owner read added when the unpack records ownership. +// +// Fedora, RHEL, Rocky, Alma, UBI and Amazon Linux ship /etc/shadow and +// /etc/gshadow at 0000, and Fedora's sudo is 04111. Put on the host as is, a +// file like that is unreadable to the user who unpacked it -- root is the only +// one DAC lets past 0000 -- and `mke2fs -d` stops on the first one with +// "Permission denied while populating file system". Block is the only rootfs +// on Linux, so that refused the whole image family. The image's mode is not +// lost by widening: the record beside the file holds it, and +// buildBlockRootfs writes it into the finished image. +// +// A root unpack keeps the mode untouched: root reads a 0000 file regardless, +// and there is no record to put the real mode back from. +func hostFileMode(mode int64) int64 { + if recordsOwnership { + return mode | 0o400 + } + return mode +} + +// hostDirMode is hostFileMode for directories, which need the owner's write +// and search bits as well as read: without them mke2fs cannot list a 0111 +// directory, and nobody but root can delete a tree that has 0555 directories +// in it (Fedora has dozens under /usr/lib), so `hull rmi` and the pull's own +// sweeps would fail on it. +func hostDirMode(mode int64) int64 { + if recordsOwnership { + return mode | 0o700 + } + return mode +} + +// recordGuestAttr stores the ownership and mode an entry has inside the image. +// Errors are the caller's to ignore, as on darwin: a filesystem with no user +// xattr support (tmpfs before 6.6, some network filesystems) costs the guest +// correct ownership, not the pull. +func recordGuestAttr(path string, mode uint32, uid, gid uint32, symlink bool) error { + if symlink { + return nil + } + var value [12]byte + binary.LittleEndian.PutUint32(value[0:4], mode&0o177777) + binary.LittleEndian.PutUint32(value[4:8], uid) + binary.LittleEndian.PutUint32(value[8:12], gid) + return unix.Setxattr(path, guestAttrXattr, value[:], 0) +} + +// GuestAttr returns the ownership and mode recorded for path by the unpack, +// and whether there was a record at all. A tree unpacked as root has none, and +// its files carry the image's ownership on disk, which is what the caller then +// gets by reading the file itself. +func GuestAttr(path string, symlink bool) (mode, uid, gid uint32, ok bool) { + if symlink { + return 0, 0, 0, false + } + value := make([]byte, 12) + n, err := unix.Getxattr(path, guestAttrXattr, value) + if err != nil || n != 12 { + return 0, 0, 0, false + } + return binary.LittleEndian.Uint32(value[0:4]), + binary.LittleEndian.Uint32(value[4:8]), + binary.LittleEndian.Uint32(value[8:12]), true +} diff --git a/pkg/ociclient/guestattr_linux_test.go b/pkg/ociclient/guestattr_linux_test.go new file mode 100644 index 00000000..16b70f42 --- /dev/null +++ b/pkg/ociclient/guestattr_linux_test.go @@ -0,0 +1,161 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ociclient + +import ( + "archive/tar" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "golang.org/x/sys/unix" +) + +// An unprivileged Linux unpack cannot chown, so the image's ownership has to +// survive beside the file until the block image is built from it. +func TestGuestAttrRoundTripsOnLinux(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "shadow") + if err := os.WriteFile(p, nil, 0o600); err != nil { + t.Fatal(err) + } + err := recordGuestAttr(p, 0o100640, 0, 42, false) + if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EOPNOTSUPP) { + t.Skipf("%s has no user xattr support", dir) + } + if err != nil { + t.Fatal(err) + } + mode, uid, gid, ok := GuestAttr(p, false) + if !ok || mode != 0o100640 || uid != 0 || gid != 42 { + t.Fatalf("GuestAttr = %o %d %d %v, want 100640 0 42 true", mode, uid, gid, ok) + } + + // No record is the root-unpack case: the caller reads the file itself. + bare := filepath.Join(dir, "bare") + if err := os.WriteFile(bare, nil, 0o644); err != nil { + t.Fatal(err) + } + if _, _, _, ok := GuestAttr(bare, false); ok { + t.Error("a file with no record reported one") + } + + // Linux refuses user.* on a symlink; recording one must not fail the + // unpack, and reading one reports no record. + link := filepath.Join(dir, "link") + if err := os.Symlink("shadow", link); err != nil { + t.Fatal(err) + } + if err := recordGuestAttr(link, 0o120777, 0, 0, true); err != nil { + t.Errorf("recording a symlink failed: %v", err) + } + if _, _, _, ok := GuestAttr(link, true); ok { + t.Error("a symlink reported a record") + } +} + +// An unprivileged unpack keeps what mke2fs and rmi need on the host -- owner +// read on files, owner rwx on directories -- while the record keeps the mode +// the image asked for, which is what the guest gets. Fedora's /etc/shadow is +// 0000 and its sudo 04111; put on the host as is, neither can be read by the +// user who owns it. +func TestUnprivilegedUnpackKeepsFilesReadableAndRecordsTheImageMode(t *testing.T) { + if !recordsOwnership { + t.Skip("a root unpack keeps the image's modes on the host") + } + rootfs := t.TempDir() + layer := rawTarball(t, + tarEntry{hdr: tar.Header{Name: "etc/", Typeflag: tar.TypeDir, Mode: 0o755}}, + tarEntry{hdr: tar.Header{Name: "etc/shadow", Typeflag: tar.TypeReg, Mode: 0o000, Gid: 42}, body: "root:!::0:::::\n"}, + tarEntry{hdr: tar.Header{Name: "usr/bin/sudo", Typeflag: tar.TypeReg, Mode: 0o4111}, body: "#!/bin/sh\n"}, + tarEntry{hdr: tar.Header{Name: "usr/lib/sealed/", Typeflag: tar.TypeDir, Mode: 0o555}}, + tarEntry{hdr: tar.Header{Name: "usr/lib/sealed/f", Typeflag: tar.TypeReg, Mode: 0o444}, body: "x"}, + ) + if err := extractTarIgnoreChown(layer, rootfs); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + hostPerm os.FileMode + recorded uint32 + }{ + {"etc/shadow", 0o400, 0o000}, + {"usr/bin/sudo", 0o511 | os.ModeSetuid, 0o4111}, + {"usr/lib/sealed", 0o755, 0o555}, + } { + p := filepath.Join(rootfs, tc.name) + fi, err := os.Lstat(p) + if err != nil { + t.Fatal(err) + } + if got := fi.Mode() & (os.ModePerm | os.ModeSetuid); got != tc.hostPerm { + t.Errorf("%s is %v on the host, want %v", tc.name, got, tc.hostPerm) + } + mode, _, _, ok := GuestAttr(p, false) + if !ok { + t.Skipf("%s has no user xattr support", rootfs) + } + if mode&0o7777 != tc.recorded { + t.Errorf("%s recorded as %o, want %o", tc.name, mode&0o7777, tc.recorded) + } + } + if _, err := os.ReadFile(filepath.Join(rootfs, "etc/shadow")); err != nil { + t.Errorf("the owner cannot read the unpacked /etc/shadow, so mke2fs could not either: %v", err) + } +} + +// Linux cannot put a user.* record on a symlink, so the unpack sets symlink +// ownership aside beside the rootfs, and a later layer's link at the same name +// wins. +func TestUnprivilegedUnpackSetsSymlinkOwnershipAside(t *testing.T) { + if !recordsOwnership { + t.Skip("a root unpack chowns the links themselves") + } + rootfs := filepath.Join(t.TempDir(), "rootfs") + if err := os.Mkdir(rootfs, 0o755); err != nil { + t.Fatal(err) + } + first := rawTarball(t, + tarEntry{hdr: tar.Header{Name: "bin/sh", Typeflag: tar.TypeSymlink, Linkname: "busybox"}}, + tarEntry{hdr: tar.Header{Name: "home/app/.profile", Typeflag: tar.TypeSymlink, Linkname: "/etc/profile", Uid: 1000, Gid: 1000}}, + ) + second := rawTarball(t, + tarEntry{hdr: tar.Header{Name: "bin/sh", Typeflag: tar.TypeSymlink, Linkname: "bash", Uid: 0, Gid: 7}}, + ) + for _, layer := range []io.Reader{first, second} { + if err := extractTarIgnoreChown(layer, rootfs); err != nil { + t.Fatal(err) + } + } + owners, err := LinkOwners(rootfs) + if err != nil { + t.Fatal(err) + } + want := map[string]LinkOwner{ + "bin/sh": {UID: 0, GID: 7}, + "home/app/.profile": {UID: 1000, GID: 1000}, + } + if !reflect.DeepEqual(owners, want) { + t.Errorf("LinkOwners = %v, want %v", owners, want) + } + // Beside the rootfs, never in it: anything in it is in the guest image. + if rel, err := filepath.Rel(rootfs, linkOwnersPath(rootfs)); err != nil || !strings.HasPrefix(rel, "..") { + t.Errorf("the symlink record is at %s, inside the rootfs", linkOwnersPath(rootfs)) + } +} diff --git a/pkg/ociclient/guestattr_other.go b/pkg/ociclient/guestattr_other.go index b789f895..f33822ef 100644 --- a/pkg/ociclient/guestattr_other.go +++ b/pkg/ociclient/guestattr_other.go @@ -12,16 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !darwin +//go:build !darwin && !linux package ociclient -// Everywhere else an unpack can hold the image's ownership in the filesystem -// itself, so there is nothing to record beside it and no reason to avoid -// containerd's extractor. See the darwin file for why macOS cannot. +// Everywhere else hull is not run, so nothing is recorded. See the darwin +// file for why macOS records ownership beside each file, and the linux file +// for why an unprivileged Linux unpack has to as well. const recordsOwnership = false +const recordsLinksAside = false + +const CopiedRecordXattr = "" + +func hostFileMode(mode int64) int64 { return mode } + +func hostDirMode(mode int64) int64 { return mode } + func recordGuestAttr(_ string, _ uint32, _, _ uint32, _ bool) error { return nil } func GuestAttr(_ string, _ bool) (mode, uid, gid uint32, ok bool) { return 0, 0, 0, false } diff --git a/pkg/ociclient/linkowners.go b/pkg/ociclient/linkowners.go new file mode 100644 index 00000000..640ce190 --- /dev/null +++ b/pkg/ociclient/linkowners.go @@ -0,0 +1,87 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ociclient + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +// LinkOwner is a symlink's ownership as the image declares it. +type LinkOwner struct { + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` +} + +// linkOwnersPath is where an unpack keeps the ownership of the symlinks in +// rootfsDir: a file beside the rootfs, never inside it, where the guest would +// see it. In the store that is images//rootfs.link-owners.json, which +// the pull publishes and rmi removes together with the rootfs. +func linkOwnersPath(rootfsDir string) string { + return filepath.Clean(rootfsDir) + ".link-owners.json" +} + +// LinkOwners returns the symlink ownership an unpack set aside for rootfsDir, +// keyed by the slash-separated path inside it. No file means nothing was set +// aside -- a root unpack, a macOS one, which records links on the link itself, +// or a tree unpacked before this existed -- and is not an error. +// +// A key whose entry is no longer a symlink is possible and harmless: a later +// layer may have removed or replaced the link. Callers consult the map only +// for entries that are symlinks in the finished tree, and the last layer to +// write a symlink at a name is also the last to write its key. +func LinkOwners(rootfsDir string) (map[string]LinkOwner, error) { + data, err := os.ReadFile(linkOwnersPath(rootfsDir)) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var owners map[string]LinkOwner + if err := json.Unmarshal(data, &owners); err != nil { + return nil, fmt.Errorf("read %s: %w", linkOwnersPath(rootfsDir), err) + } + return owners, nil +} + +// mergeLinkOwners adds one layer's symlinks to what earlier layers set aside. +// It is written to a temporary file and renamed, so an interrupted pull leaves +// the previous layers' record rather than half of one. +func mergeLinkOwners(rootfsDir string, layer map[string]LinkOwner) error { + owners, err := LinkOwners(rootfsDir) + if err != nil { + return err + } + if owners == nil { + owners = make(map[string]LinkOwner, len(layer)) + } + for name, o := range layer { + owners[name] = o + } + data, err := json.Marshal(owners) + if err != nil { + return err + } + dst := linkOwnersPath(rootfsDir) + tmp := dst + ".tmp" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + return err + } + return os.Rename(tmp, dst) +} diff --git a/pkg/ociclient/unpack.go b/pkg/ociclient/unpack.go index d876ad6e..c94a3e42 100644 --- a/pkg/ociclient/unpack.go +++ b/pkg/ociclient/unpack.go @@ -513,6 +513,9 @@ func extractTarIgnoreChown(r io.Reader, rootfsDir string) error { // The mode each directory entry asked for, applied once the layer is // through rather than as the entry is read: see applyDirModes. dirModes = map[string]int64{} + // The ownership of each symlink, where the link itself cannot hold a + // record: see recordsLinksAside. + links = map[string]LinkOwner{} ) for { @@ -672,8 +675,10 @@ func extractTarIgnoreChown(r io.Reader, rootfsDir string) error { // stops being writable. keepAttr(false) - // Set file permissions - if err := chmodPOSIX(target, header.Mode); err != nil { + // Set file permissions. hostFileMode keeps the file readable by + // its owner where the image's own mode would lock even the owner + // out; see it for why that cannot be left to the image. + if err := chmodPOSIX(target, hostFileMode(header.Mode)); err != nil { log.Warnf("failed to chmod %s: %v", target, err) } @@ -694,6 +699,9 @@ func extractTarIgnoreChown(r io.Reader, rootfsDir string) error { return fmt.Errorf("failed to create symlink %s -> %s: %w", name, header.Linkname, err) } keepAttr(true) + if recordsLinksAside { + links[filepath.ToSlash(name)] = LinkOwner{UID: uint32(header.Uid), GID: uint32(header.Gid)} + } case tar.TypeLink: // Hard link to a file extracted earlier in this rootfs. @@ -749,7 +757,7 @@ func extractTarIgnoreChown(r io.Reader, rootfsDir string) error { return fmt.Errorf("failed to copy hard link target %s: %w", name, werr) } keepAttr(false) - if cerr := chmodPOSIX(target, header.Mode); cerr != nil { + if cerr := chmodPOSIX(target, hostFileMode(header.Mode)); cerr != nil { log.Warnf("failed to chmod %s: %v", target, cerr) } continue @@ -766,6 +774,13 @@ func extractTarIgnoreChown(r io.Reader, rootfsDir string) error { } applyDirModes(root, dirModes) + if len(links) > 0 { + // Not fatal, for the same reason a failed xattr record is not: the + // links are all there, and what is lost is their owner in the guest. + if err := mergeLinkOwners(rootfsDir, links); err != nil { + log.Warnf("could not record image ownership for %d symlink(s): %v", len(links), err) + } + } return nil } @@ -790,6 +805,10 @@ func extractTarIgnoreChown(r io.Reader, rootfsDir string) error { // setuid, setgid and sticky as high bits, so the conversion has to go through // posixFileMode -- handing it a raw 01777 would set neither the sticky bit nor // anything else recognisable. +// +// hostDirMode can add the owner's rwx on top of the image's mode. Where it +// does, the record beside the directory keeps the image's own, and that is +// the mode the guest gets. func applyDirModes(root *os.Root, modes map[string]int64) { names := make([]string, 0, len(modes)) for name := range modes { @@ -804,7 +823,7 @@ func applyDirModes(root *os.Root, modes map[string]int64) { return names[i] < names[j] }) for _, name := range names { - if err := root.Chmod(name, posixFileMode(modes[name])); err != nil { + if err := root.Chmod(name, posixFileMode(hostDirMode(modes[name]))); err != nil { // Not fatal: the directory is there with the content in it, and a // filesystem that will not take the mode (or an entry a later one // replaced) costs the guest a permission bit, not the pull. diff --git a/pkg/ociclient/unpack_test.go b/pkg/ociclient/unpack_test.go index d3c2fc25..86a4b196 100644 --- a/pkg/ociclient/unpack_test.go +++ b/pkg/ociclient/unpack_test.go @@ -858,9 +858,15 @@ func TestExtractTarAppliesDirectoryModes(t *testing.T) { t.Errorf("Stat(%s): %v", name, err) continue } + // The host mode is the image's on macOS, where vz and qemu serve it to + // the guest. An unprivileged Linux unpack adds owner rwx on the host, + // and the guest gets the image's mode from the record instead. got := fi.Mode() & (fs.ModePerm | fs.ModeSetuid | fs.ModeSetgid | fs.ModeSticky) - if got != posixFileMode(want) { - t.Errorf("%s mode = %v, want %v (image asked for %#o)", name, got, posixFileMode(want), want) + if got != posixFileMode(hostDirMode(want)) { + t.Errorf("%s mode = %v, want %v (image asked for %#o)", name, got, posixFileMode(hostDirMode(want)), want) + } + if mode, _, _, ok := GuestAttr(filepath.Join(rootfs, name), false); ok && int64(mode&0o7777) != want { + t.Errorf("%s recorded as %#o, want %#o", name, mode&0o7777, want) } } From 01ed4881ac26f96c1e21223fd6314fca616bf151 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Wed, 14 Oct 2026 02:55:15 +0300 Subject: [PATCH 07/33] fix(store): remove image trees that hold read-only directories Unlinking an entry needs write permission on its directory, and an image is free to ship directories at 0555 (Fedora has dozens under /usr/lib). The unpack puts that mode on the host, so os.RemoveAll by the owner stopped with EACCES, and hull rmi, prune, instance removal and the pull's own sweeps of old and staging directories left the tree behind. RemoveTree retries a removal that failed on a permission error after adding owner rwx to every directory, walking through an os.Root so a directory symlink in the image cannot point the chmod at a host directory. Signed-off-by: Panagiotis Moustafellos --- pkg/ociclient/client.go | 2 +- pkg/store/image.go | 6 +-- pkg/store/removetree.go | 72 +++++++++++++++++++++++++++++++++++ pkg/store/removetree_test.go | 73 ++++++++++++++++++++++++++++++++++++ pkg/store/store.go | 4 +- 5 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 pkg/store/removetree.go create mode 100644 pkg/store/removetree_test.go diff --git a/pkg/ociclient/client.go b/pkg/ociclient/client.go index ebe4370a..d72ab128 100644 --- a/pkg/ociclient/client.go +++ b/pkg/ociclient/client.go @@ -267,7 +267,7 @@ func (c *Client) PullPlatform(ctx context.Context, ref, platformStr string) (*Pu defer func() { _ = staging.Close() }() defer func() { if !published { - _ = os.RemoveAll(tmpDir) + _ = store.RemoveTree(tmpDir) } }() diff --git a/pkg/store/image.go b/pkg/store/image.go index ec80ca16..038274ea 100644 --- a/pkg/store/image.go +++ b/pkg/store/image.go @@ -74,7 +74,7 @@ func (s *Store) DeleteImage(digest string) error { return err } defer func() { _ = held.Close() }() - if err := os.RemoveAll(aside); err != nil { + if err := RemoveTree(aside); err != nil { return fmt.Errorf("failed to remove image %s: %w", digest, err) } return nil @@ -152,7 +152,7 @@ func (s *Store) CommitImage(digest, stagingDir string, staging *StagingLock) err return err } if previous != nil { - _ = os.RemoveAll(oldDir) + _ = RemoveTree(oldDir) _ = previous.Close() } return nil @@ -250,7 +250,7 @@ func (s *Store) RemoveStagingDir(name string) (bool, error) { return false, err } defer func() { _ = l.Close() }() - if err := os.RemoveAll(dir); err != nil { + if err := RemoveTree(dir); err != nil { return false, fmt.Errorf("failed to remove staging directory %s: %w", name, err) } return true, nil diff --git a/pkg/store/removetree.go b/pkg/store/removetree.go new file mode 100644 index 00000000..810f1291 --- /dev/null +++ b/pkg/store/removetree.go @@ -0,0 +1,72 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "errors" + "io/fs" + "os" +) + +// RemoveTree is os.RemoveAll for a tree that an image unpack wrote. +// +// An image is free to ship directories its owner may not write into: Fedora +// has dozens of 0555 directories under /usr/lib, and the unpack puts the +// image's mode on the host directory. Unlinking an entry needs write +// permission on the directory that holds it, so a plain RemoveAll stops with +// EACCES on the first one, and `hull rmi`, prune and the pull's own sweeps +// leave the image behind -- as the user who owns every file in it. +// +// The plain removal runs first because it is what nearly every tree needs, +// and only a permission failure pays for the second walk. +func RemoveTree(path string) error { + err := os.RemoveAll(path) + if err == nil || !errors.Is(err, fs.ErrPermission) { + return err + } + if openErr := openTreeForRemoval(path); openErr != nil { + return err + } + return os.RemoveAll(path) +} + +// openTreeForRemoval adds the owner's rwx to every directory under path. +// +// The walk goes through an os.Root, as every other walk of an image tree in +// hull does: the tree is the image's, a directory in it may be an absolute +// symlink, and a chmod that followed one would loosen a host directory the +// image chose. fs.WalkDir calls back on a directory before it reads it, so +// even a 0000 directory is opened up in time to be listed. +// +// Failures on single entries are not reported here. Whatever could not be +// opened up makes the second RemoveAll fail, and that error names the entry. +func openTreeForRemoval(path string) error { + root, err := os.OpenRoot(path) + if err != nil { + return err + } + defer func() { _ = root.Close() }() + return fs.WalkDir(root.FS(), ".", func(name string, d fs.DirEntry, err error) error { + if err != nil || !d.IsDir() { + return nil + } + info, statErr := d.Info() + if statErr != nil || info.Mode().Perm()&0o700 == 0o700 { + return nil + } + _ = root.Chmod(name, info.Mode().Perm()|0o700) + return nil + }) +} diff --git a/pkg/store/removetree_test.go b/pkg/store/removetree_test.go new file mode 100644 index 00000000..57f74622 --- /dev/null +++ b/pkg/store/removetree_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "os" + "path/filepath" + "testing" +) + +// An image's 0555 and 0000 directories stop a plain RemoveAll by their owner, +// and a directory symlink in the tree must not be followed while opening the +// tree up: the chmod would land on a host directory the image chose. +func TestRemoveTreeRemovesReadOnlyDirectories(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores directory permissions, so there is nothing to open up") + } + base := t.TempDir() + tree := filepath.Join(base, "rootfs") + readOnly := filepath.Join(tree, "usr", "lib", "sealed") + if err := os.MkdirAll(readOnly, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(readOnly, "f"), []byte("x"), 0o444); err != nil { + t.Fatal(err) + } + locked := filepath.Join(tree, "locked") + if err := os.MkdirAll(filepath.Join(locked, "inner"), 0o755); err != nil { + t.Fatal(err) + } + outside := filepath.Join(base, "host-dir") + if err := os.Mkdir(outside, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(outside, 0o700) }) + if err := os.Symlink(outside, filepath.Join(tree, "escape")); err != nil { + t.Fatal(err) + } + for _, d := range []string{readOnly, filepath.Join(tree, "usr", "lib")} { + if err := os.Chmod(d, 0o555); err != nil { + t.Fatal(err) + } + } + if err := os.Chmod(locked, 0o000); err != nil { + t.Fatal(err) + } + + if err := RemoveTree(tree); err != nil { + t.Fatalf("RemoveTree: %v", err) + } + if _, err := os.Lstat(tree); !os.IsNotExist(err) { + t.Fatalf("tree is still there: %v", err) + } + fi, err := os.Stat(outside) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o500 { + t.Errorf("the directory behind a symlink in the tree went from 0500 to %o", fi.Mode().Perm()) + } +} diff --git a/pkg/store/store.go b/pkg/store/store.go index f307cdf6..d753345e 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -673,8 +673,10 @@ func (s *Store) DeleteInstance(id string) error { s.mu.Lock() defer s.mu.Unlock() + // RemoveTree, because a virtiofs instance holds a clone of the image's + // rootfs, 0555 directories and all. instanceDir := filepath.Join(s.rootDir, "instances", id) - return os.RemoveAll(instanceDir) + return RemoveTree(instanceDir) } // RootDir returns the root directory of the store From efd87a72945a2ccd81b31afb3904fe1709773b64 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Wed, 14 Oct 2026 04:05:00 +0300 Subject: [PATCH 08/33] feat(hvi): pause, resume and stop hvi over a control socket hull could only signal an hvi VMM. Every hvi run now gets --control-sock and --profile-out in its 0700 instance directory, and the record keeps both paths. hull pause and hull resume work on an hvi instance, and hull stop asks hvi to stop before it signals. The client sends one request per connection with "v":1, since hvi serves one connection at a time, and reads the reply under a timeout longer than the pause deadline hvi gives. A refusal comes back with hvi's code, and a reply that never came is told apart from a refusal, because the op may still complete in hvi. hull ps shows a paused or fenced VM, which keeps its record at running, and hull exec refuses to dial into one instead of hanging on its agent. The control socket path is checked before the image is pulled, as the agent socket path already was. Signed-off-by: Panagiotis Moustafellos --- cmd/hull/commands_darwin.go | 2 + cmd/hull/commands_linux.go | 5 +- cmd/hull/exec.go | 6 ++ cmd/hull/hvicontrol.go | 158 +++++++++++++++++++++++++++++ cmd/hull/hvicontrol_test.go | 192 ++++++++++++++++++++++++++++++++++++ cmd/hull/hvipause.go | 169 +++++++++++++++++++++++++++++++ cmd/hull/hvipause_test.go | 110 +++++++++++++++++++++ cmd/hull/hvistate_test.go | 122 +++++++++++++++++++++++ cmd/hull/inspect.go | 16 ++- cmd/hull/ps.go | 18 +++- cmd/hull/run.go | 28 ++++++ cmd/hull/stop.go | 26 +++++ docs/running.md | 15 ++- pkg/store/store.go | 50 ++++++++++ pkg/store/store_test.go | 63 ++++++++++++ 15 files changed, 972 insertions(+), 8 deletions(-) create mode 100644 cmd/hull/hvicontrol.go create mode 100644 cmd/hull/hvicontrol_test.go create mode 100644 cmd/hull/hvipause.go create mode 100644 cmd/hull/hvipause_test.go create mode 100644 cmd/hull/hvistate_test.go diff --git a/cmd/hull/commands_darwin.go b/cmd/hull/commands_darwin.go index ff8a14ce..53367c63 100644 --- a/cmd/hull/commands_darwin.go +++ b/cmd/hull/commands_darwin.go @@ -26,6 +26,8 @@ func platformCommands() []*cli.Command { execCommand(), psCommand(), stopCommand(), + pauseCommand(), + resumeCommand(), checkpointCommand(), restoreCommand(), rmCommand(), diff --git a/cmd/hull/commands_linux.go b/cmd/hull/commands_linux.go index 87b7c6b8..ac6143b8 100644 --- a/cmd/hull/commands_linux.go +++ b/cmd/hull/commands_linux.go @@ -25,7 +25,8 @@ const appUsage = "container CLI for hvi sandboxes on Linux/KVM" // platformCommands is the Linux verb set: what it takes to run an hvi sandbox // on KVM, and what brig calls to drive one (run, exec, ps, inspect, logs, -// stop, rm, assets dir, network-gateway). +// stop, rm, assets dir, network-gateway), plus pause and resume over hvi's +// control socket. // // Left out, because they drive something Linux does not have: // - checkpoint and restore: vz-runner's saved-state files, which are @@ -40,6 +41,8 @@ func platformCommands() []*cli.Command { execCommand(), psCommand(), stopCommand(), + pauseCommand(), + resumeCommand(), rmCommand(), logsCommand(), inspectCommand(), diff --git a/cmd/hull/exec.go b/cmd/hull/exec.go index d683fda9..29c386e1 100644 --- a/cmd/hull/exec.go +++ b/cmd/hull/exec.go @@ -110,6 +110,12 @@ func execInstance(_ context.Context, cmd *cli.Command) error { if state.Status != "running" { return fmt.Errorf("instance %s is not running (status: %s)", instanceID, state.Status) } + // The agent socket of a paused or fenced hvi guest accepts and then + // waits for a guest that never answers: exec hung there, deaf to + // SIGTERM, until it was killed. + if err := refuseIfGuestHalted(state); err != nil { + return err + } sockPath := s.InstanceAgentSocket(instanceID) conn, err := net.Dial("unix", sockPath) diff --git a/cmd/hull/hvicontrol.go b/cmd/hull/hvicontrol.go new file mode 100644 index 00000000..4e318447 --- /dev/null +++ b/cmd/hull/hvicontrol.go @@ -0,0 +1,158 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "time" +) + +// hviControlVersion is the protocol version every request and reply on hvi's +// control socket carries as "v". hvi answers a request without it with +// unsupported_version, so it is never optional here. +const hviControlVersion = 1 + +// hviControlTimeout bounds every control op except checkpoint. +// +// hvi gives a pause one 2 s deadline for the I/O gate and the vCPUs together +// and asks clients for a reply timeout of at least 3 s (hvi docs/cli.md, "The +// control socket"). A shorter one gives up on a pause that then succeeds, +// leaving a VM paused that hull reported as not paused. 5 s leaves room for a +// loaded host without letting a wedged VMM hang the CLI. +const hviControlTimeout = 5 * time.Second + +// hviControlMaxReply bounds a reply line. The largest is the profile, a few +// KiB; the bound keeps a misbehaving peer from growing hull's buffer without +// end. +const hviControlMaxReply = 1 << 20 + +// hviReply is one reply line. Fields not used by an op are left zero. +type hviReply struct { + V int `json:"v"` + OK bool `json:"ok"` + State string `json:"state,omitempty"` + Vcpus int `json:"vcpus,omitempty"` + VcpusParked int `json:"vcpus_parked,omitempty"` + Code string `json:"code,omitempty"` + Error string `json:"error,omitempty"` + ManifestSHA256 string `json:"manifest_sha256,omitempty"` + Stats json.RawMessage `json:"stats,omitempty"` + Profile json.RawMessage `json:"profile,omitempty"` +} + +// hviRefusal is hvi answering ok:false. Callers branch on Code, which hvi +// keeps stable (src/checkpoint/control.rs ErrorCode); Message is prose for a +// person and may change. +type hviRefusal struct { + Op string + Code string + Message string +} + +func (e *hviRefusal) Error() string { + return fmt.Sprintf("hvi refused %s: %s (%s)", e.Op, e.Message, e.Code) +} + +// hviRefusalCode returns the code of an hviRefusal in err's chain, or "". +func hviRefusalCode(err error) string { + var r *hviRefusal + if errors.As(err, &r) { + return r.Code + } + return "" +} + +// errHviNoReply marks a request that was sent and got no reply in time. For a +// state-changing op that is not a "no": the op may still complete in hvi. +var errHviNoReply = errors.New("no reply from hvi") + +// hviControl sends one request on sock and returns hvi's reply. +// +// One connection per request: hvi serves one connection at a time, so a +// client holding one open would lock every other hull command out of the VM. +// An ok:false reply comes back as an *hviRefusal with the reply alongside. +func hviControl(sock string, req map[string]any, timeout time.Duration) (*hviReply, error) { + op, _ := req["op"].(string) + msg := map[string]any{"v": hviControlVersion} + for k, v := range req { + msg[k] = v + } + line, err := json.Marshal(msg) + if err != nil { + return nil, err + } + + deadline := time.Now().Add(timeout) + conn, err := net.DialTimeout("unix", sock, timeout) + if err != nil { + return nil, fmt.Errorf("connect to hvi's control socket %s: %w", sock, err) + } + defer func() { _ = conn.Close() }() + if err := conn.SetDeadline(deadline); err != nil { + return nil, err + } + if _, err := conn.Write(append(line, '\n')); err != nil { + return nil, fmt.Errorf("send %s to hvi: %w", op, err) + } + + reply, err := readHviReply(conn) + if err != nil { + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() || errors.Is(err, os.ErrDeadlineExceeded) { + return nil, fmt.Errorf("%w to %s within %s", errHviNoReply, op, timeout) + } + return nil, fmt.Errorf("read hvi's reply to %s: %w", op, err) + } + if reply.V != hviControlVersion { + return nil, fmt.Errorf("hvi replied to %s with protocol version %d, hull speaks %d", + op, reply.V, hviControlVersion) + } + if !reply.OK { + return reply, &hviRefusal{Op: op, Code: reply.Code, Message: reply.Error} + } + return reply, nil +} + +// readHviReply reads one newline-terminated JSON object, refusing a line +// longer than hviControlMaxReply. +func readHviReply(r io.Reader) (*hviReply, error) { + br := bufio.NewReaderSize(io.LimitReader(r, hviControlMaxReply+1), 4096) + line, err := br.ReadBytes('\n') + if err != nil { + if errors.Is(err, io.EOF) { + if len(line) > hviControlMaxReply { + return nil, fmt.Errorf("reply longer than %d bytes", hviControlMaxReply) + } + return nil, errors.New("hvi closed the connection without a reply") + } + return nil, err + } + var reply hviReply + if err := json.Unmarshal(line, &reply); err != nil { + return nil, fmt.Errorf("reply is not JSON: %w", err) + } + return &reply, nil +} + +// hviOp sends a request with no arguments under the default timeout. +func hviOp(sock, op string) (*hviReply, error) { + return hviControl(sock, map[string]any{"op": op}, hviControlTimeout) +} diff --git a/cmd/hull/hvicontrol_test.go b/cmd/hull/hvicontrol_test.go new file mode 100644 index 00000000..8ad74fac --- /dev/null +++ b/cmd/hull/hvicontrol_test.go @@ -0,0 +1,192 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "encoding/json" + "errors" + "net" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// stubHvi is a stand-in for hvi's control socket: it reads one request line +// per connection, records it, and answers with whatever reply the test's +// handler returns, after the handler's delay. +type stubHvi struct { + path string + + mu sync.Mutex + requests []map[string]any +} + +type stubAnswer struct { + reply string + delay time.Duration +} + +func newStubHvi(t *testing.T, handle func(req map[string]any) stubAnswer) *stubHvi { + t.Helper() + stub := &stubHvi{path: filepath.Join(shortSocketDir(t), "c.sock")} + ln, err := net.Listen("unix", stub.path) + if err != nil { + t.Fatalf("listen: %v", err) + } + done := make(chan struct{}) + t.Cleanup(func() { + close(done) + _ = ln.Close() + }) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + defer func() { _ = conn.Close() }() + line, err := bufio.NewReader(conn).ReadBytes('\n') + if err != nil { + return + } + var req map[string]any + if err := json.Unmarshal(line, &req); err != nil { + return + } + stub.mu.Lock() + stub.requests = append(stub.requests, req) + stub.mu.Unlock() + ans := handle(req) + select { + case <-time.After(ans.delay): + case <-done: + return + } + _, _ = conn.Write([]byte(ans.reply + "\n")) + }() + } + }() + return stub +} + +func (s *stubHvi) seen() []map[string]any { + s.mu.Lock() + defer s.mu.Unlock() + return append([]map[string]any(nil), s.requests...) +} + +func TestHviControlSendsVersionAndReadsState(t *testing.T) { + stub := newStubHvi(t, func(req map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":true,"state":"paused","vcpus":2,"vcpus_parked":2}`} + }) + reply, err := hviOp(stub.path, "pause") + if err != nil { + t.Fatalf("pause: %v", err) + } + if reply.State != "paused" || reply.Vcpus != 2 || reply.VcpusParked != 2 { + t.Errorf("reply = %+v", reply) + } + reqs := stub.seen() + if len(reqs) != 1 || reqs[0]["v"] != float64(1) || reqs[0]["op"] != "pause" { + t.Errorf("requests = %v, want one {v:1, op:pause}", reqs) + } +} + +func TestHviControlRefusalCarriesTheCode(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":false,"code":"fenced","error":"the VM was fenced"}`} + }) + _, err := hviOp(stub.path, "resume") + if err == nil { + t.Fatal("resume of a fenced VM succeeded") + } + if got := hviRefusalCode(err); got != "fenced" { + t.Errorf("code = %q, want fenced (err %v)", got, err) + } + if !strings.Contains(err.Error(), "the VM was fenced") { + t.Errorf("error %q does not carry hvi's reason", err) + } +} + +// A pause whose reply comes after the client's timeout must surface as +// errHviNoReply: the caller has to tell the user the VM may now be paused, +// which a plain I/O error would not prompt. +func TestHviControlSlowPauseTimesOut(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":true,"state":"paused"}`, delay: 4 * time.Second} + }) + start := time.Now() + _, err := hviControl(stub.path, map[string]any{"op": "pause"}, 3*time.Second) + if !errors.Is(err, errHviNoReply) { + t.Fatalf("err = %v, want errHviNoReply", err) + } + if took := time.Since(start); took < 3*time.Second || took > 4*time.Second { + t.Errorf("gave up after %s, want the 3s timeout", took) + } +} + +// hvi's default client budget must stay at or above the 3 s hvi documents. +func TestHviControlTimeoutCoversHvisPauseDeadline(t *testing.T) { + if hviControlTimeout < 3*time.Second { + t.Errorf("hviControlTimeout = %s, hvi asks for at least 3s", hviControlTimeout) + } +} + +func TestHviControlRefusesAnotherProtocolVersion(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":2,"ok":true,"state":"running"}`} + }) + if _, err := hviOp(stub.path, "status"); err == nil || !strings.Contains(err.Error(), "version 2") { + t.Errorf("err = %v, want a protocol version refusal", err) + } +} + +func TestHviControlHangupWithoutReply(t *testing.T) { + dir := shortSocketDir(t) + path := filepath.Join(dir, "c.sock") + ln, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = ln.Close() }() + go func() { + conn, err := ln.Accept() + if err == nil { + _ = conn.Close() + } + }() + _, err = hviOp(path, "status") + if err == nil || errors.Is(err, errHviNoReply) { + t.Errorf("err = %v, want a closed-connection error that is not a timeout", err) + } +} + +func TestHviControlNoSocket(t *testing.T) { + _, err := hviOp(filepath.Join(shortSocketDir(t), "absent.sock"), "status") + if err == nil || !strings.Contains(err.Error(), "connect to hvi's control socket") { + t.Errorf("err = %v", err) + } +} + +func TestReadHviReplyRefusesAnOverlongLine(t *testing.T) { + long := strings.Repeat("x", hviControlMaxReply+10) + if _, err := readHviReply(strings.NewReader(long)); err == nil || !strings.Contains(err.Error(), "longer than") { + t.Errorf("err = %v, want a length refusal", err) + } +} diff --git a/cmd/hull/hvipause.go b/cmd/hull/hvipause.go new file mode 100644 index 00000000..8bea1179 --- /dev/null +++ b/cmd/hull/hvipause.go @@ -0,0 +1,169 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "time" + + "github.com/urfave/cli/v3" + + "github.com/brig-sh/hull/pkg/store" +) + +// isHviInstance reports whether the record's VMM is hvi, by the base name of +// the program it ran, as processIsAVMM matches it. +func isHviInstance(state *store.InstanceState) bool { + return len(state.CmdLine) > 0 && filepath.Base(state.CmdLine[0]) == "hvi" +} + +// requireRunningHvi refuses a verb that needs a live hvi instance with a +// control socket, saying which of the three it is not. +func requireRunningHvi(state *store.InstanceState, verb string) error { + if !isHviInstance(state) { + return fmt.Errorf("%s needs an hvi instance; %s runs under %s", verb, state.ID, backendName(state)) + } + if state.Status != "running" || state.PID <= 0 || !processIsAVMM(state.PID, state) { + return fmt.Errorf("instance %s is not running", state.ID) + } + if state.ControlSock == "" { + return fmt.Errorf("instance %s was started without a control socket, so it can only be "+ + "stopped; run it again with this version of hull", state.ID) + } + return nil +} + +// hviVMStateTimeout bounds the status ask that ps, inspect and exec make. +// Short: brig runs `hull ps` often, hvi serves one connection at a time, and +// an hvi busy writing a checkpoint should cost a listing half a second, not +// hviControlTimeout per instance. +const hviVMStateTimeout = 500 * time.Millisecond + +// hviVMState asks a running hvi instance what its VM is doing, as hvi names +// it: running, paused, fenced, restored-paused. "" when the record is not a +// running hvi with a control socket; "?" when hvi did not answer in time. +// +// The record's status says only that the VMM process is alive. A paused or +// fenced VM is alive and runs nothing, and an agent connection to it waits +// for a guest that never answers, so anything about to talk to the guest +// asks here first. +func hviVMState(state *store.InstanceState) string { + if !isHviInstance(state) || state.ControlSock == "" || state.Status != "running" || state.PID <= 0 { + return "" + } + reply, err := hviControl(state.ControlSock, map[string]any{"op": "status"}, hviVMStateTimeout) + if err != nil { + return "?" + } + return reply.State +} + +// refuseIfGuestHalted refuses to talk to the guest of an hvi instance whose +// VM is paused or fenced, saying how to get it running. An hvi that does not +// answer, or has no socket, is let through: an exec then behaves as it did +// before the socket existed. +func refuseIfGuestHalted(state *store.InstanceState) error { + id := state.ID + switch hviVMState(state) { + case "paused": + return fmt.Errorf("instance %s is paused, so its guest cannot answer; `hull resume %s` first", id, id) + case store.MigrationRestoredPaused: + return fmt.Errorf("instance %s was restored and is still paused; `hull resume %s` starts it", id, id) + case store.MigrationFenced: + return fmt.Errorf("instance %s was fenced by `hull checkpoint --then fence` and its guest "+ + "will not run again in this process; `hull stop %s`, then `hull restore %s`", id, id, id) + } + return nil +} + +// backendName names the VMM a record ran, for refusals. +func backendName(state *store.InstanceState) string { + if state.Backend != "" { + return state.Backend + } + if len(state.CmdLine) > 0 { + return filepath.Base(state.CmdLine[0]) + } + return "an unknown VMM" +} + +func pauseCommand() *cli.Command { + return &cli.Command{ + Name: "pause", + Usage: "pause a running hvi instance: its I/O threads and every vCPU stop", + ArgsUsage: "NAME", + Action: func(_ context.Context, cmd *cli.Command) error { + return hviPauseResume(cmd, "pause") + }, + } +} + +func resumeCommand() *cli.Command { + return &cli.Command{ + Name: "resume", + Usage: "resume an hvi instance paused by `hull pause`", + ArgsUsage: "NAME", + Action: func(_ context.Context, cmd *cli.Command) error { + return hviPauseResume(cmd, "resume") + }, + } +} + +func hviPauseResume(cmd *cli.Command, op string) error { + if cmd.Args().Len() == 0 { + return errors.New("instance ID required") + } + id := cmd.Args().First() + s, err := globalStore(cmd) + if err != nil { + return err + } + state, err := s.GetInstance(id) + if err != nil { + return fmt.Errorf("instance not found: %s", id) + } + if err := requireRunningHvi(state, op); err != nil { + return err + } + reply, err := hviOp(state.ControlSock, op) + switch { + case errors.Is(err, errHviNoReply) && op == "pause": + // hvi keeps working on a pause the client gave up on, so the VM may + // be paused now. Saying only "timed out" leaves a frozen sandbox + // nobody knows to resume. + return fmt.Errorf("%w; the VM may have paused after hull gave up, and `hull resume %s` "+ + "is harmless if it did not", err, id) + case hviRefusalCode(err) == "fenced": + return fmt.Errorf("instance %s was fenced by `hull checkpoint --then fence` and will not run "+ + "again in this process; stop it and `hull restore %s` from its checkpoint", id, id) + case err != nil: + return err + } + if op != "pause" { + fmt.Printf("Instance %s resumed\n", id) + return nil + } + // pause answers with the state alone; status has the vCPU counts. Best + // effort: the pause has happened whether or not this second ask works. + if st, err := hviOp(state.ControlSock, "status"); err == nil && st.Vcpus > 0 { + fmt.Printf("Instance %s %s (%d of %d vCPUs parked)\n", id, st.State, st.VcpusParked, st.Vcpus) + return nil + } + fmt.Printf("Instance %s %s\n", id, reply.State) + return nil +} diff --git a/cmd/hull/hvipause_test.go b/cmd/hull/hvipause_test.go new file mode 100644 index 00000000..09e4c442 --- /dev/null +++ b/cmd/hull/hvipause_test.go @@ -0,0 +1,110 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "path/filepath" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/brig-sh/hull/pkg/store" +) + +// fakeHviInstance records a running hvi instance whose VMM is a sleep that +// reports itself as hvi, with stub as its control socket. +func fakeHviInstance(t *testing.T, s *store.Store, id, sock string) (*store.InstanceState, <-chan struct{}) { + t.Helper() + pid, done := startProcessAs(t, "hvi", "/bin/sleep", "30") + if _, err := s.CreateInstance(id); err != nil { + t.Fatal(err) + } + state := &store.InstanceState{ + ID: id, Status: "running", PID: pid, Backend: "hvi", ControlSock: sock, + CmdLine: []string{"/x/hvi", "boot", "--kernel", "/k", "--disk", "/r.img"}, + } + if err := s.SaveInstance(state); err != nil { + t.Fatal(err) + } + return state, done +} + +// stop on an hvi instance goes through the control socket before any signal. +func TestStopAsksHviOverTheControlSocket(t *testing.T) { + var pid atomic.Int64 + stub := newStubHvi(t, func(req map[string]any) stubAnswer { + if req["op"] == "stop" { + _ = syscall.Kill(int(pid.Load()), syscall.SIGKILL) + } + return stubAnswer{reply: `{"v":1,"ok":true,"state":"stopping"}`} + }) + s := newCmdTestStore(t) + state, done := fakeHviInstance(t, s, "vm1", stub.path) + pid.Store(int64(state.PID)) + + start := time.Now() + if err := stopInstanceIn(s, "vm1", 10); err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the VMM is still running") + } + if took := time.Since(start); took > 3*time.Second { + t.Errorf("stop took %s; the socket stop should not wait out the signal budget", took) + } + reqs := stub.seen() + if len(reqs) != 1 || reqs[0]["op"] != "stop" || reqs[0]["v"] != float64(1) { + t.Errorf("requests = %v, want one stop", reqs) + } + if got := mustInstance(t, s, "vm1"); got.Status != "stopped" || got.PID != 0 { + t.Errorf("record = %s pid %d", got.Status, got.PID) + } +} + +// A socket that is not there (an hvi from before the socket, or one that +// already went away) must not stop the stop: the signals still follow. +func TestStopFallsBackToSignalsWithoutASocket(t *testing.T) { + s := newCmdTestStore(t) + _, done := fakeHviInstance(t, s, "vm1", filepath.Join(shortSocketDir(t), "gone.sock")) + if err := stopInstanceIn(s, "vm1", 4); err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the VMM is still running") + } +} + +func newCmdTestStore(t *testing.T) *store.Store { + t.Helper() + s, err := store.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return s +} + +func mustInstance(t *testing.T, s *store.Store, id string) *store.InstanceState { + t.Helper() + st, err := s.GetInstance(id) + if err != nil { + t.Fatal(err) + } + return st +} diff --git a/cmd/hull/hvistate_test.go b/cmd/hull/hvistate_test.go new file mode 100644 index 00000000..3608f2b4 --- /dev/null +++ b/cmd/hull/hvistate_test.go @@ -0,0 +1,122 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/brig-sh/hull/pkg/store" +) + +func statusStub(t *testing.T, vmState string) *stubHvi { + t.Helper() + return newStubHvi(t, func(map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":true,"state":"` + vmState + `","vcpus":1,"vcpus_parked":1}`} + }) +} + +// A paused or fenced hvi VM is a live process running nothing. ps shows it +// in a column of its own, and keeps STATUS "running" in the place brig's +// Running() and List() read it from. +func TestPsShowsAPausedOrFencedVM(t *testing.T) { + stub := statusStub(t, "fenced") + s := newCmdTestStore(t) + fakeHviInstance(t, s, "vm1", stub.path) + withMigration(t, s, "vm1", store.MigrationState{State: store.MigrationFenced, BundleDir: "/b", TxID: "t"}) + + var out bytes.Buffer + if err := renderInstances(s, &out); err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + if len(lines) != 2 { + t.Fatalf("ps:\n%s", out.String()) + } + head, row := strings.Fields(lines[0]), strings.Fields(lines[1]) + if head[0] != "ID" || head[1] != "STATUS" || row[0] != "vm1" || row[1] != "running" { + t.Errorf("ID and STATUS moved: %q / %q", lines[0], lines[1]) + } + if n := len(row); n < 2 || row[n-2] != "fenced" || row[n-1] != "fenced" { + t.Errorf("row %q does not end with VM fenced, CHECKPOINT fenced", lines[1]) + } +} + +func TestPsShowsDashesForAnInstanceWithoutASocket(t *testing.T) { + s := newCmdTestStore(t) + st, _ := fakeHviInstance(t, s, "vm1", "") + st.ControlSock = "" + _ = s.SaveInstance(st) + var out bytes.Buffer + if err := renderInstances(s, &out); err != nil { + t.Fatal(err) + } + row := strings.Fields(strings.Split(strings.TrimSpace(out.String()), "\n")[1]) + if n := len(row); row[n-2] != "-" || row[n-1] != "-" { + t.Errorf("row %v, want - for VM and CHECKPOINT", row) + } +} + +// exec into a paused hvi guest used to hang, deaf to SIGTERM, until it was +// killed. It must refuse at once and name the way out. +func TestExecRefusesAHaltedGuest(t *testing.T) { + for vmState, want := range map[string]string{ + "paused": "hull resume vm1", + "restored-paused": "hull resume vm1", + "fenced": "hull restore vm1", + "running": "", + } { + t.Run(vmState, func(t *testing.T) { + stub := statusStub(t, vmState) + s := newCmdTestStore(t) + st, _ := fakeHviInstance(t, s, "vm1", stub.path) + start := time.Now() + err := refuseIfGuestHalted(st) + if want == "" && err != nil || want != "" && (err == nil || !strings.Contains(err.Error(), want)) { + t.Errorf("err = %v, want one naming %q", err, want) + } + if took := time.Since(start); took > 2*time.Second { + t.Errorf("took %s", took) + } + }) + } + // No socket answering: let through, as before the socket existed. + s := newCmdTestStore(t) + st, _ := fakeHviInstance(t, s, "vm1", filepath.Join(shortSocketDir(t), "gone.sock")) + if err := refuseIfGuestHalted(st); err != nil { + t.Errorf("an hvi with no socket was refused: %v", err) + } +} + +func TestInspectCarriesTheVMState(t *testing.T) { + stub := statusStub(t, "paused") + s := newCmdTestStore(t) + st, _ := fakeHviInstance(t, s, "vm1", stub.path) + var out bytes.Buffer + if err := printJSON(&out, inspectView(st)); err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(out.Bytes(), &doc); err != nil { + t.Fatal(err) + } + if doc["vmState"] != "paused" || doc["status"] != "running" || doc["controlSock"] != stub.path { + t.Errorf("inspect = %s", out.String()) + } +} diff --git a/cmd/hull/inspect.go b/cmd/hull/inspect.go index 6906f4c4..69c16372 100644 --- a/cmd/hull/inspect.go +++ b/cmd/hull/inspect.go @@ -54,8 +54,20 @@ func inspectInstance(ctx context.Context, cmd *cli.Command) error { } // An instance name can carry a C1 control; printJSON keeps it off the - // terminal. - return printJSON(os.Stdout, state) + // terminal. vmState is hvi's own answer for a running hvi instance: + // "status" says the VMM is alive, not that its guest runs. + return printJSON(os.Stdout, inspectView(state)) +} + +// inspectedInstance is the record as inspect prints it, with the VM's live +// state alongside. +type inspectedInstance struct { + *store.InstanceState + VMState string `json:"vmState,omitempty"` +} + +func inspectView(state *store.InstanceState) inspectedInstance { + return inspectedInstance{InstanceState: state, VMState: hviVMState(state)} } // instanceReadError reports a GetInstance failure. diff --git a/cmd/hull/ps.go b/cmd/hull/ps.go index 132837e9..67ae6352 100644 --- a/cmd/hull/ps.go +++ b/cmd/hull/ps.go @@ -62,7 +62,11 @@ func renderInstances(s *store.Store, out io.Writer) error { // Create table output w := tabwriter.NewWriter(out, 4, 8, 2, ' ', 0) - _, _ = fmt.Fprintln(w, "ID\tSTATUS\tEXIT\tPID\tIP\tCREATED") + // VM and CHECKPOINT come last: brig reads ID and STATUS by position, + // and STATUS keeps meaning "the VMM process is alive". VM is what an + // hvi VM says it is doing (a paused or fenced one is alive and runs + // nothing); CHECKPOINT is the record's migration state. + _, _ = fmt.Fprintln(w, "ID\tSTATUS\tEXIT\tPID\tIP\tCREATED\tVM\tCHECKPOINT") for _, state := range instances { status := state.Status @@ -122,13 +126,23 @@ func renderInstances(s *store.Store, out io.Writer) error { if !state.StartTime.IsZero() { created = state.StartTime.Format("2006-01-02 15:04:05") } - _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", + vm := hviVMState(state) + if vm == "" { + vm = "-" + } + checkpoint := "-" + if state.Migration != nil && state.Migration.State != "" { + checkpoint = state.Migration.State + } + _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", state.ID, status, exitStr, pidStr, ip, created, + vm, + checkpoint, ) } diff --git a/cmd/hull/run.go b/cmd/hull/run.go index 9ba4eeb8..c0b45bb8 100644 --- a/cmd/hull/run.go +++ b/cmd/hull/run.go @@ -332,6 +332,17 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { if err := checkUnixSocketPath("agent", s.InstanceAgentSocket(instanceName)); err != nil { return err } + // hvi's control socket is two bytes longer than the agent's, so a name + // that passes the check above can still fail it, and checked only where + // it is added, that failure comes after the pull, the unpack and the + // block disk. The backend is only certain once the image's annotation + // is read; checked here when the flag or the platform default already + // says hvi, and again where the socket is added. + if early, _ := selectHypervisor(hypervisorOverride, nil, defaultHypervisor); early == string(hypervisors.HviVmm) { + if err := checkUnixSocketPath("control", s.InstanceControlSocket(instanceName)); err != nil { + return err + } + } // Track image digest (will be set based on whether we use local bundle or pull from registry) var imageDigest string @@ -1476,6 +1487,21 @@ exec %s "$@" } } + // Every hvi instance gets a control socket and a machine profile, the + // way every Vz instance gets a state dir: pause, checkpoint and a + // graceful stop all go through the socket, and a run started without one + // can only be signalled. Both live in the 0700 instance directory, since + // the socket answers anyone who can connect to it. + var controlSock, profilePath string + if vmmType == hypervisors.HviVmm { + controlSock = s.InstanceControlSocket(instanceName) + profilePath = s.InstanceProfilePath(instanceName) + if err := checkUnixSocketPath("control", controlSock); err != nil { + return err + } + cmdArgs = append(cmdArgs, "--control-sock", controlSock, "--profile-out", profilePath) + } + state := &store.InstanceState{ ID: instanceName, ImageDigest: imageDigest, @@ -1484,6 +1510,8 @@ exec %s "$@" BundleDir: bundleDir, MAC: mac, Backend: string(vmmType), + ControlSock: controlSock, + ProfilePath: profilePath, } started, err := launchVMM(ctx, cmd, s, state, cmdArgs, append(gatewayFiles, shareFiles...), vmmType, detach, netMode, gatewayIP) instanceStarted = started diff --git a/cmd/hull/stop.go b/cmd/hull/stop.go index bf622a9c..4fdc580e 100644 --- a/cmd/hull/stop.go +++ b/cmd/hull/stop.go @@ -192,6 +192,20 @@ func stopInstanceIn(s *store.Store, instanceID string, timeoutSecs int) error { return nil } + // hvi: ask over the control socket first. hvi ends the run itself + // (`guest stopped: Requested`), paused or fenced, with its own teardown, + // where a signal is a signal. Half the budget, as QMP gets; a socket that + // is missing or refuses is an hvi from before the socket and falls + // through to the signals below. + if state.ControlSock != "" && isHviInstance(state) { + if hviStopBySocket(state, time.Until(budgetDeadline)/2) { + markStopped(s, state, true) + stopped = true + fmt.Printf("Instance %s stopped\n", instanceID) + return nil + } + } + // Try graceful shutdown via QMP first if state.QMPSocket != "" { log.Debugf("Attempting graceful shutdown via QMP") @@ -247,6 +261,18 @@ func stopInstanceIn(s *store.Store, instanceID string, timeoutSecs int) error { return nil } +// hviStopBySocket sends hvi's stop op and reports whether the VMM exited +// within wait. Any failure to deliver it is logged and reported as "not +// stopped", so the caller goes on to the signals. +func hviStopBySocket(state *store.InstanceState, wait time.Duration) bool { + timeout := min(hviControlTimeout, max(wait, time.Second)) + if _, err := hviControl(state.ControlSock, map[string]any{"op": "stop"}, timeout); err != nil { + log.Debugf("instance %s: stop over the control socket: %v", state.ID, err) + return false + } + return waitForExit(state.PID, wait) +} + // waitForExit polls until the process is gone or the timeout elapses. func waitForExit(pid int, timeout time.Duration) bool { deadline := time.Now().Add(timeout) diff --git a/docs/running.md b/docs/running.md index ed69fd05..6033da6c 100644 --- a/docs/running.md +++ b/docs/running.md @@ -87,9 +87,18 @@ inspect or exec into. hull ps ``` -Columns are ID, STATUS, EXIT, PID, IP and CREATED. `hull ps` reconciles as it -prints: a record marked running whose process is gone is rewritten to stopped. -There is no JSON mode. +Columns are ID, STATUS, EXIT, PID, IP, CREATED, VM and CHECKPOINT. `hull ps` +reconciles as it prints: a record marked running whose process is gone is +rewritten to stopped. There is no JSON mode. + +STATUS says whether the VMM process is alive. For an hvi instance, VM is what +the VM itself reports over its control socket: `running`, `paused`, `fenced` +or `restored-paused`, `?` if hvi did not answer within half a second, and `-` +for any other instance. A paused or fenced instance shows STATUS `running` and +runs nothing; `hull exec` refuses it and says how to get it running. +CHECKPOINT is the record's migration state from the last `hull checkpoint` or +`hull restore` (see [checkpoint-restore.md](checkpoint-restore.md)), or `-`. +`hull inspect` prints the VM column as `vmState`. ```bash hull inspect "$ID" # always JSON diff --git a/pkg/store/store.go b/pkg/store/store.go index d753345e..9959d24b 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -92,6 +92,43 @@ type InstanceState struct { // TelemetryEndSent guards the telemetry `end` event to exactly once // across the foreground exit path and `stop`. TelemetryEndSent bool `json:"telemetryEndSent,omitempty"` + // ControlSock is the hvi control socket (--control-sock) that pauses, + // resumes, checkpoints and stops the VM. Empty for other backends and for + // hvi instances started before hull passed it: those cannot be paused or + // checkpointed, only signalled. + ControlSock string `json:"controlSock,omitempty"` + // ProfilePath is where hvi wrote its machine profile (--profile-out): the + // facts a destination host compares before it accepts a checkpoint. + ProfilePath string `json:"profilePath,omitempty"` + // Migration is where this instance stands in a checkpoint, restore or + // move. Nil until the first `hull checkpoint`. Stopping the instance + // leaves it alone: a fenced instance that was stopped is exactly the one + // a restore needs to find. + Migration *MigrationState `json:"migration,omitempty"` +} + +// Migration states. A checkpointed instance kept running after the capture, +// so its disk has moved on since; a fenced one never ran again, so its disk +// still matches the bundle. Restoring records restoring, then restored-paused +// once hvi has rebuilt the VM, then restored once it runs. +const ( + MigrationCheckpointed = "checkpointed" + MigrationFenced = "fenced" + MigrationRestoring = "restoring" + MigrationRestoredPaused = "restored-paused" + MigrationRestored = "restored" +) + +// MigrationState is the record of the last checkpoint taken of an instance +// and of what became of it. +type MigrationState struct { + // State is one of the Migration* constants. + State string `json:"state"` + // BundleDir is the absolute directory hvi wrote the checkpoint bundle to. + BundleDir string `json:"bundleDir"` + // TxID names the checkpoint. hull draws it; hvi's checkpoint op takes no + // id of its own. + TxID string `json:"txid"` } // StatusStarting marks the one moment an instance has no pid to record: the @@ -663,6 +700,19 @@ func (s *Store) InstanceAgentSocket(id string) string { return filepath.Join(s.rootDir, "instances", id, "agent.sock") } +// InstanceControlSocket returns the path hvi binds its control socket at. +// It sits in the 0700 instance directory because the socket has no +// authentication: whoever can connect to it can pause or stop the VM. +func (s *Store) InstanceControlSocket(id string) string { + return s.instancePath(id, "control.sock") +} + +// InstanceProfilePath returns where hvi writes the instance's machine +// profile. +func (s *Store) InstanceProfilePath(id string) string { + return s.instancePath(id, "profile.json") +} + // DeleteInstance removes an instance directory func (s *Store) DeleteInstance(id string) error { // This one recurses, so it validates even though callers resolve the diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index efe51891..db6a5846 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -15,8 +15,11 @@ package store import ( + "encoding/json" "os" "path/filepath" + "reflect" + "strings" "testing" "time" ) @@ -241,3 +244,63 @@ func TestInstanceStateExitStatusRoundTrip(t *testing.T) { t.Errorf("ExitCode = %v, want nil for a record without a status", plain.ExitCode) } } + +// The migration fields are read by brig and by a remote hull over ssh, so their +// JSON names are part of the record's contract: lowerCamel like the rest, and +// absent from a record that never had them. +func TestInstanceStateMigrationFieldsRoundTrip(t *testing.T) { + s := newTestStore(t) + if _, err := s.CreateInstance("vm1"); err != nil { + t.Fatal(err) + } + want := &InstanceState{ + ID: "vm1", + Status: "running", + ControlSock: s.InstanceControlSocket("vm1"), + ProfilePath: s.InstanceProfilePath("vm1"), + Migration: &MigrationState{State: MigrationFenced, BundleDir: "/b", TxID: "t1"}, + } + if err := s.SaveInstance(want); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(s.InstanceDir("vm1"), "state.json")) + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + if doc["controlSock"] != want.ControlSock || doc["profilePath"] != want.ProfilePath { + t.Errorf("state.json lacks controlSock or profilePath: %s", raw) + } + wantMigration := map[string]any{"state": "fenced", "bundleDir": "/b", "txid": "t1"} + if !reflect.DeepEqual(doc["migration"], wantMigration) { + t.Errorf("migration = %v, want %v", doc["migration"], wantMigration) + } + got, err := s.GetInstance("vm1") + if err != nil { + t.Fatal(err) + } + if got.ControlSock != want.ControlSock || got.ProfilePath != want.ProfilePath || + got.Migration == nil || *got.Migration != *want.Migration { + t.Errorf("round trip = %+v, want %+v", got, want) + } + if filepath.Dir(got.ControlSock) != s.InstanceDir("vm1") || filepath.Dir(got.ProfilePath) != s.InstanceDir("vm1") { + t.Errorf("control socket %s and profile %s must sit in the instance dir %s", + got.ControlSock, got.ProfilePath, s.InstanceDir("vm1")) + } + + if err := s.SaveInstance(&InstanceState{ID: "vm1", Status: "stopped"}); err != nil { + t.Fatal(err) + } + raw, err = os.ReadFile(filepath.Join(s.InstanceDir("vm1"), "state.json")) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"controlSock", "profilePath", "migration"} { + if strings.Contains(string(raw), key) { + t.Errorf("a record without %s still writes it: %s", key, raw) + } + } +} From e5b0e526ded0fbd4a2fb06eb77f3e4e91a1660c9 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Wed, 14 Oct 2026 04:15:48 +0300 Subject: [PATCH 09/33] feat(hvi): checkpoint hvi instances and restore a fenced checkpoint hull checkpoint and hull restore now take an hvi instance as well as a Vz one. Both verbs move to portable files and dispatch on the record, and the Vz paths are unchanged. A checkpoint sends hvi's checkpoint op with the checkpoint's txid into a 0700 bundle directory under the --checkpoint-root every hvi run now gets, since hvi's confinement lets it create files only there. --then fence leaves the guest parked for good, the source side of a move, and --then continue resumes it. Directory roots and host shares are refused, as no bundle captures them. A restore builds hvi's command line from the one the instance last ran through an allowlist, so a flag hull does not know is refused rather than dropped, binds the restore to the bundle's manifest and txid, and can adopt a later latest.json. It waits on the restore's own VMM, sees a detached hvi that exits at once, and dials the gateway. The control flags are passed only to an hvi that takes them, because the hvi hull pins answers them with "unknown boot arg" and exits. A per-instance migration lock serialises checkpoints and restores of one instance, and a restored guest is recorded as run before anything can run it, so a second restore cannot put the bundle's RAM over a disk the guest has written since. checkpoint-restore.md becomes a page for both backends and says what an hvi bundle holds, which restore states exist, what the lock covers and what was checked live. Signed-off-by: Panagiotis Moustafellos --- README.md | 8 +- cmd/hull/checkpoint.go | 59 +-- cmd/hull/checkpoint_linux.go | 32 ++ cmd/hull/commands_linux.go | 11 +- cmd/hull/hvicheckpoint.go | 470 ++++++++++++++++++++++++ cmd/hull/hvicheckpoint_test.go | 284 +++++++++++++++ cmd/hull/hvicontrol_test.go | 5 +- cmd/hull/hviflags.go | 59 +++ cmd/hull/hviflags_test.go | 55 +++ cmd/hull/hvimigration_test.go | 480 ++++++++++++++++++++++++ cmd/hull/hvipause.go | 16 +- cmd/hull/hvipause_test.go | 9 +- cmd/hull/hvirestore.go | 646 +++++++++++++++++++++++++++++++++ cmd/hull/hvirestore_test.go | 426 ++++++++++++++++++++++ cmd/hull/run.go | 33 +- docs/backends.md | 47 ++- docs/checkpoint-restore.md | 251 ++++++++++++- docs/cli.md | 30 +- docs/storage.md | 23 +- pkg/store/store.go | 9 +- 20 files changed, 2844 insertions(+), 109 deletions(-) create mode 100644 cmd/hull/checkpoint_linux.go create mode 100644 cmd/hull/hvicheckpoint.go create mode 100644 cmd/hull/hvicheckpoint_test.go create mode 100644 cmd/hull/hviflags.go create mode 100644 cmd/hull/hviflags_test.go create mode 100644 cmd/hull/hvimigration_test.go create mode 100644 cmd/hull/hvirestore.go create mode 100644 cmd/hull/hvirestore_test.go diff --git a/README.md b/README.md index d0582c5a..4a0c9fb9 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,8 @@ Three backends exist. They are not equivalent, and the gaps are real: | `--net shared` | NAT, works with no extra privilege | **refused**, see below | vmnet, needs root or a QEMU signed for `com.apple.vm.networking` | | `--gateway-sock` | yes | yes | yes | | `exec` | yes | yes | yes | -| Checkpoint and restore | yes, with a block rootfs | no | no | +| Checkpoint and restore | yes, with a block rootfs | checkpoint with a block rootfs and no shares, into a partial bundle; no restore yet ([why](docs/checkpoint-restore.md#what-a-bundle-holds)) | no | +| Pause and resume | no | yes | no | | GUI window, Rosetta | yes | no | no | **The `hvi` networking gap is the one to know.** hvi's built-in stack answers @@ -164,8 +165,9 @@ instance. [docs/architecture.md](docs/architecture.md) explains the seams. with a warning on stderr. A clean `compose config` does not mean every key in the file is honored at runtime. [docs/compose.md](docs/compose.md) lists what works. -- Checkpoint and restore are `vz` only, need a block rootfs, and make no - promise about application-level consistency. They do not move a VM to another +- Checkpoint and restore work on `vz`. `hvi` checkpoints into a bundle that + cannot be resumed yet, and its restore waits for `hvi`'s. Both need a block + rootfs, and they make no promise about application-level consistency. They do not move a VM to another host, and they cannot undo an effect the guest already sent to the outside world. See [docs/checkpoint-restore.md](docs/checkpoint-restore.md). - The `vz` and `hvi` backends need a code-signing entitlement. The Homebrew diff --git a/cmd/hull/checkpoint.go b/cmd/hull/checkpoint.go index edcabd6c..3d2fdc6d 100644 --- a/cmd/hull/checkpoint.go +++ b/cmd/hull/checkpoint.go @@ -42,28 +42,9 @@ const ( ckptError = "error" ) -func instanceCheckpointDir(s *store.Store, id string) string { - return filepath.Join(s.InstanceDir(id), "checkpoint") -} - -func checkpointCommand() *cli.Command { - return &cli.Command{ - Name: "checkpoint", - Usage: "checkpoint a running Vz instance (pause, save VM + disk state, resume)", - Flags: []cli.Flag{ - &cli.IntFlag{ - Name: "timeout", - Value: 60, - Usage: "seconds to wait for the checkpoint to complete", - }, - }, - Action: func(ctx context.Context, cmd *cli.Command) error { - return checkpointInstance(ctx, cmd) - }, - } -} - -func checkpointInstance(_ context.Context, cmd *cli.Command) error { +// vzCheckpointInstance is `hull checkpoint` for a Vz instance: vz-runner +// saves the machine state and clones the disk on SIGUSR1. +func vzCheckpointInstance(_ context.Context, cmd *cli.Command) error { args := cmd.Args() if args.Len() == 0 { return errors.New("instance ID required") @@ -297,37 +278,9 @@ func freshCheckpointFailure(ckptDir string, started time.Time) (string, bool) { return reason, true } -func restoreCommand() *cli.Command { - return &cli.Command{ - Name: "restore", - Usage: "restore a stopped Vz instance from its checkpoint", - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "detach", - Aliases: []string{"d"}, - Usage: "run in background", - }, - &cli.IntFlag{ - Name: "stop-grace", - Value: 10, - Usage: "seconds vz-runner waits for the guest to answer a stop request before forcing", - }, - &cli.BoolFlag{ - Name: "wait-ip", - Usage: "with --detach and NAT networking, wait for the DHCP lease and record the IP before returning", - }, - &cli.StringFlag{ - Name: "gateway-sock", - Usage: "re-join the user-mode network gateway at this control socket (required if the instance ran on the gateway)", - }, - }, - Action: func(ctx context.Context, cmd *cli.Command) error { - return restoreInstance(ctx, cmd) - }, - } -} - -func restoreInstance(ctx context.Context, cmd *cli.Command) error { +// vzRestoreInstance is `hull restore` for a Vz instance: vz-runner is started +// again on the same command line with --restore. +func vzRestoreInstance(ctx context.Context, cmd *cli.Command) error { args := cmd.Args() if args.Len() == 0 { return errors.New("instance ID required") diff --git a/cmd/hull/checkpoint_linux.go b/cmd/hull/checkpoint_linux.go new file mode 100644 index 00000000..9f8a097e --- /dev/null +++ b/cmd/hull/checkpoint_linux.go @@ -0,0 +1,32 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + + "github.com/urfave/cli/v3" +) + +// errVzCheckpointMacOnly answers checkpoint and restore for anything but an +// hvi instance. On macOS those go to vz-runner, whose saved state is +// Virtualization.framework's and bound to the Mac that wrote it; on Linux +// every instance is hvi, so this is only reached by a record that is not. +var errVzCheckpointMacOnly = errors.New("checkpoint and restore on Linux take hvi instances only") + +func vzCheckpointInstance(context.Context, *cli.Command) error { return errVzCheckpointMacOnly } + +func vzRestoreInstance(context.Context, *cli.Command) error { return errVzCheckpointMacOnly } diff --git a/cmd/hull/commands_linux.go b/cmd/hull/commands_linux.go index ac6143b8..cf9d54e5 100644 --- a/cmd/hull/commands_linux.go +++ b/cmd/hull/commands_linux.go @@ -25,12 +25,11 @@ const appUsage = "container CLI for hvi sandboxes on Linux/KVM" // platformCommands is the Linux verb set: what it takes to run an hvi sandbox // on KVM, and what brig calls to drive one (run, exec, ps, inspect, logs, -// stop, rm, assets dir, network-gateway), plus pause and resume over hvi's -// control socket. +// stop, rm, assets dir, network-gateway), plus pause, resume, checkpoint and +// restore over hvi's control socket. checkpoint and restore are the same +// verbs as on macOS; there they also take Vz instances, here only hvi ones. // -// Left out, because they drive something Linux does not have: -// - checkpoint and restore: vz-runner's saved-state files, which are -// Virtualization.framework's and bound to the Mac that wrote them. +// Left out, because it drives something Linux does not have: // - store: detaches the APFS sparse image the macOS store is mounted from. // // compose is listed but refuses; see linuxCompose. @@ -43,6 +42,8 @@ func platformCommands() []*cli.Command { stopCommand(), pauseCommand(), resumeCommand(), + checkpointCommand(), + restoreCommand(), rmCommand(), logsCommand(), inspectCommand(), diff --git a/cmd/hull/hvicheckpoint.go b/cmd/hull/hvicheckpoint.go new file mode 100644 index 00000000..63e74951 --- /dev/null +++ b/cmd/hull/hvicheckpoint.go @@ -0,0 +1,470 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + "time" + + "github.com/urfave/cli/v3" + + "github.com/brig-sh/hull/pkg/store" +) + +// hviCheckpointRecordName is the file in /checkpoint/ that names +// the last hvi checkpoint. vz-runner writes a file of the same name with its +// own fields into the same directory of a Vz instance; an instance is one or +// the other, and every reader dispatches on the backend before reading it. +const hviCheckpointRecordName = "latest.json" + +// Checkpoint follow-ups, as hvi spells them. +const ( + hviThenContinue = "continue" + hviThenFence = "fence" +) + +// hviCheckpoint asks hvi to write a checkpoint bundle into dir, an absolute +// path, and then either resume the guest or fence it. txid goes into the +// bundle's session record, so the bundle names the checkpoint hull recorded. +func hviCheckpoint(sock, dir, then, txid string, timeout time.Duration) (*hviReply, error) { + return hviControl(sock, map[string]any{"op": "checkpoint", "dir": dir, "then": then, "txid": txid}, timeout) +} + +// hviCheckpointable refuses an hvi command line whose guest a checkpoint +// cannot carry, naming the reason. +// +// A checkpoint is the guest's RAM and device state plus the disk it was using. +// Everything the guest depends on has to be in those two, so: +// - the root must be a disk (--disk). A virtio-fs root lives in a host +// directory and in the guest's FUSE caches, which no bundle captures. +// - no virtio-fs share (--share-ro, --share-rw). hvi refuses a profile with +// shares anyway; refusing here says so before the VM is paused. +func hviCheckpointable(cmdLine []string) error { + if len(cmdLine) == 0 || filepath.Base(cmdLine[0]) != "hvi" { + return errors.New("checkpoint of this kind needs the hvi backend") + } + if !slices.Contains(cmdLine, "--disk") { + return errors.New("checkpoint needs a block rootfs, which a checkpoint carries as a disk: " + + "start the instance with `run --rootfs-type block` (a virtiofs root lives in a host " + + "directory no checkpoint captures)") + } + for _, arg := range cmdLine { + if arg == "--share-ro" || arg == "--share-rw" { + return errors.New("checkpoint refuses an instance with a shared host directory " + + "(--shared-dir): the guest's view of it lives in host files and FUSE caches no " + + "checkpoint captures. Run it with no shares") + } + } + return nil +} + +// checkpointCommand is one verb over two backends. A Vz instance is saved by +// vz-runner on a signal (vzCheckpointInstance, macOS only); an hvi instance +// over its control socket. The record says which, so the user never has to. +func checkpointCommand() *cli.Command { + return &cli.Command{ + Name: "checkpoint", + Usage: "checkpoint a running instance: Vz saves VM and disk state and resumes; hvi " + + "writes a checkpoint bundle and continues or fences", + ArgsUsage: "NAME", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "timeout", + Value: 60, + Usage: "seconds to wait for the checkpoint to complete", + }, + &cli.StringFlag{ + Name: "then", + Value: hviThenContinue, + Usage: "hvi: what the guest does after the checkpoint, 'continue' or 'fence' " + + "(never runs again in this process; for a move or a restore)", + }, + &cli.StringFlag{ + Name: "dir", + Usage: "hvi: empty directory below /checkpoint to write the bundle to (default: a new one there)", + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Len() == 0 { + return errors.New("instance ID required") + } + s, err := globalStore(cmd) + if err != nil { + return err + } + id := cmd.Args().First() + state, err := s.GetInstance(id) + if err != nil { + return fmt.Errorf("instance not found: %s", id) + } + if !isHviInstance(state) { + if cmd.IsSet("then") || cmd.IsSet("dir") { + return errors.New("--then and --dir apply to hvi instances only") + } + return vzCheckpointInstance(ctx, cmd) + } + return hviCheckpointInstance(s, state, cmd.String("then"), cmd.String("dir"), + time.Duration(cmd.Int("timeout"))*time.Second) + }, + } +} + +// hviCheckpointStats is the stats object hvi's checkpoint reply carries. +// Pointers, so a field this hvi does not send is told apart from a zero. +type hviCheckpointStats struct { + PauseMs *int64 `json:"pause_ms"` + CaptureMs *int64 `json:"capture_ms"` + RAMPresentBytes *int64 `json:"ram_present_bytes"` + RAMTotalBytes *int64 `json:"ram_total_bytes"` +} + +// hviCheckpointRecord is /checkpoint/latest.json for an hvi +// instance: which bundle is the latest, how it was taken, and what hvi +// reported. The bundle's own manifest is the authority on its contents; +// manifestSha256 ties this record to that manifest. +type hviCheckpointRecord struct { + Dir string `json:"dir"` + ManifestSHA256 string `json:"manifestSha256"` + Then string `json:"then"` + Stats json.RawMessage `json:"stats,omitempty"` + SavedAt time.Time `json:"savedAt"` + TxID string `json:"txid"` +} + +// writeHviCheckpointRecord publishes rec as ckptDir/latest.json. +// +// Through a temp file, fsync and rename, so a reader sees the previous record +// or this one and never half of either: a restore that read a torn record +// would pick the wrong bundle or none, and the crash it would follow is the +// moment a checkpoint matters most. +func writeHviCheckpointRecord(ckptDir string, rec hviCheckpointRecord) error { + data, err := json.MarshalIndent(rec, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(ckptDir, 0o700); err != nil { + return fmt.Errorf("create %s: %w", ckptDir, err) + } + tmp, err := os.CreateTemp(ckptDir, hviCheckpointRecordName+".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer func() { _ = os.Remove(tmpName) }() + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return err + } + if _, err := tmp.Write(append(data, '\n')); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmpName, filepath.Join(ckptDir, hviCheckpointRecordName)); err != nil { + return err + } + if d, err := os.Open(ckptDir); err == nil { + _ = d.Sync() + _ = d.Close() + } + return nil +} + +// readHviCheckpointRecord reads ckptDir/latest.json. +func readHviCheckpointRecord(ckptDir string) (hviCheckpointRecord, error) { + var rec hviCheckpointRecord + data, err := os.ReadFile(filepath.Join(ckptDir, hviCheckpointRecordName)) + if err != nil { + return rec, err + } + if err := json.Unmarshal(data, &rec); err != nil { + return rec, fmt.Errorf("%s in %s does not read as a checkpoint record: %w", + hviCheckpointRecordName, ckptDir, err) + } + return rec, nil +} + +// newTxID names a checkpoint. hull draws it and hands it to hvi, which +// records it in the bundle: it names the default bundle directory and, in a +// later move, keys each step so a retried step finds its own earlier work. +func newTxID() (string, error) { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return hex.EncodeToString(b[:]), nil +} + +// instanceCheckpointDir is the instance's checkpoint directory: vz-runner's +// state dir for a Vz instance; latest.json and the bundles hull places by +// default for an hvi one. +func instanceCheckpointDir(s *store.Store, id string) string { + return filepath.Join(s.InstanceDir(id), "checkpoint") +} + +func hviCheckpointInstance(s *store.Store, state *store.InstanceState, then, dir string, timeout time.Duration) error { + id := state.ID + if then != hviThenContinue && then != hviThenFence { + return fmt.Errorf("--then must be %q or %q (got %q)", hviThenContinue, hviThenFence, then) + } + if err := requireRunningHvi(state, "checkpoint"); err != nil { + return err + } + if err := hviCheckpointable(state.CmdLine); err != nil { + return err + } + if timeout < hviControlTimeout { + timeout = hviControlTimeout + } + unlock, err := lockInstanceMigration(s, id) + if err != nil { + return err + } + defer unlock() + + txid, err := newTxID() + if err != nil { + return err + } + ckptDir := instanceCheckpointDir(s, id) + root, ok := flagValue(state.CmdLine, "--checkpoint-root") + if !ok { + return fmt.Errorf("instance %s was started without a checkpoint root, so its hvi may not "+ + "write a bundle anywhere; run it again with an hvi that takes --checkpoint-root", id) + } + if dir == "" { + dir = filepath.Join(root, txid) + } else if dir, err = bundleDirBelow(root, dir); err != nil { + return err + } + created, err := prepareBundleDir(dir) + if err != nil { + return err + } + + // A restored guest that was never resumed may be resumed by this + // checkpoint: by "continue" after the capture, and by hvi after a + // failed capture of either kind unless the fence was already applied. + // A reply that never comes says nothing either way. On success the + // record moves on to the new bundle, so this costs nothing then. + if err := markRestoredGuestRunning(s, id); err != nil { + return err + } + started := time.Now() + reply, err := hviCheckpoint(state.ControlSock, dir, then, txid, timeout) + if err != nil { + // hvi resumes the guest after a failed capture (unless the fence was + // already applied), and what it wrote is not a bundle: the manifest + // is written last and is missing. A directory hull made for it goes; + // a --dir that was there before is emptied and kept, so a failed + // attempt leaves no RAM-sized debris and the same --dir can be + // retried (prepareBundleDir refuses one that is not empty). With no + // reply hvi may still be writing, and everything stays. + if !errors.Is(err, errHviNoReply) { + if created { + _ = store.RemoveTree(dir) + } else { + emptyDir(dir) + } + } + switch { + case errors.Is(err, errHviNoReply): + return fmt.Errorf("%w; hvi may still be writing the bundle to %s with the guest "+ + "paused. Once it is done, `hull resume %s` either resumes the guest or answers "+ + "that it was fenced; retry with a longer --timeout", err, dir, id) + case hviRefusalCode(err) == "not_implemented": + return fmt.Errorf("this hvi (%s) cannot checkpoint yet: %w", state.CmdLine[0], err) + } + return err + } + + rec := hviCheckpointRecord{ + Dir: dir, + ManifestSHA256: reply.ManifestSHA256, + Then: then, + Stats: reply.Stats, + SavedAt: time.Now().UTC(), + TxID: txid, + } + if err := writeHviCheckpointRecord(ckptDir, rec); err != nil { + return fmt.Errorf("the bundle is in %s but recording it failed: %w", dir, err) + } + + fresh, err := s.GetInstance(id) + if err != nil { + return err + } + previous := fresh.Migration + migrationState := store.MigrationCheckpointed + if then == hviThenFence { + migrationState = store.MigrationFenced + } + fresh.Migration = &store.MigrationState{State: migrationState, BundleDir: dir, TxID: txid, + ManifestSHA256: reply.ManifestSHA256} + if err := s.SaveInstance(fresh); err != nil { + return fmt.Errorf("the bundle is in %s but the instance record was not updated: %w", dir, err) + } + // One bundle per instance of hull's own naming: each is the size of the + // guest's touched RAM, and only the latest is ever restored. A bundle + // placed with --dir was named by whoever chose it and is never removed + // here. + if previous != nil && previous.BundleDir != dir && previous.BundleDir == filepath.Join(root, previous.TxID) { + _ = store.RemoveTree(previous.BundleDir) + } + + fmt.Printf("Checkpoint of %s written to %s in %s (%s)\n", id, dir, + time.Since(started).Round(time.Millisecond), describeCheckpointStats(then, reply.Stats)) + return nil +} + +// hviCheckpointRoot creates /checkpoint, 0700, and returns it +// with links resolved, for --checkpoint-root. +// +// hvi is confined before the guest runs, and its confinement grants file +// writes below this one directory and nowhere else, so without it every +// checkpoint is refused. It must exist at boot. It is passed resolved, and +// every bundle path hull sends is built from or checked against this same +// resolved spelling, so the two always agree as strings: on macOS /tmp is a +// link to /private/tmp, and a check that mixed spellings would put a +// directory below the root outside it. +func hviCheckpointRoot(instanceDir string) (string, error) { + root := filepath.Join(instanceDir, "checkpoint") + if err := os.MkdirAll(root, 0o700); err != nil { + return "", fmt.Errorf("create %s: %w", root, err) + } + return filepath.EvalSymlinks(root) +} + +// bundleDirBelow resolves a --dir and keeps it below root, the checkpoint +// root the instance's hvi was started with. +// +// hvi's confinement lets it write there and nowhere else, and it refuses a +// bundle directory outside it; saying so here names the root instead of +// leaving the user an EPERM. The root on the recorded command line is +// resolved (hviCheckpointRoot), so the directory is resolved too before the +// prefix check: on macOS /tmp is a link to /private/tmp, and a --dir under +// /tmp/... would otherwise never be "below" /private/tmp/.... hvi is sent +// the same resolved path, which matches the root it was given. +func bundleDirBelow(root, dir string) (string, error) { + abs, err := filepath.Abs(dir) + if err != nil { + return "", err + } + parent, err := filepath.EvalSymlinks(filepath.Dir(abs)) + if err != nil { + return "", fmt.Errorf("--dir %s: %w", dir, err) + } + resolved := filepath.Join(parent, filepath.Base(abs)) + if !strings.HasPrefix(resolved, root+string(filepath.Separator)) { + return "", fmt.Errorf("--dir %s is not below %s, the one directory this instance's hvi "+ + "may write a bundle to", dir, root) + } + return resolved, nil +} + +// prepareBundleDir makes dir ready for hvi to write a bundle into: created +// 0700 if missing, or an existing empty directory. It reports whether it made +// the directory. A directory with files in it is refused rather than mixed: +// a bundle's manifest lists its components by name, and a stale file beside +// them would sit in a directory that no longer describes one checkpoint. +func prepareBundleDir(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + switch { + case errors.Is(err, os.ErrNotExist): + if err := os.MkdirAll(dir, 0o700); err != nil { + return false, fmt.Errorf("create %s: %w", dir, err) + } + return true, nil + case err != nil: + return false, fmt.Errorf("%s: %w", dir, err) + case len(entries) > 0: + return false, fmt.Errorf("%s is not empty; a checkpoint bundle needs a directory of its own", dir) + } + return false, nil +} + +// emptyDir removes everything inside dir and keeps dir itself. +func emptyDir(dir string) { + entries, err := os.ReadDir(dir) + if err != nil { + return + } + for _, e := range entries { + _ = store.RemoveTree(filepath.Join(dir, e.Name())) + } +} + +// lockInstanceMigration takes /migration.lock for one checkpoint +// or restore of instance id, refusing at once if another hull holds it. +// +// Each of them reads the record's migration, talks to hvi and writes the +// record back. Two at once interleave those steps: a checkpoint deletes the +// bundle it read as previous, which the other one may just have recorded in +// latest.json, and a restore could start from a bundle a checkpoint is about +// to replace. The lock sits in the instance directory, not in the checkpoint +// root hvi may write to. +func lockInstanceMigration(s *store.Store, id string) (func(), error) { + f, err := os.OpenFile(filepath.Join(s.InstanceDir(id), "migration.lock"), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("open the migration lock of %s: %w", id, err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = f.Close() + if errors.Is(err, syscall.EWOULDBLOCK) { + return nil, fmt.Errorf("another checkpoint or restore of %s is in progress", id) + } + return nil, fmt.Errorf("lock the migration of %s: %w", id, err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + _ = f.Close() + }, nil +} + +// describeCheckpointStats renders hvi's stats for a person, leaving out what +// this hvi did not report. +func describeCheckpointStats(then string, raw json.RawMessage) string { + parts := []string{"then " + then} + var st hviCheckpointStats + if len(raw) > 0 && json.Unmarshal(raw, &st) == nil { + if st.PauseMs != nil { + parts = append(parts, fmt.Sprintf("paused %d ms", *st.PauseMs)) + } + if st.CaptureMs != nil { + parts = append(parts, fmt.Sprintf("capture %d ms", *st.CaptureMs)) + } + if st.RAMPresentBytes != nil && st.RAMTotalBytes != nil { + parts = append(parts, fmt.Sprintf("RAM %.1f of %.1f MiB stored", + float64(*st.RAMPresentBytes)/(1<<20), float64(*st.RAMTotalBytes)/(1<<20))) + } + } + return strings.Join(parts, ", ") +} diff --git a/cmd/hull/hvicheckpoint_test.go b/cmd/hull/hvicheckpoint_test.go new file mode 100644 index 00000000..7002e3d8 --- /dev/null +++ b/cmd/hull/hvicheckpoint_test.go @@ -0,0 +1,284 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/brig-sh/hull/pkg/store" +) + +func TestHviCheckpointable(t *testing.T) { + cases := []struct { + name string + cmdLine []string + refusal string // "" means eligible + }{ + {"block root", []string{"/opt/hull/hvi", "boot", "--kernel", "k", "--disk", "/r.img"}, ""}, + {"block root with gateway", []string{"hvi", "boot", "--disk", "/r.img", "--net-gateway", "/g.sock"}, ""}, + {"restored instance", []string{"/x/hvi", "restore", "--from", "/b", "--disk", "/r.img"}, ""}, + {"owner flags without shares", []string{"hvi", "boot", "--fs-uid", "1000", "--fs-gid", "1000", "--disk", "/r"}, ""}, + {"virtiofs root", []string{"hvi", "boot", "--kernel", "k", "--share-rw", "/root", "rootfs"}, "block rootfs"}, + {"disk plus a shared dir", []string{"hvi", "boot", "--disk", "/r", "--share-ro", "/src", "fs1"}, "shared host directory"}, + {"disk plus a writable share", []string{"hvi", "boot", "--disk", "/r", "--share-rw", "/w", "fs1"}, "shared host directory"}, + {"vz", []string{"/x/vz-runner", "--rootfs", "/r"}, "hvi backend"}, + {"qemu", []string{"qemu-system-aarch64", "-drive", "file=/r"}, "hvi backend"}, + {"a path that merely mentions hvi", []string{"/hvi/bin/qemu", "--disk", "/r"}, "hvi backend"}, + {"empty", nil, "hvi backend"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := hviCheckpointable(tc.cmdLine) + switch { + case tc.refusal == "" && err != nil: + t.Errorf("refused: %v", err) + case tc.refusal != "" && (err == nil || !strings.Contains(err.Error(), tc.refusal)): + t.Errorf("err = %v, want a refusal naming %q", err, tc.refusal) + } + }) + } +} + +func TestWriteHviCheckpointRecord(t *testing.T) { + dir := filepath.Join(t.TempDir(), "checkpoint") + rec := hviCheckpointRecord{ + Dir: "/b1", ManifestSHA256: "aa", Then: hviThenFence, + Stats: json.RawMessage(`{"pause_ms":3}`), SavedAt: time.Unix(1700000000, 0).UTC(), TxID: "t1", + } + if err := writeHviCheckpointRecord(dir, rec); err != nil { + t.Fatal(err) + } + info, err := os.Stat(filepath.Join(dir, "latest.json")) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Errorf("latest.json mode = %v, want 0600", info.Mode().Perm()) + } + raw, _ := os.ReadFile(filepath.Join(dir, "latest.json")) + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + for _, key := range []string{"dir", "manifestSha256", "then", "stats", "savedAt", "txid"} { + if _, ok := doc[key]; !ok { + t.Errorf("latest.json lacks %s: %s", key, raw) + } + } + + rec.Dir, rec.TxID = "/b2", "t2" + if err := writeHviCheckpointRecord(dir, rec); err != nil { + t.Fatal(err) + } + got, err := readHviCheckpointRecord(dir) + if err != nil { + t.Fatal(err) + } + var stats bytes.Buffer + _ = json.Compact(&stats, got.Stats) + if got.Dir != "/b2" || got.TxID != "t2" || got.ManifestSHA256 != "aa" || stats.String() != `{"pause_ms":3}` { + t.Errorf("record = %+v", got) + } + entries, _ := os.ReadDir(dir) + if len(entries) != 1 { + t.Errorf("checkpoint dir holds %d entries, want latest.json alone (a temp file left behind?)", len(entries)) + } +} + +func TestHviCheckpointRecordsTheBundle(t *testing.T) { + stub := newStubHvi(t, func(req map[string]any) stubAnswer { + dir, _ := req["dir"].(string) + _ = os.WriteFile(filepath.Join(dir, "manifest.json"), []byte("{}"), 0o600) + return stubAnswer{reply: `{"v":1,"ok":true,"manifest_sha256":"m1","stats":{"pause_ms":4,"capture_ms":9,"ram_present_bytes":1048576,"ram_total_bytes":4194304}}`} + }) + s := newCmdTestStore(t) + state, _ := fakeHviInstance(t, s, "vm1", stub.path) + + if err := hviCheckpointInstance(s, state, hviThenContinue, "", time.Minute); err != nil { + t.Fatal(err) + } + first := mustInstance(t, s, "vm1").Migration + if first == nil || first.State != store.MigrationCheckpointed || first.TxID == "" { + t.Fatalf("migration = %+v", first) + } + ckptDir := instanceCheckpointDir(s, "vm1") + root, _ := flagValue(state.CmdLine, "--checkpoint-root") + asked, _ := stub.seen()[0]["dir"].(string) + if first.BundleDir != filepath.Join(root, first.TxID) || asked != first.BundleDir { + t.Errorf("bundle %s, hvi asked for %s, want %s", first.BundleDir, asked, filepath.Join(root, first.TxID)) + } + if info, err := os.Stat(first.BundleDir); err != nil || info.Mode().Perm() != 0o700 { + t.Errorf("bundle dir: %v %v, want a 0700 directory", info, err) + } + if stub.seen()[0]["txid"] != first.TxID { + t.Errorf("hvi was sent txid %v, the record has %s", stub.seen()[0]["txid"], first.TxID) + } + rec, err := readHviCheckpointRecord(ckptDir) + if err != nil || rec.Dir != first.BundleDir || rec.ManifestSHA256 != "m1" || rec.Then != "continue" || rec.TxID != first.TxID { + t.Errorf("latest.json = %+v, %v", rec, err) + } + + // A second checkpoint replaces hull's own previous bundle. + if err := hviCheckpointInstance(s, mustInstance(t, s, "vm1"), hviThenFence, "", time.Minute); err != nil { + t.Fatal(err) + } + second := mustInstance(t, s, "vm1").Migration + if second.State != store.MigrationFenced || second.TxID == first.TxID { + t.Errorf("migration = %+v", second) + } + if _, err := os.Stat(first.BundleDir); !os.IsNotExist(err) { + t.Errorf("the previous bundle %s is still there: %v", first.BundleDir, err) + } + + // A bundle placed with --dir is never removed by a later checkpoint. + own := filepath.Join(instanceCheckpointDir(s, "vm1"), "mine") + st := mustInstance(t, s, "vm1") + st.Migration = nil + _ = s.SaveInstance(st) + if err := hviCheckpointInstance(s, st, hviThenContinue, own, time.Minute); err != nil { + t.Fatal(err) + } + if err := hviCheckpointInstance(s, mustInstance(t, s, "vm1"), hviThenContinue, "", time.Minute); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(own, "manifest.json")); err != nil { + t.Errorf("a --dir bundle was removed: %v", err) + } +} + +func TestHviCheckpointFailureKeepsTheRecord(t *testing.T) { + stub := newStubHvi(t, func(req map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":false,"code":"timeout","error":"vCPUs did not park"}`} + }) + s := newCmdTestStore(t) + state, _ := fakeHviInstance(t, s, "vm1", stub.path) + err := hviCheckpointInstance(s, state, hviThenFence, "", time.Minute) + dir, _ := stub.seen()[0]["dir"].(string) + if hviRefusalCode(err) != "timeout" { + t.Fatalf("err = %v, want hvi's timeout refusal", err) + } + if got := mustInstance(t, s, "vm1").Migration; got != nil { + t.Errorf("a failed checkpoint recorded %+v", got) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Errorf("the directory made for the failed bundle is still there: %v", err) + } + if _, err := os.Stat(filepath.Join(instanceCheckpointDir(s, "vm1"), "latest.json")); !os.IsNotExist(err) { + t.Errorf("a failed checkpoint wrote latest.json: %v", err) + } +} + +func TestHviCheckpointRefusals(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { return stubAnswer{reply: `{"v":1,"ok":true}`} }) + s := newCmdTestStore(t) + state, _ := fakeHviInstance(t, s, "vm1", stub.path) + + shared := *state + shared.CmdLine = append(slices.Clone(state.CmdLine), "--share-rw", "/w", "fs1") + if err := hviCheckpointInstance(s, &shared, hviThenContinue, "", time.Minute); err == nil || + !strings.Contains(err.Error(), "shared host directory") { + t.Errorf("shares: %v", err) + } + noSock := *state + noSock.ControlSock = "" + if err := hviCheckpointInstance(s, &noSock, hviThenContinue, "", time.Minute); err == nil || + !strings.Contains(err.Error(), "has no control socket") { + t.Errorf("no socket: %v", err) + } + if err := hviCheckpointInstance(s, state, "later", "", time.Minute); err == nil || + !strings.Contains(err.Error(), "--then") { + t.Errorf("bad --then: %v", err) + } + full := filepath.Join(instanceCheckpointDir(s, "vm1"), "full") + _ = os.MkdirAll(full, 0o700) + _ = os.WriteFile(filepath.Join(full, "x"), nil, 0o600) + if err := hviCheckpointInstance(s, state, hviThenContinue, full, time.Minute); err == nil || + !strings.Contains(err.Error(), "not empty") { + t.Errorf("non-empty --dir: %v", err) + } + // hvi may write below its checkpoint root and nowhere else. + if err := hviCheckpointInstance(s, state, hviThenContinue, t.TempDir(), time.Minute); err == nil || + !strings.Contains(err.Error(), "not below") { + t.Errorf("--dir outside the root: %v", err) + } + rootless := *state + rootless.CmdLine = []string{"/x/hvi", "boot", "--disk", "/r.img"} + if err := hviCheckpointInstance(s, &rootless, hviThenContinue, "", time.Minute); err == nil || + !strings.Contains(err.Error(), "without a checkpoint root") { + t.Errorf("no checkpoint root: %v", err) + } + stopped := *state + stopped.Status = "stopped" + if err := hviCheckpointInstance(s, &stopped, hviThenContinue, "", time.Minute); err == nil || + !strings.Contains(err.Error(), "not running") { + t.Errorf("stopped: %v", err) + } + if n := len(stub.seen()); n != 0 { + t.Errorf("hvi was sent %d requests by checkpoints that should have been refused", n) + } +} + +func TestHviControlCheckpointRequest(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":true,"manifest_sha256":"ab12","stats":{"pause_ms":7}}`} + }) + reply, err := hviCheckpoint(stub.path, "/abs/b", hviThenFence, "t1", time.Minute) + if err != nil { + t.Fatal(err) + } + if reply.ManifestSHA256 != "ab12" || string(reply.Stats) != `{"pause_ms":7}` { + t.Errorf("reply = %+v", reply) + } + req := stub.seen()[0] + if req["op"] != "checkpoint" || req["dir"] != "/abs/b" || req["then"] != "fence" || req["txid"] != "t1" || req["v"] != float64(1) { + t.Errorf("request = %v", req) + } +} + +// The root hvi is given must exist, be private and be resolved: hvi compares +// bundle paths against the resolved root as given, so a root under a link +// (/tmp on macOS) would make every bundle "not below" it. +func TestHviCheckpointRootIsPrivateAndResolved(t *testing.T) { + base := t.TempDir() + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(base, link); err != nil { + t.Fatal(err) + } + root, err := hviCheckpointRoot(link) + if err != nil { + t.Fatal(err) + } + want, _ := filepath.EvalSymlinks(filepath.Join(base, "checkpoint")) + if root != want { + t.Errorf("root = %s, want the resolved %s", root, want) + } + if info, err := os.Stat(root); err != nil || !info.IsDir() || info.Mode().Perm() != 0o700 { + t.Errorf("root: %v %v, want a 0700 directory", info, err) + } + got, err := bundleDirBelow(root, filepath.Join(link, "checkpoint", "b1")) + if err != nil || got != filepath.Join(root, "b1") { + t.Errorf("a --dir reached through a link resolved to %q, %v; want %s", got, err, filepath.Join(root, "b1")) + } + if _, err := bundleDirBelow(root, root); err == nil { + t.Error("the root itself was accepted as a bundle directory") + } +} diff --git a/cmd/hull/hvicontrol_test.go b/cmd/hull/hvicontrol_test.go index 8ad74fac..076d91f8 100644 --- a/cmd/hull/hvicontrol_test.go +++ b/cmd/hull/hvicontrol_test.go @@ -78,7 +78,10 @@ func newStubHvi(t *testing.T, handle func(req map[string]any) stubAnswer) *stubH case <-done: return } - _, _ = conn.Write([]byte(ans.reply + "\n")) + // An empty reply is a hangup with no answer at all. + if ans.reply != "" { + _, _ = conn.Write([]byte(ans.reply + "\n")) + } }() } }() diff --git a/cmd/hull/hviflags.go b/cmd/hull/hviflags.go new file mode 100644 index 00000000..40998939 --- /dev/null +++ b/cmd/hull/hviflags.go @@ -0,0 +1,59 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "strings" + "time" +) + +// hviBootTakesControlFlags reports whether the hvi at binary accepts +// --control-sock, --profile-out and --checkpoint-root on `boot`. +// +// hvi refuses a boot argument it does not know and exits before the guest +// starts, so passing the three to an hvi from before them (the release hull +// pins, for one) fails every run, not just checkpoints. Asked of the binary +// that will run, once per run: hull is shipped beside one hvi but a store can +// outlive it, and PATH or the urunc config can name another. +// +// The probe is `boot --checkpoint-root` with no value. --checkpoint-root is +// the newest of the three, so an hvi that knows it knows the others; it fails +// while parsing the arguments, before any hypervisor, sandbox or file is +// touched. An hvi without it answers `unknown boot arg "--checkpoint-root"` +// (src/main.rs), one with it that the flag needs a directory. Any other +// answer is taken as "supported": the boot itself then fails with hvi's own +// reason, which says more than a guess here would. +func hviBootTakesControlFlags(binary string) (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var out bytes.Buffer + probe := exec.CommandContext(ctx, binary, "boot", "--checkpoint-root") + probe.Stdout = &out + probe.Stderr = &out + err := probe.Run() + if ctx.Err() != nil { + return false, fmt.Errorf("%s boot did not answer a probe within 5s", binary) + } + var exitErr *exec.ExitError + if err != nil && !errors.As(err, &exitErr) { + return false, fmt.Errorf("run %s: %w", binary, err) + } + return !strings.Contains(out.String(), `unknown boot arg "--checkpoint-root"`), nil +} diff --git a/cmd/hull/hviflags_test.go b/cmd/hull/hviflags_test.go new file mode 100644 index 00000000..d5273581 --- /dev/null +++ b/cmd/hull/hviflags_test.go @@ -0,0 +1,55 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// The pinned hvi (890f9e0) refuses --checkpoint-root as an unknown boot +// argument and so would refuse the whole boot; run must leave the three +// control flags off for it and pass them to an hvi that takes them. +func TestHviBootTakesControlFlags(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil { + t.Fatal(err) + } + return p + } + // What the hvi at 890f9e0 and at bf3b1aa print, verbatim. + pinned := write("pinned", `echo 'hvi: unknown boot arg "--checkpoint-root"' >&2; exit 1`) + current := write("current", `echo 'hvi: --checkpoint-root needs a directory' >&2; exit 1`) + // The probe must ask about --checkpoint-root alone, with nothing that + // could boot a guest; any other argv reads here as an hvi without it. + asked := write("asked", `[ "$1 $2 $#" = "boot --checkpoint-root 2" ] || { echo 'hvi: unknown boot arg "--checkpoint-root"' >&2; exit 1; } +echo 'hvi: --checkpoint-root needs a directory' >&2; exit 1`) + + if takes, err := hviBootTakesControlFlags(pinned); err != nil || takes { + t.Errorf("the pinned hvi: takes=%v err=%v", takes, err) + } + if takes, err := hviBootTakesControlFlags(current); err != nil || !takes { + t.Errorf("an hvi with the flags: takes=%v err=%v", takes, err) + } + if takes, err := hviBootTakesControlFlags(asked); err != nil || !takes { + t.Errorf("probe argv: takes=%v err=%v", takes, err) + } + if _, err := hviBootTakesControlFlags(filepath.Join(dir, "absent")); err == nil { + t.Error("a missing binary was not reported") + } +} diff --git a/cmd/hull/hvimigration_test.go b/cmd/hull/hvimigration_test.go new file mode 100644 index 00000000..b3bc9bb4 --- /dev/null +++ b/cmd/hull/hvimigration_test.go @@ -0,0 +1,480 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/brig-sh/hull/pkg/store" +) + +// withMigration records m on instance id. +func withMigration(t *testing.T, s *store.Store, id string, m store.MigrationState) { + t.Helper() + st := mustInstance(t, s, id) + st.Migration = &m + if err := s.SaveInstance(st); err != nil { + t.Fatal(err) + } +} + +func TestSetRestoreStateIsOneWay(t *testing.T) { + s := newCmdTestStore(t) + if _, err := s.CreateInstance("vm1"); err != nil { + t.Fatal(err) + } + _ = s.SaveInstance(&store.InstanceState{ID: "vm1", Status: "stopped"}) + steps := []struct{ from, to, want string }{ + {store.MigrationRestoring, store.MigrationRestoredPaused, store.MigrationRestoredPaused}, + {store.MigrationRestoredPaused, store.MigrationRestored, store.MigrationRestored}, + {store.MigrationRestored, store.MigrationRestoredPaused, store.MigrationRestored}, + {store.MigrationRestored, store.MigrationRestoring, store.MigrationRestored}, + {store.MigrationRestoredPaused, store.MigrationRestoring, store.MigrationRestoredPaused}, + // Not in a restore: left alone. + {store.MigrationFenced, store.MigrationRestored, store.MigrationFenced}, + {store.MigrationCheckpointed, store.MigrationRestoredPaused, store.MigrationCheckpointed}, + } + for _, step := range steps { + withMigration(t, s, "vm1", store.MigrationState{State: step.from, BundleDir: "/b", TxID: "t"}) + setRestoreState(s, "vm1", "t", step.to) + if got := mustInstance(t, s, "vm1").Migration.State; got != step.want { + t.Errorf("%s -> %s: record says %s, want %s", step.from, step.to, got, step.want) + } + } + // Another checkpoint's restore never writes over this one. + withMigration(t, s, "vm1", store.MigrationState{State: store.MigrationRestoring, BundleDir: "/b", TxID: "new"}) + setRestoreState(s, "vm1", "old", store.MigrationRestored) + if got := mustInstance(t, s, "vm1").Migration.State; got != store.MigrationRestoring { + t.Errorf("a stale txid moved the record to %s", got) + } +} + +// `hull restore --paused` and then `hull resume`: the restored guest runs, so +// the record must refuse another restore, whatever hvi answered. +func TestResumeMakesARestoredGuestUnrestorable(t *testing.T) { + for _, tc := range []struct { + name string + reply string + }{ + {"resumed", `{"v":1,"ok":true,"state":"running"}`}, + // hvi hangs up without a word: the resume may have happened. + {"no reply", ""}, + } { + for _, from := range []string{store.MigrationRestoredPaused, store.MigrationRestoring} { + t.Run(tc.name+"/"+from, func(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { return stubAnswer{reply: tc.reply} }) + s := newCmdTestStore(t) + fakeHviInstance(t, s, "vm1", stub.path) + withMigration(t, s, "vm1", store.MigrationState{State: from, BundleDir: "/b", TxID: "t"}) + err := hviPauseResumeIn(s, "vm1", "resume") + if (tc.reply == "") != (err != nil) { + t.Errorf("resume: %v", err) + } + m := mustInstance(t, s, "vm1").Migration + if restorableMigration(m) == nil { + t.Errorf("after a resume the record says %s, which allows a restore", m.State) + } + }) + } + } +} + +// A fenced checkpoint is no restore in progress: resume leaves it as it is +// (hvi refuses that resume anyway). +func TestResumeLeavesAFencedRecord(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":false,"code":"fenced","error":"fenced"}`} + }) + s := newCmdTestStore(t) + fakeHviInstance(t, s, "vm1", stub.path) + withMigration(t, s, "vm1", store.MigrationState{State: store.MigrationFenced, BundleDir: "/b", TxID: "t"}) + if err := hviPauseResumeIn(s, "vm1", "resume"); err == nil { + t.Error("a fenced resume was not refused") + } + if m := mustInstance(t, s, "vm1").Migration; m.State != store.MigrationFenced { + t.Errorf("record = %s, want fenced", m.State) + } +} + +// The restore's own resume: no reply, or a hangup, still records the guest +// as having run. +func TestRestoreResumeWithoutReplyIsRecordedAsRun(t *testing.T) { + for _, tc := range []struct { + name string + answer stubAnswer + noRep bool + }{ + {"hangup", stubAnswer{}, false}, + {"timeout", stubAnswer{reply: `{"v":1,"ok":true,"state":"running"}`, delay: hviControlTimeout + time.Second}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + stub := newStubHvi(t, func(req map[string]any) stubAnswer { + if req["op"] == "resume" { + return tc.answer + } + return stubAnswer{reply: `{"v":1,"ok":true,"state":"restored-paused"}`} + }) + s := newCmdTestStore(t) + fakeHviInstance(t, s, "vm1", stub.path) + withMigration(t, s, "vm1", store.MigrationState{State: store.MigrationRestoring, BundleDir: "/b", TxID: "t"}) + got, err := awaitRestoredAndResume(s, "vm1", "t", stub.path, + hviRestoreOptions{timeout: 10 * time.Second}, time.Time{}, nil) + if got != store.MigrationRestored || err == nil { + t.Errorf("result = %q, %v; want restored and an error", got, err) + } + if tc.noRep && !errors.Is(err, errHviNoReply) { + t.Errorf("err = %v, want no reply", err) + } + if m := mustInstance(t, s, "vm1").Migration; restorableMigration(m) == nil { + t.Errorf("record says %s, which allows a restore", m.State) + } + }) + } +} + +// The interleaving a foreground `hull restore --paused` meets: the wait +// records restored-paused, another terminal resumes the guest, then the VMM +// exits and launchVMM saves the record it has held since launch (migration: +// restoring), and the restore's backstop writes restored-paused. The record +// must still say the guest ran. +func TestForegroundRestoreKeepsAResumeGivenMeanwhile(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":true,"state":"running"}`} + }) + s := newCmdTestStore(t) + held, _ := fakeHviInstance(t, s, "vm1", stub.path) + held.Migration = &store.MigrationState{State: store.MigrationRestoring, BundleDir: "/b", TxID: "t"} + _ = s.SaveInstance(held) + stale := *held + staleMigration := *held.Migration + stale.Migration = &staleMigration + + setRestoreState(s, "vm1", "t", store.MigrationRestoredPaused) + if err := hviPauseResumeIn(s, "vm1", "resume"); err != nil { + t.Fatal(err) + } + // launchVMM's markStopped. + stale.Status, stale.PID = "stopped", 0 + keepRecordedMigration(s, &stale) + _ = s.SaveInstance(&stale) + // hviRestoreInstance's backstop with the wait's result. + setRestoreState(s, "vm1", "t", store.MigrationRestoredPaused) + + m := mustInstance(t, s, "vm1").Migration + if restorableMigration(m) == nil { + t.Errorf("record says %s after the restored guest ran", m.State) + } +} + +// A run's foreground launchVMM must not forget a fenced checkpoint taken +// from another terminal when its VMM exits. +func TestVMMExitKeepsAFenceTakenMeanwhile(t *testing.T) { + s := newCmdTestStore(t) + held, _ := fakeHviInstance(t, s, "vm1", "/nonexistent.sock") + stale := *held + withMigration(t, s, "vm1", store.MigrationState{State: store.MigrationFenced, BundleDir: "/b", TxID: "t"}) + stale.Status, stale.PID = "stopped", 0 + keepRecordedMigration(s, &stale) + _ = s.SaveInstance(&stale) + if m := mustInstance(t, s, "vm1").Migration; m == nil || m.State != store.MigrationFenced { + t.Errorf("record = %+v, want the fenced checkpoint", m) + } +} + +// `--then continue` may resume a restored guest that was never resumed, +// and a failed capture of either kind may resume it too, so the record must +// say it ran before the request goes out. +func TestCheckpointMarksARestoredGuestRun(t *testing.T) { + for _, then := range []string{hviThenContinue, hviThenFence} { + t.Run(then, func(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":false,"code":"capture_failed","error":"disk sync failed"}`} + }) + s := newCmdTestStore(t) + fakeHviInstance(t, s, "vm1", stub.path) + withMigration(t, s, "vm1", store.MigrationState{State: store.MigrationRestoredPaused, BundleDir: "/b", TxID: "t"}) + if err := hviCheckpointInstance(s, mustInstance(t, s, "vm1"), then, "", time.Minute); err == nil { + t.Fatal("a failed capture was not reported") + } + if m := mustInstance(t, s, "vm1").Migration; restorableMigration(m) == nil { + t.Errorf("record says %s, which allows a restore", m.State) + } + }) + } +} + +// A checkpoint that gets no reply may still be written by hvi: the +// directory stays, and the record is left as it was. +func TestHviCheckpointWithoutReplyKeepsTheDirectory(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":true,"manifest_sha256":"m"}`, delay: hviControlTimeout + 2*time.Second} + }) + s := newCmdTestStore(t) + state, _ := fakeHviInstance(t, s, "vm1", stub.path) + err := hviCheckpointInstance(s, state, hviThenFence, "", time.Second) + if !errors.Is(err, errHviNoReply) { + t.Fatalf("err = %v, want no reply", err) + } + dir, _ := stub.seen()[0]["dir"].(string) + if _, err := os.Stat(dir); err != nil { + t.Errorf("the bundle directory hvi may still be writing was removed: %v", err) + } + if m := mustInstance(t, s, "vm1").Migration; m != nil { + t.Errorf("a checkpoint with no reply recorded %+v", m) + } + if _, err := readHviCheckpointRecord(instanceCheckpointDir(s, "vm1")); !os.IsNotExist(err) { + t.Errorf("a checkpoint with no reply wrote latest.json: %v", err) + } +} + +// writeBundle makes a bundle directory below root holding a manifest.json and +// returns the directory and the manifest's sha256. +func writeBundle(t *testing.T, root, name, manifest string) (string, string) { + t.Helper() + dir := filepath.Join(root, name) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(manifest), 0o600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256([]byte(manifest)) + return dir, hex.EncodeToString(sum[:]) +} + +// stoppedHviInstance records a stopped hvi instance that ran binary with a +// checkpoint root, and returns it with the root. +func stoppedHviInstance(t *testing.T, s *store.Store, binary string) (*store.InstanceState, string) { + t.Helper() + dir, err := s.CreateInstance("vm1") + if err != nil { + t.Fatal(err) + } + root, err := hviCheckpointRoot(dir) + if err != nil { + t.Fatal(err) + } + st := &store.InstanceState{ + ID: "vm1", Status: "stopped", Backend: "hvi", + CmdLine: []string{binary, "boot", "--kernel", "/k", "--disk", "/r.img", "--checkpoint-root", root}, + } + if err := s.SaveInstance(st); err != nil { + t.Fatal(err) + } + return st, root +} + +func hviWithoutRestore(t *testing.T) string { + t.Helper() + p := filepath.Join(t.TempDir(), "hvi") + if err := os.WriteFile(p, []byte("#!/bin/sh\necho 'hvi: unknown subcommand \"restore\"; expected `boot`' >&2\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + return p +} + +// A fenced checkpoint whose record update failed is named only by +// latest.json. Restore says how to adopt it, and --adopt-latest records it +// once its manifest checks out. +func TestRestoreAdoptsAFencedLatestCheckpoint(t *testing.T) { + s, err := store.New(shortSocketDir(t)) + if err != nil { + t.Fatal(err) + } + st, root := stoppedHviInstance(t, s, hviWithoutRestore(t)) + old, oldSum := writeBundle(t, root, "t1", `{"old":1}`) + st.Migration = &store.MigrationState{State: store.MigrationFenced, BundleDir: old, TxID: "t1", ManifestSHA256: oldSum} + _ = s.SaveInstance(st) + newer, newerSum := writeBundle(t, root, "t2", `{"new":1}`) + if err := writeHviCheckpointRecord(root, hviCheckpointRecord{Dir: newer, TxID: "t2", Then: hviThenFence, ManifestSHA256: newerSum}); err != nil { + t.Fatal(err) + } + + err = hviRestoreInstance(context.Background(), nil, s, st, hviRestoreOptions{timeout: time.Second}) + if err == nil || !strings.Contains(err.Error(), "--adopt-latest") { + t.Fatalf("without --adopt-latest: %v", err) + } + if m := mustInstance(t, s, "vm1").Migration; m.TxID != "t1" { + t.Errorf("a refused restore changed the record to %+v", m) + } + + // Adopted; this hvi has no restore, so that is where it stops. + err = hviRestoreInstance(context.Background(), nil, s, mustInstance(t, s, "vm1"), hviRestoreOptions{timeout: time.Second, adoptLatest: true}) + if err == nil || !strings.Contains(err.Error(), "no restore subcommand") { + t.Fatalf("with --adopt-latest: %v", err) + } + m := mustInstance(t, s, "vm1").Migration + if m.TxID != "t2" || m.State != store.MigrationFenced || m.BundleDir != newer || m.ManifestSHA256 != newerSum { + t.Errorf("adopted record = %+v", m) + } +} + +func TestRestoreRefusesToAdoptAnUnsafeLatest(t *testing.T) { + for _, tc := range []struct { + name string + rec func(dir, sum string) hviCheckpointRecord + refusal string + }{ + {"continued", func(dir, sum string) hviCheckpointRecord { + return hviCheckpointRecord{Dir: dir, TxID: "t2", Then: hviThenContinue, ManifestSHA256: sum} + }, "neither it nor"}, + {"manifest changed", func(dir, _ string) hviCheckpointRecord { + return hviCheckpointRecord{Dir: dir, TxID: "t2", Then: hviThenFence, ManifestSHA256: strings.Repeat("0", 64)} + }, "changed since"}, + {"no manifest hash", func(dir, _ string) hviCheckpointRecord { + return hviCheckpointRecord{Dir: dir, TxID: "t2", Then: hviThenFence} + }, "no manifest sha256"}, + {"outside the root", func(_, sum string) hviCheckpointRecord { + return hviCheckpointRecord{Dir: "/etc", TxID: "t2", Then: hviThenFence, ManifestSHA256: sum} + }, "not below"}, + } { + t.Run(tc.name, func(t *testing.T) { + s, err := store.New(shortSocketDir(t)) + if err != nil { + t.Fatal(err) + } + st, root := stoppedHviInstance(t, s, hviWithoutRestore(t)) + dir, sum := writeBundle(t, root, "t2", `{"new":1}`) + if err := writeHviCheckpointRecord(root, tc.rec(dir, sum)); err != nil { + t.Fatal(err) + } + err = hviRestoreInstance(context.Background(), nil, s, st, hviRestoreOptions{timeout: time.Second, adoptLatest: true}) + if err == nil || !strings.Contains(err.Error(), tc.refusal) { + t.Errorf("err = %v, want a refusal naming %q", err, tc.refusal) + } + if m := mustInstance(t, s, "vm1").Migration; m != nil { + t.Errorf("an unsafe latest.json was adopted: %+v", m) + } + }) + } +} + +// latest.json is the file that would say a later checkpoint exists, in a +// directory hvi may write to: one that does not parse is refused, not +// skipped. +func TestRestoreRefusesAnUnreadableLatest(t *testing.T) { + s, err := store.New(shortSocketDir(t)) + if err != nil { + t.Fatal(err) + } + st, root := stoppedHviInstance(t, s, hviWithoutRestore(t)) + dir, sum := writeBundle(t, root, "t1", `{}`) + st.Migration = &store.MigrationState{State: store.MigrationFenced, BundleDir: dir, TxID: "t1", ManifestSHA256: sum} + _ = s.SaveInstance(st) + if err := os.WriteFile(filepath.Join(root, "latest.json"), []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + err = hviRestoreInstance(context.Background(), nil, s, st, hviRestoreOptions{timeout: time.Second}) + if err == nil || !strings.Contains(err.Error(), "cannot tell") { + t.Errorf("err = %v", err) + } +} + +// The record keeps the manifest's sha256 hvi reported; a bundle whose +// manifest no longer matches it is refused before anything is launched. +func TestRestoreRefusesABundleChangedSinceTheCheckpoint(t *testing.T) { + s, err := store.New(shortSocketDir(t)) + if err != nil { + t.Fatal(err) + } + st, root := stoppedHviInstance(t, s, hviWithoutRestore(t)) + dir, sum := writeBundle(t, root, "t1", `{"a":1}`) + if err := writeHviCheckpointRecord(root, hviCheckpointRecord{Dir: dir, TxID: "t1", Then: hviThenFence, ManifestSHA256: sum}); err != nil { + t.Fatal(err) + } + st.Migration = &store.MigrationState{State: store.MigrationFenced, BundleDir: dir, TxID: "t1", ManifestSHA256: sum} + _ = s.SaveInstance(st) + if err := os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(`{"a":2}`), 0o600); err != nil { + t.Fatal(err) + } + err = hviRestoreInstance(context.Background(), nil, s, st, hviRestoreOptions{timeout: time.Second}) + if err == nil || !strings.Contains(err.Error(), "changed since") { + t.Errorf("err = %v", err) + } +} + +// The record names the sha256 hvi reported, so restore can check it. +func TestHviCheckpointRecordsTheManifestHash(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":true,"manifest_sha256":"abc"}`} + }) + s := newCmdTestStore(t) + state, _ := fakeHviInstance(t, s, "vm1", stub.path) + if err := hviCheckpointInstance(s, state, hviThenFence, "", time.Minute); err != nil { + t.Fatal(err) + } + if m := mustInstance(t, s, "vm1").Migration; m.ManifestSHA256 != "abc" { + t.Errorf("record = %+v", m) + } +} + +// Two checkpoints, or a checkpoint and a restore, of one instance must not +// interleave: the second is refused at once, before hvi is asked anything. +func TestCheckpointAndRestoreRefuseWhileAnotherHoldsTheLock(t *testing.T) { + stub := newStubHvi(t, func(map[string]any) stubAnswer { return stubAnswer{reply: `{"v":1,"ok":true}`} }) + s := newCmdTestStore(t) + state, _ := fakeHviInstance(t, s, "vm1", stub.path) + unlock, err := lockInstanceMigration(s, "vm1") + if err != nil { + t.Fatal(err) + } + if err := hviCheckpointInstance(s, state, hviThenContinue, "", time.Minute); err == nil || + !strings.Contains(err.Error(), "in progress") { + t.Errorf("checkpoint: %v", err) + } + if err := hviRestoreInstance(context.Background(), nil, s, state, hviRestoreOptions{timeout: time.Second}); err == nil || + !strings.Contains(err.Error(), "in progress") { + t.Errorf("restore: %v", err) + } + if n := len(stub.seen()); n != 0 { + t.Errorf("hvi was sent %d requests", n) + } + unlock() + if err := hviCheckpointInstance(s, state, hviThenContinue, "", time.Minute); err != nil { + t.Errorf("after unlock: %v", err) + } +} + +// A failed capture into a --dir that was there before empties it and keeps +// it, so the same --dir can be retried. +func TestHviCheckpointFailureEmptiesAnExistingDir(t *testing.T) { + stub := newStubHvi(t, func(req map[string]any) stubAnswer { + dir, _ := req["dir"].(string) + _ = os.WriteFile(filepath.Join(dir, "ram.bin"), []byte("partial"), 0o600) + _ = os.MkdirAll(filepath.Join(dir, "sub"), 0o700) + return stubAnswer{reply: `{"v":1,"ok":false,"code":"capture_failed","error":"disk sync failed"}`} + }) + s := newCmdTestStore(t) + state, _ := fakeHviInstance(t, s, "vm1", stub.path) + own := filepath.Join(instanceCheckpointDir(s, "vm1"), "mine") + if err := os.MkdirAll(own, 0o700); err != nil { + t.Fatal(err) + } + if err := hviCheckpointInstance(s, state, hviThenContinue, own, time.Minute); hviRefusalCode(err) != "capture_failed" { + t.Fatalf("err = %v", err) + } + entries, err := os.ReadDir(own) + if err != nil || len(entries) != 0 { + t.Errorf("--dir after a failed capture: %v entries, %v; want it kept and empty", len(entries), err) + } +} diff --git a/cmd/hull/hvipause.go b/cmd/hull/hvipause.go index 8bea1179..3b18b024 100644 --- a/cmd/hull/hvipause.go +++ b/cmd/hull/hvipause.go @@ -42,8 +42,9 @@ func requireRunningHvi(state *store.InstanceState, verb string) error { return fmt.Errorf("instance %s is not running", state.ID) } if state.ControlSock == "" { - return fmt.Errorf("instance %s was started without a control socket, so it can only be "+ - "stopped; run it again with this version of hull", state.ID) + return fmt.Errorf("instance %s has no control socket, so it can only be stopped: the hvi "+ + "it runs (%s) predates the socket, or an older hull started it. Run it again with an "+ + "hvi that takes --control-sock", state.ID, state.CmdLine[0]) } return nil } @@ -128,11 +129,15 @@ func hviPauseResume(cmd *cli.Command, op string) error { if cmd.Args().Len() == 0 { return errors.New("instance ID required") } - id := cmd.Args().First() s, err := globalStore(cmd) if err != nil { return err } + return hviPauseResumeIn(s, cmd.Args().First(), op) +} + +// hviPauseResumeIn sends op, pause or resume, to instance id's hvi. +func hviPauseResumeIn(s *store.Store, id, op string) error { state, err := s.GetInstance(id) if err != nil { return fmt.Errorf("instance not found: %s", id) @@ -140,6 +145,11 @@ func hviPauseResume(cmd *cli.Command, op string) error { if err := requireRunningHvi(state, op); err != nil { return err } + if op == "resume" { + if err := markRestoredGuestRunning(s, id); err != nil { + return err + } + } reply, err := hviOp(state.ControlSock, op) switch { case errors.Is(err, errHviNoReply) && op == "pause": diff --git a/cmd/hull/hvipause_test.go b/cmd/hull/hvipause_test.go index 09e4c442..e31651f5 100644 --- a/cmd/hull/hvipause_test.go +++ b/cmd/hull/hvipause_test.go @@ -29,12 +29,17 @@ import ( func fakeHviInstance(t *testing.T, s *store.Store, id, sock string) (*store.InstanceState, <-chan struct{}) { t.Helper() pid, done := startProcessAs(t, "hvi", "/bin/sleep", "30") - if _, err := s.CreateInstance(id); err != nil { + dir, err := s.CreateInstance(id) + if err != nil { + t.Fatal(err) + } + root, err := hviCheckpointRoot(dir) + if err != nil { t.Fatal(err) } state := &store.InstanceState{ ID: id, Status: "running", PID: pid, Backend: "hvi", ControlSock: sock, - CmdLine: []string{"/x/hvi", "boot", "--kernel", "/k", "--disk", "/r.img"}, + CmdLine: []string{"/x/hvi", "boot", "--kernel", "/k", "--disk", "/r.img", "--checkpoint-root", root}, } if err := s.SaveInstance(state); err != nil { t.Fatal(err) diff --git a/cmd/hull/hvirestore.go b/cmd/hull/hvirestore.go new file mode 100644 index 00000000..5e9abf8f --- /dev/null +++ b/cmd/hull/hvirestore.go @@ -0,0 +1,646 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/urfave/cli/v3" + "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" + + "github.com/brig-sh/hull/pkg/store" +) + +func restoreCommand() *cli.Command { + return &cli.Command{ + Name: "restore", + Usage: "restore a stopped instance from its checkpoint: Vz from its saved state, " + + "hvi from the bundle of a fenced checkpoint", + ArgsUsage: "NAME", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "detach", + Aliases: []string{"d"}, + Usage: "run in background", + }, + &cli.IntFlag{ + Name: "stop-grace", + Value: 10, + Usage: "seconds vz-runner waits for the guest to answer a stop request before forcing", + }, + &cli.BoolFlag{ + Name: "wait-ip", + Usage: "with --detach and NAT networking, wait for the DHCP lease and record the IP before returning", + }, + &cli.StringFlag{ + Name: "gateway-sock", + Usage: "Vz: re-join the user-mode network gateway at this control socket (required if " + + "the instance ran on the gateway); hvi re-joins the gateway it ran on by itself", + }, + &cli.BoolFlag{ + Name: "paused", + Usage: "hvi: leave the restored guest paused; `hull resume` starts it", + }, + &cli.IntFlag{ + Name: "timeout", + Value: 120, + Usage: "hvi: seconds to wait for hvi to rebuild the VM from the bundle", + }, + &cli.BoolFlag{ + Name: "adopt-latest", + Usage: "hvi: restore from the fenced checkpoint latest.json names when the record " + + "missed it (a checkpoint whose record update failed)", + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Len() == 0 { + return errors.New("instance ID required") + } + s, err := globalStore(cmd) + if err != nil { + return err + } + id := cmd.Args().First() + state, err := s.GetInstance(id) + if err != nil { + return fmt.Errorf("instance not found: %s", id) + } + if !isHviInstance(state) { + if cmd.IsSet("paused") || cmd.IsSet("timeout") || cmd.IsSet("adopt-latest") { + return errors.New("--paused, --timeout and --adopt-latest apply to hvi instances only") + } + return vzRestoreInstance(ctx, cmd) + } + if cmd.IsSet("gateway-sock") { + return errors.New("an hvi instance re-joins the gateway socket it ran with " + + "(--net-gateway on its command line); --gateway-sock is for Vz instances") + } + return hviRestoreInstance(ctx, cmd, s, state, hviRestoreOptions{ + detach: cmd.Bool("detach"), + paused: cmd.Bool("paused"), + timeout: time.Duration(cmd.Int("timeout")) * time.Second, + adoptLatest: cmd.Bool("adopt-latest"), + }) + }, + } +} + +type hviRestoreOptions struct { + detach bool + paused bool + timeout time.Duration + adoptLatest bool +} + +// hviRestoreArgv turns the command line an hvi instance last ran with into +// the `hvi restore` command line that brings it back from the bundle in from. +// +// The bundle holds the guest's RAM, so it already has the kernel, initramfs +// and command line in it, and its profile fixes the memory size and vCPU +// count: those flags are dropped. What the bundle cannot hold are the host +// resources the devices were attached to, and those are kept: the disk, the +// gateway socket (so a restore on the same host re-joins the same gateway), +// the MAC the gateway's leases are keyed by, the agent socket, the sandbox id, +// the event ledger, and the checkpoint root, without which the restored guest +// could never be checkpointed again. A bare --net is the deprecated spelling of +// --net-stub and is passed under its new name, since restore takes only that. +// +// An allowlist, not a list of flags to remove: a flag this function has never +// heard of could carry a host resource restore would silently go without, so +// it is refused by name instead. The control socket and profile paths are the +// caller's, placed last, which is what makes the mapping idempotent: the +// command line a restore records is a valid input to the next one. +func hviRestoreArgv(cmdLine []string, from, controlSock, profilePath string) ([]string, error) { + if len(cmdLine) < 2 { + return nil, errors.New("the recorded hvi command line is empty") + } + if sub := cmdLine[1]; sub != "boot" && sub != "restore" { + return nil, fmt.Errorf("the recorded hvi command line runs %q, not boot or restore", sub) + } + keep := map[string]bool{ + "--disk": true, "--net-gateway": true, "--net-mac": true, + "--agent-sock": true, "--sandbox-id": true, "--events": true, + "--checkpoint-root": true, + } + drop := map[string]bool{ + "--kernel": true, "--initramfs": true, "--cmdline": true, "--mem-mib": true, + "--cpus": true, "--fs-uid": true, "--fs-gid": true, "--from": true, + "--control-sock": true, "--profile-out": true, + } + out := []string{cmdLine[0], "restore", "--from", from} + sawDisk := false + for i := 2; i < len(cmdLine); i++ { + flag := cmdLine[i] + switch { + case flag == "--net" || flag == "--net-stub": + out = append(out, "--net-stub") + case keep[flag] || drop[flag]: + if i+1 >= len(cmdLine) { + return nil, fmt.Errorf("the recorded hvi command line ends at %s, which takes a value", flag) + } + if keep[flag] { + out = append(out, flag, cmdLine[i+1]) + sawDisk = sawDisk || flag == "--disk" + } + i++ + case flag == "--share-ro" || flag == "--share-rw": + return nil, errors.New("the instance shared a host directory, which no checkpoint carries") + default: + return nil, fmt.Errorf("the recorded hvi command line has %s, which hull does not know "+ + "how to carry into a restore", flag) + } + } + if !sawDisk { + return nil, errors.New("the recorded hvi command line has no --disk; a restore needs the " + + "block rootfs the checkpoint was taken with") + } + return append(out, "--control-sock", controlSock, "--profile-out", profilePath), nil +} + +// flagValue returns the value after the first occurrence of flag in argv. +func flagValue(argv []string, flag string) (string, bool) { + for i := 0; i+1 < len(argv); i++ { + if argv[i] == flag { + return argv[i+1], true + } + } + return "", false +} + +// hviHasRestore reports whether the hvi at binary has a restore subcommand. +// +// Asked of the binary the instance ran, not of PATH: that is the hvi whose +// bundle format the checkpoint is in, and the one restore will run. hvi +// answers an unknown subcommand with a fixed sentence (src/main.rs), so its +// absence is told apart from any other failure, such as restore complaining +// that it was given no --from, which means it is there. +func hviHasRestore(binary string) (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var out bytes.Buffer + probe := exec.CommandContext(ctx, binary, "restore") + probe.Stdout = &out + probe.Stderr = &out + err := probe.Run() + if ctx.Err() != nil { + return false, fmt.Errorf("%s restore did not answer within 5s", binary) + } + var exitErr *exec.ExitError + if err != nil && !errors.As(err, &exitErr) { + return false, fmt.Errorf("run %s: %w", binary, err) + } + return !strings.Contains(out.String(), `unknown subcommand "restore"`), nil +} + +// reconcileLatestCheckpoint compares the record's checkpoint with the one +// /checkpoint/latest.json names, and adopts the latter into the +// record when asked and safe. +// +// checkpoint writes latest.json before the record, so a failure between the +// two leaves latest.json naming a later checkpoint than the record does. +// Restoring the record's would rewind past a checkpoint whose fate the +// record never learned, so a mismatch is refused unless adopt is set. Even +// then only a fenced one is adopted, since the guest never ran after it, +// and only a bundle below the instance's checkpoint root whose manifest.json +// still hashes to the sha256 latest.json recorded from hvi's reply. +// +// A latest.json that is there and does not parse is refused rather than +// skipped: it is the file that would say a later checkpoint exists, and it +// sits in the directory hvi may write to. +func reconcileLatestCheckpoint(s *store.Store, state *store.InstanceState, adopt bool) error { + latest, err := readHviCheckpointRecord(instanceCheckpointDir(s, state.ID)) + switch { + case errors.Is(err, os.ErrNotExist): + return nil + case err != nil: + return fmt.Errorf("%w; hull cannot tell whether a later checkpoint was taken", err) + } + recorded := "" + if state.Migration != nil { + recorded = state.Migration.TxID + } + if latest.TxID == recorded { + return nil + } + if latest.Then != hviThenFence { + return fmt.Errorf("latest.json names checkpoint %s in %s, a later one the record never "+ + "took in, taken with --then %s: the guest ran on after it, so neither it nor the "+ + "record's can be restored", latest.TxID, latest.Dir, latest.Then) + } + if !adopt { + return fmt.Errorf("its record names checkpoint %q, but latest.json names %s in %s, a later "+ + "one the record never took in. It was fenced, so it can be restored: `hull restore "+ + "--adopt-latest %s` checks its manifest and records it first", recorded, latest.TxID, + latest.Dir, state.ID) + } + root, ok := flagValue(state.CmdLine, "--checkpoint-root") + if !ok { + return errors.New("the instance ran without a checkpoint root, so latest.json cannot name a bundle of its own") + } + dir, err := bundleDirBelow(root, latest.Dir) + if err != nil { + return fmt.Errorf("latest.json: %w", err) + } + if latest.ManifestSHA256 == "" { + return errors.New("latest.json records no manifest sha256 to check its bundle against") + } + if err := checkBundleManifest(dir, latest.ManifestSHA256); err != nil { + return err + } + state.Migration = &store.MigrationState{State: store.MigrationFenced, BundleDir: dir, + TxID: latest.TxID, ManifestSHA256: latest.ManifestSHA256} + if err := s.SaveInstance(state); err != nil { + return fmt.Errorf("record the adopted checkpoint: %w", err) + } + fmt.Fprintf(os.Stderr, "Adopted checkpoint %s in %s from latest.json\n", latest.TxID, dir) + return nil +} + +// checkBundleManifest checks that dir/manifest.json hashes to want, the +// sha256 hvi reported when it sealed the bundle. hvi hashes the file's bytes +// as written. The manifest lists every other part with its sha256, which hvi +// checks on restore, so this one hash binds the whole bundle to the +// checkpoint hull recorded, and a bundle rewritten since, by anything that +// can write the checkpoint root, is refused. "" (a record from before hull +// kept the hash) checks nothing. +func checkBundleManifest(dir, want string) error { + if want == "" { + return nil + } + data, err := os.ReadFile(filepath.Join(dir, "manifest.json")) + if err != nil { + return fmt.Errorf("bundle %s has no readable manifest: %w", dir, err) + } + sum := sha256.Sum256(data) + if got := hex.EncodeToString(sum[:]); got != want { + return fmt.Errorf("bundle %s: manifest.json has sha256 %s, but hvi reported %s when it "+ + "sealed the checkpoint; the bundle was changed since", dir, got, want) + } + return nil +} + +// restorableMigration reports whether the record's last checkpoint can be +// restored onto the instance's disk. +// +// The disk is not in the bundle; the restore reopens the live one. That is +// only consistent if the guest never wrote to it after the capture: +// - fenced: the source never ran again. The case a restore is for. +// - restoring or restored-paused: a restore that did not get as far as +// running the guest, so the disk is still the one the bundle saw. Every +// op that may run that guest (the restore's own resume, `hull resume`, +// any `hull checkpoint`) records restored before it is sent, +// and setRestoreState never moves a record back, so these two stay true. +// +// checkpointed (then continue) means the guest kept running and writing, and +// restored means a restored guest ran: the disk has moved on from the RAM in +// the bundle, and a guest given the two together sees a filesystem that +// disagrees with its own caches. That is refused, as Vz restore refuses a +// checkpoint whose disk clone is missing. +func restorableMigration(m *store.MigrationState) error { + if m == nil || m.BundleDir == "" { + return errors.New("it has no hvi checkpoint; take one with `hull checkpoint --then fence`") + } + switch m.State { + case store.MigrationFenced, store.MigrationRestoring, store.MigrationRestoredPaused: + return nil + case store.MigrationCheckpointed: + return errors.New("its checkpoint was taken with --then continue, and the guest has " + + "written to its disk since, which the bundle does not hold. Take one with " + + "`hull checkpoint --then fence` to restore from") + case store.MigrationRestored: + return errors.New("its checkpoint was already restored and the restored guest ran, so " + + "its disk has moved on from the bundle") + } + return fmt.Errorf("its checkpoint is in an unknown state %q", m.State) +} + +func hviRestoreInstance(ctx context.Context, cmd *cli.Command, s *store.Store, state *store.InstanceState, opt hviRestoreOptions) error { + id := state.ID + // Held until the wait below ends, not until launchVMM returns: in the + // foreground that is the restored guest's whole life, and a checkpoint + // of it must not be refused for as long as it runs. + unlock, err := lockInstanceMigration(s, id) + if err != nil { + return err + } + var unlockOnce sync.Once + release := func() { unlockOnce.Do(unlock) } + defer release() + // Read again under the lock: a checkpoint that finished meanwhile + // changed the record. + if state, err = s.GetInstance(id); err != nil { + return fmt.Errorf("instance not found: %s", id) + } + if state.Status == "running" && state.PID > 0 && processIsAVMM(state.PID, state) { + return fmt.Errorf("instance %s is still running; stop it first (the checkpoint is kept)", id) + } + // As for Vz: "starting" with no pid may hide a live VMM holding the disk, + // and a second one on the same disk is the corruption restore must not + // cause. + if state.Status == store.StatusStarting && state.PID <= 0 { + if reapOrphanVMM(s, state) { + log.Warnf("instance %s had a VMM running that was never recorded; killed it "+ + "before restoring", id) + } + } + if err := reconcileLatestCheckpoint(s, state, opt.adoptLatest); err != nil { + return fmt.Errorf("cannot restore %s: %w", id, err) + } + if err := restorableMigration(state.Migration); err != nil { + return fmt.Errorf("cannot restore %s: %w", id, err) + } + bundle := state.Migration.BundleDir + if _, err := os.Stat(bundle); err != nil { + return fmt.Errorf("cannot restore %s: its bundle %s is gone", id, bundle) + } + if err := checkBundleManifest(bundle, state.Migration.ManifestSHA256); err != nil { + return fmt.Errorf("cannot restore %s: %w", id, err) + } + if err := hviCheckpointable(state.CmdLine); err != nil { + return err + } + has, err := hviHasRestore(state.CmdLine[0]) + if err != nil { + return err + } + if !has { + return fmt.Errorf("cannot restore %s: the hvi it ran (%s) has no restore subcommand yet. "+ + "The bundle in %s is kept; restore it with an hvi that has one", id, state.CmdLine[0], bundle) + } + + controlSock := s.InstanceControlSocket(id) + profilePath := s.InstanceProfilePath(id) + if err := checkUnixSocketPath("control", controlSock); err != nil { + return err + } + argv, err := hviRestoreArgv(state.CmdLine, bundle, controlSock, profilePath) + if err != nil { + return fmt.Errorf("cannot restore %s: %w", id, err) + } + // hvi relays the guest's NIC to the gateway socket by path, so a restore + // on the same host re-joins the gateway the instance left, with the same + // MAC and so the same lease. A gateway that is not there would leave the + // guest with a NIC that reaches nothing, since hvi only warns and falls + // back to its no-egress stub: refuse instead. Dialled, as run's + // pre-flight does, not just looked at: a gateway that died leaves its + // socket file behind. + if gw, ok := flagValue(argv, "--net-gateway"); ok { + conn, err := net.DialTimeout("unix", gw, 2*time.Second) + if err != nil { + return fmt.Errorf("cannot restore %s: it ran on the network gateway at %s, which is not "+ + "answering (%w); start `hull network-gateway` there first", id, gw, err) + } + _ = conn.Close() + } + + state.ControlSock = controlSock + state.ProfilePath = profilePath + state.Migration.State = store.MigrationRestoring + txid := state.Migration.TxID + + type awaited struct { + state string + err error + } + result := make(chan awaited, 1) + gone := make(chan struct{}) + since := time.Now() + go func() { + st, err := awaitRestoredAndResume(s, id, txid, controlSock, opt, since, gone) + release() + result <- awaited{st, err} + }() + // The IP the record holds stays the guest's: it resumes with its network + // state in memory and the same MAC. So no rediscovery, as for Vz. + started, launchErr := launchVMM(ctx, cmd, s, state, argv, nil, hypervisors.HviVmm, opt.detach, "none", "") + // In the foreground launchVMM returns when the VMM has exited, and on a + // failed start there never was one. Either way nothing will answer on + // the socket, and the wait must end now rather than at its deadline. + if !opt.detach || !started { + close(gone) + } + + // The wait records each state as it is reached; this is the backstop + // for one it reached and could not record, written through the same + // one-way setter, so it cannot undo a `hull resume` given meanwhile. + final := <-result + if final.state != "" { + setRestoreState(s, id, txid, final.state) + } + if launchErr != nil { + return launchErr + } + if final.err != nil { + return fmt.Errorf("restore of %s: %w", id, final.err) + } + return nil +} + +// awaitRestoredAndResume waits until hvi reports restored-paused on the +// control socket, then resumes the guest unless opt.paused. It returns the +// migration state reached, "" for none past restoring, and why it stopped +// short if it did. +// +// It runs beside launchVMM, which in the foreground holds the terminal until +// the VMM exits, so nothing after that call could ever send the resume. Until +// hvi binds the socket, a dial finds nothing or the source's stale socket and +// is refused; both mean "not yet". A VMM that died, or gone closing, ends the +// wait at once rather than at the deadline. since is when the restore began. +// +// The state it returns errs toward "the guest ran": once a resume has been +// sent, the guest may be writing to the disk, and a record that still said +// restored-paused would let a second restore put the bundle's RAM over it. +func awaitRestoredAndResume(s *store.Store, id, txid, sock string, opt hviRestoreOptions, + since time.Time, gone <-chan struct{}) (string, error) { + exited := errors.New("hvi exited before the guest was restored; its reason is in " + + "`hull logs` (detached) or on the terminal") + deadline := time.Now().Add(opt.timeout) + for { + if time.Now().After(deadline) { + return "", fmt.Errorf("hvi did not report restored-paused within %s. If it is "+ + "still running, `hull stop %s` ends it; the bundle is kept", opt.timeout, id) + } + select { + case <-gone: + return "", exited + default: + } + // Only a pid recorded by this restore says anything about its VMM. + // Until launchVMM's first save the record is the old one, and a + // source that died without `hull stop` (a crash, a reboot) left its + // dead pid there; reading that as "hvi exited" would fail a restore + // that is going fine. launchVMM stamps StartTime on that save. Until + // it has saved, the state is not written either: a write from a + // record read before that save would put back the pid-less one. + ours := false + if rec, err := s.GetInstance(id); err == nil && rec.PID > 0 && !rec.StartTime.Before(since) { + if restoreVMMExited(rec.PID, opt.detach) { + return "", hviExitedError(exited, rec.LogFile) + } + ours = true + } + reply, err := hviOp(sock, "status") + if err == nil && reply.State == store.MigrationRestoredPaused && ours { + break + } + if err == nil && (reply.State == "running" || reply.State == "stopping") { + return store.MigrationRestored, fmt.Errorf("hvi reported %s before hull resumed the "+ + "guest; it is recorded as having run", reply.State) + } + time.Sleep(100 * time.Millisecond) + } + if opt.paused { + setRestoreState(s, id, txid, store.MigrationRestoredPaused) + fmt.Fprintf(os.Stderr, "Instance %s restored and paused; `hull resume %s` starts it\n", id, id) + return store.MigrationRestoredPaused, nil + } + // Recorded before the resume is sent: after it, the guest may be + // running whatever the reply says, and a crash of hull between the two + // must not leave a record that allows another restore. + setRestoreState(s, id, txid, store.MigrationRestored) + if _, err := hviOp(sock, "resume"); err != nil { + if errors.Is(err, errHviNoReply) { + return store.MigrationRestored, fmt.Errorf("resume after restore: %w; the guest may "+ + "be running. `hull resume %s` again is harmless if it is", err, id) + } + return store.MigrationRestored, fmt.Errorf("resume after restore: %w", err) + } + fmt.Fprintf(os.Stderr, "Instance %s restored and running\n", id) + return store.MigrationRestored, nil +} + +// restoreVMMExited reports whether the VMM a restore started has exited. +// +// A detached restore starts hvi as this process's child and never waits on +// it, so an hvi that exits at once (one that refuses the bundle, or has no +// restore for this backend) stays a zombie while the wait runs, and a +// zombie answers signal 0 like a live process. The wait then ran to its +// deadline, two minutes by default, for a VMM that was gone in a +// millisecond. So a detached restore asks wait4 without blocking first, +// which also reaps the zombie. In the foreground launchVMM waits on the +// child itself and closes gone, and reaping it here would take its exit +// status from that wait, so only signal 0 is asked there. +func restoreVMMExited(pid int, detached bool) bool { + if detached { + var ws syscall.WaitStatus + switch wpid, err := syscall.Wait4(pid, &ws, syscall.WNOHANG, nil); { + case wpid == pid: + return true + case err == nil: + return false + } + // ECHILD: not this process's child after all; fall through. + } + return syscall.Kill(pid, 0) != nil +} + +// hviExitedError adds the last line of the instance log, which is where a +// detached hvi says why it stopped, to the "hvi exited" error. +func hviExitedError(exited error, logFile string) error { + if logFile == "" { + return exited + } + data, err := os.ReadFile(logFile) + if err != nil { + return exited + } + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + last := strings.TrimSpace(lines[len(lines)-1]) + if last == "" { + return exited + } + if len(last) > 400 { + last = last[:400] + "..." + } + return fmt.Errorf("%w; it said: %s", exited, last) +} + +// setRestoreState records how far the restore of checkpoint txid got. +// +// One way only: restoring, then restored-paused, then restored, and never +// back. Several hull processes write this field while one restore runs (the +// restore itself, its own backstop write, a `hull resume` from another +// terminal), and the last writer is not always the one that knows most. A +// guest recorded as having run must stay so, or a second restore would put +// the bundle's RAM over a disk that guest has written since. A record that +// has moved on to another checkpoint, or out of a restore, is left alone. +func setRestoreState(s *store.Store, id, txid, to string) { + fresh, err := s.GetInstance(id) + if err != nil || fresh.Migration == nil || fresh.Migration.TxID != txid { + return + } + rank := map[string]int{ + store.MigrationRestoring: 1, + store.MigrationRestoredPaused: 2, + store.MigrationRestored: 3, + } + from := rank[fresh.Migration.State] + if from == 0 || rank[to] <= from { + return + } + fresh.Migration.State = to + _ = s.SaveInstance(fresh) +} + +// markRestoredGuestRunning records that a restored guest is about to run, +// before the op that may run it is sent: `hull resume`, or any checkpoint, +// which resumes it after "continue" or a failed capture. From then on the +// disk may move on from the bundle, so a restore from it is refused. Sent +// first, because the reply may never come and the op may still take +// effect. A record not in a restore is left alone. +func markRestoredGuestRunning(s *store.Store, id string) error { + fresh, err := s.GetInstance(id) + if err != nil { + return err + } + m := fresh.Migration + if m == nil || (m.State != store.MigrationRestoring && m.State != store.MigrationRestoredPaused) { + return nil + } + m.State = store.MigrationRestored + if err := s.SaveInstance(fresh); err != nil { + return fmt.Errorf("record that the restored guest of %s runs: %w", id, err) + } + return nil +} + +// keepRecordedMigration replaces state.Migration with the record's on disk, +// for a caller about to save a record it has held while the VMM ran. +// +// Other hull processes write Migration while a VMM runs: `hull checkpoint` +// records a fenced bundle, `hull resume` that a restored guest ran. In the +// foreground launchVMM holds the record it started from for the VMM's whole +// life, and saving it at exit as it was would put back the migration state +// from launch: a fenced checkpoint forgotten, so it cannot be restored, or a +// restored guest that ran recorded as one that never did, so it can be +// restored again over a disk it has written. +func keepRecordedMigration(s *store.Store, state *store.InstanceState) { + if fresh, err := s.GetInstance(state.ID); err == nil { + state.Migration = fresh.Migration + } +} diff --git a/cmd/hull/hvirestore_test.go b/cmd/hull/hvirestore_test.go new file mode 100644 index 00000000..b580ec46 --- /dev/null +++ b/cmd/hull/hvirestore_test.go @@ -0,0 +1,426 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "net" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/brig-sh/hull/pkg/store" +) + +func TestHviRestoreArgv(t *testing.T) { + const from, cs, pp = "/s/i/checkpoint/tx", "/s/i/control.sock", "/s/i/profile.json" + tail := []string{"--control-sock", cs, "--profile-out", pp} + cases := []struct { + name string + cmdLine []string + want []string + refusal string + }{ + { + name: "darwin run with the stub NIC", + cmdLine: []string{"/opt/hvi", "boot", "--kernel", "/k", "--mem-mib", "512", "--cpus", "2", + "--fs-uid", "1000", "--fs-gid", "1000", "--initramfs", "/i", "--cmdline", "console=hvc0 x", + "--disk", "/s/i/rootfs.img", "--net", "--net-mac", "52:54:00:aa:bb:cc", + "--agent-sock", "/s/i/agent.sock", "--sandbox-id", "i", + "--control-sock", "/old/control.sock", "--profile-out", "/old/profile.json"}, + want: append([]string{"/opt/hvi", "restore", "--from", from, "--disk", "/s/i/rootfs.img", + "--net-stub", "--net-mac", "52:54:00:aa:bb:cc", "--agent-sock", "/s/i/agent.sock", + "--sandbox-id", "i"}, tail...), + }, + { + name: "linux run on the gateway", + cmdLine: []string{"/h/hvi", "boot", "--kernel", "/k", "--mem-mib", "256", "--cpus", "1", + "--disk", "/r", "--net-gateway", "/g.sock", "--net-mac", "m", "--events", "/e.ndjson", + "--checkpoint-root", "/s/i/checkpoint"}, + want: append([]string{"/h/hvi", "restore", "--from", from, "--disk", "/r", + "--net-gateway", "/g.sock", "--net-mac", "m", "--events", "/e.ndjson", + "--checkpoint-root", "/s/i/checkpoint"}, tail...), + }, + { + name: "a share", + cmdLine: []string{"hvi", "boot", "--disk", "/r", "--share-ro", "/src", "t"}, + refusal: "shared a host directory", + }, + { + name: "no disk", + cmdLine: []string{"hvi", "boot", "--kernel", "/k"}, + refusal: "no --disk", + }, + { + name: "a flag hull does not know", + cmdLine: []string{"hvi", "boot", "--disk", "/r", "--net-tap", "tap0"}, + refusal: "--net-tap", + }, + { + name: "a value flag cut off at the end", + cmdLine: []string{"hvi", "boot", "--disk"}, + refusal: "takes a value", + }, + { + name: "not boot or restore", + cmdLine: []string{"hvi", "dump-fdt", "--disk", "/r"}, + refusal: "not boot or restore", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := hviRestoreArgv(tc.cmdLine, from, cs, pp) + if tc.refusal != "" { + if err == nil || !strings.Contains(err.Error(), tc.refusal) { + t.Fatalf("err = %v, want a refusal naming %q", err, tc.refusal) + } + return + } + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got, tc.want) { + t.Fatalf("argv\n got %q\nwant %q", got, tc.want) + } + // A restored instance records the restore argv as its command + // line, and the next restore starts from that. + again, err := hviRestoreArgv(got, from, cs, pp) + if err != nil { + t.Fatalf("the mapping refuses its own output: %v", err) + } + if !slices.Equal(again, got) { + t.Errorf("not idempotent\nonce %q\ntwice %q", got, again) + } + // Another bundle replaces --from rather than adding a second. + other, _ := hviRestoreArgv(got, "/other", cs, pp) + if v, _ := flagValue(other, "--from"); v != "/other" || strings.Count(strings.Join(other, " "), "--from") != 1 { + t.Errorf("--from not replaced: %q", other) + } + }) + } +} + +// restoring and restored-paused are restorable only because nothing has +// run the restored guest yet; TestResumeMakesARestoredGuestUnrestorable and +// the tests after it check that whatever runs it says so first. +func TestRestorableMigration(t *testing.T) { + for state, ok := range map[string]bool{ + store.MigrationFenced: true, + store.MigrationRestoring: true, + store.MigrationRestoredPaused: true, + store.MigrationCheckpointed: false, + store.MigrationRestored: false, + "bogus": false, + } { + err := restorableMigration(&store.MigrationState{State: state, BundleDir: "/b"}) + if (err == nil) != ok { + t.Errorf("%s: err = %v, want restorable=%v", state, err, ok) + } + } + if err := restorableMigration(nil); err == nil { + t.Error("an instance with no checkpoint is restorable") + } +} + +func TestAwaitRestoredAndResume(t *testing.T) { + var statusCalls atomic.Int32 + stub := newStubHvi(t, func(req map[string]any) stubAnswer { + switch req["op"] { + case "status": + if statusCalls.Add(1) < 3 { + return stubAnswer{reply: `{"v":1,"ok":true,"state":"restoring"}`} + } + return stubAnswer{reply: `{"v":1,"ok":true,"state":"restored-paused","vcpus":1,"vcpus_parked":1}`} + case "resume": + return stubAnswer{reply: `{"v":1,"ok":true,"state":"running"}`} + } + return stubAnswer{reply: `{"v":1,"ok":false,"code":"unknown_op","error":"?"}`} + }) + s := newCmdTestStore(t) + state, _ := fakeHviInstance(t, s, "vm1", stub.path) + state.Migration = &store.MigrationState{State: store.MigrationRestoring, BundleDir: "/b", TxID: "t"} + _ = s.SaveInstance(state) + + got, err := awaitRestoredAndResume(s, "vm1", "t", stub.path, hviRestoreOptions{timeout: 10 * time.Second}, time.Time{}, nil) + if err != nil || got != store.MigrationRestored { + t.Fatalf("result = %q, %v", got, err) + } + var ops []string + for _, r := range stub.seen() { + ops = append(ops, r["op"].(string)) + } + if want := []string{"status", "status", "status", "resume"}; !slices.Equal(ops, want) { + t.Errorf("ops = %v, want %v", ops, want) + } + if m := mustInstance(t, s, "vm1").Migration; m.State != store.MigrationRestored { + t.Errorf("migration = %+v", m) + } +} + +func TestAwaitRestoredPausedSendsNoResume(t *testing.T) { + stub := newStubHvi(t, func(req map[string]any) stubAnswer { + return stubAnswer{reply: `{"v":1,"ok":true,"state":"restored-paused"}`} + }) + s := newCmdTestStore(t) + state, _ := fakeHviInstance(t, s, "vm1", stub.path) + state.Migration = &store.MigrationState{State: store.MigrationRestoring, BundleDir: "/b", TxID: "t"} + _ = s.SaveInstance(state) + got, err := awaitRestoredAndResume(s, "vm1", "t", stub.path, hviRestoreOptions{paused: true, timeout: 5 * time.Second}, time.Time{}, nil) + if err != nil || got != store.MigrationRestoredPaused { + t.Fatalf("result = %q, %v", got, err) + } + if m := mustInstance(t, s, "vm1").Migration; m.State != store.MigrationRestoredPaused { + t.Errorf("migration = %+v", m) + } + for _, r := range stub.seen() { + if r["op"] == "resume" { + t.Error("--paused sent a resume") + } + } +} + +// Until hvi binds the socket there is nothing to dial; a VMM that is gone ends +// the wait at once instead of at the deadline. +func TestAwaitRestoredEndsWhenTheVMMIsGone(t *testing.T) { + s := newCmdTestStore(t) + if _, err := s.CreateInstance("vm1"); err != nil { + t.Fatal(err) + } + gone := make(chan struct{}) + time.AfterFunc(300*time.Millisecond, func() { close(gone) }) + start := time.Now() + got, err := awaitRestoredAndResume(s, "vm1", "t", filepath.Join(shortSocketDir(t), "none.sock"), + hviRestoreOptions{timeout: time.Minute}, time.Now(), gone) + if got != "" || err == nil || !strings.HasPrefix(err.Error(), "hvi exited") { + t.Errorf("result = %q, %v", got, err) + } + if took := time.Since(start); took > 5*time.Second { + t.Errorf("waited %s after the VMM was gone", took) + } +} + +// A detached restore never waits on the hvi it started, so an hvi that +// refuses the bundle and exits at once is a zombie child: alive to signal 0. +// The wait must see it exit (and say what it printed) at once, not at its +// deadline. Seen live on KVM, where hvi restore refused and hull waited the +// full two minutes. +func TestAwaitRestoredSeesADetachedVMMExit(t *testing.T) { + s := newCmdTestStore(t) + dir, err := s.CreateInstance("vm1") + if err != nil { + t.Fatal(err) + } + logFile := filepath.Join(dir, "vmm.log") + child := exec.Command("/bin/sh", "-c", `echo "restoring /b ..." > "$1"; `+ + `echo "hvi: the KVM restore is not built yet" >> "$1"; exit 1`, "sh", logFile) + if err := child.Start(); err != nil { + t.Fatal(err) + } + since := time.Now() + state := &store.InstanceState{ID: "vm1", Status: "running", PID: child.Process.Pid, Backend: "hvi", + LogFile: logFile, StartTime: since, + Migration: &store.MigrationState{State: store.MigrationRestoring, BundleDir: "/b", TxID: "t"}} + if err := s.SaveInstance(state); err != nil { + t.Fatal(err) + } + time.Sleep(200 * time.Millisecond) // the child is a zombie by now + if syscall.Kill(child.Process.Pid, 0) != nil { + t.Fatal("the exited child is not a zombie; the test proves nothing") + } + start := time.Now() + got, err := awaitRestoredAndResume(s, "vm1", "t", filepath.Join(shortSocketDir(t), "none.sock"), + hviRestoreOptions{detach: true, timeout: time.Minute}, since, make(chan struct{})) + if got != "" || err == nil || !strings.HasPrefix(err.Error(), "hvi exited") || + !strings.Contains(err.Error(), "the KVM restore is not built yet") { + t.Errorf("result = %q, %v", got, err) + } + if took := time.Since(start); took > 5*time.Second { + t.Errorf("waited %s for a VMM that had exited", took) + } +} + +func TestHviHasRestore(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil { + t.Fatal(err) + } + return p + } + // What the hvi at d80837c..b8ad111 prints, verbatim. + old := write("old", `echo 'hvi: unknown subcommand "restore"; expected `+"`boot`, `dump-fdt`"+`' >&2; exit 1`) + newer := write("new", `echo 'hvi: restore needs --from ' >&2; exit 1`) + if has, err := hviHasRestore(old); err != nil || has { + t.Errorf("an hvi without restore: has=%v err=%v", has, err) + } + if has, err := hviHasRestore(newer); err != nil || !has { + t.Errorf("an hvi with restore: has=%v err=%v", has, err) + } + if _, err := hviHasRestore(filepath.Join(dir, "absent")); err == nil { + t.Error("a missing binary was not reported") + } +} + +// Every refusal restore makes comes before anything is launched, so each is +// checked with no command and no VMM. +func TestHviRestoreRefusals(t *testing.T) { + binDir := t.TempDir() + withRestore := filepath.Join(binDir, "hvi") + if err := os.WriteFile(withRestore, []byte("#!/bin/sh\necho 'hvi: restore needs --from ' >&2\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + oldDir := t.TempDir() + withoutRestore := filepath.Join(oldDir, "hvi") + if err := os.WriteFile(withoutRestore, []byte("#!/bin/sh\necho 'hvi: unknown subcommand \"restore\"; expected `boot`' >&2\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + + // A short root, so the control socket path passes its length check and + // the gateway case reaches the gateway. + s, err := store.New(shortSocketDir(t)) + if err != nil { + t.Fatal(err) + } + bundle := t.TempDir() + // A socket file whose listener is gone, as a gateway that died leaves. + deadSocket := filepath.Join(shortSocketDir(t), "gw.sock") + if ln, err := net.Listen("unix", deadSocket); err == nil { + ln.(*net.UnixListener).SetUnlinkOnClose(false) + _ = ln.Close() + } else { + t.Fatal(err) + } + base := func() *store.InstanceState { + return &store.InstanceState{ + ID: "vm1", Status: "stopped", Backend: "hvi", + CmdLine: []string{withRestore, "boot", "--kernel", "/k", "--disk", "/r.img"}, + Migration: &store.MigrationState{State: store.MigrationFenced, BundleDir: bundle, TxID: "t1"}, + } + } + if _, err := s.CreateInstance("vm1"); err != nil { + t.Fatal(err) + } + cases := []struct { + name string + mutate func(*store.InstanceState) + latest string // txid latest.json names; "" writes none + refusal string + }{ + {"never checkpointed", func(st *store.InstanceState) { st.Migration = nil }, "", "no hvi checkpoint"}, + {"continued after the checkpoint", func(st *store.InstanceState) { + st.Migration.State = store.MigrationCheckpointed + }, "", "--then continue"}, + {"already restored and run", func(st *store.InstanceState) { + st.Migration.State = store.MigrationRestored + }, "", "already restored"}, + {"bundle deleted", func(st *store.InstanceState) { + st.Migration.BundleDir = filepath.Join(bundle, "gone") + }, "", "is gone"}, + {"latest.json names a later checkpoint", nil, "t2", "never took in"}, + {"a shared directory", func(st *store.InstanceState) { + st.CmdLine = append(st.CmdLine, "--share-ro", "/src", "fs1") + }, "", "shared host directory"}, + {"an hvi without restore", func(st *store.InstanceState) { + st.CmdLine[0] = withoutRestore + }, "", "no restore subcommand"}, + {"a gateway that is not running", func(st *store.InstanceState) { + st.CmdLine = append(st.CmdLine, "--net-gateway", filepath.Join(bundle, "gw.sock")) + }, "t1", "not answering"}, + {"a gateway that died and left its socket", func(st *store.InstanceState) { + st.CmdLine = append(st.CmdLine, "--net-gateway", deadSocket) + }, "t1", "not answering"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + st := base() + if tc.mutate != nil { + tc.mutate(st) + } + ckpt := instanceCheckpointDir(s, "vm1") + _ = os.Remove(filepath.Join(ckpt, "latest.json")) + if tc.latest != "" { + if err := writeHviCheckpointRecord(ckpt, hviCheckpointRecord{Dir: bundle, TxID: tc.latest, Then: hviThenFence}); err != nil { + t.Fatal(err) + } + } + if err := s.SaveInstance(st); err != nil { + t.Fatal(err) + } + err := hviRestoreInstance(context.Background(), nil, s, st, hviRestoreOptions{timeout: time.Second}) + if err == nil || !strings.Contains(err.Error(), tc.refusal) { + t.Errorf("err = %v, want a refusal naming %q", err, tc.refusal) + } + }) + } + + // A fenced source that is still alive holds the disk: restore must wait + // for its stop, or two VMMs would write one disk. + stub := newStubHvi(t, func(map[string]any) stubAnswer { return stubAnswer{reply: `{"v":1,"ok":true}`} }) + live, _ := fakeHviInstance(t, s, "live", stub.path) + live.Migration = &store.MigrationState{State: store.MigrationFenced, BundleDir: bundle, TxID: "t1"} + if err := hviRestoreInstance(context.Background(), nil, s, live, hviRestoreOptions{timeout: time.Second}); err == nil || + !strings.Contains(err.Error(), "still running") { + t.Errorf("a running source: %v", err) + } +} + +// A source that died without `hull stop` leaves "running" and its dead pid in +// the record. Until the restore's own launch replaces them, that pid must not +// be read as the new VMM exiting. +func TestAwaitRestoredIgnoresTheSourcesStalePid(t *testing.T) { + ready := time.Now().Add(300 * time.Millisecond) + stub := newStubHvi(t, func(req map[string]any) stubAnswer { + if req["op"] == "resume" { + return stubAnswer{reply: `{"v":1,"ok":true,"state":"running"}`} + } + if time.Now().Before(ready) { + return stubAnswer{reply: `{"v":1,"ok":true,"state":"restoring"}`} + } + return stubAnswer{reply: `{"v":1,"ok":true,"state":"restored-paused"}`} + }) + s := newCmdTestStore(t) + pid, done := startProcess(t, "/bin/sleep", "30") + _ = syscall.Kill(pid, syscall.SIGKILL) + <-done + if _, err := s.CreateInstance("vm1"); err != nil { + t.Fatal(err) + } + if err := s.SaveInstance(&store.InstanceState{ + ID: "vm1", Status: "running", PID: pid, StartTime: time.Now().Add(-time.Hour), + Migration: &store.MigrationState{State: store.MigrationRestoring, BundleDir: "/b", TxID: "t"}, + }); err != nil { + t.Fatal(err) + } + since := time.Now() + // launchVMM's save of the restore's own VMM, a little later. + newPid, _ := startProcessAs(t, "hvi", "/bin/sleep", "30") + time.AfterFunc(200*time.Millisecond, func() { + _ = s.SaveInstance(&store.InstanceState{ + ID: "vm1", Status: "running", PID: newPid, StartTime: time.Now(), + Migration: &store.MigrationState{State: store.MigrationRestoring, BundleDir: "/b", TxID: "t"}, + }) + }) + got, err := awaitRestoredAndResume(s, "vm1", "t", stub.path, hviRestoreOptions{timeout: 10 * time.Second}, since, nil) + if err != nil || got != store.MigrationRestored { + t.Errorf("result = %q, %v, want restored: the source's dead pid ended the wait", got, err) + } +} diff --git a/cmd/hull/run.go b/cmd/hull/run.go index c0b45bb8..f1ca9d7b 100644 --- a/cmd/hull/run.go +++ b/cmd/hull/run.go @@ -1491,15 +1491,34 @@ exec %s "$@" // way every Vz instance gets a state dir: pause, checkpoint and a // graceful stop all go through the socket, and a run started without one // can only be signalled. Both live in the 0700 instance directory, since - // the socket answers anyone who can connect to it. + // the socket answers anyone who can connect to it. An hvi that predates + // the flags would refuse to boot with them, so it runs without: it can + // still be stopped, and pause and checkpoint say why they cannot. var controlSock, profilePath string - if vmmType == hypervisors.HviVmm { - controlSock = s.InstanceControlSocket(instanceName) - profilePath = s.InstanceProfilePath(instanceName) - if err := checkUnixSocketPath("control", controlSock); err != nil { + if vmmType == hypervisors.HviVmm && len(cmdArgs) > 0 { + takesFlags, err := hviBootTakesControlFlags(cmdArgs[0]) + if err != nil { return err } - cmdArgs = append(cmdArgs, "--control-sock", controlSock, "--profile-out", profilePath) + if takesFlags { + controlSock = s.InstanceControlSocket(instanceName) + profilePath = s.InstanceProfilePath(instanceName) + if err := checkUnixSocketPath("control", controlSock); err != nil { + return err + } + root, err := hviCheckpointRoot(instanceDir) + if err != nil { + return err + } + cmdArgs = append(cmdArgs, "--control-sock", controlSock, "--profile-out", profilePath, + "--checkpoint-root", root) + } else { + // Debug, not a warning: this is every run on the hvi hull + // ships with until that one has the socket, and nobody asked + // for a pause yet. requireRunningHvi says it when one does. + log.Debugf("%s takes no --control-sock; instance %s will not be pausable", + cmdArgs[0], instanceName) + } } state := &store.InstanceState{ @@ -2572,6 +2591,7 @@ func launchVMM(ctx context.Context, cmd *cli.Command, s *store.Store, state *sto state.PID = vmmCmd.Process.Pid state.Status = "running" + keepRecordedMigration(s, state) if err := s.SaveInstance(state); err != nil { log.WithError(err).Warn("failed to save instance state") } @@ -2649,6 +2669,7 @@ func launchVMM(ctx context.Context, cmd *cli.Command, s *store.Store, state *sto if state.ExitedAt.IsZero() { state.ExitedAt = time.Now() } + keepRecordedMigration(s, state) _ = s.SaveInstance(state) sendEndOnce(s, state) if detach { diff --git a/docs/backends.md b/docs/backends.md index 07e99886..3a6b160f 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -30,8 +30,8 @@ Start with `vz`. Move to another backend only for a reason on this list. | Choose | When | |---|---| -| `vz` | the default. You want NAT with no setup, a GUI window, Rosetta, or checkpoint and restore | -| `hvi` | you want a persistent writable root without a disk image, correct Linux file modes over a share, or the same device model hull's guests get on Linux | +| `vz` | the default. You want NAT with no setup, a GUI window, Rosetta, or a checkpoint you can restore any number of times | +| `hvi` | you want a persistent writable root without a disk image, correct Linux file modes over a share, the same device model hull's guests get on Linux, or pause and a checkpoint bundle | | `qemu` | you need a device or a machine option only QEMU has | ## Capability comparison @@ -50,7 +50,8 @@ Start with `vz`. Move to another backend only for a reason on this list. | `--net shared` | Apple NAT, no extra privilege | **refused**, see below | vmnet, needs root or a signed QEMU | **refused** without `--gateway-sock` | | `--gateway-sock` | yes | yes | yes | yes | | `exec` | yes | yes | yes | yes | -| Checkpoint and restore | yes, with a block rootfs | **no** | **no** | **no** | +| Checkpoint and restore | yes, with a block rootfs | checkpoint with a block rootfs and no shares, into a [partial bundle](checkpoint-restore.md#what-a-bundle-holds); **no restore yet** | **no** | as on macOS | +| Pause and resume | **no** | yes | **no** | yes | | `--gui`, `--gui-title` | yes | no | no | no | | `--rosetta` | yes | no | no | no | @@ -114,8 +115,9 @@ pull platform, because the platform is decided before the image is pulled. to shut down and forces a stop after `--stop-grace` seconds, 10 by default. A second signal forces the stop at once. -**Checkpoint and restore.** Only here, and only with a block rootfs. A virtiofs -root goes stale across a restore, so hull refuses the combination. See +**Checkpoint and restore.** Only with a block rootfs. A virtiofs root goes +stale across a restore, so hull refuses the combination. `hvi` checkpoints +too, by a different mechanism and with different rules. See [checkpoint-restore.md](checkpoint-restore.md). One operational note, recorded from CI rather than derived from code: machine @@ -205,12 +207,18 @@ refused with an error naming the gateway flag, and the MAC flag has no effect. **Exec** works. `hvi` stands up a host Unix listener and bridges it over virtio-vsock to an in-guest agent, and hull wires that up on every run. -**No checkpoint or restore.** `hvi` has no code that saves or reloads VM state -at this pin. Its subcommands are `boot`, `dump-fdt`, `smoke`, -`smoke-shm-verify`, `sandbox-selftest`, `seccomp-selftest` and `version`. The -`Quiesce::checkpoint` name inside `hvi` is a vCPU park primitive used so a -plugin can read registers consistently, not a VM snapshot. hull's `checkpoint` -and `restore` commands refuse any backend but `vz`. +**Pause, checkpoint and restore** go through the control socket hull gives +every `hvi` run (`--control-sock`, with the machine profile at +`--profile-out`). `hull pause` and `hull resume` stop and restart the I/O +threads and every vCPU. `hull checkpoint` writes a bundle, then continues or +fences the guest. Today that bundle is partial: RAM and device state, without +the vCPU, interrupt controller and timer state a resume needs. `hull restore` +would restore a fenced bundle onto the disk the guest stopped with, and +refuses until `hvi` has a `restore` subcommand, which none has yet. `hull +stop` asks over the socket before it signals. The `hvi` at hull's submodule +pin predates the socket; hull asks each `hvi` before a run and leaves the +flags off one that does not take them, so it boots without pause or +checkpoint. See [checkpoint-restore.md](checkpoint-restore.md#hvi). **Other limits `hvi` states about itself.** Eight vCPUs on a GICv2 arm64 Linux host, which does not apply on macOS because Apple's in-kernel GIC is a GICv3. @@ -227,11 +235,13 @@ long-running `hvi` guest has no host time source to correct drift against. hull also builds for arm64 Linux, where it runs `hvi` on KVM and nothing else: `hvi`, a block rootfs, and either no network or a gateway. That is the -configuration a sandbox needs to be moved between two KVM hosts, which is -planned and not built yet; no checkpoint, restore or migration exists on -Linux today. brig drives it with the same verbs it uses on macOS: `run`, -`exec`, `ps`, `inspect`, `logs`, `stop`, `rm`, `assets dir` and -`network-gateway`. `pull`, `images`, `rmi`, `prune`, `telemetry` and `version` +configuration a sandbox needs to be moved between two KVM hosts. `pause`, +`resume` and `checkpoint` work as on macOS, and `restore` is refused as on +macOS until `hvi` has one (see +[checkpoint-restore.md](checkpoint-restore.md#hvi)); moving a sandbox to +another host is planned and not built yet. brig drives it with the same verbs +it uses on macOS: `run`, `exec`, `ps`, `inspect`, `logs`, `stop`, `rm`, +`assets dir` and `network-gateway`. `pull`, `images`, `rmi`, `prune`, `telemetry` and `version` work too. A sandbox started on a Mac cannot be moved to Linux, and none of the planned @@ -286,8 +296,9 @@ What is different from macOS: argument` on the first file owned by an id the namespace does not map (`/etc/shadow`, gid 42, in alpine). The pull stops there. Run hull as an ordinary user or as real root instead. -- **Not on Linux:** `compose`, `checkpoint`, `restore`, `store`, `--gui` and - `--rosetta`. +- **Not on Linux:** `compose`, `store`, `--gui` and `--rosetta`. + `checkpoint` and `restore` take `hvi` instances only, which on Linux is + every instance. The user running hull needs read-write access to `/dev/kvm`, usually through the `kvm` group. A group added to a logged-in user reaches only new sessions. diff --git a/docs/checkpoint-restore.md b/docs/checkpoint-restore.md index d4085fa5..6ca57834 100644 --- a/docs/checkpoint-restore.md +++ b/docs/checkpoint-restore.md @@ -1,11 +1,31 @@ -# Checkpoint / restore (Vz backend) +# Checkpoint and restore + +Two backends can save a running guest and bring it back later without a cold +boot: `vz` on macOS, and `hvi` on macOS and Linux. The verbs are the same, and +hull picks the path from the instance's record: + +| | `vz` | `hvi` | +|---|---|---| +| How | vz-runner, on a signal | hvi's control socket | +| Captures | Virtualization.framework's machine state, plus an APFS clone of the disk | today a [partial bundle](#what-a-bundle-holds): RAM and device state, not the vCPU, interrupt controller or timer state a resume needs; not the disk | +| After `checkpoint` | the guest keeps running | `--then continue` (the default) or `--then fence` | +| Restore from | the latest checkpoint, any number of times | **not yet**: needs an `hvi` with a `restore` subcommand, which no `hvi` has. hull's side takes a fenced checkpoint, onto the disk the guest stopped with | +| Also | | `hull pause` and `hull resume` | +| Hosts | macOS 14+, Apple silicon | macOS (HVF) and Linux (KVM) | + +Both need a block rootfs (`run --rootfs-type block`) and no shared host +directory. For `hvi`, pause, resume and checkpoint work today; restore is +built in hull and refuses until `hvi` gains `restore`, because no bundle `hvi` +writes yet can be resumed. + +## Vz The Vz backend can checkpoint a running microVM, meaning pause, save the machine state and resume, then later restore it, resuming the guest exactly where it was instead of cold-booting. Built on Virtualization.framework's `saveMachineStateTo` / `restoreMachineStateFrom` (macOS 14+, Apple Silicon). -## Usage +### Usage Checkpointable instances must use a **block rootfs** (see below): @@ -37,7 +57,7 @@ large the state file is relative to guest RAM, and how fast a restore is. None of them had a source in the tree, so they have been removed rather than restated. Measure on your own hardware if you need a figure. -## What is captured +### What is captured Artifacts live in `/instances//checkpoint/`: @@ -76,7 +96,7 @@ Artifacts live in `/instances//checkpoint/`: ("Stale file handle") and the guest effectively dies. The CLI errors out early instead of producing a broken restore. -## Semantics and limits +### Semantics and limits - **Treat this as suspend and resume on one machine, not migration.** hull records the machine identifier and requires the identical identity at @@ -128,7 +148,7 @@ Artifacts live in `/instances//checkpoint/`: automatically. Instances created by an older hull (no state dir) must be re-run once before they can be checkpointed. -## e2e coverage +### e2e coverage `test/pty-checkpoint-test.py` boots the ubuntu test image on a block rootfs, starts a tick counter in the guest shell, checkpoints mid-count, verifies the VM @@ -149,3 +169,224 @@ successes in 4 attempts on one machine against 15 in 16 on another, while ordinary boots on the same machine pass. SIP state is not the discriminator, and the reason is unresolved. CI pins the checkpoint job to a runner with a console session. + +## hvi + +hull drives `hvi` through the control socket hvi serves with `--control-sock`. +Every `hvi` run gets one at `/control.sock`, a machine profile at +`/profile.json` (`--profile-out`), and `--checkpoint-root +/checkpoint`, the one directory hvi's confinement lets it write a +bundle under. The record keeps the first two as `controlSock` and +`profilePath`, which `hull inspect` prints; the root is on the recorded +command line. The socket has +no authentication: anyone who can connect to it can pause or stop the VM. It +sits in the instance directory, which is 0700. + +The control socket arrived in hvi after the commit hull's `hvi-vmm` submodule +pins (890f9e0), which answers these flags with `unknown boot arg` and would +refuse the boot. So before each run hull asks the `hvi` it is about to start, +with `hvi boot --checkpoint-root` and no value: that fails while parsing on +every `hvi`, and names the flag as unknown only on one without it. An `hvi` +without the flags runs as before, with no control socket: `hull stop` works, +and `pause` and `checkpoint` say that this `hvi` predates the socket. For +pause and checkpoint, put an `hvi` built from hvi's `feat/teleport` branch +(bf3b1aa or later) next to hull. + +### Usage + +```bash +hull run --detach --hypervisor hvi --rootfs-type block --net none --name demo alpine + +hull pause demo # the I/O threads and every vCPU stop +hull resume demo + +hull checkpoint demo # --then continue: write a bundle, keep running +hull checkpoint --then fence demo # write a bundle; the guest never runs again here +hull stop demo +hull restore demo # from the fenced bundle, then resume +hull restore --paused --detach demo # from the bundle, left paused for `hull resume` +``` + +`pause` and `resume` are idempotent. A paused guest's counter does not move, +and neither does anything else in it. `hull ps` shows such an instance as +STATUS `running` (its VMM is alive) and VM `paused` or `fenced`, and `hull +exec` refuses it with the command that gets it running. hull waits 5 s for each reply. hvi gives a pause 2 s in +all and asks clients for at least 3 s; a client that gives up sooner reports +"not paused" for a VM that then pauses. If a pause times out anyway, hull says +the VM may now be paused, and `hull resume` is harmless when it is not. + +`hull stop` on an `hvi` instance sends `stop` on the socket first and waits up +to half of `--timeout` for hvi to end the run (it logs `guest stopped: +Requested`), then falls back to SIGTERM and SIGKILL as for any VMM. An `hvi` +from before the socket goes straight to the signals. + +### Checkpoint + +`hull checkpoint` needs a running `hvi` instance with a control socket, booted +from a block rootfs (`--disk`) with no shared host directory. A virtio-fs root +or share lives in host files and the guest's FUSE caches, which no bundle +holds, so those are refused by name before the VM is paused. + +hull sends `{"v":1,"op":"checkpoint","dir":DIR,"then":"continue"|"fence"}` +and waits up to `--timeout` seconds (60). hvi pauses the guest, writes a +bundle into `DIR`, and then either resumes the guest or, with `fence`, keeps +it parked for good: hvi refuses a later `resume` with `fenced`, and only +`stop` ends it. A failed capture resumes the guest (a guest that was paused +stays paused) and leaves no bundle: hull removes a directory it made for one, +and empties a `--dir` you gave so the same one can be retried. With no reply +in time hull leaves everything in place, since hvi may still be writing. + +One checkpoint or restore of an instance runs at a time: each takes +`/migration.lock`, and a second one is refused at once. + +`DIR` is a new 0700 directory `/checkpoint/`, where the txid +is a random id hull draws, or an empty directory you name with `--dir`, which +has to be below `/checkpoint` too. hull resolves links in it and in +the root it passes at boot, so the two are compared in one spelling (on macOS +`/tmp` is a link to `/private/tmp`), and sends hvi that resolved path. An instance started without a checkpoint root +is refused. On +success hull writes `/checkpoint/latest.json`: + +```json +{"dir": "...", "manifestSha256": "...", "then": "fence", + "stats": {"pause_ms": 0, "capture_ms": 0, "ram_present_bytes": 0, "ram_total_bytes": 0}, + "savedAt": "...", "txid": "..."} +``` + +and sets the record's `migration` to `{"state": "checkpointed"|"fenced", +"bundleDir": "...", "txid": "...", "manifestSha256": "..."}`. The record keeps +the manifest's sha256 too, outside the directory hvi may write to, and a +restore checks the bundle's `manifest.json` against it. `stats` is what hvi reported. hull keeps +one bundle of its own per instance and removes the previous one after a new +checkpoint succeeds; a bundle placed with `--dir` is never removed by hull. + +An `hvi` from before the checkpoint op answers `not_implemented`, and hull +reports that this `hvi` cannot checkpoint yet. + +### What a bundle holds + +A bundle (checkpoint format 1) is a directory of `manifest.json`, written +last with the size and sha256 of every other part, `profile.json`, `ram.map` +and `ram.bin` (the guest's RAM, sparse: all-zero pages take no space), +`cpu.json`, `platform.json`, `devices.json` (the virtio-blk, net and vsock +devices and the UART), `disk.json` and `session.json`, which carries hull's +txid. `hvi checkpoint-verify DIR` checks one. + +**Today the bundle is partial.** The hvi that serves the op today +records only part of each vCPU's registers and none of the interrupt +controller or timer state, and `checkpoint-verify` says so: "vCPU registers, +interrupt controller and timer not captured, so no restore can resume this +bundle". The full vCPU, GIC and timer capture comes with hvi's `restore`, and until +then no bundle can be restored. The disk is not in the +bundle: hvi syncs it and records its size and write count, and the restore +reopens the same file. hvi's own +[docs/checkpoint.md](../hvi-vmm/docs/checkpoint.md) describes the format and +the clock rules once that page lands in the commit hull pins. In short: the +guest's monotonic clock continues from the captured value, so the downtime is +invisible to it, timers that expired in the meantime fire on resume, and the +wall clock is behind by the downtime until the guest corrects it. Agent +connections over vsock are reset by a restore. + +### Restore + +No `hvi` has a `restore` subcommand yet, so `hull restore` on an `hvi` +instance ends at the last refusal below. What follows is what hull does once +one does. + +`hull restore` on an `hvi` instance starts `hvi restore` from the bundle the +record names. It refuses, with the reason: + +- while the instance's VMM is alive. A fenced source stays alive until `hull + stop`, and two VMMs must never share one disk. +- unless the checkpoint was fenced, or an earlier restore of it never ran the + guest. The disk is not in the bundle, so it has to be the disk the guest + stopped with. After `--then continue` the guest kept writing to it, and + after a restored guest has run it has moved on: a guest given that disk with + the older RAM sees a filesystem that disagrees with its own caches. +- when `latest.json` names a later checkpoint than the record. That is a + checkpoint whose record update failed. If it was fenced, the guest never ran + after it, and `hull restore --adopt-latest` records it and restores from it, + once its bundle is below the checkpoint root and its `manifest.json` hashes + to the sha256 in `latest.json`. One taken with `--then continue` cannot be + restored, and neither can the record's older one, since the guest ran on + after both. A `latest.json` that does not parse is refused too. +- when the bundle's `manifest.json` no longer hashes to the sha256 the record + kept from hvi's reply. +- when the gateway the instance ran on does not answer on its socket. +- when the `hvi` the instance ran has no `restore` subcommand. + +`hull rm` removes the instance directory, and the bundles hull placed are +inside it. + +The `hvi restore` command line is built from the one the instance last ran. +hull drops `--kernel`, `--initramfs`, `--cmdline`, `--mem-mib`, `--cpus`, +`--fs-uid` and `--fs-gid`, because the bundle's RAM and profile fix them. It +keeps `--disk`, `--net-gateway`, `--net-mac`, `--agent-sock`, `--sandbox-id` +and `--events`, the host resources the devices attach to, and +`--checkpoint-root`, so the restored guest can be checkpointed again. A bare `--net` +becomes `--net-stub`. hull then adds `--from` and the instance's control +socket and profile paths. Any other flag is refused by name rather than +dropped. + +The record goes `starting`, then `running`, as for any run, while +`migration.state` goes `restoring`, then `restored-paused` once hvi reports it +on the socket, then `restored` when hull sends `resume`. With `--paused`, +hull stops at `restored-paused` and `hull resume` starts the guest. Whatever +may run the restored guest (the restore's own resume, `hull resume`, any +`hull checkpoint`, since a failed capture resumes the guest) records +`restored` before it is sent, and a resume with no reply counts as one that +ran. The state never moves back, whichever hull process writes it last, so a +guest that may have run is never restored over again. hull waits up to +`--timeout` seconds (120) for hvi to rebuild the VM. + +**The gateway.** hvi reaches the gateway by the socket path on its command +line, so a restore on the same host re-joins the gateway the instance left, +with the same MAC and so the same lease. `--gateway-sock` is for Vz only. A +restore on another host needs a gateway there with the same subnet and gateway +address, because the guest resumes with its address and routes in memory. The +planned `hull migrate` sets that up; nothing does yet. + +**Other hosts.** A bundle restores only where the machine profile matches: +same hvi commit, backend, CPU identity and ID registers, counter frequency, +interrupt controller layout, RAM layout and devices. hvi checks this before it +creates anything. A Mac to Linux move is refused: the guest's counter runs at +24 MHz on Apple silicon and 54 MHz on a Raspberry Pi 5, and KVM cannot change +it. + +### Verified + +On this Mac (HVF) and on a Raspberry Pi 5 (KVM, seccomp on), with an alpine +guest on a block rootfs printing a counter every second, hull run with `hvi` +built from hvi's `feat/teleport` at bf3b1aa: + +- `hull pause` held the counter for 5 s with every vCPU parked, a second + `pause` succeeded, and `hull resume` restarted it. While paused, `hull ps` + showed STATUS `running` and VM `paused`, and `hull exec` refused in under + 0.1 s; it worked after resume. +- Two `hull checkpoint` runs started together: one wrote its bundle, the + other was refused as in progress. The sha256 of the bundle's + `manifest.json` matched the one in `latest.json` and in the record. +- `hull checkpoint --then continue` stored about 57 MiB of a 512 MiB + guest's RAM in 0.3 s (Mac) and 0.6 to 1.0 s (Pi), `hvi checkpoint-verify` + accepted the bundle (and called it partial), and the counter kept going. +- `hull restore` on the running instance was refused. +- `hull checkpoint --then fence` wrote a second bundle, removed the first, + froze the counter, and recorded `fenced`. `hull ps` showed VM and + CHECKPOINT `fenced`, and `hull exec` and `hull resume` were refused. +- `hull stop` ended the run through the socket (`guest stopped: Requested`) + with no `hvi` left running, and the record kept `fenced`. +- `hull restore` was refused, because that `hvi` has no `restore` + subcommand yet. With the record's txid rewritten to stand in for a failed + record update, restore pointed at `--adopt-latest`, which adopted the + fenced checkpoint. After one byte was added to its `manifest.json`, restore + was refused as changed since the checkpoint. + +With the `hvi` hull pins (890f9e0), on both hosts: the run left the control +flags off, the guest booted and counted, `pause` and `checkpoint` said the +`hvi` predates the socket, and `exec` and `stop` worked. On this Mac, +`test/hvi-boot-test.py`, the boot check CI runs, passed with that `hvi`; with +the hull from before this check it failed, because hull passed the flags and +`hvi` exited with `unknown boot arg "--control-sock"`. + +A live restore waits for hvi's `restore`; until then the restore path is +covered by unit tests against a stub control socket. diff --git a/docs/cli.md b/docs/cli.md index f360de7f..6639b1c2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -27,7 +27,7 @@ for the one build that reports `dev`. ## Commands -There are nineteen top-level commands. `network-gateway` is the only hidden one: +There are twenty-one top-level commands. `network-gateway` is the only hidden one: it does not appear in `hull --help`, but it runs and prints its own help when named. No flag anywhere in the tree is hidden. @@ -38,8 +38,10 @@ named. No flag anywhere in the tree is hidden. | `exec ` | run a command in a running instance | | `ps` | list instances | | `stop ` | stop a running instance | -| `checkpoint ` | pause a running `vz` instance, save VM and disk state, resume it | -| `restore ` | restore a stopped `vz` instance from its checkpoint | +| `pause ` | pause a running `hvi` instance: its I/O threads and every vCPU stop | +| `resume ` | resume an `hvi` instance paused by `pause` | +| `checkpoint ` | `vz`: pause, save VM and disk state, resume. `hvi`: write a checkpoint bundle, then continue or fence | +| `restore ` | restore a stopped instance: `vz` from its saved state; `hvi` from a fenced bundle, refused until `hvi` has a `restore` subcommand | | `rm ` | remove a stopped instance | | `logs ` | show an instance's log | | `inspect ` | print instance details as JSON | @@ -145,8 +147,9 @@ as a default that an explicit `--env TERM` beats. | `images` | `--json` prints the store's records with full digests | | `rmi ...` | `--force`, `-f` remove one a stopped instance refers to; `--platform

` narrow to one platform | | `prune` | `--all` also remove every image no instance refers to; `--dry-run` print and remove nothing | -| `checkpoint ` | `--timeout `, default 60 | -| `restore ` | `--detach`, `-d`; `--stop-grace `, default 10; `--wait-ip`; `--gateway-sock ` | +| `pause `, `resume ` | none | +| `checkpoint ` | `--timeout `, default 60; `hvi` only: `--then continue\|fence`, default `continue`; `--dir `, an empty directory for the bundle, below `/checkpoint` | +| `restore ` | `--detach`, `-d`; `--stop-grace `, default 10; `--wait-ip`; `--gateway-sock ` (`vz` only); `hvi` only: `--paused`, `--timeout `, default 120, `--adopt-latest` | | `assets show` | none | | `assets dir` | none. Prints the directory and nothing else, for scripts | | `assets pull [REF]` | `--force` download even when the assets are present | @@ -158,9 +161,20 @@ as a default that an explicit `--env TERM` beats. `-t` means two different things: a pseudo-terminal on `exec`, and a force-kill timeout in seconds on `stop`. -`hull checkpoint` refuses an instance that is not running, one whose VMM is not -`vz-runner`, one started without a checkpoint state directory, and one that is -not on a block rootfs. +`hull checkpoint` refuses an instance that is not running and one that is not +on a block rootfs. For a `vz` instance it also refuses one started without a +checkpoint state directory. For an `hvi` instance it also refuses one started +without a control socket (an `hvi` from before the socket gets none) and one +with a shared host directory, and `--then` and `--dir` are refused on any +other backend. + +`hull pause` and `hull resume` take only `hvi` instances. `hull restore` on an +`hvi` instance refuses unless its last checkpoint was fenced, while its VMM is +still running, and when the `hvi` it ran has no `restore` subcommand. See +[checkpoint-restore.md](checkpoint-restore.md#hvi). + +`hull stop` on an `hvi` instance asks hvi over its control socket first, then +signals as for any VMM. `hull images` prints `No images found` and `hull ps` prints `No instances found` on an empty store. Both exit 0. `hull prune` prints diff --git a/docs/storage.md b/docs/storage.md index 8f72cba3..f33149cb 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -193,7 +193,11 @@ in the name only keeps two pulls' directories apart. ├── state.json 0600 ├── bundle/ config.json plus the rootfs symlink or image ├── log 0600, detached runs only - ├── checkpoint/ machine-id, vm.vzstate, rootfs.img, latest.json + ├── checkpoint/ vz: machine-id, vm.vzstate, rootfs.img, latest.json + │ hvi: latest.json, / (the bundle); hvi's --checkpoint-root + ├── control.sock hvi only: the control socket (pause, checkpoint, stop) + ├── profile.json hvi only: the machine profile hvi writes at boot + ├── migration.lock hvi only: held by one checkpoint or restore at a time └── (QMP socket, agent socket, staged boot files) ``` @@ -500,8 +504,21 @@ renamed over `state.json`, so a reader never sees a half-written record. ## Checkpoints -Checkpoint artifacts live in `/instances//checkpoint/` as -`machine-id`, `vm.vzstate`, `rootfs.img`, `latest.json` and a failure marker. +Checkpoint artifacts live in `/instances//checkpoint/`. For a `vz` +instance they are `machine-id`, `vm.vzstate`, `rootfs.img`, `latest.json` and +a failure marker. + +For an `hvi` instance they are `latest.json`, 0600, which names the latest +bundle (`dir`, `manifestSha256`, `then`, `stats`, `savedAt`, `txid`), and the +bundle itself in a 0700 directory named by its txid, or by `--dir`. hvi is +started with this directory as its `--checkpoint-root`, created 0700 at run +time: its confinement lets it write below it and nowhere else. hull writes `latest.json` through a temp file and a rename, and +keeps one bundle of its own per instance. The record's `migration` field +(`state`, `bundleDir`, `txid`, `manifestSha256`) says which checkpoint is +current, whether it was fenced, and which manifest a restore must find. Every `hvi` instance also has `control.sock` and `profile.json` +beside `state.json`, recorded as `controlSock` and `profilePath`. The socket +has no authentication, so the 0700 instance directory is what keeps other +accounts away from it. Every `vz` instance is started with a checkpoint state directory, because that directory holds the machine identifier a later restore needs. So diff --git a/pkg/store/store.go b/pkg/store/store.go index 9959d24b..3b959dde 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -126,9 +126,14 @@ type MigrationState struct { State string `json:"state"` // BundleDir is the absolute directory hvi wrote the checkpoint bundle to. BundleDir string `json:"bundleDir"` - // TxID names the checkpoint. hull draws it; hvi's checkpoint op takes no - // id of its own. + // TxID names the checkpoint. hull draws it and hvi records it in the + // bundle's session record. TxID string `json:"txid"` + // ManifestSHA256 is the sha256 of the bundle's manifest.json as hvi + // reported it on the checkpoint. Kept here, outside the checkpoint root + // hvi may write to, so a restore can check the bundle is the one hull + // recorded. + ManifestSHA256 string `json:"manifestSha256,omitempty"` } // StatusStarting marks the one moment an instance has no pid to record: the From 17280e3393ca89e9f36fcc4c990271781e7a3424 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Wed, 14 Oct 2026 06:22:09 +0300 Subject: [PATCH 10/33] feat(hvi): move a running hvi instance to another host `hull migrate NAME --to USER@HOST` drives a move: preflight on both hosts, reserve, fence, transfer, restore paused on the destination, then commit. The fence comes before the transfer because the bundle is the fenced checkpoint; there is no pre-copy in this form. Both hosts keep /migration.json, written through a synced rename before the step that relies on it, with a transition table per side so no record can go back past the fence or the commit. The instance record gains the migrating, migrated and incoming states and an owner epoch that hvi records with the fence, so a destination can refuse a commit meant for another move. A move's txid is 128 random bits, migrate.lock serialises its steps, and the plain verbs refuse an instance a move holds. The source frames the bundle and a sparse disk into its own stream, since archive/tar cannot write sparse entries, and runs the destination's hidden verbs (migrate-probe, migrate-receive, migrate-prepare, migrate-commit, migrate-abort) over ssh in batch mode with every argument shell-quoted. Each step finds its effect already there when run again, so `hull migrate NAME --txid T` resumes from the record. A refused probe leaves no record behind, and the probe says how to move an instance back to the host it left. migration.md describes the steps, the states, the stream, a failure table and what ran live through the commit on HVF. Signed-off-by: Panagiotis Moustafellos --- cmd/hull/commands_darwin.go | 5 +- cmd/hull/commands_linux.go | 5 +- cmd/hull/compose_down_test.go | 5 +- cmd/hull/hvicheckpoint.go | 65 +- cmd/hull/hvicheckpoint_test.go | 12 +- cmd/hull/hvipause.go | 3 + cmd/hull/hvirestore.go | 62 +- cmd/hull/migrate_dest.go | 831 ++++++++++++++++++++++++ cmd/hull/migrate_dest_test.go | 168 +++++ cmd/hull/migrate_fixture_test.go | 390 ++++++++++++ cmd/hull/migrate_host_darwin.go | 32 + cmd/hull/migrate_host_linux.go | 29 + cmd/hull/migrate_record.go | 448 +++++++++++++ cmd/hull/migrate_record_test.go | 127 ++++ cmd/hull/migrate_source.go | 1023 ++++++++++++++++++++++++++++++ cmd/hull/migrate_ssh.go | 243 +++++++ cmd/hull/migrate_ssh_test.go | 84 +++ cmd/hull/migrate_stream.go | 553 ++++++++++++++++ cmd/hull/migrate_stream_test.go | 183 ++++++ cmd/hull/migrate_test.go | 573 +++++++++++++++++ cmd/hull/rm.go | 6 + cmd/hull/telemetry.go | 10 +- cmd/hull/testmain_linux_test.go | 32 + docs/README.md | 1 + docs/checkpoint-restore.md | 6 + docs/cli.md | 32 +- docs/migration.md | 341 ++++++++++ docs/storage.md | 17 + pkg/store/store.go | 37 ++ 29 files changed, 5288 insertions(+), 35 deletions(-) create mode 100644 cmd/hull/migrate_dest.go create mode 100644 cmd/hull/migrate_dest_test.go create mode 100644 cmd/hull/migrate_fixture_test.go create mode 100644 cmd/hull/migrate_host_darwin.go create mode 100644 cmd/hull/migrate_host_linux.go create mode 100644 cmd/hull/migrate_record.go create mode 100644 cmd/hull/migrate_record_test.go create mode 100644 cmd/hull/migrate_source.go create mode 100644 cmd/hull/migrate_ssh.go create mode 100644 cmd/hull/migrate_ssh_test.go create mode 100644 cmd/hull/migrate_stream.go create mode 100644 cmd/hull/migrate_stream_test.go create mode 100644 cmd/hull/migrate_test.go create mode 100644 cmd/hull/testmain_linux_test.go create mode 100644 docs/migration.md diff --git a/cmd/hull/commands_darwin.go b/cmd/hull/commands_darwin.go index 53367c63..4e9a0d06 100644 --- a/cmd/hull/commands_darwin.go +++ b/cmd/hull/commands_darwin.go @@ -20,7 +20,7 @@ const appUsage = "native macOS container CLI for unikernels" // platformCommands is every verb hull has on macOS. func platformCommands() []*cli.Command { - return []*cli.Command{ + return append([]*cli.Command{ pullCommand(), runCommand(), execCommand(), @@ -30,6 +30,7 @@ func platformCommands() []*cli.Command { resumeCommand(), checkpointCommand(), restoreCommand(), + migrateCommand(), rmCommand(), logsCommand(), inspectCommand(), @@ -42,5 +43,5 @@ func platformCommands() []*cli.Command { networkGatewayCommand(), telemetryCommand(), versionCommand(), - } + }, migrateRemoteCommands()...) } diff --git a/cmd/hull/commands_linux.go b/cmd/hull/commands_linux.go index cf9d54e5..f031de65 100644 --- a/cmd/hull/commands_linux.go +++ b/cmd/hull/commands_linux.go @@ -34,7 +34,7 @@ const appUsage = "container CLI for hvi sandboxes on Linux/KVM" // // compose is listed but refuses; see linuxCompose. func platformCommands() []*cli.Command { - return []*cli.Command{ + return append([]*cli.Command{ pullCommand(), runCommand(), execCommand(), @@ -44,6 +44,7 @@ func platformCommands() []*cli.Command { resumeCommand(), checkpointCommand(), restoreCommand(), + migrateCommand(), rmCommand(), logsCommand(), inspectCommand(), @@ -55,7 +56,7 @@ func platformCommands() []*cli.Command { networkGatewayCommand(), telemetryCommand(), versionCommand(), - } + }, migrateRemoteCommands()...) } // errComposeMacOnly is what every compose subcommand answers on Linux. diff --git a/cmd/hull/compose_down_test.go b/cmd/hull/compose_down_test.go index adf40fc9..1991cd3d 100644 --- a/cmd/hull/compose_down_test.go +++ b/cmd/hull/compose_down_test.go @@ -69,7 +69,10 @@ func TestMain(m *testing.M) { } os.Exit(0) } - os.Exit(m.Run()) + testBinaryAsHull() + code := m.Run() + removeFakeHvi() + os.Exit(code) } // ReadSelfExecTrace reads back the argv of every intercepted selfExec call, diff --git a/cmd/hull/hvicheckpoint.go b/cmd/hull/hvicheckpoint.go index 63e74951..9b1845bd 100644 --- a/cmd/hull/hvicheckpoint.go +++ b/cmd/hull/hvicheckpoint.go @@ -48,8 +48,14 @@ const ( // hviCheckpoint asks hvi to write a checkpoint bundle into dir, an absolute // path, and then either resume the guest or fence it. txid goes into the // bundle's session record, so the bundle names the checkpoint hull recorded. -func hviCheckpoint(sock, dir, then, txid string, timeout time.Duration) (*hviReply, error) { - return hviControl(sock, map[string]any{"op": "checkpoint", "dir": dir, "then": then, "txid": txid}, timeout) +// epoch, when not 0, is the owner epoch a move hands over with this bundle; +// hvi records it beside the txid. +func hviCheckpoint(sock, dir, then, txid string, epoch uint64, timeout time.Duration) (*hviReply, error) { + req := map[string]any{"op": "checkpoint", "dir": dir, "then": then, "txid": txid} + if epoch != 0 { + req["epoch"] = epoch + } + return hviControl(sock, req, timeout) } // hviCheckpointable refuses an hvi command line whose guest a checkpoint @@ -241,8 +247,8 @@ func hviCheckpointInstance(s *store.Store, state *store.InstanceState, then, dir if err := hviCheckpointable(state.CmdLine); err != nil { return err } - if timeout < hviControlTimeout { - timeout = hviControlTimeout + if err := refuseDuringMove(s, id, "checkpoint"); err != nil { + return err } unlock, err := lockInstanceMigration(s, id) if err != nil { @@ -254,20 +260,43 @@ func hviCheckpointInstance(s *store.Store, state *store.InstanceState, then, dir if err != nil { return err } + started := time.Now() + rec, err := hviCheckpointLocked(s, state, then, dir, txid, 0, timeout) + if err != nil { + return err + } + fmt.Printf("Checkpoint of %s written to %s in %s (%s)\n", id, rec.Dir, + time.Since(started).Round(time.Millisecond), describeCheckpointStats(then, rec.Stats)) + return nil +} + +// hviCheckpointLocked takes checkpoint txid of a running hvi instance into +// dir (a new directory below the checkpoint root when ""), records it in +// latest.json and the instance record, and returns what it recorded. The +// caller holds the instance's migration lock and has checked that the +// instance can be checkpointed; a move calls it with its own txid and the +// epoch it hands over. +func hviCheckpointLocked(s *store.Store, state *store.InstanceState, then, dir, txid string, + epoch uint64, timeout time.Duration) (*hviCheckpointRecord, error) { + id := state.ID + if timeout < hviControlTimeout { + timeout = hviControlTimeout + } ckptDir := instanceCheckpointDir(s, id) root, ok := flagValue(state.CmdLine, "--checkpoint-root") if !ok { - return fmt.Errorf("instance %s was started without a checkpoint root, so its hvi may not "+ + return nil, fmt.Errorf("instance %s was started without a checkpoint root, so its hvi may not "+ "write a bundle anywhere; run it again with an hvi that takes --checkpoint-root", id) } + var err error if dir == "" { dir = filepath.Join(root, txid) } else if dir, err = bundleDirBelow(root, dir); err != nil { - return err + return nil, err } created, err := prepareBundleDir(dir) if err != nil { - return err + return nil, err } // A restored guest that was never resumed may be resumed by this @@ -276,10 +305,9 @@ func hviCheckpointInstance(s *store.Store, state *store.InstanceState, then, dir // A reply that never comes says nothing either way. On success the // record moves on to the new bundle, so this costs nothing then. if err := markRestoredGuestRunning(s, id); err != nil { - return err + return nil, err } - started := time.Now() - reply, err := hviCheckpoint(state.ControlSock, dir, then, txid, timeout) + reply, err := hviCheckpoint(state.ControlSock, dir, then, txid, epoch, timeout) if err != nil { // hvi resumes the guest after a failed capture (unless the fence was // already applied), and what it wrote is not a bundle: the manifest @@ -297,13 +325,13 @@ func hviCheckpointInstance(s *store.Store, state *store.InstanceState, then, dir } switch { case errors.Is(err, errHviNoReply): - return fmt.Errorf("%w; hvi may still be writing the bundle to %s with the guest "+ + return nil, fmt.Errorf("%w; hvi may still be writing the bundle to %s with the guest "+ "paused. Once it is done, `hull resume %s` either resumes the guest or answers "+ "that it was fenced; retry with a longer --timeout", err, dir, id) case hviRefusalCode(err) == "not_implemented": - return fmt.Errorf("this hvi (%s) cannot checkpoint yet: %w", state.CmdLine[0], err) + return nil, fmt.Errorf("this hvi (%s) cannot checkpoint yet: %w", state.CmdLine[0], err) } - return err + return nil, err } rec := hviCheckpointRecord{ @@ -315,12 +343,12 @@ func hviCheckpointInstance(s *store.Store, state *store.InstanceState, then, dir TxID: txid, } if err := writeHviCheckpointRecord(ckptDir, rec); err != nil { - return fmt.Errorf("the bundle is in %s but recording it failed: %w", dir, err) + return nil, fmt.Errorf("the bundle is in %s but recording it failed: %w", dir, err) } fresh, err := s.GetInstance(id) if err != nil { - return err + return nil, err } previous := fresh.Migration migrationState := store.MigrationCheckpointed @@ -330,7 +358,7 @@ func hviCheckpointInstance(s *store.Store, state *store.InstanceState, then, dir fresh.Migration = &store.MigrationState{State: migrationState, BundleDir: dir, TxID: txid, ManifestSHA256: reply.ManifestSHA256} if err := s.SaveInstance(fresh); err != nil { - return fmt.Errorf("the bundle is in %s but the instance record was not updated: %w", dir, err) + return nil, fmt.Errorf("the bundle is in %s but the instance record was not updated: %w", dir, err) } // One bundle per instance of hull's own naming: each is the size of the // guest's touched RAM, and only the latest is ever restored. A bundle @@ -339,10 +367,7 @@ func hviCheckpointInstance(s *store.Store, state *store.InstanceState, then, dir if previous != nil && previous.BundleDir != dir && previous.BundleDir == filepath.Join(root, previous.TxID) { _ = store.RemoveTree(previous.BundleDir) } - - fmt.Printf("Checkpoint of %s written to %s in %s (%s)\n", id, dir, - time.Since(started).Round(time.Millisecond), describeCheckpointStats(then, reply.Stats)) - return nil + return &rec, nil } // hviCheckpointRoot creates /checkpoint, 0700, and returns it diff --git a/cmd/hull/hvicheckpoint_test.go b/cmd/hull/hvicheckpoint_test.go index 7002e3d8..79c23a59 100644 --- a/cmd/hull/hvicheckpoint_test.go +++ b/cmd/hull/hvicheckpoint_test.go @@ -241,7 +241,7 @@ func TestHviControlCheckpointRequest(t *testing.T) { stub := newStubHvi(t, func(map[string]any) stubAnswer { return stubAnswer{reply: `{"v":1,"ok":true,"manifest_sha256":"ab12","stats":{"pause_ms":7}}`} }) - reply, err := hviCheckpoint(stub.path, "/abs/b", hviThenFence, "t1", time.Minute) + reply, err := hviCheckpoint(stub.path, "/abs/b", hviThenFence, "t1", 0, time.Minute) if err != nil { t.Fatal(err) } @@ -252,6 +252,16 @@ func TestHviControlCheckpointRequest(t *testing.T) { if req["op"] != "checkpoint" || req["dir"] != "/abs/b" || req["then"] != "fence" || req["txid"] != "t1" || req["v"] != float64(1) { t.Errorf("request = %v", req) } + if _, ok := req["epoch"]; ok { + t.Errorf("a checkpoint outside a move sent an epoch: %v", req) + } + // A move's fence carries the epoch it hands over. + if _, err := hviCheckpoint(stub.path, "/abs/c", hviThenFence, "t2", 7, time.Minute); err != nil { + t.Fatal(err) + } + if req := stub.seen()[1]; req["epoch"] != float64(7) || req["txid"] != "t2" { + t.Errorf("request = %v, want epoch 7", req) + } } // The root hvi is given must exist, be private and be resolved: hvi compares diff --git a/cmd/hull/hvipause.go b/cmd/hull/hvipause.go index 3b18b024..0c48349d 100644 --- a/cmd/hull/hvipause.go +++ b/cmd/hull/hvipause.go @@ -146,6 +146,9 @@ func hviPauseResumeIn(s *store.Store, id, op string) error { return err } if op == "resume" { + if err := refuseDuringMove(s, id, "resume"); err != nil { + return err + } if err := markRestoredGuestRunning(s, id); err != nil { return err } diff --git a/cmd/hull/hvirestore.go b/cmd/hull/hvirestore.go index 5e9abf8f..2b55a2de 100644 --- a/cmd/hull/hvirestore.go +++ b/cmd/hull/hvirestore.go @@ -71,6 +71,12 @@ func restoreCommand() *cli.Command { Value: 120, Usage: "hvi: seconds to wait for hvi to rebuild the VM from the bundle", }, + &cli.StringFlag{ + Name: "move", + Hidden: true, + Usage: "hvi: the txid of the move (`hull migrate`) whose received checkpoint this " + + "restores; with --paused, which a move's restore always is", + }, &cli.BoolFlag{ Name: "adopt-latest", Usage: "hvi: restore from the fenced checkpoint latest.json names when the record " + @@ -91,7 +97,7 @@ func restoreCommand() *cli.Command { return fmt.Errorf("instance not found: %s", id) } if !isHviInstance(state) { - if cmd.IsSet("paused") || cmd.IsSet("timeout") || cmd.IsSet("adopt-latest") { + if cmd.IsSet("paused") || cmd.IsSet("timeout") || cmd.IsSet("adopt-latest") || cmd.IsSet("move") { return errors.New("--paused, --timeout and --adopt-latest apply to hvi instances only") } return vzRestoreInstance(ctx, cmd) @@ -105,6 +111,7 @@ func restoreCommand() *cli.Command { paused: cmd.Bool("paused"), timeout: time.Duration(cmd.Int("timeout")) * time.Second, adoptLatest: cmd.Bool("adopt-latest"), + move: cmd.String("move"), }) }, } @@ -115,6 +122,10 @@ type hviRestoreOptions struct { paused bool timeout time.Duration adoptLatest bool + // move is the txid of the move whose received checkpoint this restores + // on its destination. Only that move's own restore may bring up an + // incoming instance, and only paused. + move string } // hviRestoreArgv turns the command line an hvi instance last ran with into @@ -335,6 +346,14 @@ func restorableMigration(m *store.MigrationState) error { case store.MigrationRestored: return errors.New("its checkpoint was already restored and the restored guest ran, so " + "its disk has moved on from the bundle") + case store.MigrationMigrating: + return fmt.Errorf("its fenced checkpoint belongs to move %s to %s, which decides what becomes "+ + "of it: `hull migrate --abort %s` brings the instance back here", m.MoveTxID, m.Peer, m.MoveTxID) + case store.MigrationMigrated: + return fmt.Errorf("it moved to %s (move %s) and runs there; its checkpoint here is stale. "+ + "Moving it back is a new `hull migrate` run on %s", m.Peer, m.MoveTxID, m.Peer) + case store.MigrationIncoming: + return fmt.Errorf("it is arriving from %s by move %s, and only that move restores it", m.Peer, m.MoveTxID) } return fmt.Errorf("its checkpoint is in an unknown state %q", m.State) } @@ -368,11 +387,16 @@ func hviRestoreInstance(ctx context.Context, cmd *cli.Command, s *store.Store, s "before restoring", id) } } - if err := reconcileLatestCheckpoint(s, state, opt.adoptLatest); err != nil { + if err := checkMoveRestore(s, state, opt); err != nil { return fmt.Errorf("cannot restore %s: %w", id, err) } - if err := restorableMigration(state.Migration); err != nil { - return fmt.Errorf("cannot restore %s: %w", id, err) + if opt.move == "" { + if err := reconcileLatestCheckpoint(s, state, opt.adoptLatest); err != nil { + return fmt.Errorf("cannot restore %s: %w", id, err) + } + if err := restorableMigration(state.Migration); err != nil { + return fmt.Errorf("cannot restore %s: %w", id, err) + } } bundle := state.Migration.BundleDir if _, err := os.Stat(bundle); err != nil { @@ -644,3 +668,33 @@ func keepRecordedMigration(s *store.Store, state *store.InstanceState) { state.Migration = fresh.Migration } } + +// checkMoveRestore holds a restore to the rules of a move. Without --move it +// refuses an instance a move has in hand (refuseDuringMove). With it, the +// restore must be that move's own, on its destination, paused, of the +// received checkpoint, before the commit: the move's restore step. +func checkMoveRestore(s *store.Store, state *store.InstanceState, opt hviRestoreOptions) error { + if opt.move == "" { + return refuseDuringMove(s, state.ID, "restore") + } + if !opt.paused { + return errors.New("a move's restore is always --paused; only its commit resumes the guest") + } + rec, err := readMigrationRecord(s.InstanceMigrationRecord(state.ID)) + if err != nil { + return fmt.Errorf("move %s: %w", opt.move, err) + } + m := state.Migration + switch { + case rec.TxID != opt.move || rec.Role != roleDestination: + return fmt.Errorf("the instance is not the destination of move %s", opt.move) + case rec.State != dstVerified && rec.State != dstRestoredPaused: + return fmt.Errorf("move %s is %s here, and only a verified one is restored", opt.move, rec.State) + case m == nil || m.MoveTxID != opt.move || m.BundleDir == "": + return fmt.Errorf("the instance record holds no checkpoint of move %s", opt.move) + case m.State != store.MigrationIncoming && m.State != store.MigrationRestoring && + m.State != store.MigrationRestoredPaused: + return fmt.Errorf("the checkpoint of move %s is %s, so it may have run already", opt.move, m.State) + } + return nil +} diff --git a/cmd/hull/migrate_dest.go b/cmd/hull/migrate_dest.go new file mode 100644 index 00000000..a719df0a --- /dev/null +++ b/cmd/hull/migrate_dest.go @@ -0,0 +1,831 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + "github.com/urfave/cli/v3" + "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" + "golang.org/x/sys/unix" + + "github.com/brig-sh/hull/pkg/store" +) + +// The destination's half of a move: the hidden verbs `hull migrate` runs +// over ssh. Each prints one migrateReply on stdout and nothing else, and +// each is safe to run again: a step that finds its effect already there +// answers as if it had just done it. + +// moveLockWait is how long a destination verb waits for another step of the +// same move to finish before it answers busy. +const moveLockWait = 30 * time.Second + +// destRestore brings the received checkpoint of instance name up paused, for +// move txid. A variable so the tests can stand in for hvi. +var destRestore = runDestRestore + +// destResume resumes the restored guest at commit. A variable for the same +// reason. +var destResume = func(s *store.Store, name string) error { + state, err := s.GetInstance(name) + if err != nil { + return err + } + if err := markRestoredGuestRunning(s, name); err != nil { + return err + } + _, err = hviOp(state.ControlSock, "resume") + return err +} + +// destHviPath is the hvi a destination uses: the one the operator named, or +// the one `hull run` would. +func destHviPath(override string) (string, error) { + if override != "" { + info, err := os.Stat(override) + if err != nil { + return "", fmt.Errorf("hvi %s: %w", override, err) + } + if info.IsDir() || info.Mode()&0o111 == 0 { + return "", fmt.Errorf("hvi %s is not an executable", override) + } + if filepath.Base(override) != "hvi" { + return "", fmt.Errorf("hvi %s must be named hvi: hull tells an hvi instance by its program's name", override) + } + return override, nil + } + vmm, err := newVMM(hypervisors.HviVmm, nil) + if err != nil { + return "", err + } + return vmm.Path(), nil +} + +// hviVersion asks an hvi for its version line. +func hviVersion(binary string) string { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, binary, "--version").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) +} + +// storeFreeBytes is the space left for an unprivileged user under dir. +func storeFreeBytes(dir string) int64 { + var st unix.Statfs_t + if err := unix.Statfs(dir, &st); err != nil { + return -1 + } + return int64(st.Bavail) * int64(st.Bsize) //nolint:gosec,unconvert // block counts fit; Bsize's type differs by OS +} + +// destProbe reports what a source needs to know before it fences anything. +// It changes nothing. +func destProbe(s *store.Store, name, hviOverride, gatewaySock string) *migrateReply { + p := &migrateProbe{ + HullVersion: build.String(), + OS: runtime.GOOS, + Arch: runtime.GOARCH, + StoreDir: s.RootDir(), + FreeBytes: storeFreeBytes(s.RootDir()), + } + p.Hypervisor, p.HypervisorDetail = hypervisorAvailable() + if hvi, err := destHviPath(hviOverride); err != nil { + p.HviError = err.Error() + } else { + p.Hvi = hvi + p.HviVersion = hviVersion(hvi) + if ok, err := hviBootTakesControlFlags(hvi); err != nil { + p.HviError = err.Error() + } else { + p.HviControlFlags = ok + } + if ok, err := hviHasRestore(hvi); err != nil { + p.HviError = err.Error() + } else { + p.HviRestore = ok + } + } + if name != "" { + if err := store.ValidateInstanceID(name); err != nil { + return refuse(codeFailed, err) + } + if _, err := os.Stat(s.InstanceDir(name)); err == nil { + p.NameTaken = true + if rec, err := readMigrationRecord(s.InstanceMigrationRecord(name)); err == nil { + p.NameTxID = rec.TxID + p.NameMove = rec.Role + " " + rec.State + } + } + } + if gatewaySock != "" { + conn, err := net.DialTimeout("unix", gatewaySock, 2*time.Second) + if err != nil { + p.GatewayDetail = err.Error() + } else { + _ = conn.Close() + p.GatewayOK = true + } + } + return &migrateReply{OK: true, Probe: p} +} + +// refuse is a definite no. +func refuse(code string, err error) *migrateReply { + return &migrateReply{OK: false, Code: code, Error: err.Error()} +} + +// refuseWith is a definite no that carries the record it is about. +func refuseWith(code string, err error, rec *migrationRecord) *migrateReply { + r := refuse(code, err) + r.Record = rec + return r +} + +// destReserve claims name for move txid: the instance directory and its +// migration.json, state reserved. Nothing is received yet, so a reservation +// costs the destination nothing but the name. +func destReserve(s *store.Store, txid, name string, req migrateReserveRequest) *migrateReply { + if err := checkTxID(txid); err != nil { + return refuse(codeFailed, err) + } + if err := store.ValidateInstanceID(name); err != nil { + return refuse(codeFailed, err) + } + if rec, _, err := findMigrationByTxID(s, txid); err == nil { + switch { + case rec.State == migAborted: + return refuseWith(codeAborted, fmt.Errorf("move %s was aborted here", txid), rec) + case rec.Name != name || rec.Role != roleDestination: + return refuseWith(codeFailed, fmt.Errorf("move %s is recorded here for %s as %s", txid, rec.Name, rec.Role), rec) + } + return &migrateReply{OK: true, Record: rec} + } + if _, err := destHviPath(req.Spec.Hvi); err != nil { + return refuse(codeFailed, err) + } + for _, sock := range []struct{ what, path string }{ + {"agent", s.InstanceAgentSocket(name)}, {"control", s.InstanceControlSocket(name)}, + } { + if err := checkUnixSocketPath(sock.what, sock.path); err != nil { + return refuse(codeFailed, err) + } + } + dir, err := s.CreateInstance(name) + if errors.Is(err, store.ErrInstanceExists) { + return refuse(codeNameTaken, fmt.Errorf("an instance named %s already exists here", name)) + } + if err != nil { + return refuse(codeFailed, err) + } + spec := req.Spec + rec := &migrationRecord{ + TxID: txid, Role: roleDestination, Name: name, Epoch: req.Epoch, State: dstReserved, + Peer: spec.SourceHost, Spec: &spec, + History: []migrateHistory{{State: dstReserved, At: time.Now().UTC()}}, + } + if err := writeMigrationRecord(s.InstanceMigrationRecord(name), rec); err != nil { + _ = store.RemoveTree(dir) + return refuse(codeFailed, err) + } + return &migrateReply{OK: true, Record: rec} +} + +// destRecord finds move txid's record for a destination verb, locked. The +// unlock is nil when there is nothing to unlock. +func destRecord(s *store.Store, txid string) (*migrationRecord, string, func(), *migrateReply) { + rec, path, err := findMigrationByTxID(s, txid) + if errors.Is(err, os.ErrNotExist) { + return nil, "", nil, refuse(codeUnknownTxID, fmt.Errorf("no move %s is recorded here", txid)) + } + if err != nil { + return nil, "", nil, refuse(codeFailed, err) + } + if rec.Role != roleDestination { + return nil, "", nil, refuseWith(codeFailed, fmt.Errorf("move %s is recorded here as its %s", txid, rec.Role), rec) + } + if rec.State == migAborted { + return rec, path, nil, nil + } + unlock, err := lockMove(filepath.Dir(path), moveLockWait) + if err != nil { + return nil, "", nil, refuseWith(codeBusy, err, rec) + } + // Read again under the lock: the step that held it may have moved it. + if rec, err = readMigrationRecord(path); err != nil { + unlock() + return nil, "", nil, refuse(codeFailed, err) + } + return rec, path, unlock, nil +} + +// destReceive reads the stream on in into the instance reserved for txid, +// checks it against its own trailer and the bundle's manifest, puts the +// bundle and the disk where `hull restore` will look, and has hvi verify the +// bundle. It ends in verified, with a record of the instance written in the +// incoming state that only this move can restore and resume. +func destReceive(s *store.Store, txid, name string, in io.Reader) *migrateReply { + rec, path, unlock, no := destRecord(s, txid) + if no != nil { + return no + } + if unlock != nil { + defer unlock() + } + if rec.State == migAborted { + return refuseWith(codeAborted, fmt.Errorf("move %s was aborted here", txid), rec) + } + if rec.Name != name { + return refuseWith(codeFailed, fmt.Errorf("move %s is for %s here, not %s", txid, rec.Name, name), rec) + } + if migrateStateOrder(roleDestination, rec.State) >= migrateStateOrder(roleDestination, dstVerified) { + return &migrateReply{OK: true, Record: rec} + } + if err := rec.advance(path, dstReceiving); err != nil { + return refuseWith(codeFailed, err, rec) + } + instDir := s.InstanceDir(name) + staging := filepath.Join(instDir, "incoming") + _ = store.RemoveTree(staging) + if err := os.MkdirAll(filepath.Join(staging, "bundle"), 0o700); err != nil { + return refuseWith(codeFailed, rec.fail(path, err), rec) + } + started := time.Now() + res, err := readMigrateStream(in, func(fname string, _ int64) (*os.File, error) { + return os.OpenFile(filepath.Join(staging, filepath.FromSlash(fname)), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + }) + if err == nil && res.Header.TxID != txid { + err = fmt.Errorf("the stream is for move %s, not %s", res.Header.TxID, txid) + } + if err != nil { + _ = store.RemoveTree(staging) + return refuseWith(codeFailed, rec.fail(path, fmt.Errorf("receive: %w", err)), rec) + } + took := time.Since(started) + disk, err := checkReceivedBundle(filepath.Join(staging, "bundle"), res) + if err != nil { + _ = store.RemoveTree(staging) + return refuseWith(codeVerifyFailed, rec.fail(path, err), rec) + } + + // Into place: the bundle below the checkpoint root its hvi will be + // given, the disk where a run keeps it. + root, err := hviCheckpointRoot(instDir) + if err == nil { + err = placeReceived(staging, root, txid, instDir) + } + if err != nil { + return refuseWith(codeFailed, rec.fail(path, err), rec) + } + rec.BundleDir = filepath.Join(root, txid) + rec.ManifestSHA256 = res.Header.ManifestSHA256 + rec.Disk = &migrateDisk{Path: filepath.Join(instDir, "bundle", "rootfs.ext4"), Size: disk.Size, + SHA256: disk.SHA256, DataBytes: disk.DataBytes} + rec.Transfer = &migrateTransfer{Bytes: res.Bytes, Seconds: took.Seconds(), StartedAt: started.UTC(), + BytesPerSec: int64(float64(res.Bytes) / max(took.Seconds(), 1e-3))} + if err := rec.advance(path, dstReceived); err != nil { + return refuseWith(codeFailed, err, rec) + } + + hvi, err := destHviPath(rec.Spec.Hvi) + if err != nil { + return refuseWith(codeFailed, rec.fail(path, err), rec) + } + verdict, err := hviCheckpointVerify(hvi, rec.BundleDir) + if err != nil { + return refuseWith(codeVerifyFailed, rec.fail(path, err), rec) + } + rec.Verify = verdict + if err := writeIncomingInstance(s, rec, hvi, root); err != nil { + return refuseWith(codeFailed, rec.fail(path, err), rec) + } + if err := rec.advance(path, dstVerified); err != nil { + return refuseWith(codeFailed, err, rec) + } + return &migrateReply{OK: true, Record: rec} +} + +// checkReceivedBundle checks the received bundle against its manifest: +// manifest.json hashes to what the stream header says hvi reported at the +// fence, every component the manifest lists arrived with its size and +// sha256, nothing else did, and the disk is the size disk.json says. It +// returns the disk's line from the stream. +func checkReceivedBundle(dir string, res *migrateStreamResult) (streamSum, error) { + got := map[string]streamSum{} + for _, f := range res.Files { + got[f.Name] = f + } + disk, ok := got["disk"] + if !ok { + return streamSum{}, errors.New("the stream carried no disk") + } + man, ok := got["bundle/manifest.json"] + if !ok { + return streamSum{}, errors.New("the stream carried no manifest.json") + } + if res.Header.ManifestSHA256 == "" || man.SHA256 != res.Header.ManifestSHA256 { + return streamSum{}, fmt.Errorf("manifest.json arrived with sha256 %s, but hvi reported %s at the fence", + man.SHA256, res.Header.ManifestSHA256) + } + comps, err := readManifestComponents(dir) + if err != nil { + return streamSum{}, err + } + want := map[string]bool{"disk": true, "bundle/manifest.json": true} + for _, c := range comps { + name := "bundle/" + c.Path + want[name] = true + f, ok := got[name] + if !ok { + return streamSum{}, fmt.Errorf("the manifest lists %s, which did not arrive", c.Path) + } + if f.Size != c.Size || f.SHA256 != c.SHA256 { + return streamSum{}, fmt.Errorf("%s arrived with %d bytes and sha256 %s; the manifest says %d and %s", + c.Path, f.Size, f.SHA256, c.Size, c.SHA256) + } + } + for name := range got { + if !want[name] { + return streamSum{}, fmt.Errorf("the stream carried %s, which the manifest does not list", name) + } + } + if size, ok, err := bundleDiskSize(dir); err != nil { + return streamSum{}, err + } else if ok && size != disk.Size { + return streamSum{}, fmt.Errorf("the disk arrived as %d bytes, but disk.json says the guest's was %d", disk.Size, size) + } + return disk, nil +} + +// manifestComponent is one entry of a bundle's manifest.json. +type manifestComponent struct { + Name string `json:"name"` + Path string `json:"path"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` +} + +// readManifestComponents reads the component list of dir/manifest.json. The +// rest of the manifest is hvi's to read. +func readManifestComponents(dir string) ([]manifestComponent, error) { + data, err := os.ReadFile(filepath.Join(dir, "manifest.json")) + if err != nil { + return nil, fmt.Errorf("read the bundle's manifest: %w", err) + } + var m struct { + Components []manifestComponent `json:"components"` + } + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("the bundle's manifest.json: %w", err) + } + if len(m.Components) == 0 || len(m.Components) > migrateMaxFiles-2 { + return nil, fmt.Errorf("the bundle's manifest lists %d components", len(m.Components)) + } + for _, c := range m.Components { + if !migrateStreamName.MatchString("bundle/"+c.Path) || c.Path == "manifest.json" { + return nil, fmt.Errorf("the bundle's manifest lists a component at %q", c.Path) + } + } + return m.Components, nil +} + +// bundleDiskSize reads size_bytes from dir/disk.json, when the bundle has +// one (a guest without a disk has none, and a move needs a disk anyway). +func bundleDiskSize(dir string) (int64, bool, error) { + data, err := os.ReadFile(filepath.Join(dir, "disk.json")) + if errors.Is(err, os.ErrNotExist) { + return 0, false, nil + } + if err != nil { + return 0, false, err + } + var d struct { + SizeBytes *int64 `json:"size_bytes"` + } + if err := json.Unmarshal(data, &d); err != nil || d.SizeBytes == nil { + return 0, false, fmt.Errorf("the bundle's disk.json has no size_bytes: %v", err) + } + return *d.SizeBytes, true, nil +} + +// placeReceived renames the staged bundle to root/txid and the staged disk to +// instDir/bundle/rootfs.ext4, replacing what an earlier attempt left, and +// syncs both directories. +func placeReceived(staging, root, txid, instDir string) error { + bundleDir := filepath.Join(root, txid) + _ = store.RemoveTree(bundleDir) + if err := os.Rename(filepath.Join(staging, "bundle"), bundleDir); err != nil { + return fmt.Errorf("place the bundle: %w", err) + } + diskDir := filepath.Join(instDir, "bundle") + if err := os.MkdirAll(diskDir, 0o700); err != nil { + return err + } + if err := os.Rename(filepath.Join(staging, "disk"), filepath.Join(diskDir, "rootfs.ext4")); err != nil { + return fmt.Errorf("place the disk: %w", err) + } + _ = store.RemoveTree(staging) + if err := syncDir(root); err != nil { + return err + } + return syncDir(diskDir) +} + +// hviCheckpointVerify runs `hvi checkpoint-verify dir` and returns its +// verdict, the last line it printed. +func hviCheckpointVerify(hvi, dir string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + var out bytes.Buffer + c := exec.CommandContext(ctx, hvi, "checkpoint-verify", dir) + c.Stdout = &out + c.Stderr = &out + err := c.Run() + text := strings.TrimSpace(out.String()) + if i := strings.LastIndexByte(text, '\n'); i >= 0 { + text = text[i+1:] + } + if err != nil { + return "", fmt.Errorf("%s checkpoint-verify %s: %v: %s", hvi, dir, err, text) + } + return text, nil +} + +// destCmdLine is the hvi command line the received instance is recorded +// with: a restore from the received bundle, with the destination's own disk, +// sockets and checkpoint root. hviRestoreArgv maps it to the command `hull +// restore` runs, as it does for a record of a local run. +func destCmdLine(s *store.Store, rec *migrationRecord, hvi, root string) []string { + spec := rec.Spec + instDir := s.InstanceDir(rec.Name) + argv := []string{hvi, "restore", "--from", rec.BundleDir, "--disk", rec.Disk.Path} + switch spec.Net { + case "gateway": + argv = append(argv, "--net-gateway", spec.GatewaySock) + case "stub": + argv = append(argv, "--net-stub") + } + if spec.MAC != "" { + argv = append(argv, "--net-mac", spec.MAC) + } + if spec.Agent { + argv = append(argv, "--agent-sock", s.InstanceAgentSocket(rec.Name)) + } + if spec.SandboxID != "" { + argv = append(argv, "--sandbox-id", spec.SandboxID) + } + if spec.Events { + argv = append(argv, "--events", filepath.Join(instDir, "events")) + } + return append(argv, "--checkpoint-root", root) +} + +// writeIncomingInstance writes the received instance's state.json: stopped, +// with the move's fenced bundle as its checkpoint in the incoming state. +func writeIncomingInstance(s *store.Store, rec *migrationRecord, hvi, root string) error { + spec := rec.Spec + state := &store.InstanceState{ + ID: rec.Name, + ImageDigest: spec.ImageDigest, + Status: "stopped", + LogFile: s.InstanceLogFile(rec.Name), + CmdLine: destCmdLine(s, rec, hvi, root), + BundleDir: s.InstanceBundleDir(rec.Name), + MAC: spec.MAC, + IP: spec.IP, + Backend: string(hypervisors.HviVmm), + StartTime: time.Now(), + Migration: &store.MigrationState{State: store.MigrationIncoming, BundleDir: rec.BundleDir, + TxID: rec.TxID, ManifestSHA256: rec.ManifestSHA256, MoveTxID: rec.TxID, Peer: rec.Peer}, + } + return s.SaveInstance(state) +} + +// runDestRestore runs `hull restore --detach --paused --move TXID NAME` as a +// child, so the restore takes the instance's migration lock and records its +// progress exactly as a restore an operator ran would. +func runDestRestore(s *store.Store, name, txid string) error { + self, err := os.Executable() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + var out bytes.Buffer + c := exec.CommandContext(ctx, self, "--store-dir", s.RootDir(), "restore", "--detach", "--paused", + "--move", txid, name) + c.Stdout = &out + c.Stderr = &out + if err := c.Run(); err != nil { + text := strings.TrimSpace(out.String()) + if i := strings.LastIndex(text, "error: "); i >= 0 { + text = text[i+len("error: "):] + } + return fmt.Errorf("hull restore %s: %s", name, text) + } + return nil +} + +// destRestoredPaused reports whether the instance's VM is up and paused by +// the restore, as hvi says. +func destRestoredPaused(s *store.Store, name string) bool { + state, err := s.GetInstance(name) + if err != nil { + return false + } + return hviVMState(state) == store.MigrationRestoredPaused +} + +// destPrepareLocked restores the verified checkpoint paused, unless it +// already is. The caller holds the move lock. +func destPrepareLocked(s *store.Store, rec *migrationRecord, path string) *migrateReply { + switch rec.State { + case dstVerified: + case dstRestoredPaused: + if destRestoredPaused(s, rec.Name) { + return nil + } + // The paused VM went away (a reboot, a kill): the checkpoint is + // still the fenced one and nothing ran it, so restore it again. + default: + return refuseWith(codeNotReady, fmt.Errorf("move %s is %s here; it has to be verified to be restored", + rec.TxID, rec.State), rec) + } + started := time.Now() + if err := destRestore(s, rec.Name, rec.TxID); err != nil { + return refuseWith(codeRestoreRefused, rec.fail(path, err), rec) + } + if rec.Timings == nil { + rec.Timings = map[string]int64{} + } + rec.Timings["restore"] = time.Since(started).Milliseconds() + if err := rec.advance(path, dstRestoredPaused); err != nil { + return refuseWith(codeFailed, err, rec) + } + return nil +} + +// destPrepare is migrate-prepare: restore the received checkpoint paused, +// before any commit. A restore that hvi refuses (an incompatible host, an +// hvi with no restore) shows here, while the source can still abort. +func destPrepare(s *store.Store, txid string) *migrateReply { + rec, path, unlock, no := destRecord(s, txid) + if no != nil { + return no + } + if unlock != nil { + defer unlock() + } + switch rec.State { + case migAborted: + return refuseWith(codeAborted, fmt.Errorf("move %s was aborted here", txid), rec) + case dstRunning: + return &migrateReply{OK: true, Record: rec, VM: "running"} + } + if no := destPrepareLocked(s, rec, path); no != nil { + return no + } + return &migrateReply{OK: true, Record: rec, VM: store.MigrationRestoredPaused} +} + +// destCommit is migrate-commit: the source has recorded fenced and +// committing, and lets the destination run the VM. It records running +// before it sends the resume, so that from the moment the guest may run, an +// abort is refused; and it takes epoch as the instance's owner epoch. +func destCommit(s *store.Store, txid string, epoch uint64) *migrateReply { + rec, path, unlock, no := destRecord(s, txid) + if no != nil { + return no + } + if unlock != nil { + defer unlock() + } + if rec.Epoch != epoch { + return refuseWith(codeWrongEpoch, fmt.Errorf("move %s hands over epoch %d, not %d", txid, rec.Epoch, epoch), rec) + } + switch rec.State { + case migAborted: + return refuseWith(codeAborted, fmt.Errorf("move %s was aborted here, so it cannot commit", txid), rec) + case dstRunning: + return &migrateReply{OK: true, Record: rec, VM: "running"} + } + if no := destPrepareLocked(s, rec, path); no != nil { + return no + } + started := time.Now() + state, err := s.GetInstance(rec.Name) + if err != nil { + return refuseWith(codeFailed, rec.fail(path, err), rec) + } + state.OwnerEpoch = epoch + if err := s.SaveInstance(state); err != nil { + return refuseWith(codeFailed, rec.fail(path, err), rec) + } + // The destination's commit point: from here the guest may run, and + // nothing may abort the move. + if err := rec.advance(path, dstRunning); err != nil { + return refuseWith(codeFailed, err, rec) + } + if err := destResume(s, rec.Name); err != nil { + // The move committed, whatever the resume did: the record says + // running, and the source, reading that, records the commit too. + return refuseWith(codeFailed, rec.fail(path, fmt.Errorf("committed, but the resume failed: %w; "+ + "`hull resume %s` here", err, rec.Name)), rec) + } + if rec.Timings == nil { + rec.Timings = map[string]int64{} + } + rec.Timings["resume"] = time.Since(started).Milliseconds() + _ = writeMigrationRecord(path, rec) + return &migrateReply{OK: true, Record: rec, VM: "running"} +} + +// destAbort is migrate-abort: refused once the move has committed here; +// otherwise it records aborted first (so a commit that arrives later is +// refused), then stops any paused VM the move restored, keeps the record as +// a tombstone and frees the name. A move this store never heard of +// answers ok with code unknown_txid: nothing here can run it. +func destAbort(s *store.Store, txid string) *migrateReply { + rec, path, unlock, no := destRecord(s, txid) + if no != nil { + if no.Code == codeUnknownTxID { + no.OK = true + } + return no + } + if unlock != nil { + defer unlock() + } + switch rec.State { + case migAborted: + return &migrateReply{OK: true, Record: rec} + case dstRunning: + return refuseWith(codeRunning, fmt.Errorf("move %s committed here and %s runs here, at epoch %d", + txid, rec.Name, rec.Epoch), rec) + } + if err := rec.advance(path, migAborted); err != nil { + return refuseWith(codeFailed, err, rec) + } + if err := os.MkdirAll(s.MigrationTombstoneDir(), 0o700); err == nil { + _ = writeMigrationRecord(filepath.Join(s.MigrationTombstoneDir(), txid+".json"), rec) + } + var cleanup error + if _, err := s.GetInstance(rec.Name); err == nil { + cleanup = removeInstanceIn(s, rec.Name, true) + } else { + cleanup = s.DeleteInstance(rec.Name) + } + reply := &migrateReply{OK: true, Record: rec} + if cleanup != nil { + reply.Error = fmt.Sprintf("aborted, but removing %s failed: %v", rec.Name, cleanup) + } + return reply +} + +// destStatus is migrate-status: the record, and what hvi says the VM is +// doing. +func destStatus(s *store.Store, txid string) *migrateReply { + rec, _, err := findMigrationByTxID(s, txid) + if errors.Is(err, os.ErrNotExist) { + return refuse(codeUnknownTxID, fmt.Errorf("no move %s is recorded here", txid)) + } + if err != nil { + return refuse(codeFailed, err) + } + reply := &migrateReply{OK: true, Record: rec} + if state, err := s.GetInstance(rec.Name); err == nil && rec.State != migAborted { + reply.VM = hviVMState(state) + } + return reply +} + +// migrateRemoteVerb wraps a destination verb: stdout carries its one reply +// and nothing else, so everything the verb prints on the way (stop and +// resume print to stdout) goes to stderr instead. +func migrateRemoteVerb(name, usage string, flags []cli.Flag, + do func(cmd *cli.Command, s *store.Store) *migrateReply) *cli.Command { + return &cli.Command{ + Name: name, + Usage: usage, + Hidden: true, + Flags: flags, + Action: func(_ context.Context, cmd *cli.Command) error { + out := os.Stdout + os.Stdout = os.Stderr + defer func() { os.Stdout = out }() + var reply *migrateReply + if s, err := globalStore(cmd); err != nil { + reply = refuse(codeFailed, err) + } else { + reply = do(cmd, s) + } + reply.Protocol = migrateProtocol + return json.NewEncoder(out).Encode(reply) + }, + } +} + +// migrateRemoteCommands are the destination's verbs, hidden: `hull migrate` +// runs them over ssh, and docs/migration.md describes them. +func migrateRemoteCommands() []*cli.Command { + txidFlag := &cli.StringFlag{Name: "txid", Required: true, Usage: "the move's txid"} + nameFlag := &cli.StringFlag{Name: "name", Usage: "the instance's name on this host"} + return []*cli.Command{ + migrateRemoteVerb("migrate-probe", "report whether this host can take a moved instance", + []cli.Flag{nameFlag, &cli.StringFlag{Name: "hvi"}, &cli.StringFlag{Name: "gateway-sock"}}, + func(cmd *cli.Command, s *store.Store) *migrateReply { + return destProbe(s, cmd.String("name"), cmd.String("hvi"), cmd.String("gateway-sock")) + }), + migrateRemoteVerb("migrate-receive", "reserve a name for a move, or receive its stream on stdin", + []cli.Flag{txidFlag, nameFlag, &cli.BoolFlag{Name: "reserve"}}, + func(cmd *cli.Command, s *store.Store) *migrateReply { + if cmd.Bool("reserve") { + var req migrateReserveRequest + if err := json.NewDecoder(io.LimitReader(os.Stdin, 1<<20)).Decode(&req); err != nil { + return refuse(codeFailed, fmt.Errorf("read the reservation: %w", err)) + } + return destReserve(s, cmd.String("txid"), cmd.String("name"), req) + } + return destReceive(s, cmd.String("txid"), cmd.String("name"), os.Stdin) + }), + migrateRemoteVerb("migrate-prepare", "restore a received move paused", + []cli.Flag{txidFlag}, + func(cmd *cli.Command, s *store.Store) *migrateReply { return destPrepare(s, cmd.String("txid")) }), + migrateRemoteVerb("migrate-commit", "take over a moved instance and run it", + []cli.Flag{txidFlag, &cli.Uint64Flag{Name: "epoch", Required: true}}, + func(cmd *cli.Command, s *store.Store) *migrateReply { + return destCommit(s, cmd.String("txid"), cmd.Uint64("epoch")) + }), + migrateRemoteVerb("migrate-status", "report a move's record on this host", + []cli.Flag{txidFlag}, + func(cmd *cli.Command, s *store.Store) *migrateReply { return destStatus(s, cmd.String("txid")) }), + migrateRemoteVerb("migrate-abort", "abort a move on this host unless it committed", + []cli.Flag{txidFlag}, + func(cmd *cli.Command, s *store.Store) *migrateReply { return destAbort(s, cmd.String("txid")) }), + } +} + +// fileSHA256 hashes the file at path. +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// fileModNs is a file's size and modification time in nanoseconds. +func fileModNs(path string) (int64, int64, error) { + info, err := os.Stat(path) + if err != nil { + return 0, 0, err + } + return info.Size(), info.ModTime().UnixNano(), nil +} + +// fileAllocated is the bytes a file occupies on disk, holes left out. +func fileAllocated(path string) int64 { + info, err := os.Stat(path) + if err != nil { + return 0 + } + if st, ok := info.Sys().(*syscall.Stat_t); ok { + return st.Blocks * 512 + } + return info.Size() +} diff --git a/cmd/hull/migrate_dest_test.go b/cmd/hull/migrate_dest_test.go new file mode 100644 index 00000000..7c3ab580 --- /dev/null +++ b/cmd/hull/migrate_dest_test.go @@ -0,0 +1,168 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/brig-sh/hull/pkg/store" +) + +// fakeHviProgram returns an executable named hvi that answers the probes +// hull makes of an hvi binary: --version, the boot flag probe, restore +// (present or not), and checkpoint-verify (passes a directory with a +// manifest.json). +// +// One of each kind per test binary, run once before it is handed out: macOS +// scans an executable the first time it runs, which on a busy machine has +// taken over 30 s, longer than hull's 5 s bound on a probe. +func fakeHviProgram(t *testing.T, hasRestore bool) string { + t.Helper() + fakeHviOnce.Do(func() { + dir, err := os.MkdirTemp("", "hull-fake-hvi-") + if err != nil { + fakeHviErr = err + return + } + fakeHviDir = dir + for _, has := range []bool{false, true} { + restore := `echo 'error: unknown subcommand "restore"' >&2; exit 2` + if has { + restore = `echo 'error: restore needs --from' >&2; exit 2` + } + script := fmt.Sprintf(`#!/bin/sh +case "$1" in +--version) echo "hvi 0.0.0-test" ;; +boot) echo "error: --checkpoint-root needs a directory" >&2; exit 2 ;; +restore) %s ;; +checkpoint-verify) [ -f "$2/manifest.json" ] || { echo "no manifest in $2" >&2; exit 1; }; echo "ok: test bundle" ;; +*) echo "unknown" >&2; exit 2 ;; +esac +`, restore) + path := filepath.Join(dir, fmt.Sprintf("restore-%v", has), "hvi") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + fakeHviErr = err + return + } + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + fakeHviErr = err + return + } + _ = exec.Command(path, "--version").Run() + } + }) + if fakeHviErr != nil { + t.Fatal(fakeHviErr) + } + return filepath.Join(fakeHviDir, fmt.Sprintf("restore-%v", hasRestore), "hvi") +} + +var ( + fakeHviOnce sync.Once + fakeHviDir string + fakeHviErr error +) + +// shortStore is a store under /tmp, so the destination's socket paths fit +// in sun_path. +func shortStore(t *testing.T) *store.Store { + t.Helper() + dir := shortSocketDir(t) + s, err := store.New(dir) + if err != nil { + t.Fatal(err) + } + return s +} + +// The destination's verbs are idempotent and keep to their record. +func TestDestinationVerbs(t *testing.T) { + s := shortStore(t) + hvi := fakeHviProgram(t, true) + txid := strings.Repeat("ab", 16) + req := migrateReserveRequest{Epoch: 4, Spec: migrateSpec{SourceHost: "src", Net: "none", Hvi: hvi}} + if r := destReserve(s, txid, "vm1", req); !r.OK || r.Record.State != dstReserved { + t.Fatalf("reserve: %+v", r) + } + if r := destReserve(s, txid, "vm1", req); !r.OK { + t.Errorf("a second reserve of the same move: %+v", r) + } + if r := destReserve(s, strings.Repeat("cd", 16), "vm1", req); r.OK || r.Code != codeNameTaken { + t.Errorf("another move for the same name: %+v", r) + } + if r := destProbe(s, "vm1", hvi, ""); !r.OK || !r.Probe.NameTaken || r.Probe.NameTxID != txid || + !r.Probe.HviRestore || !r.Probe.HviControlFlags { + t.Errorf("probe: %+v", r.Probe) + } + // The gateway the source names is dialled, not just looked for. + gw := filepath.Join(s.RootDir(), "gw.sock") + if r := destProbe(s, "", hvi, gw); r.Probe.GatewayOK || r.Probe.GatewayDetail == "" { + t.Errorf("probe of a gateway that is not there: %+v", r.Probe) + } + ln, err := net.Listen("unix", gw) + if err != nil { + t.Fatal(err) + } + defer func() { _ = ln.Close() }() + if r := destProbe(s, "", hvi, gw); !r.Probe.GatewayOK { + t.Errorf("probe of a listening gateway: %+v", r.Probe) + } + if r := destCommit(s, txid, 4); r.OK || r.Code != codeNotReady { + t.Errorf("commit of a reserved move: %+v", r) + } + if r := destCommit(s, txid, 3); r.OK || r.Code != codeWrongEpoch { + t.Errorf("commit with the wrong epoch: %+v", r) + } + if r := destStatus(s, strings.Repeat("ef", 16)); r.OK || r.Code != codeUnknownTxID { + t.Errorf("status of an unknown move: %+v", r) + } + if r := destAbort(s, strings.Repeat("ef", 16)); !r.OK || r.Code != codeUnknownTxID { + t.Errorf("abort of an unknown move: %+v", r) + } + // A stream that is not one is refused, and the record says why. + if r := destReceive(s, txid, "vm1", strings.NewReader("garbage")); r.OK { + t.Errorf("receive of garbage: %+v", r) + } + rec, _, _ := findMigrationByTxID(s, txid) + if rec.State != dstReceiving || rec.Error == "" { + t.Errorf("after a bad stream: %s %q", rec.State, rec.Error) + } + if r := destAbort(s, txid); !r.OK || r.Record.State != migAborted { + t.Errorf("abort: %+v", r) + } + if r := destAbort(s, txid); !r.OK { + t.Errorf("a second abort: %+v", r) + } + if r := destReserve(s, txid, "vm1", req); r.OK || r.Code != codeAborted { + t.Errorf("reserve after the abort: %+v", r) + } + if _, err := os.Stat(s.InstanceDir("vm1")); !os.IsNotExist(err) { + t.Errorf("the aborted instance directory: %v", err) + } +} + +// removeFakeHvi removes the fake hvi programs once every test has run. +func removeFakeHvi() { + if fakeHviDir != "" { + _ = os.RemoveAll(fakeHviDir) + } +} diff --git a/cmd/hull/migrate_fixture_test.go b/cmd/hull/migrate_fixture_test.go new file mode 100644 index 00000000..0f177df5 --- /dev/null +++ b/cmd/hull/migrate_fixture_test.go @@ -0,0 +1,390 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync" + "syscall" + "testing" + + "github.com/brig-sh/hull/pkg/store" +) + +// The fixture for move tests: two stores in one process, a source instance +// whose VMM is a process named hvi with a stub control socket that writes a +// bundle on checkpoint, a fake hvi program for the probes and +// checkpoint-verify, and a peer that runs the destination's verbs against +// the second store, which a test can make drop a reply or answer for it. + +// writeFakeBundle writes a bundle with a manifest that lists its parts with +// their sizes and hashes, as hvi seals one, and a disk.json naming diskSize. +// It returns the manifest's sha256. +func writeFakeBundle(t *testing.T, dir string, diskSize int64) string { + t.Helper() + parts := map[string][]byte{ + "profile.json": []byte(`{"profile_version":1}`), + "ram.bin": bytes.Repeat([]byte("ram!"), 4096), + "disk.json": []byte(fmt.Sprintf(`{"version":1,"path_hint":"/d","size_bytes":%d,"sha256_sparse":null,`+ + `"generation":3,"synced":true}`, diskSize)), + "session.json": []byte(`{"version":1}`), + } + var comps []manifestComponent + for _, name := range []string{"profile.json", "ram.bin", "disk.json", "session.json"} { + if err := os.WriteFile(filepath.Join(dir, name), parts[name], 0o600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(parts[name]) + comps = append(comps, manifestComponent{Name: name, Path: name, Size: int64(len(parts[name])), + SHA256: hex.EncodeToString(sum[:])}) + } + man, _ := json.Marshal(map[string]any{"checkpoint_format": 1, "components": comps}) + if err := os.WriteFile(filepath.Join(dir, "manifest.json"), man, 0o600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(man) + return hex.EncodeToString(sum[:]) +} + +// moveVM is the source VM's stub hvi: it fences on a fence checkpoint and +// records what it was asked. +type moveVM struct { + stub *stubHvi + pid int + done <-chan struct{} + + mu sync.Mutex + fenced bool + stopped bool + failCheckpoint bool +} + +func (v *moveVM) ops() []string { + var out []string + for _, r := range v.stub.seen() { + out = append(out, fmt.Sprint(r["op"])) + } + return out +} + +// moveFixture is a source store with one running hvi instance, and a +// destination store. +type moveFixture struct { + src, dst *store.Store + hvi string + vm *moveVM + disk string + name string + restores int +} + +const fixtureDiskSize = 64 << 20 + +func newMoveFixture(t *testing.T, hasRestore bool) *moveFixture { + t.Helper() + f := &moveFixture{src: shortStore(t), dst: shortStore(t), hvi: fakeHviProgram(t, hasRestore), name: "vm1"} + dir, err := f.src.CreateInstance(f.name) + if err != nil { + t.Fatal(err) + } + root, err := hviCheckpointRoot(dir) + if err != nil { + t.Fatal(err) + } + f.disk = filepath.Join(dir, "bundle", "rootfs.ext4") + writeSparseDisk(t, f.disk, fixtureDiskSize) + + vm := &moveVM{} + vm.pid, vm.done = startProcessAs(t, "hvi", "/bin/sleep", "60") + vm.stub = newStubHvi(t, func(req map[string]any) stubAnswer { + vm.mu.Lock() + defer vm.mu.Unlock() + state := "running" + if vm.fenced { + state = "fenced" + } + switch req["op"] { + case "status": + return stubAnswer{reply: `{"v":1,"ok":true,"state":"` + state + `","vcpus":1,"vcpus_parked":0}`} + case "checkpoint": + if vm.fenced { + return stubAnswer{reply: `{"v":1,"ok":false,"code":"fenced","error":"fenced"}`} + } + if vm.failCheckpoint { + return stubAnswer{reply: `{"v":1,"ok":false,"code":"capture_failed","error":"disk sync failed"}`} + } + sum := writeFakeBundle(t, req["dir"].(string), fixtureDiskSize) + if req["then"] == "fence" { + vm.fenced = true + } + return stubAnswer{reply: `{"v":1,"ok":true,"state":"fenced","manifest_sha256":"` + sum + + `","stats":{"pause_ms":1,"capture_ms":2,"ram_present_bytes":16384,"ram_total_bytes":65536}}`} + case "resume": + if vm.fenced { + return stubAnswer{reply: `{"v":1,"ok":false,"code":"fenced","error":"the VM was fenced"}`} + } + return stubAnswer{reply: `{"v":1,"ok":true,"state":"running"}`} + case "stop": + vm.stopped = true + _ = syscall.Kill(vm.pid, syscall.SIGKILL) + return stubAnswer{reply: `{"v":1,"ok":true,"state":"stopping"}`} + } + return stubAnswer{reply: `{"v":1,"ok":false,"code":"unknown_op","error":"?"}`} + }) + f.vm = vm + state := &store.InstanceState{ + ID: f.name, Status: "running", PID: vm.pid, Backend: "hvi", ControlSock: vm.stub.path, + MAC: "52:54:00:12:34:56", IP: "10.0.2.15", ImageDigest: "sha256:abc", + CmdLine: []string{f.hvi, "boot", "--kernel", "/k", "--mem-mib", "64", "--cpus", "1", "--disk", f.disk, + "--net-mac", "52:54:00:12:34:56", "--agent-sock", filepath.Join(dir, "agent.sock"), + "--sandbox-id", f.name, "--checkpoint-root", root}, + } + if err := f.src.SaveInstance(state); err != nil { + t.Fatal(err) + } + // The destination's restore: a process named hvi whose stub reports + // restored-paused until it is resumed, recorded as `hull restore + // --paused` records one. + prevRestore, prevResume := destRestore, destResume + t.Cleanup(func() { destRestore, destResume = prevRestore, prevResume }) + destRestore = func(s *store.Store, name, txid string) error { + if !hasRestore { + return errors.New(`the hvi it ran has no restore subcommand yet`) + } + f.restores++ + var mu sync.Mutex + resumed, fenced := false, false + stub := newStubHvi(t, func(req map[string]any) stubAnswer { + mu.Lock() + defer mu.Unlock() + switch req["op"] { + case "resume": + if fenced { + return stubAnswer{reply: `{"v":1,"ok":false,"code":"fenced","error":"fenced"}`} + } + resumed = true + return stubAnswer{reply: `{"v":1,"ok":true,"state":"running"}`} + case "status": + switch { + case fenced: + return stubAnswer{reply: `{"v":1,"ok":true,"state":"fenced"}`} + case resumed: + return stubAnswer{reply: `{"v":1,"ok":true,"state":"running"}`} + } + return stubAnswer{reply: `{"v":1,"ok":true,"state":"restored-paused"}`} + case "checkpoint": + // A moved instance can move on: back, for one. + sum := writeFakeBundle(t, req["dir"].(string), fixtureDiskSize) + fenced = req["then"] == "fence" + return stubAnswer{reply: `{"v":1,"ok":true,"state":"fenced","manifest_sha256":"` + sum + `"}`} + case "stop": + return stubAnswer{reply: `{"v":1,"ok":true,"state":"stopping"}`} + } + return stubAnswer{reply: `{"v":1,"ok":false,"code":"unknown_op","error":"?"}`} + }) + pid, _ := startProcessAs(t, "hvi", "/bin/sleep", "60") + st, err := s.GetInstance(name) + if err != nil { + return err + } + if err := checkMoveRestore(s, st, hviRestoreOptions{move: txid, paused: true}); err != nil { + return err + } + st.Status, st.PID, st.ControlSock = "running", pid, stub.path + st.Migration.State = store.MigrationRestoredPaused + return s.SaveInstance(st) + } + prevLocal := localRestore + t.Cleanup(func() { localRestore = prevLocal }) + localRestore = func(*store.Store, string) error { return nil } + return f +} + +// options is a move to the fixture's destination. +func (f *moveFixture) options(allowNoRestore bool) migrateOptions { + return migrateOptions{ + to: "pm@dest", + remote: migrateRemote{Target: "pm@dest", Hvi: f.hvi, AllowNoRestore: allowNoRestore}, + fenceTimout: 0, + out: io.Discard, + } +} + +// srcRecord and dstRecord read the two sides' records of the move. +func (f *moveFixture) srcRecord(t *testing.T) *migrationRecord { + t.Helper() + rec, err := readMigrationRecord(f.src.InstanceMigrationRecord(f.name)) + if err != nil { + t.Fatal(err) + } + return rec +} + +func (f *moveFixture) dstRecord(t *testing.T, txid string) *migrationRecord { + t.Helper() + rec, _, err := findMigrationByTxID(f.dst, txid) + if err != nil { + t.Fatal(err) + } + return rec +} + +// localPeer runs the destination's verbs in this process against a store. +// drop names verbs whose reply is lost after they ran; down makes every +// call fail without running anything; answer replaces a verb's reply. +type localPeer struct { + s *store.Store + mu sync.Mutex + drop map[string]int + down bool + answer map[string]*migrateReply + calls []string +} + +func newLocalPeer(s *store.Store) *localPeer { + return &localPeer{s: s, drop: map[string]int{}, answer: map[string]*migrateReply{}} +} + +func (p *localPeer) call(_ context.Context, verb string, args []string, stdin io.Reader) (*migrateReply, error) { + p.mu.Lock() + p.calls = append(p.calls, verb) + down, canned := p.down, p.answer[verb] + dropped := p.drop[verb] > 0 + if dropped { + p.drop[verb]-- + } + p.mu.Unlock() + if down { + if stdin != nil { + _, _ = io.Copy(io.Discard, stdin) + } + return nil, fmt.Errorf("%w to %s: ssh: connect to host dest: Connection refused", errMoveNoAnswer, verb) + } + if canned != nil { + canned.Protocol = migrateProtocol + if !canned.OK { + return canned, &migrateRefusal{Verb: verb, Reply: canned} + } + return canned, nil + } + flags := map[string]string{} + for i := 0; i < len(args); i++ { + if i+1 < len(args) && args[i] != "--reserve" { + flags[args[i]] = args[i+1] + i++ + } else { + flags[args[i]] = "true" + } + } + var reply *migrateReply + switch verb { + case "migrate-probe": + reply = destProbe(p.s, flags["--name"], flags["--hvi"], flags["--gateway-sock"]) + case "migrate-receive": + if flags["--reserve"] == "true" { + var req migrateReserveRequest + if err := json.NewDecoder(stdin).Decode(&req); err != nil { + return nil, err + } + reply = destReserve(p.s, flags["--txid"], flags["--name"], req) + } else { + reply = destReceive(p.s, flags["--txid"], flags["--name"], stdin) + _, _ = io.Copy(io.Discard, stdin) + } + case "migrate-prepare": + reply = destPrepare(p.s, flags["--txid"]) + case "migrate-commit": + epoch, _ := strconv.ParseUint(flags["--epoch"], 10, 64) + reply = destCommit(p.s, flags["--txid"], epoch) + case "migrate-status": + reply = destStatus(p.s, flags["--txid"]) + case "migrate-abort": + reply = destAbort(p.s, flags["--txid"]) + default: + return nil, fmt.Errorf("no verb %s", verb) + } + reply.Protocol = migrateProtocol + // Through JSON, as over ssh. + data, _ := json.Marshal(reply) + reply, err := parseMigrateReply(data) + if err != nil { + return nil, err + } + if dropped { + return nil, fmt.Errorf("%w to %s: connection reset after the verb ran", errMoveNoAnswer, verb) + } + if !reply.OK { + return reply, &migrateRefusal{Verb: verb, Reply: reply} + } + return reply, nil +} + +func (p *localPeer) called(verb string) int { + p.mu.Lock() + defer p.mu.Unlock() + n := 0 + for _, c := range p.calls { + if c == verb { + n++ + } + } + return n +} + +// runHullEnv makes a re-exec'd test binary run hull's main, and fakeSSHEnv +// makes it stand in for ssh (see TestMoveOverFakeSSH). +const ( + runHullEnv = "HULL_TEST_RUN_HULL" + fakeSSHEnv = "HULL_TEST_FAKE_SSH" +) + +// testBinaryAsHull runs this test binary as ssh or as hull when the +// environment asks for it, and returns otherwise. Each TestMain calls it +// first. +func testBinaryAsHull() { + if os.Getenv(fakeSSHEnv) != "" { + // As ssh: the remote command, the last argument, runs through + // sh as a remote login shell would run it, and the hull it names + // is this binary again, as hull. + _ = os.Unsetenv(fakeSSHEnv) + _ = os.Setenv(runHullEnv, "1") + sh := exec.Command("/bin/sh", "-c", os.Args[len(os.Args)-1]) + sh.Stdin, sh.Stdout, sh.Stderr = os.Stdin, os.Stdout, os.Stderr + if err := sh.Run(); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + os.Exit(exitErr.ExitCode()) + } + os.Exit(255) + } + os.Exit(0) + } + if os.Getenv(runHullEnv) != "" { + // As hull: the destination's verbs, and their `hull restore`. + main() + os.Exit(0) + } +} diff --git a/cmd/hull/migrate_host_darwin.go b/cmd/hull/migrate_host_darwin.go new file mode 100644 index 00000000..206b0651 --- /dev/null +++ b/cmd/hull/migrate_host_darwin.go @@ -0,0 +1,32 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "golang.org/x/sys/unix" + +// hypervisorAvailable reports whether this Mac can run hvi guests: +// Hypervisor.framework answers kern.hv_support with 1 when the CPU and the +// OS support it. Whether this hull may use it is a matter of hvi's +// entitlement, which the restore itself will find out. +func hypervisorAvailable() (bool, string) { + v, err := unix.SysctlUint32("kern.hv_support") + if err != nil { + return false, "kern.hv_support: " + err.Error() + } + if v != 1 { + return false, "kern.hv_support is 0: Hypervisor.framework is not available" + } + return true, "Hypervisor.framework" +} diff --git a/cmd/hull/migrate_host_linux.go b/cmd/hull/migrate_host_linux.go new file mode 100644 index 00000000..ff81d907 --- /dev/null +++ b/cmd/hull/migrate_host_linux.go @@ -0,0 +1,29 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "os" + +// hypervisorAvailable reports whether this user can open /dev/kvm read-write, +// which is what hvi does first. A user not in the kvm group finds out here, +// before the source fences anything, rather than at the restore. +func hypervisorAvailable() (bool, string) { + f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) + if err != nil { + return false, "/dev/kvm: " + err.Error() + } + _ = f.Close() + return true, "/dev/kvm" +} diff --git a/cmd/hull/migrate_record.go b/cmd/hull/migrate_record.go new file mode 100644 index 00000000..9cf149b7 --- /dev/null +++ b/cmd/hull/migrate_record.go @@ -0,0 +1,448 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "syscall" + "time" + + "github.com/brig-sh/hull/pkg/store" +) + +// migrateRecordVersion is the version of migration.json this hull writes. +const migrateRecordVersion = 1 + +// The two sides of a move. +const ( + roleSource = "source" + roleDestination = "destination" +) + +// Source states, in the order a move passes them. docs/migration.md has the +// table. The fence comes before the transfer: in this version the bundle a +// move carries is the fenced checkpoint itself (there is no pre-copy), so +// there is nothing to send until the source has stopped for good. +const ( + srcPreflight = "preflight" + srcReserved = "reserved" + srcFenced = "fenced" + srcTransferring = "transferring" + srcTransferred = "transferred" + srcCommitting = "committing" + srcCommitted = "committed" +) + +// Destination states, in order. +const ( + dstReserved = "reserved" + dstReceiving = "receiving" + dstReceived = "received" + dstVerified = "verified" + dstRestoredPaused = "restored-paused" + dstRunning = "running" +) + +// migAborted ends a move on either side. +const migAborted = "aborted" + +// migrateTransitions is every state change a record may make, by role. A +// step that is re-run finds its record already in the target state and +// moves nothing; anything not listed here is refused, so a bug or a stale +// process cannot walk a record backwards past the fence or the commit. +var migrateTransitions = map[string]map[string][]string{ + roleSource: { + srcPreflight: {srcReserved, migAborted}, + srcReserved: {srcFenced, migAborted}, + srcFenced: {srcTransferring, migAborted}, + srcTransferring: {srcTransferred, migAborted}, + srcTransferred: {srcCommitting, migAborted}, + // committing may still abort: only once the destination has + // confirmed it is not running and has recorded the abort, which + // also makes it refuse a commit that arrives late. + srcCommitting: {srcCommitted, migAborted}, + }, + roleDestination: { + dstReserved: {dstReceiving, migAborted}, + dstReceiving: {dstReceived, migAborted}, + // A received copy that failed verification is received again. + dstReceived: {dstVerified, dstReceiving, migAborted}, + dstVerified: {dstRestoredPaused, migAborted}, + dstRestoredPaused: {dstRunning, migAborted}, + }, +} + +// migrateStateOrder ranks the states of role, for "has the move got this far +// yet". aborted ranks with none of them. +func migrateStateOrder(role, state string) int { + order := []string{srcPreflight, srcReserved, srcFenced, srcTransferring, srcTransferred, srcCommitting, srcCommitted} + if role == roleDestination { + order = []string{dstReserved, dstReceiving, dstReceived, dstVerified, dstRestoredPaused, dstRunning} + } + for i, s := range order { + if s == state { + return i + 1 + } + } + return 0 +} + +// migrationRecord is /migration.json, on both sides of a move. +type migrationRecord struct { + Version int `json:"version"` + TxID string `json:"txid"` + Role string `json:"role"` + Name string `json:"name"` + // Epoch is the owner epoch this move hands over: the source's + // OwnerEpoch plus one. The commit carries it, and the destination + // takes it as its own when it commits. + Epoch uint64 `json:"epoch"` + State string `json:"state"` + // Peer is the other host: USER@HOST of the destination on the source, + // the source's host name on the destination. + Peer string `json:"peer"` + + // Source only: how to reach the destination's hull again on a retry. + Remote *migrateRemote `json:"remote,omitempty"` + + // BundleDir and ManifestSHA256 name the fenced checkpoint the move + // carries, on either side once it is there. + BundleDir string `json:"bundleDir,omitempty"` + ManifestSHA256 string `json:"manifestSha256,omitempty"` + // Disk is the guest's disk as fenced (source) or received (destination). + Disk *migrateDisk `json:"disk,omitempty"` + // Transfer is what the stream carried and how fast. + Transfer *migrateTransfer `json:"transfer,omitempty"` + // Spec is what the source asked the destination to build (destination). + Spec *migrateSpec `json:"spec,omitempty"` + // Verify is hvi checkpoint-verify's verdict on the received bundle. + Verify string `json:"verify,omitempty"` + // Timings are the milliseconds each step took, by step name. + Timings map[string]int64 `json:"timingsMs,omitempty"` + + // Error is the last step's failure, kept for `hull migrate --status`. + Error string `json:"error,omitempty"` + History []migrateHistory `json:"history"` + Created time.Time `json:"createdAt"` + Updated time.Time `json:"updatedAt"` +} + +// migrateRemote is how the source reaches the destination's hull. +type migrateRemote struct { + Target string `json:"target"` + Store string `json:"store,omitempty"` + Hull string `json:"hull,omitempty"` + Hvi string `json:"hvi,omitempty"` + GatewaySock string `json:"gatewaySock,omitempty"` + // AllowNoRestore lets the move go on although an hvi on either side + // has no restore subcommand: for testing the transfer only. A move + // with it can fence the source and then have no way to bring the VM + // back on either host. + AllowNoRestore bool `json:"allowNoRestore,omitempty"` +} + +// migrateSpec is what the source asks the destination to build: the parts of +// its instance record and hvi command line that are not in the bundle. +type migrateSpec struct { + SourceHost string `json:"sourceHost"` + ImageDigest string `json:"imageDigest,omitempty"` + MAC string `json:"mac,omitempty"` + IP string `json:"ip,omitempty"` + SandboxID string `json:"sandboxId,omitempty"` + Agent bool `json:"agent,omitempty"` + Events bool `json:"events,omitempty"` + // Net is how the destination's guest NIC is backed: none, stub, or + // gateway at GatewaySock, a socket on the destination. + Net string `json:"net"` + GatewaySock string `json:"gatewaySock,omitempty"` + // Hvi is the destination's hvi, when the operator named one. + Hvi string `json:"hvi,omitempty"` +} + +// migrateReserveRequest is migrate-receive --reserve's standard input. +type migrateReserveRequest struct { + Epoch uint64 `json:"epoch"` + Spec migrateSpec `json:"spec"` +} + +// migrateDisk describes the disk image a move carries. +type migrateDisk struct { + Path string `json:"path"` + Size int64 `json:"size"` + // ModNs is the source file's modification time at the fence, in + // nanoseconds. A transfer that finds it changed refuses: something + // wrote the disk after the guest was fenced. + ModNs int64 `json:"modNs,omitempty"` + // SHA256 is over the sparse stream (see docs/migration.md), the same + // on both ends; DataBytes is how much of the image the stream carried. + SHA256 string `json:"sha256,omitempty"` + DataBytes int64 `json:"dataBytes,omitempty"` +} + +// migrateTransfer is the stream's size and speed. +type migrateTransfer struct { + Bytes int64 `json:"bytes"` + Seconds float64 `json:"seconds"` + BytesPerSec int64 `json:"bytesPerSec"` + StartedAt time.Time `json:"startedAt"` +} + +// migrateHistory is one state change. +type migrateHistory struct { + State string `json:"state"` + At time.Time `json:"at"` +} + +// txidPattern is a move's txid: 128 random bits in lower-case hex. +var txidPattern = regexp.MustCompile(`^[0-9a-f]{32}$`) + +// newMoveTxID draws a move's txid. 128 bits, not the 64 of a checkpoint's: +// a move's txid is matched across two hosts' stores and lives on in their +// tombstones, and a collision there would let one move's commit act on +// another's record. +func newMoveTxID() (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return hex.EncodeToString(b[:]), nil +} + +// checkTxID refuses anything that is not a move's txid. It goes into file +// names and remote command lines. +func checkTxID(txid string) error { + if !txidPattern.MatchString(txid) { + return fmt.Errorf("txid %q is not a move's txid (32 lower-case hex digits)", txid) + } + return nil +} + +// advance moves rec to state to, if the transition table allows it, and +// writes it. Already being in to is not a move and succeeds. +func (rec *migrationRecord) advance(path, to string) error { + if rec.State == to { + return nil + } + allowed := false + for _, next := range migrateTransitions[rec.Role][rec.State] { + if next == to { + allowed = true + break + } + } + if !allowed { + return fmt.Errorf("move %s: %s cannot go from %s to %s", rec.TxID, rec.Role, rec.State, to) + } + rec.State = to + rec.Error = "" + rec.History = append(rec.History, migrateHistory{State: to, At: time.Now().UTC()}) + return writeMigrationRecord(path, rec) +} + +// fail records why the current step stopped, without moving the state. +func (rec *migrationRecord) fail(path string, err error) error { + rec.Error = err.Error() + if werr := writeMigrationRecord(path, rec); werr != nil { + return errors.Join(err, fmt.Errorf("record the failure: %w", werr)) + } + return err +} + +// writeMigrationRecord publishes rec at path: temp file, fsync, rename, then +// an fsync of the directory. +// +// Each state is a promise to the other host. "fenced" says the source will +// not run the guest again; "committing" says it has let the destination +// take over. A record that a crash could lose or tear would break the +// promise after it was made, so a state counts as recorded only once it is +// on stable storage, and only then does the step that relies on it go on. +func writeMigrationRecord(path string, rec *migrationRecord) error { + rec.Version = migrateRecordVersion + rec.Updated = time.Now().UTC() + if rec.Created.IsZero() { + rec.Created = rec.Updated + } + data, err := json.MarshalIndent(rec, "", " ") + if err != nil { + return err + } + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, filepath.Base(path)+".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer func() { _ = os.Remove(tmpName) }() + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return err + } + if _, err := tmp.Write(append(data, '\n')); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmpName, path); err != nil { + return err + } + return syncDir(dir) +} + +// syncDir fsyncs a directory, so a rename or a create in it is durable. +func syncDir(dir string) error { + d, err := os.Open(dir) + if err != nil { + return err + } + defer func() { _ = d.Close() }() + if err := d.Sync(); err != nil && !errors.Is(err, syscall.EINVAL) && !errors.Is(err, syscall.ENOTSUP) { + return err + } + return nil +} + +// readMigrationRecord reads the record at path. +func readMigrationRecord(path string) (*migrationRecord, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var rec migrationRecord + if err := json.Unmarshal(data, &rec); err != nil { + return nil, fmt.Errorf("%s does not read as a move record: %w", path, err) + } + if rec.Version != migrateRecordVersion { + return nil, fmt.Errorf("%s is a version %d move record; this hull reads version %d", + path, rec.Version, migrateRecordVersion) + } + return &rec, nil +} + +// findMigrationByTxID finds the record of move txid in s: in an instance +// directory, or among the tombstones of aborted moves. It returns the +// record's path too. os.ErrNotExist when this store never saw the move. +func findMigrationByTxID(s *store.Store, txid string) (*migrationRecord, string, error) { + if err := checkTxID(txid); err != nil { + return nil, "", err + } + entries, err := os.ReadDir(filepath.Join(s.RootDir(), "instances")) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, "", err + } + for _, e := range entries { + if !e.IsDir() || store.ValidateInstanceID(e.Name()) != nil { + continue + } + path := s.InstanceMigrationRecord(e.Name()) + rec, err := readMigrationRecord(path) + if err == nil && rec.TxID == txid { + return rec, path, nil + } + } + path := filepath.Join(s.MigrationTombstoneDir(), txid+".json") + rec, err := readMigrationRecord(path) + if err != nil { + return nil, "", err + } + return rec, path, nil +} + +// lockMove takes /migrate.lock, which one step of a move holds on +// its host. wait > 0 waits that long for it; 0 refuses at once. +// +// This is not the migration.lock of checkpoint and restore: a move's steps +// run `hull restore` as a child process, which takes that one, and a move +// must keep everything else out while it does. The two nest in one order +// only: migrate.lock first. +func lockMove(dir string, wait time.Duration) (func(), error) { + f, err := os.OpenFile(filepath.Join(dir, "migrate.lock"), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("open the move lock: %w", err) + } + deadline := time.Now().Add(wait) + for { + err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if err == nil { + break + } + if !errors.Is(err, syscall.EWOULDBLOCK) { + _ = f.Close() + return nil, fmt.Errorf("lock the move: %w", err) + } + if time.Now().After(deadline) { + _ = f.Close() + return nil, errMoveBusy + } + time.Sleep(100 * time.Millisecond) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + _ = f.Close() + }, nil +} + +// errMoveBusy is another step of the same move holding its lock. +var errMoveBusy = errors.New("another step of this move is running on this host") + +// moveRefusal is the refusal for an instance that a move has in hand, for a +// verb that would run it or change what a move carries: "" when none does. +func moveRefusal(s *store.Store, id, verb string) error { + rec, err := readMigrationRecord(s.InstanceMigrationRecord(id)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("%s of %s refused: %w", verb, id, err) + } + switch { + case rec.Role == roleSource && rec.State == srcCommitted: + return fmt.Errorf("instance %s moved to %s (move %s) and runs there at epoch %d; what is left "+ + "of it here is stale. Moving it back is a new `hull migrate` run on %s", id, rec.Peer, + rec.TxID, rec.Epoch, rec.Peer) + case rec.Role == roleSource && rec.State != migAborted: + return fmt.Errorf("instance %s is being moved to %s (move %s, %s): no %s until the move "+ + "commits, or `hull migrate --abort %s` ends it", id, rec.Peer, rec.TxID, rec.State, verb, rec.TxID) + case rec.Role == roleDestination && rec.State != dstRunning && rec.State != migAborted: + return fmt.Errorf("instance %s is arriving from %s (move %s, %s): only the move's commit "+ + "runs it, and `hull migrate --abort %s` on the source ends it", id, rec.Peer, rec.TxID, + rec.State, rec.TxID) + } + return nil +} + +// refuseDuringMove refuses verb on an instance a move has in hand. +func refuseDuringMove(s *store.Store, id, verb string) error { + return moveRefusal(s, id, verb) +} + +// moveCommittedAway reports whether the instance is the stale source of a +// committed move, which may be removed like any stopped instance. +func moveCommittedAway(s *store.Store, id string) bool { + rec, err := readMigrationRecord(s.InstanceMigrationRecord(id)) + return err == nil && rec.Role == roleSource && rec.State == srcCommitted +} diff --git a/cmd/hull/migrate_record_test.go b/cmd/hull/migrate_record_test.go new file mode 100644 index 00000000..3657c01c --- /dev/null +++ b/cmd/hull/migrate_record_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +// Every transition the table allows is taken, every other one refused, and +// re-entering the current state is a no-op that still succeeds. +func TestMoveTransitions(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "migration.json") + states := map[string][]string{ + roleSource: {srcPreflight, srcReserved, srcFenced, srcTransferring, srcTransferred, srcCommitting, + srcCommitted, migAborted}, + roleDestination: {dstReserved, dstReceiving, dstReceived, dstVerified, dstRestoredPaused, dstRunning, + migAborted}, + } + for role, all := range states { + for _, from := range all { + for _, to := range all { + rec := &migrationRecord{TxID: strings.Repeat("a", 32), Role: role, State: from} + err := rec.advance(path, to) + want := from == to || slices.Contains(migrateTransitions[role][from], to) + if (err == nil) != want { + t.Errorf("%s %s -> %s: err %v, want allowed=%v", role, from, to, err, want) + } + if err == nil { + got, rerr := readMigrationRecord(path) + if from != to && (rerr != nil || got.State != to) { + t.Errorf("%s %s -> %s: recorded %+v %v", role, from, to, got, rerr) + } + } + } + } + } + // The ones that matter most, spelled out. + for _, c := range []struct{ role, from, to string }{ + {roleSource, srcCommitted, migAborted}, // no abort after the commit + {roleSource, srcCommitting, srcTransferred}, // no way back past the commit point + {roleSource, srcFenced, srcReserved}, // nor past the fence + {roleDestination, dstRunning, migAborted}, // a running destination is never aborted + {roleDestination, migAborted, dstRunning}, // an aborted one never runs + {roleSource, srcReserved, srcTransferring}, // no transfer without a fence + } { + rec := &migrationRecord{Role: c.role, State: c.from} + if err := rec.advance(path, c.to); err == nil { + t.Errorf("%s %s -> %s was allowed", c.role, c.from, c.to) + } + } +} + +func TestMoveRecordJSON(t *testing.T) { + path := filepath.Join(t.TempDir(), "migration.json") + rec := &migrationRecord{TxID: strings.Repeat("0f", 16), Role: roleSource, Name: "vm1", Epoch: 3, + State: srcFenced, Peer: "pm@rp2", Remote: &migrateRemote{Target: "pm@rp2", Store: "/s"}, + Disk: &migrateDisk{Path: "/d", Size: 10, ModNs: 5, SHA256: "ab", DataBytes: 4}, + Transfer: &migrateTransfer{Bytes: 100, Seconds: 0.5, BytesPerSec: 200}, + Timings: map[string]int64{"fence": 12}, + } + if err := writeMigrationRecord(path, rec); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("record mode: %v %v", info, err) + } + data, _ := os.ReadFile(path) + var raw map[string]any + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatal(err) + } + for _, key := range []string{"version", "txid", "role", "name", "epoch", "state", "peer", "remote", "disk", + "transfer", "timingsMs", "history", "createdAt", "updatedAt"} { + if _, ok := raw[key]; !ok { + t.Errorf("migration.json has no %q: %s", key, data) + } + } + got, err := readMigrationRecord(path) + if err != nil || got.TxID != rec.TxID || got.Epoch != 3 || got.Disk.SHA256 != "ab" || got.Remote.Store != "/s" { + t.Errorf("read back %+v, %v", got, err) + } + // Another version is refused, not misread. + _ = os.WriteFile(path, bytes.Replace(data, []byte(`"version": 1`), []byte(`"version": 2`), 1), 0o600) + if _, err := readMigrationRecord(path); err == nil { + t.Error("a version 2 record was read") + } + entries, _ := os.ReadDir(filepath.Dir(path)) + if len(entries) != 1 { + t.Errorf("the record's directory has %d entries (a temp file left behind?)", len(entries)) + } +} + +func TestMoveTxID(t *testing.T) { + a, err := newMoveTxID() + if err != nil || checkTxID(a) != nil { + t.Fatalf("txid %q: %v", a, err) + } + b, _ := newMoveTxID() + if a == b { + t.Error("two txids are the same") + } + for _, bad := range []string{"", "abc", strings.Repeat("A", 32), strings.Repeat("a", 31) + "/", "../../x"} { + if checkTxID(bad) == nil { + t.Errorf("txid %q accepted", bad) + } + } +} diff --git a/cmd/hull/migrate_source.go b/cmd/hull/migrate_source.go new file mode 100644 index 00000000..bb26a73e --- /dev/null +++ b/cmd/hull/migrate_source.go @@ -0,0 +1,1023 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strconv" + "strings" + "time" + + "github.com/urfave/cli/v3" + + "github.com/brig-sh/hull/pkg/store" +) + +// `hull migrate NAME --to USER@HOST` moves a running hvi instance to another +// host over ssh. This file is the source's half: it drives every step, and +// the destination answers through the hidden verbs in migrate_dest.go. +// docs/migration.md is the protocol, its states and its failure table. + +// migrateOptions are the flags of one `hull migrate` run. +type migrateOptions struct { + to string + remote migrateRemote + txid string + dryRun bool + fenceTimout time.Duration + // out takes the progress lines. + out io.Writer +} + +func migrateCommand() *cli.Command { + return &cli.Command{ + Name: "migrate", + Usage: "move a running hvi instance to another host over ssh; the source is fenced " + + "before the destination may run it, and never runs it again once the move commits", + ArgsUsage: "NAME | TXID", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "to", Usage: "USER@HOST to move the instance to, over ssh"}, + &cli.StringFlag{Name: "remote-store", Usage: "the destination hull's --store-dir (default: its own default)"}, + &cli.StringFlag{Name: "remote-hull", Usage: "path of hull on the destination (default: hull on its PATH)"}, + &cli.StringFlag{Name: "remote-hvi", Usage: "path of hvi on the destination (default: the one its hull would run)"}, + &cli.StringFlag{Name: "remote-gateway-sock", Usage: "a network gateway on the destination, with the " + + "source's subnet, for the moved guest's NIC (default: none; a guest that ran on a gateway gets " + + "hvi's no-egress stub NIC instead)"}, + &cli.StringFlag{Name: "txid", Usage: "resume the move with this txid from where it stopped, or start one with it"}, + &cli.BoolFlag{Name: "dry-run", Usage: "check both hosts and print the plan; change nothing on either"}, + &cli.BoolFlag{Name: "abort", Usage: "abort the move NAME or TXID names, if it has not committed, and run the instance here again"}, + &cli.BoolFlag{Name: "status", Usage: "print the move's record on both hosts and what it means"}, + &cli.IntFlag{Name: "timeout", Value: 120, Usage: "seconds to wait for hvi to write the fenced checkpoint"}, + &cli.BoolFlag{Name: "allow-no-restore", Usage: "testing only: go on although an hvi on either side has no " + + "restore subcommand. The source is fenced and nothing can run the guest again until both can restore"}, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Len() != 1 { + return errors.New("name the instance (or, for --abort and --status, the move's txid)") + } + s, err := globalStore(cmd) + if err != nil { + return err + } + arg := cmd.Args().First() + if cmd.Bool("abort") || cmd.Bool("status") { + if cmd.Bool("abort") && cmd.Bool("status") { + return errors.New("--abort and --status do not go together") + } + name, err := moveInstanceName(s, arg) + if err != nil { + return err + } + if cmd.Bool("status") { + return sourceStatus(ctx, s, name, os.Stdout, nil) + } + return sourceAbort(ctx, s, name, os.Stderr, nil) + } + opt := migrateOptions{ + to: cmd.String("to"), + remote: migrateRemote{ + Target: cmd.String("to"), Store: cmd.String("remote-store"), Hull: cmd.String("remote-hull"), + Hvi: cmd.String("remote-hvi"), GatewaySock: cmd.String("remote-gateway-sock"), + AllowNoRestore: cmd.Bool("allow-no-restore"), + }, + txid: cmd.String("txid"), + dryRun: cmd.Bool("dry-run"), + fenceTimout: time.Duration(cmd.Int("timeout")) * time.Second, + out: os.Stderr, + } + return sourceMigrate(ctx, s, arg, opt, nil) + }, + } +} + +// moveInstanceName resolves the argument of --abort or --status: a move's +// txid recorded in this store, or an instance name. +func moveInstanceName(s *store.Store, arg string) (string, error) { + if txidPattern.MatchString(arg) { + if rec, _, err := findMigrationByTxID(s, arg); err == nil { + return rec.Name, nil + } + } + if err := store.ValidateInstanceID(arg); err != nil { + return "", err + } + return arg, nil +} + +// peerFor is the destination of rec, over ssh unless the caller gave one. +func peerFor(rec *migrationRecord, given movePeer) movePeer { + if given != nil { + return given + } + return &sshPeer{target: rec.Remote.Target, remote: *rec.Remote} +} + +// sourceMigrate is `hull migrate NAME --to TARGET`: a new move, or the move +// --txid names resumed from its recorded state. peer is nil outside tests. +func sourceMigrate(ctx context.Context, s *store.Store, name string, opt migrateOptions, peer movePeer) error { + if err := store.ValidateInstanceID(name); err != nil { + return err + } + state, err := s.GetInstance(name) + if err != nil { + return fmt.Errorf("instance not found: %s", name) + } + unlock, err := lockMove(s.InstanceDir(name), 0) + if errors.Is(err, errMoveBusy) { + return fmt.Errorf("another `hull migrate` of %s is running", name) + } + if err != nil { + return err + } + defer unlock() + + path := s.InstanceMigrationRecord(name) + rec, err := readMigrationRecord(path) + switch { + case errors.Is(err, os.ErrNotExist): + rec = nil + case err != nil: + return err + case rec.Role != roleSource: + if rec.State != dstRunning { + return moveRefusal(s, name, "migrate") + } + // It arrived here by a move that committed: moving it on is a new + // move, which replaces that record. + rec = nil + case rec.State == migAborted: + rec = nil + case opt.txid != "" && opt.txid != rec.TxID: + return fmt.Errorf("instance %s has move %s on record (%s), not %s", name, rec.TxID, rec.State, opt.txid) + case opt.txid == "" && rec.State == srcCommitted: + return moveRefusal(s, name, "migrate") + case opt.txid == "": + return fmt.Errorf("instance %s is in move %s to %s, stopped at %s%s. Resume it with "+ + "`hull migrate %s --txid %s`, or end it with `hull migrate --abort %s`", name, rec.TxID, + rec.Peer, rec.State, describeRecordError(rec), name, rec.TxID, rec.TxID) + case opt.to != "" && opt.to != rec.Remote.Target: + return fmt.Errorf("move %s goes to %s, not %s", rec.TxID, rec.Remote.Target, opt.to) + } + + probed := false + if rec == nil { + if rec, err = startMove(ctx, s, state, opt, peer); err != nil || rec == nil { + return err + } + probed = true + } else { + _, _ = fmt.Fprintf(opt.out, "Resuming move %s of %s to %s at %s\n", rec.TxID, name, rec.Peer, rec.State) + } + m := &moveSource{s: s, name: name, path: path, rec: rec, peer: peerFor(rec, peer), out: opt.out, + fenceTimeout: opt.fenceTimout, probed: probed} + return m.run(ctx) +} + +// describeRecordError is ", because ..." for a record whose last step +// failed. +func describeRecordError(rec *migrationRecord) string { + if rec.Error == "" { + return "" + } + return " (" + rec.Error + ")" +} + +// startMove checks the instance and the destination and, unless this is a +// dry run, writes the new move's record in preflight. It returns nil and no +// error for a dry run. +func startMove(ctx context.Context, s *store.Store, state *store.InstanceState, opt migrateOptions, + peer movePeer) (*migrationRecord, error) { + if opt.to == "" { + return nil, errors.New("--to USER@HOST is required") + } + if err := checkSSHTarget(opt.to); err != nil { + return nil, err + } + txid := opt.txid + if txid == "" { + var err error + if txid, err = newMoveTxID(); err != nil { + return nil, err + } + } else if err := checkTxID(txid); err != nil { + return nil, err + } + spec, err := sourcePreflight(s, state, opt) + if err != nil { + return nil, fmt.Errorf("cannot move %s: %w", state.ID, err) + } + rec := &migrationRecord{ + TxID: txid, Role: roleSource, Name: state.ID, Epoch: state.OwnerEpoch + 1, State: srcPreflight, + Peer: opt.to, Remote: &opt.remote, Spec: spec, + } + // Probed before the record is written: a destination that cannot take + // the instance leaves nothing behind on either host to resume or abort. + probe, err := probeDestination(ctx, peerFor(rec, peer), rec, s, state) + if err != nil { + return nil, fmt.Errorf("cannot move %s: %w", state.ID, err) + } + if opt.dryRun { + printMovePlan(opt.out, rec, state, probe) + return nil, nil + } + rec.History = []migrateHistory{{State: srcPreflight, At: time.Now().UTC()}} + if err := writeMigrationRecord(s.InstanceMigrationRecord(state.ID), rec); err != nil { + return nil, err + } + _, _ = fmt.Fprintf(opt.out, "Move %s of %s to %s, epoch %d\n", txid, state.ID, opt.to, rec.Epoch) + return rec, nil +} + +// sourcePreflight checks that the instance can be moved at all, and returns +// what the destination must build. Nothing has changed anywhere when it +// fails. +func sourcePreflight(s *store.Store, state *store.InstanceState, opt migrateOptions) (*migrateSpec, error) { + if err := requireRunningHvi(state, "migrate"); err != nil { + return nil, err + } + if err := hviCheckpointable(state.CmdLine); err != nil { + return nil, err + } + if m := state.Migration; m != nil && m.State == store.MigrationIncoming { + return nil, errors.New("it is arriving by a move that has not committed") + } + switch vm := hviVMState(state); vm { + case "running", "paused": + default: + return nil, fmt.Errorf("its VM is %s, and a move takes a running or paused one", vm) + } + // A command line restore cannot carry cannot be moved either: the + // destination brings the instance up with a restore. + if _, err := hviRestoreArgv(state.CmdLine, "/check", "/check", "/check"); err != nil { + return nil, err + } + if !opt.remote.AllowNoRestore { + has, err := hviHasRestore(state.CmdLine[0]) + if err != nil { + return nil, err + } + if !has { + return nil, fmt.Errorf("its hvi (%s) has no restore subcommand, so an aborted move could "+ + "never run it here again. Use an hvi that can restore (or, to test the transfer "+ + "alone, --allow-no-restore)", state.CmdLine[0]) + } + } + host, _ := os.Hostname() + spec := &migrateSpec{SourceHost: host, ImageDigest: state.ImageDigest, MAC: state.MAC, IP: state.IP, + Hvi: opt.remote.Hvi, Net: "none"} + argv := state.CmdLine + if v, ok := flagValue(argv, "--net-mac"); ok { + spec.MAC = v + } + if v, ok := flagValue(argv, "--sandbox-id"); ok { + spec.SandboxID = v + } + _, spec.Agent = flagValue(argv, "--agent-sock") + _, spec.Events = flagValue(argv, "--events") + switch { + case slices.Contains(argv, "--net-gateway"): + if opt.remote.GatewaySock != "" { + spec.Net, spec.GatewaySock = "gateway", opt.remote.GatewaySock + } else { + spec.Net = "stub" + _, _ = fmt.Fprintf(opt.out, "warning: %s runs on a network gateway here, and no --remote-gateway-sock "+ + "was given: on the destination its NIC is hvi's stub, which reaches nothing\n", state.ID) + } + case slices.Contains(argv, "--net-stub") || slices.Contains(argv, "--net"): + spec.Net = "stub" + } + return spec, nil +} + +// probeDestination runs migrate-probe and holds the answer to what the move +// needs. +func probeDestination(ctx context.Context, peer movePeer, rec *migrationRecord, s *store.Store, + state *store.InstanceState) (*migrateProbe, error) { + args := []string{"--name", rec.Name} + if rec.Remote.Hvi != "" { + args = append(args, "--hvi", rec.Remote.Hvi) + } + if rec.Remote.GatewaySock != "" { + args = append(args, "--gateway-sock", rec.Remote.GatewaySock) + } + reply, err := peer.call(ctx, "migrate-probe", args, nil) + if err != nil { + return nil, fmt.Errorf("probe %s: %w", rec.Peer, err) + } + p := reply.Probe + if p == nil { + return nil, fmt.Errorf("probe %s: the reply has no probe", rec.Peer) + } + var why []string + if p.OS != runtime.GOOS || p.Arch != runtime.GOARCH { + why = append(why, fmt.Sprintf("it is %s/%s and this host %s/%s: a checkpoint restores only on "+ + "the same kind of host", p.OS, p.Arch, runtime.GOOS, runtime.GOARCH)) + } + if !p.Hypervisor { + why = append(why, "it cannot run a VM: "+p.HypervisorDetail) + } + switch { + case p.Hvi == "" || p.HviError != "": + why = append(why, "its hvi: "+p.HviError) + case !p.HviControlFlags: + why = append(why, fmt.Sprintf("its hvi (%s) predates the control socket", p.Hvi)) + case !p.HviRestore && !rec.Remote.AllowNoRestore: + why = append(why, fmt.Sprintf("its hvi (%s) has no restore subcommand", p.Hvi)) + } + switch { + case p.NameTaken && p.NameTxID != rec.TxID && p.NameMove == roleSource+" "+srcCommitted: + // Moving an instance back to where it ran before: that host keeps + // the stale copy the first move left, under the same name. + why = append(why, fmt.Sprintf("it still has the stale copy of %s that move %s left when it moved "+ + "away; `hull rm %s` there removes it", rec.Name, p.NameTxID, rec.Name)) + case p.NameTaken && p.NameTxID != rec.TxID: + why = append(why, fmt.Sprintf("an instance named %s already exists there", rec.Name)) + } + if rec.Remote.GatewaySock != "" && !p.GatewayOK { + why = append(why, fmt.Sprintf("its gateway %s does not answer: %s", rec.Remote.GatewaySock, p.GatewayDetail)) + } + if need := moveSpaceNeeded(s, state); p.FreeBytes >= 0 && p.FreeBytes < need { + why = append(why, fmt.Sprintf("its store %s has %s free and the move needs about %s", + p.StoreDir, humanBytes(p.FreeBytes), humanBytes(need))) + } + if len(why) > 0 { + return p, fmt.Errorf("%s cannot take %s: %s", rec.Peer, rec.Name, strings.Join(why, "; ")) + } + return p, nil +} + +// moveSpaceNeeded estimates what a move writes on the destination: the +// disk's allocated bytes and the guest's whole RAM (the bundle holds at most +// that), plus a tenth. +func moveSpaceNeeded(s *store.Store, state *store.InstanceState) int64 { + var need int64 + if disk, ok := flagValue(state.CmdLine, "--disk"); ok { + need += fileAllocated(disk) + } + if mib, ok := flagValue(state.CmdLine, "--mem-mib"); ok { + if n, err := strconv.ParseInt(mib, 10, 64); err == nil { + need += n << 20 + } + } else if data, err := os.ReadFile(s.InstanceProfilePath(state.ID)); err == nil { + var p struct { + RAMRegions []struct { + Size int64 `json:"size"` + } `json:"ram_regions"` + } + if json.Unmarshal(data, &p) == nil { + for _, r := range p.RAMRegions { + need += r.Size + } + } + } + return need + need/10 +} + +// humanBytes renders n for a person. +func humanBytes(n int64) string { + switch { + case n >= 1<<30: + return fmt.Sprintf("%.1f GiB", float64(n)/(1<<30)) + case n >= 1<<20: + return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20)) + } + return fmt.Sprintf("%d bytes", n) +} + +// printMovePlan is the dry run's answer. +func printMovePlan(w io.Writer, rec *migrationRecord, state *store.InstanceState, p *migrateProbe) { + _, _ = fmt.Fprintf(w, "Dry run: %s can move to %s\n", state.ID, rec.Peer) + _, _ = fmt.Fprintf(w, " destination: hull %s on %s/%s, hvi %s (%s), store %s with %s free\n", + p.HullVersion, p.OS, p.Arch, p.Hvi, p.HviVersion, p.StoreDir, humanBytes(p.FreeBytes)) + _, _ = fmt.Fprintf(w, " network: %s", rec.Spec.Net) + if rec.Spec.GatewaySock != "" { + _, _ = fmt.Fprintf(w, " at %s", rec.Spec.GatewaySock) + } + _, _ = fmt.Fprintf(w, ", MAC %s\n", rec.Spec.MAC) + _, _ = fmt.Fprintf(w, " epoch %d -> %d\n", state.OwnerEpoch, rec.Epoch) + _, _ = fmt.Fprintln(w, " steps: reserve the name there, fence this VM (it never runs here again unless the "+ + "move is aborted), stream the checkpoint and the disk, verify, restore paused there, commit") + if !p.HviRestore { + _, _ = fmt.Fprintln(w, " warning: the destination's hvi has no restore subcommand; the move will stop "+ + "before the commit with the source fenced") + } +} + +// moveSource drives one move from its recorded state to committed. +type moveSource struct { + s *store.Store + name string + path string + rec *migrationRecord + peer movePeer + out io.Writer + fenceTimeout time.Duration + // probed is set when this run has already probed the destination. + probed bool +} + +func (m *moveSource) logf(format string, args ...any) { + _, _ = fmt.Fprintf(m.out, format+"\n", args...) +} + +func (m *moveSource) timing(step string, since time.Time) { + if m.rec.Timings == nil { + m.rec.Timings = map[string]int64{} + } + m.rec.Timings[step] = time.Since(since).Milliseconds() +} + +// run takes the move from its recorded state to committed, one step at a +// time. Each step records its state only once its effect is durable, and +// each finds its effect already there when it is run again, so a run that +// stops anywhere can be run again with the same txid. +func (m *moveSource) run(ctx context.Context) error { + for { + var err error + switch m.rec.State { + case srcPreflight: + err = m.reserve(ctx) + case srcReserved: + err = m.fence() + case srcFenced: + err = m.rec.advance(m.path, srcTransferring) + case srcTransferring: + err = m.transfer(ctx) + case srcTransferred: + err = m.prepare(ctx) + case srcCommitting: + err = m.commit(ctx) + case srcCommitted: + return m.finish() + default: + return fmt.Errorf("move %s is %s", m.rec.TxID, m.rec.State) + } + if err != nil { + return m.stopped(err) + } + } +} + +// stopped records why the move stopped and says what the operator can do. +func (m *moveSource) stopped(err error) error { + err = m.rec.fail(m.path, err) + fenced := migrateStateOrder(roleSource, m.rec.State) >= migrateStateOrder(roleSource, srcFenced) + next := fmt.Sprintf("`hull migrate %s --txid %s` resumes it from here", m.name, m.rec.TxID) + if m.rec.State == srcCommitting { + return fmt.Errorf("move %s stopped at %s: %w\nThe source stays fenced. The destination may "+ + "have committed; %s and records the commit, or `hull migrate --abort %s` ends the move "+ + "if the destination confirms it is not running", m.rec.TxID, m.rec.State, err, next, m.rec.TxID) + } + if fenced { + return fmt.Errorf("move %s stopped at %s: %w\nThe source stays fenced and will not run "+ + "until the move commits or is aborted. %s; `hull migrate --abort %s` brings the instance "+ + "back here", m.rec.TxID, m.rec.State, err, next, m.rec.TxID) + } + return fmt.Errorf("move %s stopped at %s: %w\nThe instance still runs here. %s; `hull migrate "+ + "--abort %s` releases the destination", m.rec.TxID, m.rec.State, err, next, m.rec.TxID) +} + +// reserve probes the destination and claims the instance's name there. +func (m *moveSource) reserve(ctx context.Context) error { + state, err := m.s.GetInstance(m.name) + if err != nil { + return err + } + if !m.probed { + if _, err := probeDestination(ctx, m.peer, m.rec, m.s, state); err != nil { + return err + } + } + req, err := json.Marshal(migrateReserveRequest{Epoch: m.rec.Epoch, Spec: *m.rec.Spec}) + if err != nil { + return err + } + if _, err := m.peer.call(ctx, "migrate-receive", []string{"--txid", m.rec.TxID, "--name", m.name, "--reserve"}, + bytes.NewReader(req)); err != nil { + return fmt.Errorf("reserve %s on %s: %w", m.name, m.rec.Peer, err) + } + m.logf("Reserved %s on %s", m.name, m.rec.Peer) + return m.rec.advance(m.path, srcReserved) +} + +// fence takes the checkpoint that stops the guest for good, with the move's +// txid and epoch, and hands it to the move: the instance record says +// migrating, which a restore refuses. A fence already taken (a re-run after +// a crash) is found and adopted. +func (m *moveSource) fence() error { + started := time.Now() + state, err := m.s.GetInstance(m.name) + if err != nil { + return err + } + if !fencedFor(state, m.rec.TxID) { + if err := m.fenceNow(state); err != nil { + return err + } + m.timing("fence", started) + } + state, err = m.s.GetInstance(m.name) + if err != nil { + return err + } + state.Migration.State = store.MigrationMigrating + state.Migration.MoveTxID = m.rec.TxID + state.Migration.Peer = m.rec.Peer + if err := m.s.SaveInstance(state); err != nil { + return err + } + disk, ok := flagValue(state.CmdLine, "--disk") + if !ok { + return errors.New("the instance has no --disk") + } + size, mod, err := fileModNs(disk) + if err != nil { + return err + } + if want, ok, err := bundleDiskSize(state.Migration.BundleDir); err != nil { + return err + } else if ok && want != size { + return fmt.Errorf("the disk %s is %d bytes, but the fenced checkpoint says %d", disk, size, want) + } + m.rec.BundleDir = state.Migration.BundleDir + m.rec.ManifestSHA256 = state.Migration.ManifestSHA256 + m.rec.Disk = &migrateDisk{Path: disk, Size: size, ModNs: mod} + m.logf("Fenced %s: checkpoint %s", m.name, m.rec.BundleDir) + return m.rec.advance(m.path, srcFenced) +} + +// fencedFor reports whether the record already holds the fenced checkpoint +// of move txid. +func fencedFor(state *store.InstanceState, txid string) bool { + mig := state.Migration + return mig != nil && mig.TxID == txid && + (mig.State == store.MigrationFenced || mig.State == store.MigrationMigrating) +} + +// fenceNow takes the fenced checkpoint, or adopts one hvi took for this move +// when hull did not live to record it. +func (m *moveSource) fenceNow(state *store.InstanceState) error { + if err := requireRunningHvi(state, "migrate"); err != nil { + return fmt.Errorf("%w, and no fenced checkpoint for move %s is recorded", err, m.rec.TxID) + } + unlock, err := lockInstanceMigration(m.s, m.name) + if err != nil { + return err + } + defer unlock() + root, ok := flagValue(state.CmdLine, "--checkpoint-root") + if !ok { + return errors.New("the instance has no checkpoint root") + } + dir := filepath.Join(root, m.rec.TxID) + switch vm := hviVMState(state); vm { + case store.MigrationFenced: + // Fenced, and not on record: either this move's fence, whose + // reply hull did not live to read, or someone else's. + return m.adoptFence(state, dir) + case "running", "paused": + // A capture that failed or was cut short leaves no manifest; its + // leftovers would make the directory unusable for the retry. + if _, err := os.Stat(filepath.Join(dir, "manifest.json")); errors.Is(err, os.ErrNotExist) { + emptyDir(dir) + } + default: + return fmt.Errorf("its VM is %s; a move fences a running or paused one", vm) + } + m.logf("Fencing %s (checkpoint with then=fence)", m.name) + ck, err := hviCheckpointLocked(m.s, state, hviThenFence, "", m.rec.TxID, m.rec.Epoch, m.fenceTimeout) + if err != nil { + return err + } + m.logf("Checkpoint written in %s", describeCheckpointStats(hviThenFence, ck.Stats)) + return nil +} + +// adoptFence records the fenced bundle hvi wrote for this move into dir when +// the checkpoint's reply was lost, after hvi has verified it. +func (m *moveSource) adoptFence(state *store.InstanceState, dir string) error { + sum, err := fileSHA256(filepath.Join(dir, "manifest.json")) + if err != nil { + return fmt.Errorf("the VM is fenced, but not by this move (no bundle at %s): `hull migrate --abort %s`", + dir, m.rec.TxID) + } + if _, err := hviCheckpointVerify(state.CmdLine[0], dir); err != nil { + return err + } + state.Migration = &store.MigrationState{State: store.MigrationFenced, BundleDir: dir, TxID: m.rec.TxID, + ManifestSHA256: sum} + if err := m.s.SaveInstance(state); err != nil { + return err + } + m.logf("Adopted the fenced checkpoint %s", dir) + return nil +} + +// checkDiskUnchanged refuses a disk that is not the one fenced. +func (m *moveSource) checkDiskUnchanged() error { + d := m.rec.Disk + size, mod, err := fileModNs(d.Path) + if err != nil { + return err + } + if size != d.Size || mod != d.ModNs { + return fmt.Errorf("the disk %s changed after the fence (size %d -> %d, mtime %d -> %d): something "+ + "wrote it while the guest was fenced, and the checkpoint no longer matches it", d.Path, d.Size, + size, d.ModNs, mod) + } + return nil +} + +// bundleFiles lists what of the bundle a move sends: manifest.json and +// every component it lists, by bare name. +func bundleFiles(dir string) ([]string, error) { + comps, err := readManifestComponents(dir) + if err != nil { + return nil, err + } + files := []string{"manifest.json"} + for _, c := range comps { + files = append(files, c.Path) + } + return files, nil +} + +// transfer streams the bundle and the disk to migrate-receive, which ends +// verified on the destination. A destination already verified is not sent +// it again. +func (m *moveSource) transfer(ctx context.Context) error { + if reply, err := m.peer.call(ctx, "migrate-status", []string{"--txid", m.rec.TxID}, nil); err == nil && + reply.Record != nil && + migrateStateOrder(roleDestination, reply.Record.State) >= migrateStateOrder(roleDestination, dstVerified) { + if err := m.compareReceived(reply.Record); err != nil { + return err + } + return m.rec.advance(m.path, srcTransferred) + } + if err := m.checkDiskUnchanged(); err != nil { + return err + } + files, err := bundleFiles(m.rec.BundleDir) + if err != nil { + return err + } + comps, err := readManifestComponents(m.rec.BundleDir) + if err != nil { + return err + } + m.logf("Sending %s to %s", m.name, m.rec.Peer) + started := time.Now() + pr, pw := io.Pipe() + sw := newMigrateStreamWriter(pw) + var diskSum streamSum + sent := make(chan error, 1) + go func() { + err := func() error { + if err := sw.header(streamHeader{TxID: m.rec.TxID, ManifestSHA256: m.rec.ManifestSHA256}); err != nil { + return err + } + for _, f := range files { + sum, err := sw.file("bundle/"+f, filepath.Join(m.rec.BundleDir, f)) + if err != nil { + return err + } + // The source checks its own bundle against the manifest + // as it reads it: a bundle damaged here is not sent. + if f == "manifest.json" && sum.SHA256 != m.rec.ManifestSHA256 { + return fmt.Errorf("manifest.json has sha256 %s, but hvi reported %s at the fence", sum.SHA256, m.rec.ManifestSHA256) + } + for _, c := range comps { + if c.Path == f && (c.SHA256 != sum.SHA256 || c.Size != sum.Size) { + return fmt.Errorf("the bundle's %s does not match its manifest", f) + } + } + } + var err error + if diskSum, err = sw.sparse("disk", m.rec.Disk.Path); err != nil { + return err + } + return sw.finish() + }() + _ = pw.CloseWithError(err) + sent <- err + }() + reply, err := m.peer.call(ctx, "migrate-receive", []string{"--txid", m.rec.TxID, "--name", m.name}, pr) + _ = pr.CloseWithError(errors.New("the destination stopped reading")) + sendErr := <-sent + if sendErr != nil && !errors.Is(sendErr, io.ErrClosedPipe) && err == nil { + err = fmt.Errorf("send: %w", sendErr) + } + if err != nil { + if sendErr != nil && !errors.Is(sendErr, io.ErrClosedPipe) && !strings.Contains(err.Error(), sendErr.Error()) { + err = fmt.Errorf("%w (sending: %v)", err, sendErr) + } + return fmt.Errorf("transfer to %s: %w", m.rec.Peer, err) + } + took := time.Since(started) + if err := m.checkDiskUnchanged(); err != nil { + return err + } + if reply.Record == nil { + return errors.New("the destination's receive reply has no record") + } + if sendErr == nil { + m.rec.Disk.SHA256 = diskSum.SHA256 + m.rec.Disk.DataBytes = diskSum.DataBytes + m.rec.Transfer = &migrateTransfer{Bytes: sw.written(), Seconds: took.Seconds(), StartedAt: started.UTC(), + BytesPerSec: int64(float64(sw.written()) / max(took.Seconds(), 1e-3))} + m.timing("transfer", started) + } + if err := m.compareReceived(reply.Record); err != nil { + return err + } + if t := m.rec.Transfer; t != nil { + m.logf("Sent %s in %.1f s (%s/s); disk %s of %s carried; destination: %s", humanBytes(t.Bytes), + t.Seconds, humanBytes(t.BytesPerSec), humanBytes(m.rec.Disk.DataBytes), humanBytes(m.rec.Disk.Size), + reply.Record.Verify) + } + return m.rec.advance(m.path, srcTransferred) +} + +// compareReceived checks the destination's record of what it received +// against what the source sent. +func (m *moveSource) compareReceived(dst *migrationRecord) error { + if dst.TxID != m.rec.TxID || dst.Epoch != m.rec.Epoch { + return fmt.Errorf("the destination recorded move %s epoch %d, not %s epoch %d", dst.TxID, dst.Epoch, + m.rec.TxID, m.rec.Epoch) + } + if dst.ManifestSHA256 != m.rec.ManifestSHA256 { + return fmt.Errorf("the destination has manifest sha256 %s, the source %s", dst.ManifestSHA256, m.rec.ManifestSHA256) + } + if dst.Disk == nil || dst.Disk.Size != m.rec.Disk.Size || + (m.rec.Disk.SHA256 != "" && dst.Disk.SHA256 != m.rec.Disk.SHA256) { + return errors.New("the destination's disk is not the one the source sent") + } + if m.rec.Disk.SHA256 == "" { + m.rec.Disk.SHA256 = dst.Disk.SHA256 + m.rec.Disk.DataBytes = dst.Disk.DataBytes + } + return nil +} + +// prepare has the destination restore the checkpoint paused. Only once that +// worked does the source record committing: a restore the destination +// refuses is learned while an abort can still bring the instance back here. +func (m *moveSource) prepare(ctx context.Context) error { + started := time.Now() + m.logf("Restoring %s paused on %s", m.name, m.rec.Peer) + if _, err := m.peer.call(ctx, "migrate-prepare", []string{"--txid", m.rec.TxID}, nil); err != nil { + if refusalCode(err) == codeRestoreRefused { + return fmt.Errorf("%s could not restore the checkpoint: %w", m.rec.Peer, err) + } + return err + } + m.timing("prepare", started) + // The source's commit point. Durable before the commit is sent: + // from here an abort needs the destination's word that it is not + // running. + return m.rec.advance(m.path, srcCommitting) +} + +// commit tells the destination to run the VM, and records the commit once +// the destination says it has taken over this move at this epoch. A reply +// that never comes leaves the source fenced in committing; running the move +// again sends the commit again, which the destination answers the same way. +func (m *moveSource) commit(ctx context.Context) error { + started := time.Now() + reply, err := m.peer.call(ctx, "migrate-commit", + []string{"--txid", m.rec.TxID, "--epoch", strconv.FormatUint(m.rec.Epoch, 10)}, nil) + if reply == nil || reply.Record == nil || reply.Record.State != dstRunning || + reply.Record.TxID != m.rec.TxID || reply.Record.Epoch != m.rec.Epoch { + if err == nil { + err = errors.New("the destination's reply does not say it runs this move") + } + return fmt.Errorf("commit on %s: %w", m.rec.Peer, err) + } + if err != nil { + m.logf("warning: %s took the move over, but: %v", m.rec.Peer, err) + } + m.timing("commit", started) + return m.rec.advance(m.path, srcCommitted) +} + +// finish is what follows the commit on the source, and is safe to run +// again: the instance record marks the checkpoint stale, and the fenced hvi, +// which nothing will run again, is stopped. +func (m *moveSource) finish() error { + state, err := m.s.GetInstance(m.name) + if err != nil { + return err + } + if state.Migration != nil && state.Migration.State != store.MigrationMigrated { + state.Migration.State = store.MigrationMigrated + state.Migration.MoveTxID = m.rec.TxID + state.Migration.Peer = m.rec.Peer + if err := m.s.SaveInstance(state); err != nil { + return err + } + } + if state.Status == "running" && state.PID > 0 && processIsAVMM(state.PID, state) { + if err := stopInstanceIn(m.s, m.name, 10); err != nil { + m.logf("warning: stopping the fenced VMM here: %v", err) + } + } + m.logf("Moved %s to %s: it runs there at epoch %d. Its checkpoint here is stale", m.name, m.rec.Peer, + m.rec.Epoch) + if t := m.rec.Timings; t != nil { + m.logf("Timings (ms): fence %d, transfer %d, restore paused %d, commit %d", t["fence"], t["transfer"], + t["prepare"], t["commit"]) + } + return nil +} + +// sourceAbort is `hull migrate --abort`: before the commit, and only once +// the destination has confirmed that it is not running the VM and will +// refuse the commit, it ends the move and brings the instance back here. +func sourceAbort(ctx context.Context, s *store.Store, name string, w io.Writer, peer movePeer) error { + unlock, err := lockMove(s.InstanceDir(name), 0) + if errors.Is(err, errMoveBusy) { + return fmt.Errorf("a `hull migrate` of %s is running; stop it first", name) + } + if err != nil { + return err + } + defer unlock() + path := s.InstanceMigrationRecord(name) + rec, err := readMigrationRecord(path) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("instance %s has no move on record", name) + } + if err != nil { + return err + } + if rec.Role != roleSource { + return fmt.Errorf("instance %s arrived by move %s; abort a move on its source host", name, rec.TxID) + } + switch rec.State { + case srcCommitted: + return fmt.Errorf("move %s committed: %s runs on %s at epoch %d. Moving it back is a new "+ + "`hull migrate` run on %s", rec.TxID, name, rec.Peer, rec.Epoch, rec.Peer) + case migAborted: + _, _ = fmt.Fprintf(w, "Move %s was already aborted\n", rec.TxID) + return nil + } + state, err := s.GetInstance(name) + if err != nil { + return err + } + fenced := migrateStateOrder(roleSource, rec.State) >= migrateStateOrder(roleSource, srcFenced) || + fencedFor(state, rec.TxID) + p := peerFor(rec, peer) + reply, err := p.call(ctx, "migrate-abort", []string{"--txid", rec.TxID}, nil) + switch { + case err != nil && refusalCode(err) == codeRunning: + return fmt.Errorf("the destination %s says move %s committed there and runs %s: it cannot be "+ + "aborted. `hull migrate %s --txid %s` records the commit here", rec.Peer, rec.TxID, name, name, rec.TxID) + case err != nil && fenced: + return rec.fail(path, fmt.Errorf("cannot abort move %s: %w\nWithout the destination's word that it "+ + "is not running %s and will refuse the commit, the source stays fenced: running it here could "+ + "put it on two hosts. Retry when %s answers", rec.TxID, err, name, rec.Peer)) + case err != nil: + _, _ = fmt.Fprintf(w, "warning: %s did not confirm the abort (%v); it may still hold a reservation for %s, "+ + "which `hull migrate --abort %s` again releases\n", rec.Peer, err, name, rec.TxID) + } + if reply != nil && reply.Error != "" { + _, _ = fmt.Fprintf(w, "warning: %s: %s\n", rec.Peer, reply.Error) + } + if err := rec.advance(path, migAborted); err != nil { + return err + } + if !fenced { + _, _ = fmt.Fprintf(w, "Aborted move %s; %s never stopped running here\n", rec.TxID, name) + return nil + } + // The destination is out of it. The instance is the fenced checkpoint + // again, and is restored from it here. + if state.Migration != nil && state.Migration.TxID == rec.TxID { + state.Migration.State = store.MigrationFenced + if err := s.SaveInstance(state); err != nil { + return err + } + } + if state.Status == "running" && state.PID > 0 && processIsAVMM(state.PID, state) { + if err := stopInstanceIn(s, name, 10); err != nil { + return fmt.Errorf("aborted move %s, but stopping the fenced VMM failed: %w; `hull stop %s`, "+ + "then `hull restore %s`", rec.TxID, err, name, name) + } + } + _, _ = fmt.Fprintf(w, "Aborted move %s; restoring %s here from its fenced checkpoint\n", rec.TxID, name) + if err := localRestore(s, name); err != nil { + return fmt.Errorf("aborted move %s, and %s stays stopped with its fenced checkpoint: %w. `hull "+ + "restore %s` brings it back once that is fixed", rec.TxID, name, err, name) + } + return nil +} + +// localRestore runs `hull restore --detach NAME` here, as a child for the +// same reason as on the destination. +var localRestore = func(s *store.Store, name string) error { + self, err := os.Executable() + if err != nil { + return err + } + var buf bytes.Buffer + c := exec.Command(self, "--store-dir", s.RootDir(), "restore", "--detach", name) + c.Stdout = &buf + c.Stderr = &buf + if err := c.Run(); err != nil { + return fmt.Errorf("hull restore: %s", strings.TrimSpace(buf.String())) + } + return nil +} + +// sourceStatus is `hull migrate --status`: the record here, what it means, +// and what the destination says. +func sourceStatus(ctx context.Context, s *store.Store, name string, w io.Writer, peer movePeer) error { + rec, err := readMigrationRecord(s.InstanceMigrationRecord(name)) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("instance %s has no move on record", name) + } + if err != nil { + return err + } + _, _ = fmt.Fprintf(w, "Move %s of %s: %s, %s %s, epoch %d\n", rec.TxID, rec.Name, rec.State, + map[string]string{roleSource: "to", roleDestination: "from"}[rec.Role], rec.Peer, rec.Epoch) + for _, h := range rec.History { + _, _ = fmt.Fprintf(w, " %s %s\n", h.At.Local().Format("2006-01-02 15:04:05.000"), h.State) + } + if rec.Error != "" { + _, _ = fmt.Fprintf(w, " last error: %s\n", rec.Error) + } + if t := rec.Transfer; t != nil { + _, _ = fmt.Fprintf(w, " transfer: %s in %.1f s (%s/s)\n", humanBytes(t.Bytes), t.Seconds, humanBytes(t.BytesPerSec)) + } + _, _ = fmt.Fprintf(w, " %s\n", explainMoveState(rec)) + if rec.Role != roleSource || rec.Remote == nil { + return nil + } + reply, err := peerFor(rec, peer).call(ctx, "migrate-status", []string{"--txid", rec.TxID}, nil) + switch { + case err != nil && refusalCode(err) == codeUnknownTxID: + _, _ = fmt.Fprintf(w, " destination %s: has no record of this move\n", rec.Peer) + case err != nil: + _, _ = fmt.Fprintf(w, " destination %s: %v\n", rec.Peer, err) + default: + _, _ = fmt.Fprintf(w, " destination %s: %s", rec.Peer, reply.Record.State) + if reply.VM != "" { + _, _ = fmt.Fprintf(w, ", VM %s", reply.VM) + } + if reply.Record.Error != "" { + _, _ = fmt.Fprintf(w, ", last error: %s", reply.Record.Error) + } + _, _ = fmt.Fprintln(w) + } + return nil +} + +// explainMoveState says what a record's state means for the instance. +func explainMoveState(rec *migrationRecord) string { + if rec.Role == roleDestination { + switch rec.State { + case dstRunning: + return "The move committed: the instance runs here, at this epoch." + case migAborted: + return "The move was aborted; nothing of it runs here." + } + return "The instance is arriving; only the source's commit runs it here." + } + switch rec.State { + case srcPreflight, srcReserved: + return "The instance still runs here; nothing was fenced." + case srcCommitting: + return "The source is fenced and has let the destination take over. Run the move again to " + + "learn the commit, or abort it if the destination confirms it is not running." + case srcCommitted: + return fmt.Sprintf("The instance runs on %s at epoch %d. Its checkpoint here is stale and nothing "+ + "here may run it; moving it back is a new move from %s.", rec.Peer, rec.Epoch, rec.Peer) + case migAborted: + return "The move was aborted; the instance was handed back to this host." + } + return "The source is fenced and stays so until the move commits or is aborted." +} diff --git a/cmd/hull/migrate_ssh.go b/cmd/hull/migrate_ssh.go new file mode 100644 index 00000000..d9eaf5ba --- /dev/null +++ b/cmd/hull/migrate_ssh.go @@ -0,0 +1,243 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "regexp" + "strings" + "time" + "unicode" +) + +// migrateSSHEnv names a program to run instead of ssh, with the same +// arguments. The tests point it at a script that runs the remote command +// locally against a second store, so a whole move runs on one machine with +// no network (docs/migration.md, "Testing a move without a second host"). +const migrateSSHEnv = "HULL_MIGRATE_SSH" + +// migrateProtocol is the version of the remote verbs' requests and replies. +// The probe reports it, and a source refuses a destination that speaks +// another. +const migrateProtocol = 1 + +// migrateReply is what every remote verb prints on stdout, and nothing else: +// one JSON object. ok:false is a definite answer from the destination, with +// a code the source branches on; an exit, an ssh failure or output that is +// not one of these is no answer at all, and the source must not read +// anything into it. +type migrateReply struct { + Protocol int `json:"protocol"` + OK bool `json:"ok"` + Code string `json:"code,omitempty"` + Error string `json:"error,omitempty"` + Record *migrationRecord `json:"record,omitempty"` + Probe *migrateProbe `json:"probe,omitempty"` + // VM is what the destination's hvi says its VM is doing, when asked. + VM string `json:"vm,omitempty"` +} + +// migrateProbe is what migrate-probe reports about the destination. +type migrateProbe struct { + HullVersion string `json:"hullVersion"` + OS string `json:"os"` + Arch string `json:"arch"` + Hvi string `json:"hvi,omitempty"` + HviVersion string `json:"hviVersion,omitempty"` + HviControlFlags bool `json:"hviControlFlags"` + HviRestore bool `json:"hviRestore"` + HviError string `json:"hviError,omitempty"` + Hypervisor bool `json:"hypervisor"` + HypervisorDetail string `json:"hypervisorDetail,omitempty"` + StoreDir string `json:"storeDir"` + FreeBytes int64 `json:"freeBytes"` + NameTaken bool `json:"nameTaken"` + NameTxID string `json:"nameTxid,omitempty"` + // NameMove is the role and state of that move here, "source + // committed" for the stale copy a move away left behind. + NameMove string `json:"nameMove,omitempty"` + GatewayOK bool `json:"gatewayOk,omitempty"` + GatewayDetail string `json:"gatewayDetail,omitempty"` +} + +// Reply codes a source branches on. +const ( + codeUnknownTxID = "unknown_txid" + codeAborted = "aborted" + codeRunning = "running" + codeNameTaken = "name_taken" + codeBusy = "busy" + codeRestoreRefused = "restore_refused" + codeWrongEpoch = "wrong_epoch" + codeNotReady = "not_ready" + codeVerifyFailed = "verify_failed" + codeFailed = "failed" +) + +// errMoveNoAnswer is a remote verb that produced no reply: ssh could not +// connect, the remote hull is missing or crashed, or the connection broke. +// For a step that changes state on the other side that is not a "no": the +// step may have happened. +var errMoveNoAnswer = errors.New("no answer from the destination") + +// migrateRefusal is the destination answering ok:false. +type migrateRefusal struct { + Verb string + Reply *migrateReply +} + +func (e *migrateRefusal) Error() string { + return fmt.Sprintf("the destination refused %s: %s (%s)", e.Verb, e.Reply.Error, e.Reply.Code) +} + +// refusalCode returns the code of a migrateRefusal in err's chain, or "". +func refusalCode(err error) string { + var r *migrateRefusal + if errors.As(err, &r) { + return r.Reply.Code + } + return "" +} + +// movePeer runs one remote verb on the destination. stdin, when not nil, is +// copied to the verb's standard input. The source drives every step through +// this, so the tests can put the destination in the same process, drop a +// reply, or answer for it. +type movePeer interface { + call(ctx context.Context, verb string, args []string, stdin io.Reader) (*migrateReply, error) +} + +// sshPeer runs `hull ` on the destination over ssh. +type sshPeer struct { + target string + remote migrateRemote +} + +// validSSHTarget is [USER@]HOST, nothing ssh could read as an option. +var validSSHTarget = regexp.MustCompile(`^([A-Za-z0-9._][A-Za-z0-9._-]*@)?[A-Za-z0-9._:\[\]-]+$`) + +// checkSSHTarget refuses a --to that is not [USER@]HOST. One that starts +// with '-' would be an ssh option (-oProxyCommand=... runs a program), and +// one with spaces or control characters is not a host. +func checkSSHTarget(target string) error { + if strings.HasPrefix(target, "-") || !validSSHTarget.MatchString(target) { + return fmt.Errorf("--to %q is not USER@HOST", target) + } + return nil +} + +// shellQuote quotes s for a POSIX shell. ssh hands the remote side one +// string, which the remote login shell splits, so every argument hull sends +// is quoted: a store path with a space, or a name a shell would expand, +// arrives as the one argument it was. +func shellQuote(s string) string { + if s != "" && strings.IndexFunc(s, func(r rune) bool { + return r >= unicode.MaxASCII || (!unicode.IsLetter(r) && !unicode.IsDigit(r) && !strings.ContainsRune("@%+=:,./_-", r)) + }) < 0 { + return s + } + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + +// remoteCommand is the command line the destination's shell runs for verb. +func (p *sshPeer) remoteCommand(verb string, args []string) string { + hull := p.remote.Hull + if hull == "" { + hull = "hull" + } + argv := []string{hull} + if p.remote.Store != "" { + argv = append(argv, "--store-dir", p.remote.Store) + } + argv = append(argv, verb) + argv = append(argv, args...) + quoted := make([]string, len(argv)) + for i, a := range argv { + quoted[i] = shellQuote(a) + } + return strings.Join(quoted, " ") +} + +// sshArgv is the ssh command line for verb: batch mode (a password prompt +// would hang a move with the source fenced), keepalives so a dead link ends +// the step instead of blocking it, no tty, and "--" so the target can never +// be read as an option. +func (p *sshPeer) sshArgv(verb string, args []string) []string { + ssh := os.Getenv(migrateSSHEnv) + if ssh == "" { + ssh = "ssh" + } + return []string{ssh, "-T", "-o", "BatchMode=yes", "-o", "ServerAliveInterval=15", + "-o", "ServerAliveCountMax=4", "--", p.target, p.remoteCommand(verb, args)} +} + +func (p *sshPeer) call(ctx context.Context, verb string, args []string, stdin io.Reader) (*migrateReply, error) { + argv := p.sshArgv(verb, args) + c := exec.CommandContext(ctx, argv[0], argv[1:]...) + var stdout, stderr bytes.Buffer + c.Stdout = &stdout + c.Stderr = &stderr + c.Stdin = stdin + // A destination that dies mid-stream must not leave the copy blocked + // on a pipe nobody reads. + c.WaitDelay = 5 * time.Second + runErr := c.Run() + reply, perr := parseMigrateReply(stdout.Bytes()) + if perr != nil { + detail := strings.TrimSpace(stderr.String()) + if len(detail) > 2000 { + detail = detail[len(detail)-2000:] + } + if runErr == nil { + runErr = perr + } + if detail != "" { + detail = "; it said: " + detail + } + return nil, fmt.Errorf("%w to %s on %s: %v%s", errMoveNoAnswer, verb, p.target, runErr, detail) + } + if !reply.OK { + return reply, &migrateRefusal{Verb: verb, Reply: reply} + } + return reply, nil +} + +// parseMigrateReply reads the one JSON object a remote verb prints. Output +// that is anything else is not a reply. +func parseMigrateReply(out []byte) (*migrateReply, error) { + out = bytes.TrimSpace(out) + if len(out) == 0 { + return nil, errors.New("it printed nothing") + } + var reply migrateReply + dec := json.NewDecoder(bytes.NewReader(out)) + if err := dec.Decode(&reply); err != nil { + return nil, fmt.Errorf("its output is not a reply: %w", err) + } + if dec.More() { + return nil, errors.New("its output has more than one reply") + } + if reply.Protocol != migrateProtocol { + return nil, fmt.Errorf("it speaks move protocol %d, this hull %d", reply.Protocol, migrateProtocol) + } + return &reply, nil +} diff --git a/cmd/hull/migrate_ssh_test.go b/cmd/hull/migrate_ssh_test.go new file mode 100644 index 00000000..fe76437f --- /dev/null +++ b/cmd/hull/migrate_ssh_test.go @@ -0,0 +1,84 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// The remote command line survives the remote shell: every argument +// arrives as the one it was, whatever it holds. +func TestSSHRemoteCommandQuoting(t *testing.T) { + dir := t.TempDir() + args := []string{"--name", "it's $HOME `id` \"q\" *", "--txid", "", "a\nb", "--x=$(touch " + dir + "/pwned)"} + // "hull" is a shell function that prints each argument it gets: a + // new script would cost a first-run scan on macOS. + p := &sshPeer{target: "pm@h", remote: migrateRemote{Hull: "printargs", Store: dir + "/my store"}} + cmdline := p.remoteCommand("migrate-status", args) + out, err := exec.Command("/bin/sh", "-c", + `printargs() { for a; do printf '%s\0' "$a"; done; }; `+cmdline).Output() + if err != nil { + t.Fatalf("%s: %v", cmdline, err) + } + got := strings.Split(strings.TrimSuffix(string(out), "\x00"), "\x00") + want := append([]string{"--store-dir", dir + "/my store", "migrate-status"}, args...) + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("the remote shell saw %q, want %q", got, want) + } + if _, err := os.Stat(filepath.Join(dir, "pwned")); err == nil { + t.Error("an argument ran a command") + } + argv := p.sshArgv("migrate-status", nil) + if argv[0] != "ssh" || !strings.Contains(strings.Join(argv, " "), "BatchMode=yes") || + argv[len(argv)-3] != "--" || argv[len(argv)-2] != "pm@h" { + t.Errorf("ssh argv %q", argv) + } + t.Setenv(migrateSSHEnv, "/x/fake-ssh") + if argv := p.sshArgv("migrate-status", nil); argv[0] != "/x/fake-ssh" { + t.Errorf("%s is not used: %q", migrateSSHEnv, argv) + } +} + +func TestSSHTarget(t *testing.T) { + for _, ok := range []string{"pm@rp2", "rp2", "pm@10.0.0.2", "pm@rp2.tail-net.ts.net", "pm@[fe80::1]"} { + if err := checkSSHTarget(ok); err != nil { + t.Errorf("%s: %v", ok, err) + } + } + for _, bad := range []string{"", "-oProxyCommand=sh", "pm@-oProxyCommand=sh", "pm@host;rm -rf /", "pm @host", + "pm@host\n", "pm@host'"} { + if err := checkSSHTarget(bad); err == nil { + t.Errorf("%q accepted", bad) + } + } +} + +// Anything that is not exactly one reply is no answer. +func TestParseMigrateReply(t *testing.T) { + for _, bad := range []string{"", "Instance vm1 resumed\n{}", `{"protocol":1,"ok":true}{"protocol":1}`, + `{"protocol":2,"ok":true}`, "bash: hull: command not found"} { + if _, err := parseMigrateReply([]byte(bad)); err == nil { + t.Errorf("%q parsed as a reply", bad) + } + } + r, err := parseMigrateReply([]byte(`{"protocol":1,"ok":false,"code":"running","error":"x"}` + "\n")) + if err != nil || r.OK || r.Code != codeRunning { + t.Errorf("%+v %v", r, err) + } +} diff --git a/cmd/hull/migrate_stream.go b/cmd/hull/migrate_stream.go new file mode 100644 index 00000000..9ce32a3b --- /dev/null +++ b/cmd/hull/migrate_stream.go @@ -0,0 +1,553 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "io" + "os" + "regexp" + "strings" + "syscall" + + "golang.org/x/sys/unix" +) + +// The move stream is what `hull migrate` writes into ssh and `hull +// migrate-receive` reads on the other end: the bundle's files, then the +// disk, then a trailer with every file's sha256. +// +// It is hull's own, not tar. Go's archive/tar cannot write a sparse entry +// (its writer drops the GNU sparse records), and the disk is a mostly empty +// image of several GiB: written dense, a move would carry every hole as +// zeros. GNU tar and bsdtar each do sparse files, but not the same way on +// both OSes, and hull would be parsing their output to learn what arrived. +// So the format is a few framed records (docs/migration.md, "The stream"): +// +// "HULLMIG1" +// 'H' u32 len, JSON {txid, manifestSha256} +// 'F' u32 len, JSON {name, size}, then size bytes (a bundle file) +// 'S' u32 len, JSON {name, size}, then extents: (the disk) +// u64 offset, u64 length, length bytes ... u64 0, u64 0 +// 'T' u32 len, JSON {files: [{name, size, sha256}]} +// +// Integers are big-endian. The receiver checks the trailer against what it +// hashed itself and refuses a stream that ends without one, so a cut +// connection never passes for a short file. +const migrateStreamMagic = "HULLMIG1" + +// Limits the receiver holds a stream to before it allocates or writes +// anything for a record. +const ( + migrateMaxHeader = 64 << 10 + migrateMaxFiles = 32 + migrateMaxFileSize = 64 << 30 // hvi's own bound on a bundle component + migrateMaxDiskSize = 4 << 40 + // migrateChunk is the zero-detection grain: the sender leaves out a + // chunk that is all zeros and the receiver leaves one as a hole. + migrateChunk = 64 << 10 +) + +// migrateStreamName is a name a stream may carry: "disk", or a bare file +// name below "bundle/". Anything else, a path above all, is refused. +var migrateStreamName = regexp.MustCompile(`^(disk|bundle/[A-Za-z0-9_-][A-Za-z0-9._-]{0,127})$`) + +// streamHeader is the 'H' record. +type streamHeader struct { + TxID string `json:"txid"` + ManifestSHA256 string `json:"manifestSha256"` +} + +// streamEntry is the JSON of an 'F' or 'S' record. +type streamEntry struct { + Name string `json:"name"` + Size int64 `json:"size"` +} + +// streamSum is one file's line in the trailer, and what a side computed. +type streamSum struct { + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + // DataBytes is, for the disk, the bytes its extents carried. + DataBytes int64 `json:"dataBytes,omitempty"` +} + +// streamTrailer is the 'T' record. +type streamTrailer struct { + Files []streamSum `json:"files"` +} + +// countingWriter counts what passes through it. +type countingWriter struct { + w io.Writer + n int64 +} + +func (c *countingWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.n += int64(n) + return n, err +} + +// migrateStreamWriter writes one stream. +type migrateStreamWriter struct { + cw *countingWriter + bw *bufio.Writer + sums []streamSum +} + +func newMigrateStreamWriter(w io.Writer) *migrateStreamWriter { + cw := &countingWriter{w: w} + return &migrateStreamWriter{cw: cw, bw: bufio.NewWriterSize(cw, 1<<20)} +} + +// written is the bytes handed to the underlying writer so far. +func (sw *migrateStreamWriter) written() int64 { return sw.cw.n + int64(sw.bw.Buffered()) } + +func (sw *migrateStreamWriter) record(kind byte, v any) error { + js, err := json.Marshal(v) + if err != nil { + return err + } + var hdr [5]byte + hdr[0] = kind + binary.BigEndian.PutUint32(hdr[1:], uint32(len(js))) + if _, err := sw.bw.Write(hdr[:]); err != nil { + return err + } + _, err = sw.bw.Write(js) + return err +} + +// header writes the magic and the 'H' record. +func (sw *migrateStreamWriter) header(h streamHeader) error { + if _, err := sw.bw.WriteString(migrateStreamMagic); err != nil { + return err + } + return sw.record('H', h) +} + +// file writes the regular file at path as name. The size is taken when the +// file is opened, and a file that then yields a different number of bytes +// fails the stream rather than sending a record that lies about its length. +func (sw *migrateStreamWriter) file(name, path string) (streamSum, error) { + f, err := os.Open(path) + if err != nil { + return streamSum{}, err + } + defer func() { _ = f.Close() }() + info, err := f.Stat() + if err != nil { + return streamSum{}, err + } + if !info.Mode().IsRegular() { + return streamSum{}, fmt.Errorf("%s is not a regular file", path) + } + if err := sw.record('F', streamEntry{Name: name, Size: info.Size()}); err != nil { + return streamSum{}, err + } + h := sha256.New() + n, err := io.Copy(sw.bw, io.TeeReader(io.LimitReader(f, info.Size()), h)) + if err != nil { + return streamSum{}, err + } + if n != info.Size() { + return streamSum{}, fmt.Errorf("%s shrank from %d to %d bytes while it was sent", path, info.Size(), n) + } + sum := streamSum{Name: name, Size: n, SHA256: hex.EncodeToString(h.Sum(nil))} + sw.sums = append(sw.sums, sum) + return sum, nil +} + +// sparse writes the file at path as name, carrying only its data: the +// extents SEEK_DATA finds, less every 64 KiB chunk in them that is all +// zeros. The sha256 is over the extent records as sent (offset, length and +// bytes, then the terminator), which both ends compute without reading the +// holes; docs/migration.md says why it is not the image's plain sha256. +func (sw *migrateStreamWriter) sparse(name, path string) (streamSum, error) { + f, err := os.Open(path) + if err != nil { + return streamSum{}, err + } + defer func() { _ = f.Close() }() + info, err := f.Stat() + if err != nil { + return streamSum{}, err + } + if !info.Mode().IsRegular() { + return streamSum{}, fmt.Errorf("%s is not a regular file", path) + } + size := info.Size() + extents, err := dataExtents(f, size) + if err != nil { + return streamSum{}, err + } + if err := sw.record('S', streamEntry{Name: name, Size: size}); err != nil { + return streamSum{}, err + } + h := sha256.New() + var data int64 + buf := make([]byte, 1<<20) + for _, ext := range extents { + for off := ext.off; off < ext.end; { + n := min(int64(len(buf)), ext.end-off) + if _, err := f.ReadAt(buf[:n], off); err != nil { + return streamSum{}, fmt.Errorf("read %s at %d: %w", path, off, err) + } + for c := int64(0); c < n; c += migrateChunk { + chunk := buf[c:min(c+migrateChunk, n)] + if isZero(chunk) { + continue + } + if err := sw.extent(h, off+c, chunk); err != nil { + return streamSum{}, err + } + data += int64(len(chunk)) + } + off += n + } + } + if err := sw.extent(h, 0, nil); err != nil { + return streamSum{}, err + } + // A disk that changed size while it was read was not the fenced one. + if after, err := f.Stat(); err != nil || after.Size() != size { + return streamSum{}, fmt.Errorf("%s changed size while it was sent", path) + } + sum := streamSum{Name: name, Size: size, SHA256: hex.EncodeToString(h.Sum(nil)), DataBytes: data} + sw.sums = append(sw.sums, sum) + return sum, nil +} + +// extent writes one extent record, data nil being the terminator. +func (sw *migrateStreamWriter) extent(h hash.Hash, off int64, data []byte) error { + var hdr [16]byte + binary.BigEndian.PutUint64(hdr[:8], uint64(off)) + binary.BigEndian.PutUint64(hdr[8:], uint64(len(data))) + h.Write(hdr[:]) + h.Write(data) + if _, err := sw.bw.Write(hdr[:]); err != nil { + return err + } + _, err := sw.bw.Write(data) + return err +} + +// finish writes the trailer and flushes. +func (sw *migrateStreamWriter) finish() error { + if err := sw.record('T', streamTrailer{Files: sw.sums}); err != nil { + return err + } + return sw.bw.Flush() +} + +// isZero reports whether b is all zero bytes. +func isZero(b []byte) bool { + for len(b) >= 8 { + if binary.NativeEndian.Uint64(b) != 0 { + return false + } + b = b[8:] + } + for _, c := range b { + if c != 0 { + return false + } + } + return true +} + +// extent is a byte range [off, end) of a file. +type extent struct{ off, end int64 } + +// dataExtents lists the ranges of f that hold data, by SEEK_DATA and +// SEEK_HOLE (APFS and ext4 both answer them). A filesystem that does not +// know them gets the whole file as one extent, and the zero-chunk test +// still keeps the holes out of the stream; it just has to read them. +func dataExtents(f *os.File, size int64) ([]extent, error) { + fd := int(f.Fd()) + var out []extent + for off := int64(0); off < size; { + start, err := unix.Seek(fd, off, unix.SEEK_DATA) + if err != nil { + if errors.Is(err, syscall.ENXIO) { + break // only a hole is left + } + if errors.Is(err, syscall.EINVAL) && off == 0 { + return []extent{{0, size}}, nil + } + return nil, fmt.Errorf("find data in %s: %w", f.Name(), err) + } + end, err := unix.Seek(fd, start, unix.SEEK_HOLE) + if err != nil { + return nil, fmt.Errorf("find a hole in %s: %w", f.Name(), err) + } + end = min(end, size) + if end <= start { + break + } + out = append(out, extent{start, end}) + off = end + } + return out, nil +} + +// streamSink tells the reader where each named file goes. It returns an open +// file the reader writes into and closes. +type streamSink func(name string, size int64) (*os.File, error) + +// migrateStreamResult is what a reader received. +type migrateStreamResult struct { + Header streamHeader + Files []streamSum + // Bytes is the whole stream's length, magic and trailer included. + Bytes int64 +} + +// countingReader counts what passes through it. +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + +// readMigrateStream reads a stream from r, writing each file through sink, +// and checks the trailer against what it computed. Nothing in the stream is +// trusted: names, sizes and extents are checked against the limits above +// before anything is written, and a stream with no trailer, a trailer that +// disagrees, or bytes after it is refused. +func readMigrateStream(r io.Reader, sink streamSink) (*migrateStreamResult, error) { + cr := &countingReader{r: r} + br := bufio.NewReaderSize(cr, 1<<20) + magic := make([]byte, len(migrateStreamMagic)) + if _, err := io.ReadFull(br, magic); err != nil { + return nil, fmt.Errorf("the stream ended before it began: %w", err) + } + if string(magic) != migrateStreamMagic { + return nil, fmt.Errorf("the stream does not start with %s; is the other side a hull of this version?", migrateStreamMagic) + } + res := &migrateStreamResult{} + kind, js, err := readStreamRecord(br) + if err != nil { + return nil, err + } + if kind != 'H' { + return nil, fmt.Errorf("the stream starts with a %q record, not its header", kind) + } + if err := strictUnmarshal(js, &res.Header); err != nil { + return nil, fmt.Errorf("stream header: %w", err) + } + seen := map[string]bool{} + for { + kind, js, err := readStreamRecord(br) + if err != nil { + return nil, err + } + switch kind { + case 'F', 'S': + var e streamEntry + if err := strictUnmarshal(js, &e); err != nil { + return nil, fmt.Errorf("stream entry: %w", err) + } + if err := checkStreamEntry(e, kind, seen); err != nil { + return nil, err + } + out, err := sink(e.Name, e.Size) + if err != nil { + return nil, err + } + var sum streamSum + if kind == 'F' { + sum, err = receiveFile(br, out, e) + } else { + sum, err = receiveSparse(br, out, e) + } + if cerr := out.Close(); err == nil && cerr != nil { + err = cerr + } + if err != nil { + return nil, err + } + res.Files = append(res.Files, sum) + case 'T': + var t streamTrailer + if err := strictUnmarshal(js, &t); err != nil { + return nil, fmt.Errorf("stream trailer: %w", err) + } + if err := compareTrailer(t.Files, res.Files); err != nil { + return nil, err + } + if _, err := br.ReadByte(); !errors.Is(err, io.EOF) { + return nil, errors.New("the stream goes on after its trailer") + } + res.Bytes = cr.n + return res, nil + default: + return nil, fmt.Errorf("the stream has a record of unknown kind %q", kind) + } + } +} + +// readStreamRecord reads one record's kind and JSON. +func readStreamRecord(br *bufio.Reader) (byte, []byte, error) { + var hdr [5]byte + if _, err := io.ReadFull(br, hdr[:]); err != nil { + return 0, nil, fmt.Errorf("the stream ended without its trailer: %w", err) + } + n := binary.BigEndian.Uint32(hdr[1:]) + if n > migrateMaxHeader { + return 0, nil, fmt.Errorf("a stream record header of %d bytes is over the %d limit", n, migrateMaxHeader) + } + js := make([]byte, n) + if _, err := io.ReadFull(br, js); err != nil { + return 0, nil, fmt.Errorf("the stream ended inside a record header: %w", err) + } + return hdr[0], js, nil +} + +// strictUnmarshal decodes js into v, refusing fields v does not have. +func strictUnmarshal(js []byte, v any) error { + dec := json.NewDecoder(bytes.NewReader(js)) + dec.DisallowUnknownFields() + return dec.Decode(v) +} + +// checkStreamEntry holds one entry to the name and size limits. +func checkStreamEntry(e streamEntry, kind byte, seen map[string]bool) error { + if !migrateStreamName.MatchString(e.Name) || strings.Contains(e.Name, "..") { + return fmt.Errorf("the stream names a file %q, which is not a bundle file or the disk", e.Name) + } + if seen[e.Name] { + return fmt.Errorf("the stream carries %s twice", e.Name) + } + if len(seen) >= migrateMaxFiles { + return fmt.Errorf("the stream carries more than %d files", migrateMaxFiles) + } + seen[e.Name] = true + limit := int64(migrateMaxFileSize) + if e.Name == "disk" { + if kind != 'S' { + return errors.New("the stream sends the disk as a plain file") + } + limit = migrateMaxDiskSize + } else if kind != 'F' { + return fmt.Errorf("the stream sends %s as a disk", e.Name) + } + if e.Size < 0 || e.Size > limit { + return fmt.Errorf("the stream says %s is %d bytes, outside 0 to %d", e.Name, e.Size, limit) + } + return nil +} + +// receiveFile copies e.Size bytes into out, hashing them. +func receiveFile(br *bufio.Reader, out *os.File, e streamEntry) (streamSum, error) { + h := sha256.New() + n, err := io.Copy(io.MultiWriter(out, h), io.LimitReader(br, e.Size)) + if err != nil { + return streamSum{}, fmt.Errorf("receive %s: %w", e.Name, err) + } + if n != e.Size { + return streamSum{}, fmt.Errorf("the stream ended %d bytes into %s, which is %d", n, e.Name, e.Size) + } + if err := out.Sync(); err != nil { + return streamSum{}, err + } + return streamSum{Name: e.Name, Size: n, SHA256: hex.EncodeToString(h.Sum(nil))}, nil +} + +// receiveSparse writes the extents of e into out, which it first sizes to +// e.Size so that everything no extent covers is a hole. Extents must come in +// order and stay inside the file. A zero chunk inside an extent is skipped +// too, so the copy stays sparse even if the sender's filesystem reported +// zeros as data. +func receiveSparse(br *bufio.Reader, out *os.File, e streamEntry) (streamSum, error) { + if err := out.Truncate(e.Size); err != nil { + return streamSum{}, fmt.Errorf("size %s: %w", e.Name, err) + } + h := sha256.New() + var data, next int64 + buf := make([]byte, 1<<20) + for { + var hdr [16]byte + if _, err := io.ReadFull(br, hdr[:]); err != nil { + return streamSum{}, fmt.Errorf("the stream ended inside %s: %w", e.Name, err) + } + h.Write(hdr[:]) + off := int64(binary.BigEndian.Uint64(hdr[:8])) + length := int64(binary.BigEndian.Uint64(hdr[8:])) + if length == 0 { + if off != 0 { + return streamSum{}, fmt.Errorf("%s: a malformed end of extents", e.Name) + } + break + } + if off < next || off < 0 || length < 0 || length > e.Size || off > e.Size-length { + return streamSum{}, fmt.Errorf("%s: an extent of %d bytes at %d is out of order or outside its %d bytes", + e.Name, length, off, e.Size) + } + for done := int64(0); done < length; { + n := min(int64(len(buf)), length-done) + if _, err := io.ReadFull(br, buf[:n]); err != nil { + return streamSum{}, fmt.Errorf("the stream ended inside %s: %w", e.Name, err) + } + h.Write(buf[:n]) + for c := int64(0); c < n; c += migrateChunk { + chunk := buf[c:min(c+migrateChunk, n)] + if isZero(chunk) { + continue + } + if _, err := out.WriteAt(chunk, off+done+c); err != nil { + return streamSum{}, fmt.Errorf("write %s: %w", e.Name, err) + } + } + done += n + } + data += length + next = off + length + } + if err := out.Sync(); err != nil { + return streamSum{}, err + } + return streamSum{Name: e.Name, Size: e.Size, SHA256: hex.EncodeToString(h.Sum(nil)), DataBytes: data}, nil +} + +// compareTrailer checks the sender's sums against the receiver's. +func compareTrailer(sent, got []streamSum) error { + if len(sent) != len(got) { + return fmt.Errorf("the trailer lists %d files, the stream carried %d", len(sent), len(got)) + } + for i := range sent { + s, g := sent[i], got[i] + if s.Name != g.Name || s.Size != g.Size || s.SHA256 != g.SHA256 { + return fmt.Errorf("%s: the sender sent %d bytes with sha256 %s, %d bytes with sha256 %s arrived", + g.Name, s.Size, s.SHA256, g.Size, g.SHA256) + } + } + return nil +} diff --git a/cmd/hull/migrate_stream_test.go b/cmd/hull/migrate_stream_test.go new file mode 100644 index 00000000..76ac189c --- /dev/null +++ b/cmd/hull/migrate_stream_test.go @@ -0,0 +1,183 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/binary" + "os" + "path/filepath" + "strings" + "syscall" + "testing" +) + +// streamOf writes a stream of the given bundle files and disk. +func streamOf(t *testing.T, files map[string]string, disk string) []byte { + t.Helper() + var buf bytes.Buffer + sw := newMigrateStreamWriter(&buf) + if err := sw.header(streamHeader{TxID: "t", ManifestSHA256: "m"}); err != nil { + t.Fatal(err) + } + for _, name := range []string{"manifest.json", "ram.bin"} { + if path, ok := files[name]; ok { + if _, err := sw.file("bundle/"+name, path); err != nil { + t.Fatal(err) + } + } + } + if disk != "" { + if _, err := sw.sparse("disk", disk); err != nil { + t.Fatal(err) + } + } + if err := sw.finish(); err != nil { + t.Fatal(err) + } + if int64(buf.Len()) != sw.written() { + t.Errorf("written() = %d, the buffer holds %d", sw.written(), buf.Len()) + } + return buf.Bytes() +} + +// receiveInto reads a stream into dir. +func receiveInto(dir string, stream []byte) (*migrateStreamResult, error) { + _ = os.MkdirAll(filepath.Join(dir, "bundle"), 0o700) + return readMigrateStream(bytes.NewReader(stream), func(name string, _ int64) (*os.File, error) { + return os.OpenFile(filepath.Join(dir, filepath.FromSlash(name)), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + }) +} + +func allocated(t *testing.T, path string) int64 { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return info.Sys().(*syscall.Stat_t).Blocks * 512 +} + +// A disk with holes arrives byte for byte the same, with its holes still +// holes, and the stream carries its data and not its size. +func TestMigrateStreamKeepsHoles(t *testing.T) { + src := t.TempDir() + disk := filepath.Join(src, "disk") + const size = 256 << 20 + writeSparseDisk(t, disk, size) + // A run of zeros written as data is left out too. + f, _ := os.OpenFile(disk, os.O_WRONLY, 0) + _, _ = f.WriteAt(make([]byte, 1<<20), 100<<20) + _ = f.Close() + man := filepath.Join(src, "manifest.json") + _ = os.WriteFile(man, []byte(`{"components":[]}`), 0o600) + + stream := streamOf(t, map[string]string{"manifest.json": man}, disk) + if len(stream) > 1<<20 { + t.Errorf("the stream of a %d-byte disk with 12 KiB of data is %d bytes", size, len(stream)) + } + dst := t.TempDir() + res, err := receiveInto(dst, stream) + if err != nil { + t.Fatal(err) + } + assertSameContent(t, disk, filepath.Join(dst, "disk")) + assertSameContent(t, man, filepath.Join(dst, "bundle", "manifest.json")) + if got := allocated(t, filepath.Join(dst, "disk")); got > 4<<20 { + t.Errorf("the received disk occupies %d bytes; its holes were filled", got) + } + // Data goes in 64 KiB chunks: three 4 KiB writes carry at most three. + if d := res.Files[1].DataBytes; res.Bytes != int64(len(stream)) || len(res.Files) != 2 || d < 3*4096 || d > 3*migrateChunk { + t.Errorf("result %+v", res) + } + if res.Header.TxID != "t" || res.Header.ManifestSHA256 != "m" { + t.Errorf("header %+v", res.Header) + } +} + +// A stream is refused whole when it is cut, altered, padded or names a path. +func TestMigrateStreamRefusals(t *testing.T) { + src := t.TempDir() + disk := filepath.Join(src, "disk") + writeSparseDisk(t, disk, 8<<20) + ram := filepath.Join(src, "ram.bin") + _ = os.WriteFile(ram, bytes.Repeat([]byte("x"), 10000), 0o600) + good := streamOf(t, map[string]string{"ram.bin": ram}, disk) + if _, err := receiveInto(t.TempDir(), good); err != nil { + t.Fatalf("the good stream: %v", err) + } + flip := append([]byte(nil), good...) + i := bytes.Index(flip, []byte("xxxx")) + flip[i] = 'y' + record := func(kind byte, js string) []byte { + var hdr [5]byte + hdr[0] = kind + binary.BigEndian.PutUint32(hdr[1:], uint32(len(js))) + return append(hdr[:], js...) + } + head := append([]byte(migrateStreamMagic), record('H', `{"txid":"t","manifestSha256":"m"}`)...) + cases := map[string]struct { + stream []byte + want string + }{ + "cut before the trailer": {good[:len(good)-20], "ended"}, + "cut inside a file": {good[:200], "ended"}, + "a changed byte": {flip, "sha256"}, + "bytes after the trailer": {append(append([]byte(nil), good...), 0), "after its trailer"}, + "not a stream": {[]byte("GET / HTTP/1.1\r\n"), "does not start"}, + "a path above the bundle": {append(append([]byte(nil), head...), record('F', `{"name":"bundle/../../x","size":1}`)...), "not a bundle file"}, + "an absolute path": {append(append([]byte(nil), head...), record('F', `{"name":"/etc/passwd","size":1}`)...), "not a bundle file"}, + "the disk as a plain file": {append(append([]byte(nil), head...), record('F', `{"name":"disk","size":1}`)...), "plain file"}, + "a negative size": {append(append([]byte(nil), head...), record('F', `{"name":"bundle/a","size":-1}`)...), "outside"}, + "an unknown field": {append(append([]byte(nil), head...), record('F', `{"name":"bundle/a","size":1,"mode":511}`)...), "unknown field"}, + "an oversized header": {append(append([]byte(nil), head...), 'F', 0xff, 0xff, 0xff, 0xff), "over the"}, + "an extent outside the disk": {append(append(append([]byte(nil), head...), record('S', `{"name":"disk","size":4096}`)...), + 0, 0, 0, 0, 0, 0, 0x10, 0, 0, 0, 0, 0, 0, 0, 0x10, 0), "outside"}, + } + for name, c := range cases { + _, err := receiveInto(t.TempDir(), c.stream) + if err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: err = %v, want %q", name, err, c.want) + } + } +} + +// writeSparseDisk makes a sparse file of size with data at a few offsets. +func writeSparseDisk(t *testing.T, path string, size int64) { + t.Helper() + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + if err := f.Truncate(size); err != nil { + t.Fatal(err) + } + for _, off := range []int64{0, 3 << 20, size - 4096} { + if _, err := f.WriteAt(bytes.Repeat([]byte{byte(off>>12) | 1}, 4096), off); err != nil { + t.Fatal(err) + } + } +} + +// assertSameContent compares two files byte for byte. +func assertSameContent(t *testing.T, a, b string) { + t.Helper() + da, err1 := os.ReadFile(a) + db, err2 := os.ReadFile(b) + if err1 != nil || err2 != nil || !bytes.Equal(da, db) { + t.Errorf("%s and %s differ (%v %v)", a, b, err1, err2) + } +} diff --git a/cmd/hull/migrate_test.go b/cmd/hull/migrate_test.go new file mode 100644 index 00000000..a575aade --- /dev/null +++ b/cmd/hull/migrate_test.go @@ -0,0 +1,573 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/brig-sh/hull/pkg/store" +) + +// The whole move, both sides in one process: it commits, the source is +// fenced before the destination may run, the destination takes the epoch, +// and the source's checkpoint ends up stale with its VMM stopped. +func TestMoveCommits(t *testing.T) { + f := newMoveFixture(t, true) + peer := newLocalPeer(f.dst) + if err := sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer); err != nil { + t.Fatal(err) + } + src := f.srcRecord(t) + if src.State != srcCommitted || src.Epoch != 1 { + t.Fatalf("source record %s epoch %d", src.State, src.Epoch) + } + var states []string + for _, h := range src.History { + states = append(states, h.State) + } + if want := []string{srcPreflight, srcReserved, srcFenced, srcTransferring, srcTransferred, srcCommitting, + srcCommitted}; !slices.Equal(states, want) { + t.Errorf("source went %v, want %v", states, want) + } + if src.Transfer == nil || src.Transfer.Bytes == 0 || src.Disk.SHA256 == "" || + src.Disk.DataBytes == 0 || src.Disk.DataBytes >= fixtureDiskSize { + t.Errorf("transfer %+v disk %+v", src.Transfer, src.Disk) + } + dst := f.dstRecord(t, src.TxID) + if dst.State != dstRunning || dst.Epoch != 1 || dst.Disk.SHA256 != src.Disk.SHA256 { + t.Errorf("destination record %+v", dst) + } + // The fence: a checkpoint with then=fence, the move's txid and the + // epoch it hands over. The source was never resumed. + var fence map[string]any + for _, r := range f.vm.stub.seen() { + if r["op"] == "checkpoint" { + fence = r + } + if r["op"] == "resume" { + t.Error("the source was resumed") + } + } + if fence == nil || fence["then"] != "fence" || fence["txid"] != src.TxID || fence["epoch"] != float64(1) { + t.Errorf("fence request %v", fence) + } + if !f.vm.stopped { + t.Error("the source's fenced VMM was not stopped after the commit") + } + srcState := mustInstance(t, f.src, f.name) + if srcState.Migration.State != store.MigrationMigrated || srcState.Migration.MoveTxID != src.TxID { + t.Errorf("source instance migration %+v", srcState.Migration) + } + if err := restorableMigration(srcState.Migration); err == nil { + t.Error("the stale source checkpoint is restorable") + } + dstState := mustInstance(t, f.dst, f.name) + if dstState.OwnerEpoch != 1 || dstState.MAC != "52:54:00:12:34:56" || dstState.Migration.State != store.MigrationRestored { + t.Errorf("destination instance epoch %d mac %s migration %+v", dstState.OwnerEpoch, dstState.MAC, dstState.Migration) + } + // The destination's command line is its own. + for _, want := range []string{f.dst.InstanceAgentSocket(f.name), dst.BundleDir, "--net-mac"} { + if !slices.Contains(dstState.CmdLine, want) { + t.Errorf("destination command line %v has no %s", dstState.CmdLine, want) + } + } + if disk, _ := flagValue(dstState.CmdLine, "--disk"); !strings.HasPrefix(disk, f.dst.RootDir()) { + t.Errorf("destination disk %s is not in its store", disk) + } + assertSameContent(t, f.disk, dst.Disk.Path) + // Moving again from the stale source is refused; a plain `hull migrate` + // re-run with the txid is a no-op. + if err := sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer); err == nil { + t.Error("a new move of the stale source was allowed") + } + opt := f.options(false) + opt.txid = src.TxID + if err := sourceMigrate(context.Background(), f.src, f.name, opt, peer); err != nil { + t.Errorf("re-running a committed move: %v", err) + } +} + +// The commit's reply is lost: the destination runs, the source stays +// fenced in committing and is never resumed, and running the move again +// sends the commit again and records it. +func TestMoveLostCommitAcknowledgement(t *testing.T) { + f := newMoveFixture(t, true) + peer := newLocalPeer(f.dst) + peer.drop["migrate-commit"] = 1 + err := sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer) + if err == nil || !strings.Contains(err.Error(), "stays fenced") { + t.Fatalf("err = %v", err) + } + src := f.srcRecord(t) + if src.State != srcCommitting { + t.Fatalf("source is %s, want committing", src.State) + } + if dst := f.dstRecord(t, src.TxID); dst.State != dstRunning { + t.Fatalf("destination is %s, want running", dst.State) + } + if slices.Contains(f.vm.ops(), "resume") { + t.Fatal("the source was resumed") + } + // An abort now must be refused: the destination says it runs. + if err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer); err == nil || + !strings.Contains(err.Error(), "cannot be aborted") { + t.Errorf("abort after a lost acknowledgement: %v", err) + } + if f.srcRecord(t).State != srcCommitting { + t.Error("the refused abort changed the source record") + } + // A re-run without --txid says how to resume. + if err := sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer); err == nil || + !strings.Contains(err.Error(), "--txid "+src.TxID) { + t.Errorf("re-run without the txid: %v", err) + } + opt := f.options(false) + opt.txid = src.TxID + if err := sourceMigrate(context.Background(), f.src, f.name, opt, peer); err != nil { + t.Fatal(err) + } + if got := f.srcRecord(t).State; got != srcCommitted { + t.Errorf("after the retry the source is %s", got) + } + if peer.called("migrate-commit") != 2 { + t.Errorf("commit sent %d times, want 2", peer.called("migrate-commit")) + } +} + +// The destination's hvi cannot restore: the move stops before the commit, +// with the source fenced; an abort, once the destination confirms, frees +// the destination and restores the instance here. +func TestMoveRestoreRefusedThenAbort(t *testing.T) { + f := newMoveFixture(t, false) + peer := newLocalPeer(f.dst) + restored := 0 + localRestore = func(_ *store.Store, name string) error { + restored++ + st := mustInstance(t, f.src, name) + return restorableMigration(st.Migration) + } + err := sourceMigrate(context.Background(), f.src, f.name, f.options(true), peer) + if err == nil || !strings.Contains(err.Error(), "could not restore") { + t.Fatalf("err = %v", err) + } + src := f.srcRecord(t) + if src.State != srcTransferred { + t.Fatalf("source is %s, want transferred", src.State) + } + dst := f.dstRecord(t, src.TxID) + if dst.State != dstVerified || !strings.Contains(dst.Error, "no restore subcommand") { + t.Fatalf("destination %s, error %q", dst.State, dst.Error) + } + // A plain restore of the source is refused while the move holds it. + if err := restorableMigration(mustInstance(t, f.src, f.name).Migration); err == nil { + t.Error("the source's checkpoint is restorable while a move holds it") + } + if err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer); err != nil { + t.Fatal(err) + } + if got := f.srcRecord(t).State; got != migAborted { + t.Errorf("source is %s", got) + } + if restored != 1 { + t.Errorf("the source was restored %d times, want 1", restored) + } + if !f.vm.stopped { + t.Error("the fenced VMM was not stopped before the restore") + } + if _, err := os.Stat(f.dst.InstanceDir(f.name)); !os.IsNotExist(err) { + t.Errorf("the destination kept the instance: %v", err) + } + if rec := f.dstRecord(t, src.TxID); rec.State != migAborted { + t.Errorf("tombstone %s", rec.State) + } + // A commit that arrives late is refused. + if r := destCommit(f.dst, src.TxID, src.Epoch); r.OK || r.Code != codeAborted { + t.Errorf("late commit: %+v", r) + } + // The aborted move's record does not hold the instance: a new move is + // refused only because the fixture's VMM is stopped now (the stub + // restore here does not start one), not by the old move. + if err := sourceMigrate(context.Background(), f.src, f.name, f.options(true), newLocalPeer(shortStore(t))); err == nil { + t.Error("expected the fixture's stopped VMM to refuse a new move") + } else if strings.Contains(err.Error(), "in move") { + t.Errorf("an aborted move blocks a new one: %v", err) + } +} + +// Without the destination's answer an abort after the fence refuses to +// bring the source back. +func TestMoveAbortWithoutAnAnswerKeepsTheSourceFenced(t *testing.T) { + f := newMoveFixture(t, false) + peer := newLocalPeer(f.dst) + _ = sourceMigrate(context.Background(), f.src, f.name, f.options(true), peer) + restored := 0 + localRestore = func(*store.Store, string) error { restored++; return nil } + peer.down = true + err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer) + if err == nil || !strings.Contains(err.Error(), "stays fenced") { + t.Fatalf("err = %v", err) + } + if restored != 0 || f.vm.stopped { + t.Error("the source was touched without the destination's word") + } + if got := f.srcRecord(t); got.State != srcTransferred || got.Error == "" { + t.Errorf("source record %s %q", got.State, got.Error) + } +} + +// A destination that answers running refuses the abort, whatever the +// source's record says. +func TestMoveAbortRefusedByARunningDestination(t *testing.T) { + f := newMoveFixture(t, false) + peer := newLocalPeer(f.dst) + _ = sourceMigrate(context.Background(), f.src, f.name, f.options(true), peer) + peer.answer["migrate-abort"] = &migrateReply{OK: false, Code: codeRunning, Error: "it runs here"} + restored := 0 + localRestore = func(*store.Store, string) error { restored++; return nil } + if err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer); err == nil || + !strings.Contains(err.Error(), "cannot be aborted") { + t.Fatalf("err = %v", err) + } + if restored != 0 || f.vm.stopped || f.srcRecord(t).State == migAborted { + t.Error("the abort went ahead against a running destination") + } +} + +// After the commit, an abort is refused. +func TestMoveAbortAfterCommit(t *testing.T) { + f := newMoveFixture(t, true) + peer := newLocalPeer(f.dst) + if err := sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer); err != nil { + t.Fatal(err) + } + err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer) + if err == nil || !strings.Contains(err.Error(), "new `hull migrate`") { + t.Errorf("err = %v", err) + } + if peer.called("migrate-abort") != 0 { + t.Error("the destination was asked to abort a committed move") + } + // The destination refuses it too. + if r := destAbort(f.dst, f.srcRecord(t).TxID); r.OK || r.Code != codeRunning { + t.Errorf("destination abort of a running move: %+v", r) + } +} + +// An abort before the fence leaves the source running and frees the +// destination's reservation. +func TestMoveAbortBeforeTheFence(t *testing.T) { + f := newMoveFixture(t, true) + peer := newLocalPeer(f.dst) + // The move stops right after the reservation: the capture fails, and + // hvi runs the guest on. + f.vm.mu.Lock() + f.vm.failCheckpoint = true + f.vm.mu.Unlock() + err := sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer) + if err == nil || !strings.Contains(err.Error(), "still runs here") { + t.Fatalf("err = %v", err) + } + src := f.srcRecord(t) + if src.State != srcReserved { + t.Fatalf("source %s", src.State) + } + if _, err := os.Stat(f.dst.InstanceDir(f.name)); err != nil { + t.Fatalf("no reservation on the destination: %v", err) + } + if err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(f.dst.InstanceDir(f.name)); !os.IsNotExist(err) { + t.Errorf("the reservation is still there: %v", err) + } + if f.vm.stopped { + t.Error("an abort before the fence stopped the source") + } +} + +// A transfer cut off midway leaves the source in transferring; the re-run +// sends again and goes on. The destination's restore is never run twice. +func TestMoveTransferRetry(t *testing.T) { + f := newMoveFixture(t, true) + peer := newLocalPeer(f.dst) + // The reserve goes through; the stream's connection fails. + failing := &failVerbPeer{localPeer: peer, verb: "migrate-receive", times: 1, skipFirst: 1} + err := sourceMigrate(context.Background(), f.src, f.name, f.options(false), failing) + if err == nil { + t.Fatal("the cut transfer succeeded") + } + src := f.srcRecord(t) + if src.State != srcTransferring { + t.Fatalf("source %s", src.State) + } + opt := f.options(false) + opt.txid = src.TxID + if err := sourceMigrate(context.Background(), f.src, f.name, opt, peer); err != nil { + t.Fatal(err) + } + if f.srcRecord(t).State != srcCommitted || f.restores != 1 { + t.Errorf("source %s, %d restores", f.srcRecord(t).State, f.restores) + } +} + +// failVerbPeer fails one verb a number of times without running it, after +// letting skipFirst calls of it through. +type failVerbPeer struct { + *localPeer + verb string + times int + skipFirst int +} + +func (p *failVerbPeer) call(ctx context.Context, verb string, args []string, stdin io.Reader) (*migrateReply, error) { + if verb == p.verb { + if p.skipFirst > 0 { + p.skipFirst-- + } else if p.times > 0 { + p.times-- + p.down = true + defer func() { p.down = false }() + } + } + return p.localPeer.call(ctx, verb, args, stdin) +} + +// A disk written after the fence is not sent. +func TestMoveRefusesADiskChangedAfterTheFence(t *testing.T) { + f := newMoveFixture(t, true) + peer := newLocalPeer(f.dst) + failing := &failVerbPeer{localPeer: peer, verb: "migrate-receive", times: 1, skipFirst: 1} + _ = sourceMigrate(context.Background(), f.src, f.name, f.options(false), failing) + if f.srcRecord(t).State != srcTransferring { + t.Fatalf("source %s", f.srcRecord(t).State) + } + later := time.Now().Add(time.Hour) + if err := os.Chtimes(f.disk, later, later); err != nil { + t.Fatal(err) + } + opt := f.options(false) + opt.txid = f.srcRecord(t).TxID + err := sourceMigrate(context.Background(), f.src, f.name, opt, peer) + if err == nil || !strings.Contains(err.Error(), "changed after the fence") { + t.Errorf("err = %v", err) + } +} + +// What a move holds is out of reach of the plain verbs on both hosts. +func TestMoveGuards(t *testing.T) { + f := newMoveFixture(t, false) + peer := newLocalPeer(f.dst) + _ = sourceMigrate(context.Background(), f.src, f.name, f.options(true), peer) + // Source: checkpoint, resume, rm. + if err := hviCheckpointInstance(f.src, mustInstance(t, f.src, f.name), hviThenContinue, "", time.Second); err == nil || + !strings.Contains(err.Error(), "being moved") { + t.Errorf("checkpoint of a source in a move: %v", err) + } + if err := refuseDuringMove(f.src, f.name, "resume"); err == nil { + t.Error("resume of a source in a move was allowed") + } + if err := removeInstanceIn(f.src, f.name, true); err == nil || !strings.Contains(err.Error(), "being moved") { + t.Errorf("rm of a source in a move: %v", err) + } + // Destination: the received instance restores only through its move, + // and only paused. + st := mustInstance(t, f.dst, f.name) + if err := checkMoveRestore(f.dst, st, hviRestoreOptions{}); err == nil { + t.Error("a plain restore of an incoming instance was allowed") + } + txid := f.srcRecord(t).TxID + if err := checkMoveRestore(f.dst, st, hviRestoreOptions{move: txid}); err == nil { + t.Error("a move's restore without --paused was allowed") + } + if err := checkMoveRestore(f.dst, st, hviRestoreOptions{move: strings.Repeat("e", 32), paused: true}); err == nil { + t.Error("another move's restore was allowed") + } + if err := checkMoveRestore(f.dst, st, hviRestoreOptions{move: txid, paused: true}); err != nil { + t.Errorf("the move's own restore: %v", err) + } + if err := refuseDuringMove(f.dst, f.name, "resume"); err == nil || !strings.Contains(err.Error(), "arriving") { + t.Errorf("resume of an incoming instance: %v", err) + } +} + +// A dry run changes nothing on either side. +func TestMoveDryRun(t *testing.T) { + f := newMoveFixture(t, true) + peer := newLocalPeer(f.dst) + opt := f.options(false) + opt.dryRun = true + var out bytes.Buffer + opt.out = &out + if err := sourceMigrate(context.Background(), f.src, f.name, opt, peer); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "Dry run") { + t.Errorf("output: %s", out.String()) + } + if _, err := os.Stat(f.src.InstanceMigrationRecord(f.name)); !os.IsNotExist(err) { + t.Error("a dry run wrote a record") + } + if _, err := os.Stat(f.dst.InstanceDir(f.name)); !os.IsNotExist(err) { + t.Error("a dry run reserved the name") + } + if len(f.vm.ops()) > 0 && slices.Contains(f.vm.ops(), "checkpoint") { + t.Error("a dry run fenced the source") + } + if peer.called("migrate-probe") != 1 || len(peer.calls) != 1 { + t.Errorf("calls %v", peer.calls) + } +} + +// Preflight refuses what a move cannot carry before anything changes. +func TestMovePreflightRefusals(t *testing.T) { + f := newMoveFixture(t, false) + peer := newLocalPeer(f.dst) + // No restore on this hvi, and no --allow-no-restore. + err := sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer) + if err == nil || !strings.Contains(err.Error(), "no restore subcommand") { + t.Errorf("err = %v", err) + } + if len(peer.calls) != 0 { + t.Errorf("the destination was called: %v", peer.calls) + } + // A name that is taken there. + if _, err := f.dst.CreateInstance(f.name); err != nil { + t.Fatal(err) + } + err = sourceMigrate(context.Background(), f.src, f.name, f.options(true), peer) + if err == nil || !strings.Contains(err.Error(), "already exists there") { + t.Errorf("err = %v", err) + } + if slices.Contains(f.vm.ops(), "checkpoint") { + t.Error("the source was fenced for a destination that refused") + } + // A bad target. + opt := f.options(true) + opt.to = "-oProxyCommand=touch /tmp/x" + if err := sourceMigrate(context.Background(), f.src, f.name, opt, peer); err == nil { + t.Error("an option as a target was accepted") + } +} + +// The whole move over the ssh transport, with this test binary standing in +// for ssh and for the destination's hull: the remote command line goes +// through a shell, the destination's verbs run in another process against +// the second store, and its restore is a real `hull restore --move` child, +// which the fake hvi (no restore subcommand) refuses. That is today's +// outcome with a real hvi too, until hvi has restore. +func TestMoveOverFakeSSH(t *testing.T) { + f := newMoveFixture(t, false) + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + // The destination's store is used as it is: on macOS hull would + // otherwise mount a case-sensitive image over a temporary directory. + if err := os.WriteFile(filepath.Join(f.dst.RootDir(), ".case-sensitive-apfs"), nil, 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(migrateSSHEnv, self) + t.Setenv(fakeSSHEnv, "1") + t.Setenv("DO_NOT_TRACK", "1") + opt := f.options(true) + opt.remote.Hull = self + opt.remote.Store = f.dst.RootDir() + var progress bytes.Buffer + opt.out = &progress + + err = sourceMigrate(context.Background(), f.src, f.name, opt, nil) + // The refusal is the real `hull restore`'s, from the other process. + if err == nil || !strings.Contains(err.Error(), "hull restore vm1: cannot restore vm1: the hvi it ran") { + t.Fatalf("err = %v\n%s", err, progress.String()) + } + src := f.srcRecord(t) + if src.State != srcTransferred || src.Transfer == nil || src.Disk.SHA256 == "" { + t.Fatalf("source %s %+v", src.State, src.Transfer) + } + dst := f.dstRecord(t, src.TxID) + if dst.State != dstVerified || dst.Verify != "ok: test bundle" || dst.Disk.SHA256 != src.Disk.SHA256 { + t.Fatalf("destination %+v", dst) + } + assertSameContent(t, f.disk, dst.Disk.Path) + for _, name := range []string{"manifest.json", "ram.bin", "disk.json"} { + assertSameContent(t, filepath.Join(src.BundleDir, name), filepath.Join(dst.BundleDir, name)) + } + if st := mustInstance(t, f.dst, f.name); st.Migration.State != store.MigrationIncoming || st.Status != "stopped" { + t.Errorf("destination instance %s %+v", st.Status, st.Migration) + } + + var status bytes.Buffer + if err := sourceStatus(context.Background(), f.src, f.name, &status, nil); err != nil { + t.Fatal(err) + } + if !strings.Contains(status.String(), "destination pm@dest: verified") { + t.Errorf("status:\n%s", status.String()) + } + + if err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, nil); err != nil { + t.Fatal(err) + } + if got := f.srcRecord(t).State; got != migAborted { + t.Errorf("source %s", got) + } + if _, err := os.Stat(f.dst.InstanceDir(f.name)); !os.IsNotExist(err) { + t.Errorf("the destination kept the instance: %v", err) + } + if rec := f.dstRecord(t, src.TxID); rec.State != migAborted { + t.Errorf("destination tombstone %s", rec.State) + } +} + +// Moving back is a new move from where the instance runs now, to the host +// it left, once the stale copy there is removed; the owner epoch goes on +// counting. +func TestMoveThereAndBack(t *testing.T) { + f := newMoveFixture(t, true) + ctx := context.Background() + if err := sourceMigrate(ctx, f.src, f.name, f.options(false), newLocalPeer(f.dst)); err != nil { + t.Fatal(err) + } + if got := mustInstance(t, f.dst, f.name).OwnerEpoch; got != 1 { + t.Fatalf("epoch after the first move %d", got) + } + back := f.options(false) + back.to, back.remote.Target = "pm@src", "pm@src" + err := sourceMigrate(ctx, f.dst, f.name, back, newLocalPeer(f.src)) + if err == nil || !strings.Contains(err.Error(), "`hull rm vm1` there") { + t.Fatalf("moving back over the stale copy: %v", err) + } + if err := removeInstanceIn(f.src, f.name, false); err != nil { + t.Fatalf("rm of the stale copy: %v", err) + } + if err := sourceMigrate(ctx, f.dst, f.name, back, newLocalPeer(f.src)); err != nil { + t.Fatal(err) + } + home := mustInstance(t, f.src, f.name) + if home.OwnerEpoch != 2 || home.Migration.State != store.MigrationRestored { + t.Errorf("back home: epoch %d, migration %+v", home.OwnerEpoch, home.Migration) + } + away, err := readMigrationRecord(f.dst.InstanceMigrationRecord(f.name)) + if err != nil || away.Role != roleSource || away.State != srcCommitted || away.Epoch != 2 { + t.Errorf("the second move's source record: %+v %v", away, err) + } +} diff --git a/cmd/hull/rm.go b/cmd/hull/rm.go index 4cf60040..1e65d1d8 100644 --- a/cmd/hull/rm.go +++ b/cmd/hull/rm.go @@ -72,6 +72,12 @@ func removeInstanceIn(s *store.Store, instanceID string, force bool) error { if err != nil { return removeWedgedInstance(s, instanceID, err) } + // A move in progress needs what is in the directory: on the source its + // fenced checkpoint, the only copy until the destination has one; on + // the destination, what it received. The move's abort ends it first. + if err := moveRefusal(s, instanceID, "rm"); err != nil && !moveCommittedAway(s, instanceID) { + return err + } // Check if running and force flag is set. "starting" counts as running the // moment a pid is on the record, because by then there really is a VMM. diff --git a/cmd/hull/telemetry.go b/cmd/hull/telemetry.go index 2a92dd9b..48e8edf4 100644 --- a/cmd/hull/telemetry.go +++ b/cmd/hull/telemetry.go @@ -85,10 +85,16 @@ func topLevelCommand() string { // initTelemetry runs the consent flow. Invoked from the root // Before hook, except for the telemetry subcommand itself (prompting for // consent on the way to `telemetry off` would be absurd, so that path -// manages state directly) and the network-gateway daemon (a supervised -// child; its parent's command already counts). +// manages state directly), the network-gateway daemon (a supervised +// child; its parent's command already counts) and the destination's +// migrate-* verbs (steps of a `hull migrate` on another host, which counts +// there; their stdout is a reply the source parses). func initTelemetry(cmd *cli.Command) { telemetryCmdName = topLevelCommand() + if strings.HasPrefix(telemetryCmdName, "migrate-") { + telemetryCmdName = "" + return + } if telemetryCmdName == "telemetry" || telemetryCmdName == "network-gateway" { return } diff --git a/cmd/hull/testmain_linux_test.go b/cmd/hull/testmain_linux_test.go new file mode 100644 index 00000000..e88acd16 --- /dev/null +++ b/cmd/hull/testmain_linux_test.go @@ -0,0 +1,32 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package main + +import ( + "os" + "testing" +) + +// TestMain on Linux. The macOS one is in compose_down_test.go, with the +// compose tests that only build there; both let the move tests run this +// binary as ssh and as hull. +func TestMain(m *testing.M) { + testBinaryAsHull() + code := m.Run() + removeFakeHvi() + os.Exit(code) +} diff --git a/docs/README.md b/docs/README.md index 8e66a8f1..ce5e8ef1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -23,6 +23,7 @@ have never run hull. | [network-egress.md](network-egress.md) | Restricting what a guest may reach, and what the filter does not cover | | [compose.md](compose.md) | Running several services together, one VM per service | | [checkpoint-restore.md](checkpoint-restore.md) | Saving a running VM and resuming it later | +| [migration.md](migration.md) | Moving a running `hvi` VM to another host over ssh, and recovering a move that stopped | | [images.md](images.md) | Pull policy, digests, platforms, annotations, boot assets, private registries | | [troubleshooting.md](troubleshooting.md) | Errors you are likely to hit, and what each one means | diff --git a/docs/checkpoint-restore.md b/docs/checkpoint-restore.md index 6ca57834..f2ca2432 100644 --- a/docs/checkpoint-restore.md +++ b/docs/checkpoint-restore.md @@ -18,6 +18,12 @@ directory. For `hvi`, pause, resume and checkpoint work today; restore is built in hull and refuses until `hvi` gains `restore`, because no bundle `hvi` writes yet can be resumed. +An `hvi` checkpoint taken with `--then fence` is also what moves an instance +to another host: `hull migrate` takes one, sends it and the disk over ssh, and +restores it there. See [migration.md](migration.md). While a move holds an +instance, its checkpoint is `migrating` and a plain `hull restore` of it is +refused; after the move commits it is `migrated`, stale for good. + ## Vz The Vz backend can checkpoint a running microVM, meaning pause, save the machine diff --git a/docs/cli.md b/docs/cli.md index 6639b1c2..c936976d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -27,9 +27,11 @@ for the one build that reports `dev`. ## Commands -There are twenty-one top-level commands. `network-gateway` is the only hidden one: -it does not appear in `hull --help`, but it runs and prints its own help when -named. No flag anywhere in the tree is hidden. +There are twenty-two top-level commands you call, plus hidden ones. +`network-gateway` is hidden: it does not appear in `hull --help`, but it runs +and prints its own help when named. The six `migrate-*` verbs are hidden too: +`hull migrate` runs them on the destination over ssh (see below). The one +hidden flag is `restore --move`, which only a move's own restore passes. | Command | What it does | |---|---| @@ -42,6 +44,7 @@ named. No flag anywhere in the tree is hidden. | `resume ` | resume an `hvi` instance paused by `pause` | | `checkpoint ` | `vz`: pause, save VM and disk state, resume. `hvi`: write a checkpoint bundle, then continue or fence | | `restore ` | restore a stopped instance: `vz` from its saved state; `hvi` from a fenced bundle, refused until `hvi` has a `restore` subcommand | +| `migrate --to USER@HOST` | move a running `hvi` instance to another host over ssh; see [migration.md](migration.md) | | `rm ` | remove a stopped instance | | `logs ` | show an instance's log | | `inspect ` | print instance details as JSON | @@ -54,6 +57,7 @@ named. No flag anywhere in the tree is hidden. | `telemetry` | control usage and crash telemetry | | `version` | print the version and the build behind this binary | | `network-gateway` | run the user-mode network gateway daemon (hidden) | +| `migrate-probe`, `migrate-receive`, `migrate-prepare`, `migrate-commit`, `migrate-status`, `migrate-abort` | the destination's half of a move (hidden) | ## `hull run` @@ -150,6 +154,7 @@ as a default that an explicit `--env TERM` beats. | `pause `, `resume ` | none | | `checkpoint ` | `--timeout `, default 60; `hvi` only: `--then continue\|fence`, default `continue`; `--dir `, an empty directory for the bundle, below `/checkpoint` | | `restore ` | `--detach`, `-d`; `--stop-grace `, default 10; `--wait-ip`; `--gateway-sock ` (`vz` only); `hvi` only: `--paused`, `--timeout `, default 120, `--adopt-latest` | +| `migrate ` | `--to USER@HOST`; `--remote-store

`; `--remote-hull `; `--remote-hvi `; `--remote-gateway-sock `; `--txid ` resume a move; `--dry-run`; `--abort` and `--status` (with a name or a txid); `--timeout `, default 120, for the fence; `--allow-no-restore`, testing only | | `assets show` | none | | `assets dir` | none. Prints the directory and nothing else, for scripts | | `assets pull [REF]` | `--force` download even when the assets are present | @@ -176,6 +181,26 @@ still running, and when the `hvi` it ran has no `restore` subcommand. See `hull stop` on an `hvi` instance asks hvi over its control socket first, then signals as for any VMM. +`hull migrate` moves a running `hvi` instance to `--to USER@HOST`: it fences +the guest here, streams the checkpoint and the disk over ssh, has the +destination restore it paused, and only then commits, after which the +destination resumes it and the copy here is stale. While a move holds an +instance, `checkpoint`, `resume`, `restore` and `rm` of it are refused on +both hosts. The protocol, its states and its recovery are in +[migration.md](migration.md). + +The destination's verbs each print one JSON object on stdout: +`{"protocol":1,"ok":bool,"code":...,"error":...,"record":{...}}`. They are: + +| Verb | Flags | What it does | +|---|---|---| +| `migrate-probe` | `--name`, `--hvi`, `--gateway-sock` | reports hull and hvi versions, whether hvi takes the control flags and has `restore`, the hypervisor, free space, whether the name is taken, and whether the gateway answers | +| `migrate-receive` | `--txid`, `--name`, `--reserve` | with `--reserve`, claims the name (a JSON request on stdin); without, reads the move stream on stdin and ends `verified` | +| `migrate-prepare` | `--txid` | restores the received checkpoint paused | +| `migrate-commit` | `--txid`, `--epoch` | records `running`, then resumes the guest | +| `migrate-status` | `--txid` | prints the move's record and the VM's state | +| `migrate-abort` | `--txid` | refused once running; otherwise records `aborted`, stops what the move restored and frees the name | + `hull images` prints `No images found` and `hull ps` prints `No instances found` on an empty store. Both exit 0. `hull prune` prints `Nothing to prune` and exits 0 the same way. @@ -311,6 +336,7 @@ See [telemetry.md](telemetry.md). | `CI` | counts the session as non-interactive, so the consent prompt never shows | | `TERM` | forwarded into the guest for a pty `exec` | | `COMPOSE_FILE`, `COMPOSE_PROJECT_NAME`, `COMPOSE_PROFILES`, `COMPOSE_DISABLE_ENV_FILE` | compose equivalents of the flags above | +| `HULL_MIGRATE_SSH` | a program `hull migrate` runs instead of `ssh`, with the same arguments; for testing a move on one machine (see [migration.md](migration.md#testing-a-move)) | A session counts as interactive only when `CI` is unset **and** both stdin and stderr are terminals. diff --git a/docs/migration.md b/docs/migration.md new file mode 100644 index 00000000..6eef4ea0 --- /dev/null +++ b/docs/migration.md @@ -0,0 +1,341 @@ +# Moving an instance to another host + +`hull migrate NAME --to USER@HOST` moves a running `hvi` instance to another +host over ssh. The guest keeps its memory, its disk, its MAC and its open +files. It does not reboot. + +This page is the protocol: the steps, the states each host records, the +rules that keep the VM from running on two hosts at once, what you trust, +what can go wrong, and how to recover. + +> **Status, 2026-10-14.** On macOS (HVF) a move runs end to end with the +> `hvi` on `feat/teleport` at 4ea836f, the first with a `restore` +> subcommand. On Linux (KVM) `hvi` has no restore yet (H5), so a move stops +> before the commit with the destination's answer "restore refused" (see +> [Verified](#verified)). Without `--allow-no-restore`, `hull migrate` +> refuses at preflight on an `hvi` with no restore, before anything is +> fenced. + +## Usage + +```bash +hull migrate vm1 --to pm@rp2 # move vm1 to rp2 +hull migrate vm1 --to pm@rp2 --dry-run # check both hosts, change nothing +hull migrate vm1 --to pm@rp2 \ + --remote-store /srv/hull --remote-hull /usr/local/bin/hull +hull migrate vm1 --txid 3f...e1 # resume a move that stopped +hull migrate --status vm1 # this host's record, and the other's +hull migrate --abort 3f...e1 # end a move that has not committed +``` + +| Flag | Meaning | +|---|---| +| `--to USER@HOST` | the destination, as ssh names it | +| `--remote-store DIR` | the destination hull's `--store-dir`; default its own default | +| `--remote-hull PATH` | hull on the destination; default `hull` on the remote PATH | +| `--remote-hvi PATH` | hvi on the destination; default the one its hull would run | +| `--remote-gateway-sock PATH` | a network gateway on the destination for the guest's NIC | +| `--txid TXID` | resume this move from its recorded state, or start a move with this txid | +| `--dry-run` | run every check on both hosts and print the plan; write nothing anywhere | +| `--abort` | with NAME or TXID: abort the move, if it has not committed | +| `--status` | with NAME or TXID: print the move's record here and on the other host | +| `--timeout SECS` | how long hvi may take to write the fenced checkpoint; default 120 | +| `--allow-no-restore` | testing only: go on although an `hvi` on either host cannot restore | + +The instance must be an `hvi` instance on a block rootfs, with no shared +host directory, running or paused, started with a control socket. Both hosts +must be the same OS and architecture. Everything else about the host is left +to `hvi restore` on the destination, which by its specification (SPEC H4) +compares the bundle's machine profile with the host's (CPU, counter +frequency, interrupt controller, hvi build) and refuses before it creates +anything. The moves below ran between identical hosts, so that refusal has +not been seen to run. + +## The steps + +The source drives every step. Each one is a hidden hull verb on the +destination, run as `ssh USER@HOST hull ...`, which prints one JSON +reply on stdout and nothing else. + +| # | Source state after it | What happens | Destination state after it | +|---|---|---|---| +| 1 | `preflight` | local checks; `migrate-probe` reports the destination's hull, hvi, hypervisor, free space and whether the name is free. A refusal here leaves no record on either host | (nothing) | +| 2 | `reserved` | `migrate-receive --reserve` creates the instance directory and its `migration.json` | `reserved` | +| 3 | `fenced` | `hull checkpoint --then fence` with the move's txid and epoch: the guest stops for good on this host | `reserved` | +| 4 | `transferring`, then `transferred` | the bundle and the disk go over ssh to `migrate-receive`, which checks every sha256, puts them in place and runs `hvi checkpoint-verify` | `receiving`, `received`, `verified` | +| 5 | `committing` | `migrate-prepare` runs `hull restore --paused` on the destination; only when that worked does the source record `committing` | `restored-paused` | +| 6 | `committed` | `migrate-commit --txid T --epoch E`: the destination records `running`, then resumes the guest | `running` | +| 7 | | the source marks its checkpoint stale and stops its fenced VMM | | + +Either side can also end in `aborted`. + +The plan listed the transfer before the fence. In this version the fence +comes first, because the bundle a move carries is the fenced checkpoint: hvi +has no pre-copy yet, so there is nothing to send until the guest has stopped. +Pre-copy (hvi H7) would send most of the RAM and the disk while the guest +runs, then fence and send what changed. + +### Epochs and txids + +A move's **txid** is 128 random bits in hex. It names the move on both hosts, +the bundle directory, and the checkpoint inside the bundle (`session.json`). + +Every instance has an **owner epoch** in its `state.json` (`ownerEpoch`), 0 +where it was first started. A move hands over the next epoch. The source +sends it in the fence checkpoint, in the reservation and in the commit. The +destination takes it as its own when it commits, and refuses a commit for any +other epoch. Moving the instance back is a new move from the new host, with +the next epoch again. + +## The rules + +**At most one host runs the VM.** + +1. The source fences before the destination may resume. The fence is a + checkpoint taken with `then=fence`: hvi parks every vCPU and refuses every + later `resume`. +2. The destination resumes only on `migrate-commit` with the move's txid and + epoch. Before that, its instance is `incoming`: a plain `hull restore`, + `hull resume` or `hull checkpoint` of it is refused, and the move's own + restore is always `--paused`. +3. The source sends the commit only after it has recorded `fenced` and then + `committing` on stable storage (temp file, fsync, rename, directory fsync). +4. The destination records `running` before it sends the resume. From then on + it refuses an abort. +5. A source that never hears back from the commit stays fenced, in + `committing`. Nothing resumes it automatically. Running the move again + sends the commit again; the destination answers it the same way. +6. `hull migrate --abort` works only before the commit. After the fence it + first asks the destination, which records `aborted` (so a commit that + arrives later is refused) and stops any paused VM it restored. Only on that + answer does the source restore the instance locally. With no answer, or the + answer `running`, the source stays fenced and says why. +7. After the commit the source's checkpoint is `migrated` (stale): it cannot + be restored, resumed, checkpointed or moved. `hull migrate --status` + explains. + +**Every step can be run again.** A re-run of `hull migrate NAME --txid T` +starts from the recorded state. A step that finds its effect already there +succeeds: a reserve of a name this move reserved, a fence the record already +holds (or one hvi took but hull did not live to record, which is adopted +after `hvi checkpoint-verify`), a transfer the destination has verified, a +restore that is up and paused, a commit that is running. + +**Locks.** On each host a move takes `/migrate.lock` for each step. +The source holds it for the whole run of `hull migrate`, and the fence also +takes the checkpoint lock `/migration.lock` that `hull checkpoint` +and `hull restore` use. The move's own restore runs as a child `hull restore`, +which takes `migration.lock` itself, so the two locks nest in one order only: +`migrate.lock` first. + +## Trust + +- **ssh authenticates both ends.** The source runs `ssh -T -o BatchMode=yes` + with your ssh configuration: the destination's host key is checked against + `known_hosts`, and the destination's `authorized_keys` decides who may run + hull there. hull adds no authentication of its own. Whoever can run hull as + that user on the destination can receive the guest's RAM, which holds + whatever the guest held, credentials included. +- **The stream is checked, not trusted.** The destination checks every file + name, size and extent against fixed limits before it writes, every sha256 + against the stream's trailer and against the bundle's manifest, and the + manifest's own sha256 against the one hvi reported at the fence. Then hvi + verifies the bundle again. +- **The records are the authority on each host.** `migration.json` and + `state.json` live in the 0700 instance directory, outside the checkpoint + root hvi may write to. +- **A partition never leaves two hosts running.** If the network fails during + the commit, the source stays fenced in `committing` and the destination is + either `restored-paused` (paused, not running) or `running`. Both are + visible with `hull migrate --status` and the hidden `migrate-status` verb. + Either the commit is sent again, or the abort is confirmed by the + destination. Neither side guesses. +- **Operators can still break it.** Deleting a `migration.json` by hand, or + running hvi directly on a bundle, is outside these rules. + +## Network and sockets + +The destination gets its own `agent.sock` and `control.sock` in its instance +directory, its own checkpoint root, and its copy of the disk at +`/bundle/rootfs.ext4`. The MAC is kept. + +A guest that ran on a network gateway needs a gateway on the destination +with the same subnet, so that its address, its routes and its DNS stay valid. +Start one there (`hull network-gateway`, or brig) and pass its socket with +`--remote-gateway-sock`; the probe dials it before anything is fenced. Without +it, the moved guest gets `hvi`'s stub NIC, which reaches nothing, and the +source prints a warning. SPEC H4 says `hvi restore` refuses a net mode that +differs from the bundle's, so this fallback will likely end in "restore +refused" once hvi has restore; the move can then be aborted. Neither gateway +case has been run live. + +Agent connections over vsock do not survive a move: the guest sees them reset +on restore. External connections may reconnect, since the guest keeps its +address only if the new gateway gives it the same one. + +## The stream + +`migrate-receive` reads a hull-specific stream on stdin. Go's `archive/tar` +cannot write sparse entries, and GNU tar and bsdtar do sparse files +differently, so hull frames its own: + +``` +"HULLMIG1" +'H' u32 len {"txid", "manifestSha256"} +'F' u32 len {"name": "bundle/", "size"} then size bytes (each bundle file) +'S' u32 len {"name": "disk", "size"} then extents: (the disk) + u64 offset, u64 length, length bytes ... u64 0, u64 0 +'T' u32 len {"files": [{"name", "size", "sha256", "dataBytes"}]} +``` + +Integers are big-endian. The disk's extents are what `SEEK_DATA` and +`SEEK_HOLE` find (on APFS and ext4), less every 64 KiB chunk that is all +zeros. The receiver sizes the file first and writes only the extents, so the +holes stay holes. The disk's sha256 is over the extent records as sent +(offset, length and bytes, then the terminator), not over the image with its +holes read as zeros: both ends compute it without reading the holes, which +on a mostly empty image is most of it. It is recorded in `migration.json` on +both hosts. A stream with no trailer, a trailer that disagrees, or bytes after +it is refused. + +The source checks the disk's size and modification time against the ones it +recorded at the fence before and after it sends, and refuses a disk that +changed: something wrote it while the guest was fenced. It also checks each +bundle file against the manifest as it reads it. + +`migration.json` records the stream's bytes, its duration and its rate. + +## Records + +Both hosts keep `/instances//migration.json`, 0600: + +| Field | Meaning | +|---|---| +| `version` | 1 | +| `txid`, `role` (`source` or `destination`), `name`, `epoch`, `state`, `peer` | the move | +| `remote` | source only: `target`, `store`, `hull`, `hvi`, `gatewaySock`, `allowNoRestore` | +| `bundleDir`, `manifestSha256` | the fenced checkpoint, on either host | +| `disk` | `path`, `size`, `modNs` (at the fence), `sha256`, `dataBytes` | +| `transfer` | `bytes`, `seconds`, `bytesPerSec`, `startedAt` | +| `spec` | destination only: what the source asked it to build (MAC, IP, sandbox id, network) | +| `verify` | destination only: `hvi checkpoint-verify`'s verdict | +| `timingsMs` | milliseconds per step: `fence`, `transfer`, `prepare`, `commit`, `restore`, `resume` | +| `error` | why the last step stopped | +| `history` | every state, with its time | + +A destination that aborts a move removes the instance directory, so the name +is free again, and keeps the record at `/migrations/.json`. + +The instance record's `migration` field (see [storage.md](storage.md)) gains +three states: `migrating` (the source's fenced checkpoint, held by a move), +`migrated` (the source after the commit: stale) and `incoming` (the +destination before the commit), and two fields, `moveTxid` and `peer`. + +## Failures + +| Where it stops | Source | Destination | What to do | +|---|---|---|---| +| preflight or the probe refuses | running, nothing written | nothing | fix what it names | +| the reserve gets no answer | running, `preflight` | maybe `reserved` | run it again, or `--abort` (it warns if the destination does not confirm) | +| the fence fails | running (hvi resumes after a failed capture), `reserved` | `reserved` | run it again, or `--abort` | +| the transfer breaks | fenced, `transferring` | `receiving`, staging removed | run it again; it sends everything again | +| a checksum differs | fenced, `transferring` | `receiving` with the error | find out why; run it again or `--abort` | +| the disk changed after the fence | fenced, `transferring` | | `--abort`; something wrote the disk | +| `hvi checkpoint-verify` fails | fenced, `transferring` | `received` with the error | `--abort` | +| restore refused | fenced, `transferred` | `verified` with the error | fix the destination and run it again, or `--abort` | +| the commit's reply is lost | fenced, `committing` | `restored-paused` or `running` | run it again: it learns the commit | +| commit refused (wrong epoch, aborted) | fenced, `committing` | `aborted` or other | `--abort` | +| the resume fails after the commit | `committed` (the destination owns it) | `running` with the error | `hull resume NAME` on the destination | +| `--abort` gets no answer | fenced, unchanged | unknown | retry when the destination answers | +| `--abort` answered `running` | fenced, unchanged | `running` | `hull migrate NAME --txid T` records the commit | +| the local restore after an abort fails | `aborted`, stopped, `fenced` checkpoint | `aborted` | `hull restore NAME` once hvi can | + +## Operator recovery + +- **Where is a move?** `hull migrate --status NAME` (or TXID) on the source + prints the record, its history, the last error, what the state means and the + destination's state. On the destination, + `hull --store-dir DIR migrate-status --txid T` prints its record as JSON. +- **The source host is gone for good, before the commit.** The destination + never runs the VM on its own. Abort it there with + `hull migrate-abort --txid T`, which removes the received copy. The fenced + checkpoint on the dead source was the only runnable copy. +- **The destination is gone for good, after the fence.** `--abort` cannot be + confirmed, so it refuses. If you know the destination will never come back, + stop the source's fenced VMM, set the source `migration.json` state to + `aborted` and the instance record's `migration.state` to `fenced` by hand, + and `hull restore NAME`. This is the one recovery that relies on your word + instead of the other host's. +- **A committed source.** `hull rm NAME` removes what is left. `hull migrate` + of it is refused: move it back from where it runs. +- **Moving back.** A move from B back to A is a new move, run on B. A still + has the stale copy the first move left under the same name, so the probe + refuses until `hull rm NAME` on A removes it. The owner epoch goes on + counting: 0 on A, 1 on B, 2 back on A. + +## Testing a move + +`HULL_MIGRATE_SSH` names a program to run instead of `ssh`, with the same +arguments. A script that runs its last argument with `sh -c` runs the +destination's verbs locally, so with `--remote-store` naming a second store a +whole move runs on one machine without a network: + +```sh +#!/bin/sh +for a; do last=$a; done +exec sh -c "$last" +``` + +The unit tests (`cmd/hull/migrate_test.go`) run every step in one process +with the destination behind an interface that can drop a reply or answer for +it: a commit whose acknowledgement is lost, an abort before and after the +commit, a destination that answers `running` to an abort, a transfer cut off +midway, a disk changed after the fence. `TestMoveOverFakeSSH` runs the +destination as a separate hull process through that transport, with a fake +hvi that has no restore, and ends in "restore refused" and an abort. + +Live runs are recorded in [Verified](#verified). + +## Verified + +2026-10-14, hull at this branch. Guest: alpine, block rootfs (15 GiB image, +about 13 MiB of it data), 512 MiB, 2 vCPUs, `--net none`, printing a counter +every second. This Mac has no sshd, so the Mac runs used the +`HULL_MIGRATE_SSH` stand-in with two stores; rp1 to rp2 used real ssh. + +**Through the commit, Mac to Mac (HVF), hvi 4ea836f, without +`--allow-no-restore`**, run twice: + +- `hull migrate` took 1.5 s: fence 300 to 308 ms (capture 269 ms, 56.6 of + 512 MiB RAM stored), stream 69.8 MiB in 0.39 to 0.40 s with 13.0 MiB of + the disk, restore paused 352 to 362 ms, commit 59 to 64 ms. +- The guest went on counting on the destination (TICK 10, 11, 12, ...) and + never printed its boot line there again. A file written and synced before + the move read back on the destination with `hull exec`. +- Afterwards: the source stopped its fenced VMM; its record says + `committed` and its checkpoint `migrated`; a new move of it and an abort + were refused; the destination says `running`, owner epoch 1. +- Moving it back was refused while the stale copy was there, with the + `hull rm` remedy; after `hull rm` the move back committed, the guest + counted on at home (TICK 17, 19), the file was still there, owner epoch 2. + +**Up to the restore, hvi 263713c (no restore subcommand)**, with +`--allow-no-restore`: + +| | Mac to Mac (HVF) | rp1 to rp2 (KVM, Raspberry Pi 5) | +|---|---|---| +| Preflight without `--allow-no-restore` | refused: the source hvi has no restore | | +| Fence | capture 266 ms, 56.4 of 512 MiB RAM stored | capture 634 ms, 56.8 MiB | +| Counter after the fence | held (TICK 10 to 10 over 3 s) | held (TICK 11 to 11) | +| Stream | 69.6 MiB in 0.36 s | 69.5 MiB in 1.45 s (48 MiB/s) | +| Disk carried | 13.0 MiB of 15.0 GiB | 12.7 MiB of 15.0 GiB | +| Disk allocated, source / destination (`du`) | 151 / 13 MiB | 235 / 13 MiB | +| Bundle and disk on both ends | `cmp` and `diff -r` identical | sha256 identical | +| Destination's `hvi checkpoint-verify` | `complete: every part a restore needs is captured` | `partial: vCPU registers, interrupt controller and timer not captured` (H5 not done) | +| Restore on the destination | refused: no `restore` subcommand; destination `verified`, source `transferred` | the same | +| Plain `hull restore` of either side | refused by the move | refused by the move | +| `--abort` | destination removed the instance, kept the tombstone; source stopped its fenced VMM and tried `hull restore`, refused for the same reason | the same | + +Not run live: a lost commit acknowledgement, an abort answered `running`, a +transfer cut midway and a disk changed after the fence (the unit tests cover +each), any gateway case, and a move on KVM past the restore. diff --git a/docs/storage.md b/docs/storage.md index f33149cb..ea6def8b 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -189,6 +189,7 @@ in the name only keeps two pulls' directories apart. │ └── provenance.json ├── volumes/_/ compose named volumes ├── compose/.json compose project state, plus its gateway sockets and log +├── migrations/.json the records of moves this store aborted as their destination └── instances// ├── state.json 0600 ├── bundle/ config.json plus the rootfs symlink or image @@ -198,6 +199,8 @@ in the name only keeps two pulls' directories apart. ├── control.sock hvi only: the control socket (pause, checkpoint, stop) ├── profile.json hvi only: the machine profile hvi writes at boot ├── migration.lock hvi only: held by one checkpoint or restore at a time + ├── migration.json hvi only: the record of a move to or from another host + ├── migrate.lock hvi only: held by one step of a move at a time └── (QMP socket, agent socket, staged boot files) ``` @@ -529,3 +532,17 @@ refused: Virtualization.framework does not rehydrate the guest's FUSE state in a new VMM process, so every inode would go stale at restore. See [checkpoint-restore.md](checkpoint-restore.md). + +## Moves + +`hull migrate` keeps `/migration.json` (0600) on both hosts of a +move: its txid, role, owner epoch, state, the other host, the fenced bundle, +the disk's size, sha256 and modification time at the fence, the stream's +bytes and rate, the time each step took, the last error and the history of +states. It is written through a temp file, an fsync, a rename and an fsync of +the directory, because each state is a promise to the other host. The +instance record's `migration` field gains `moveTxid` and `peer`, and three +states (`migrating`, `migrated`, `incoming`); `state.json` gains +`ownerEpoch`. A destination that aborts a move removes the instance directory +and keeps the record as `/migrations/.json`. See +[migration.md](migration.md#records). diff --git a/pkg/store/store.go b/pkg/store/store.go index 3b959dde..ce052f46 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -105,18 +105,35 @@ type InstanceState struct { // leaves it alone: a fenced instance that was stopped is exactly the one // a restore needs to find. Migration *MigrationState `json:"migration,omitempty"` + // OwnerEpoch counts the moves (`hull migrate`) that brought this + // instance to this host: 0 for one that was started here, and one more + // than the source's at every committed move. A move hands over the + // next epoch, and the destination accepts its commit only for that + // epoch, so a commit meant for an earlier move is refused. + OwnerEpoch uint64 `json:"ownerEpoch,omitempty"` } // Migration states. A checkpointed instance kept running after the capture, // so its disk has moved on since; a fenced one never ran again, so its disk // still matches the bundle. Restoring records restoring, then restored-paused // once hvi has rebuilt the VM, then restored once it runs. +// +// A move (`hull migrate`) adds three. On the source, migrating is a fenced +// checkpoint that a move has taken over: only the move, or its abort, may +// decide what becomes of it, so a plain restore refuses it. migrated is a +// source whose move committed: the VM runs on another host now, and this +// checkpoint is stale for good. On the destination, incoming is a received +// checkpoint that only the move's own restore may bring up, and only its +// commit may resume. const ( MigrationCheckpointed = "checkpointed" MigrationFenced = "fenced" MigrationRestoring = "restoring" MigrationRestoredPaused = "restored-paused" MigrationRestored = "restored" + MigrationMigrating = "migrating" + MigrationMigrated = "migrated" + MigrationIncoming = "incoming" ) // MigrationState is the record of the last checkpoint taken of an instance @@ -134,6 +151,12 @@ type MigrationState struct { // hvi may write to, so a restore can check the bundle is the one hull // recorded. ManifestSHA256 string `json:"manifestSha256,omitempty"` + // MoveTxID names the move (`hull migrate`) this checkpoint belongs to, + // when it belongs to one; its record is /migration.json. + MoveTxID string `json:"moveTxid,omitempty"` + // Peer is the other host of that move: where a migrated instance went, + // or where an incoming one came from. + Peer string `json:"peer,omitempty"` } // StatusStarting marks the one moment an instance has no pid to record: the @@ -718,6 +741,20 @@ func (s *Store) InstanceProfilePath(id string) string { return s.instancePath(id, "profile.json") } +// InstanceMigrationRecord returns the path of the record of the instance's +// move between hosts (`hull migrate`), on either side of it. +func (s *Store) InstanceMigrationRecord(id string) string { + return s.instancePath(id, "migration.json") +} + +// MigrationTombstoneDir is where a destination keeps the records of moves it +// aborted. Aborting removes the instance directory, so the name is free for +// another run, and the record moves here so a late commit or a status query +// for its txid still gets an answer. +func (s *Store) MigrationTombstoneDir() string { + return filepath.Join(s.rootDir, "migrations") +} + // DeleteInstance removes an instance directory func (s *Store) DeleteInstance(id string) error { // This one recurses, so it validates even though callers resolve the From d52b752c1e6a9bc8b66ddac2d7c4addb12a91114 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Wed, 14 Oct 2026 07:07:37 +0300 Subject: [PATCH 11/33] feat(hvi): keep a ledger of committed moves and confirm every abort A destination kept a move's record only in the instance's migration.json. When the source lost the commit's acknowledgement and the destination then moved the instance on or removed it, nothing there remembered the commit, and an abort from the source could run the guest on two hosts. The destination now keeps a ledger of the moves it committed, with the state change each entry records, and an abort that the ledger refuses says whether the instance moved on or was removed. A committing source stays fenced unless the destination confirms the abort, and takes "unknown txid" as no answer. A destination answers an abort only once nothing of the move can run there. A retry that finds the copy verified checks the disk against the fence, a second --abort finishes what the first left undone, and a txid resolves only to the instance that holds it. migration.md describes the ledger, what an abort needs, and the live runs of a lost commit acknowledgement. Signed-off-by: Panagiotis Moustafellos --- cmd/hull/migrate_dest.go | 100 ++++++-- cmd/hull/migrate_dest_test.go | 3 +- cmd/hull/migrate_record.go | 20 +- cmd/hull/migrate_source.go | 186 ++++++++++++-- cmd/hull/migrate_ssh.go | 10 + cmd/hull/migrate_test.go | 450 ++++++++++++++++++++++++++++++++++ docs/checkpoint-restore.md | 20 +- docs/cli.md | 5 +- docs/migration.md | 120 +++++++-- docs/storage.md | 7 +- pkg/store/store.go | 10 +- 11 files changed, 840 insertions(+), 91 deletions(-) diff --git a/cmd/hull/migrate_dest.go b/cmd/hull/migrate_dest.go index a719df0a..e6ec37e4 100644 --- a/cmd/hull/migrate_dest.go +++ b/cmd/hull/migrate_dest.go @@ -28,6 +28,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strings" "syscall" "time" @@ -234,7 +235,9 @@ func destRecord(s *store.Store, txid string) (*migrationRecord, string, func(), if rec.Role != roleDestination { return nil, "", nil, refuseWith(codeFailed, fmt.Errorf("move %s is recorded here as its %s", txid, rec.Role), rec) } - if rec.State == migAborted { + if path != s.InstanceMigrationRecord(rec.Name) { + // From the ledger: the move ended here, and its record there is + // never written again but by an abort of the instance it names. return rec, path, nil, nil } unlock, err := lockMove(filepath.Dir(path), moveLockWait) @@ -651,6 +654,11 @@ func destCommit(s *store.Store, txid string, epoch uint64) *migrateReply { if err := s.SaveInstance(state); err != nil { return refuseWith(codeFailed, rec.fail(path, err), rec) } + // The ledger first: once the instance record says running, the store + // must answer for this txid whatever later happens to the instance. + if err := writeMoveLedger(s, rec, dstRunning); err != nil { + return refuseWith(codeFailed, rec.fail(path, fmt.Errorf("record the commit in the ledger: %w", err)), rec) + } // The destination's commit point: from here the guest may run, and // nothing may abort the move. if err := rec.advance(path, dstRunning); err != nil { @@ -672,50 +680,91 @@ func destCommit(s *store.Store, txid string, epoch uint64) *migrateReply { // destAbort is migrate-abort: refused once the move has committed here; // otherwise it records aborted first (so a commit that arrives later is -// refused), then stops any paused VM the move restored, keeps the record as -// a tombstone and frees the name. A move this store never heard of -// answers ok with code unknown_txid: nothing here can run it. +// refused), keeps the record in the ledger, then stops any paused VM the +// move restored and frees the name. It answers ok only once all of that is +// done: a source that hears ok runs the guest again, so a copy left here +// that something could still resume is a refusal, and running the abort +// again finishes the job. A move this store has no record of is refused +// with code unknown_txid: whether that is enough for the source to run the +// guest again depends on whether it ever sent the commit, which only the +// source knows. func destAbort(s *store.Store, txid string) *migrateReply { rec, path, unlock, no := destRecord(s, txid) if no != nil { - if no.Code == codeUnknownTxID { - no.OK = true - } return no } if unlock != nil { defer unlock() } switch rec.State { - case migAborted: - return &migrateReply{OK: true, Record: rec} case dstRunning: + if path != s.InstanceMigrationRecord(rec.Name) { + return refuseWith(codeRunning, fmt.Errorf("move %s committed here at epoch %d; %s has since been "+ + "removed here or moved on", txid, rec.Epoch, rec.Name), rec) + } return refuseWith(codeRunning, fmt.Errorf("move %s committed here and %s runs here, at epoch %d", txid, rec.Name, rec.Epoch), rec) + case migAborted: + if path != s.InstanceMigrationRecord(rec.Name) { + // From the ledger: the instance is gone already. + return &migrateReply{OK: true, Record: rec} + } + default: + if err := rec.advance(path, migAborted); err != nil { + return refuseWith(codeFailed, err, rec) + } } - if err := rec.advance(path, migAborted); err != nil { - return refuseWith(codeFailed, err, rec) + // The ledger before the name is freed: once the instance directory is + // gone, it is what refuses a late commit. + if err := writeMoveLedger(s, rec, migAborted); err != nil { + return refuseWith(codeFailed, fmt.Errorf("move %s is aborted here, but recording it in the ledger "+ + "failed: %w; run the abort again", txid, err), rec) } - if err := os.MkdirAll(s.MigrationTombstoneDir(), 0o700); err == nil { - _ = writeMigrationRecord(filepath.Join(s.MigrationTombstoneDir(), txid+".json"), rec) + err := destCleanup(s, rec.Name) + if err == nil { + if _, serr := os.Stat(s.InstanceDir(rec.Name)); serr == nil { + err = errors.New("its directory is still there") + } } - var cleanup error - if _, err := s.GetInstance(rec.Name); err == nil { - cleanup = removeInstanceIn(s, rec.Name, true) - } else { - cleanup = s.DeleteInstance(rec.Name) + if err != nil { + return refuseWith(codeFailed, fmt.Errorf("move %s is aborted here, but removing %s failed: %w; "+ + "run the abort again", txid, rec.Name, err), rec) } - reply := &migrateReply{OK: true, Record: rec} - if cleanup != nil { - reply.Error = fmt.Sprintf("aborted, but removing %s failed: %v", rec.Name, cleanup) + return &migrateReply{OK: true, Record: rec} +} + +// destCleanup removes what an aborted move left of instance name: the VM +// its restore started, if any, and the directory. A variable so the tests +// can make it fail. +var destCleanup = func(s *store.Store, name string) error { + if _, err := s.GetInstance(name); err == nil { + return removeInstanceIn(s, name, true) } - return reply + return s.DeleteInstance(name) +} + +// writeMoveLedger writes rec, in state, to /migrations/.json: +// the ledger of the moves that ended on this store. A commit writes it +// before the instance record says running, an abort before it frees the +// name. Neither `hull rm` nor a later move of the instance removes it, so a +// source that asks about the move later, after its commit acknowledgement +// was lost, still gets the answer: the move committed here, or it did not. +func writeMoveLedger(s *store.Store, rec *migrationRecord, state string) error { + if err := os.MkdirAll(s.MigrationTombstoneDir(), 0o700); err != nil { + return err + } + entry := *rec + if entry.State != state { + entry.State = state + entry.History = append(slices.Clone(rec.History), migrateHistory{State: state, At: time.Now().UTC()}) + } + return writeMigrationRecord(filepath.Join(s.MigrationTombstoneDir(), rec.TxID+".json"), &entry) } // destStatus is migrate-status: the record, and what hvi says the VM is // doing. func destStatus(s *store.Store, txid string) *migrateReply { - rec, _, err := findMigrationByTxID(s, txid) + rec, path, err := findMigrationByTxID(s, txid) if errors.Is(err, os.ErrNotExist) { return refuse(codeUnknownTxID, fmt.Errorf("no move %s is recorded here", txid)) } @@ -723,6 +772,11 @@ func destStatus(s *store.Store, txid string) *migrateReply { return refuse(codeFailed, err) } reply := &migrateReply{OK: true, Record: rec} + // Only for the instance this move holds: a ledger entry's name may by + // now be another instance, or one that moved on. + if path != s.InstanceMigrationRecord(rec.Name) { + return reply + } if state, err := s.GetInstance(rec.Name); err == nil && rec.State != migAborted { reply.VM = hviVMState(state) } diff --git a/cmd/hull/migrate_dest_test.go b/cmd/hull/migrate_dest_test.go index 7c3ab580..f1a9ae14 100644 --- a/cmd/hull/migrate_dest_test.go +++ b/cmd/hull/migrate_dest_test.go @@ -135,7 +135,8 @@ func TestDestinationVerbs(t *testing.T) { if r := destStatus(s, strings.Repeat("ef", 16)); r.OK || r.Code != codeUnknownTxID { t.Errorf("status of an unknown move: %+v", r) } - if r := destAbort(s, strings.Repeat("ef", 16)); !r.OK || r.Code != codeUnknownTxID { + // Refused, not ok: only the source knows whether that is enough. + if r := destAbort(s, strings.Repeat("ef", 16)); r.OK || r.Code != codeUnknownTxID { t.Errorf("abort of an unknown move: %+v", r) } // A stream that is not one is refused, and the record says why. diff --git a/cmd/hull/migrate_record.go b/cmd/hull/migrate_record.go index 9cf149b7..52c02da2 100644 --- a/cmd/hull/migrate_record.go +++ b/cmd/hull/migrate_record.go @@ -123,6 +123,11 @@ type migrationRecord struct { // Source only: how to reach the destination's hull again on a retry. Remote *migrateRemote `json:"remote,omitempty"` + // DestReleased is set on the source of an aborted move once the + // destination has confirmed it keeps nothing of it. An abort that ran + // without that word (before the fence, with no answer) leaves it unset, + // and running the abort again asks again. + DestReleased bool `json:"destReleased,omitempty"` // BundleDir and ManifestSHA256 name the fenced checkpoint the move // carries, on either side once it is there. @@ -217,7 +222,7 @@ var txidPattern = regexp.MustCompile(`^[0-9a-f]{32}$`) // newMoveTxID draws a move's txid. 128 bits, not the 64 of a checkpoint's: // a move's txid is matched across two hosts' stores and lives on in their -// tombstones, and a collision there would let one move's commit act on +// ledgers, and a collision there would let one move's commit act on // another's record. func newMoveTxID() (string, error) { var b [16]byte @@ -344,8 +349,11 @@ func readMigrationRecord(path string) (*migrationRecord, error) { } // findMigrationByTxID finds the record of move txid in s: in an instance -// directory, or among the tombstones of aborted moves. It returns the -// record's path too. os.ErrNotExist when this store never saw the move. +// directory, or in the ledger of moves that ended here (committed or +// aborted). The instance directory wins: it is the record a move's steps +// write, and the ledger keeps the move's outcome once the instance has gone +// or moved on. It returns the record's path too. os.ErrNotExist when this +// store never saw the move. func findMigrationByTxID(s *store.Store, txid string) (*migrationRecord, string, error) { if err := checkTxID(txid); err != nil { return nil, "", err @@ -427,6 +435,12 @@ func moveRefusal(s *store.Store, id, verb string) error { case rec.Role == roleSource && rec.State != migAborted: return fmt.Errorf("instance %s is being moved to %s (move %s, %s): no %s until the move "+ "commits, or `hull migrate --abort %s` ends it", id, rec.Peer, rec.TxID, rec.State, verb, rec.TxID) + case rec.Role == roleDestination && rec.State == migAborted && verb != "rm": + // The abort's cleanup did not finish: whatever is left may be the + // restored, paused guest, which the source now runs again. + return fmt.Errorf("instance %s is what is left here of move %s from %s, which was aborted: no %s. "+ + "`hull migrate-abort --txid %s` here, or `hull rm --force %s`, removes it", id, rec.TxID, rec.Peer, + verb, rec.TxID, id) case rec.Role == roleDestination && rec.State != dstRunning && rec.State != migAborted: return fmt.Errorf("instance %s is arriving from %s (move %s, %s): only the move's commit "+ "runs it, and `hull migrate --abort %s` on the source ends it", id, rec.Peer, rec.TxID, diff --git a/cmd/hull/migrate_source.go b/cmd/hull/migrate_source.go index bb26a73e..050c89f5 100644 --- a/cmd/hull/migrate_source.go +++ b/cmd/hull/migrate_source.go @@ -113,10 +113,17 @@ func migrateCommand() *cli.Command { } // moveInstanceName resolves the argument of --abort or --status: a move's -// txid recorded in this store, or an instance name. +// txid recorded in this store, or an instance name. A txid must name the +// move the instance holds now: one found only in the ledger, or one the +// instance has since been moved on from, is not that instance's move, and +// acting on the instance's current move instead would act on another one. func moveInstanceName(s *store.Store, arg string) (string, error) { if txidPattern.MatchString(arg) { - if rec, _, err := findMigrationByTxID(s, arg); err == nil { + if rec, path, err := findMigrationByTxID(s, arg); err == nil { + if path != s.InstanceMigrationRecord(rec.Name) { + return "", fmt.Errorf("move %s ended here as its %s (%s) and no instance holds it now; "+ + "`hull migrate-status --txid %s` prints its record", arg, rec.Role, rec.State, arg) + } return rec.Name, nil } } @@ -666,11 +673,19 @@ func bundleFiles(dir string) ([]string, error) { // transfer streams the bundle and the disk to migrate-receive, which ends // verified on the destination. A destination already verified is not sent -// it again. +// it again, but the disk is checked as after a stream: unchanged since the +// fence, and the same as what the destination holds by the source's own +// hash. func (m *moveSource) transfer(ctx context.Context) error { if reply, err := m.peer.call(ctx, "migrate-status", []string{"--txid", m.rec.TxID}, nil); err == nil && reply.Record != nil && migrateStateOrder(roleDestination, reply.Record.State) >= migrateStateOrder(roleDestination, dstVerified) { + if err := m.checkDiskUnchanged(); err != nil { + return err + } + if err := m.sourceDiskSum(); err != nil { + return err + } if err := m.compareReceived(reply.Record); err != nil { return err } @@ -726,6 +741,16 @@ func (m *moveSource) transfer(ctx context.Context) error { reply, err := m.peer.call(ctx, "migrate-receive", []string{"--txid", m.rec.TxID, "--name", m.name}, pr) _ = pr.CloseWithError(errors.New("the destination stopped reading")) sendErr := <-sent + if sendErr == nil { + // The disk's hash as sent, on record before the reply is looked + // at: a reply that is lost leaves a re-run to compare the + // destination's copy with it. + m.rec.Disk.SHA256 = diskSum.SHA256 + m.rec.Disk.DataBytes = diskSum.DataBytes + if werr := writeMigrationRecord(m.path, m.rec); werr != nil { + return werr + } + } if sendErr != nil && !errors.Is(sendErr, io.ErrClosedPipe) && err == nil { err = fmt.Errorf("send: %w", sendErr) } @@ -742,9 +767,12 @@ func (m *moveSource) transfer(ctx context.Context) error { if reply.Record == nil { return errors.New("the destination's receive reply has no record") } - if sendErr == nil { - m.rec.Disk.SHA256 = diskSum.SHA256 - m.rec.Disk.DataBytes = diskSum.DataBytes + if sendErr != nil { + // The destination answered without reading it all: it had it. + if err := m.sourceDiskSum(); err != nil { + return err + } + } else { m.rec.Transfer = &migrateTransfer{Bytes: sw.written(), Seconds: took.Seconds(), StartedAt: started.UTC(), BytesPerSec: int64(float64(sw.written()) / max(took.Seconds(), 1e-3))} m.timing("transfer", started) @@ -770,14 +798,28 @@ func (m *moveSource) compareReceived(dst *migrationRecord) error { if dst.ManifestSHA256 != m.rec.ManifestSHA256 { return fmt.Errorf("the destination has manifest sha256 %s, the source %s", dst.ManifestSHA256, m.rec.ManifestSHA256) } - if dst.Disk == nil || dst.Disk.Size != m.rec.Disk.Size || - (m.rec.Disk.SHA256 != "" && dst.Disk.SHA256 != m.rec.Disk.SHA256) { + if m.rec.Disk.SHA256 == "" { + return errors.New("the source has no hash of its disk to compare the destination's with") + } + if dst.Disk == nil || dst.Disk.Size != m.rec.Disk.Size || dst.Disk.SHA256 != m.rec.Disk.SHA256 { return errors.New("the destination's disk is not the one the source sent") } - if m.rec.Disk.SHA256 == "" { - m.rec.Disk.SHA256 = dst.Disk.SHA256 - m.rec.Disk.DataBytes = dst.Disk.DataBytes + return nil +} + +// sourceDiskSum hashes the source's disk as the stream would carry it, when +// the record has no hash of it yet: the stream was cut before its end, or +// hull did not live to record it. +func (m *moveSource) sourceDiskSum() error { + if m.rec.Disk.SHA256 != "" { + return nil + } + sum, err := newMigrateStreamWriter(io.Discard).sparse("disk", m.rec.Disk.Path) + if err != nil { + return fmt.Errorf("hash the disk: %w", err) } + m.rec.Disk.SHA256 = sum.SHA256 + m.rec.Disk.DataBytes = sum.DataBytes return nil } @@ -855,6 +897,8 @@ func (m *moveSource) finish() error { // sourceAbort is `hull migrate --abort`: before the commit, and only once // the destination has confirmed that it is not running the VM and will // refuse the commit, it ends the move and brings the instance back here. +// Running it again on an aborted move finishes what the first run left +// undone: a release the destination did not confirm, or the local restore. func sourceAbort(ctx context.Context, s *store.Store, name string, w io.Writer, peer movePeer) error { unlock, err := lockMove(s.InstanceDir(name), 0) if errors.Is(err, errMoveBusy) { @@ -880,32 +924,57 @@ func sourceAbort(ctx context.Context, s *store.Store, name string, w io.Writer, return fmt.Errorf("move %s committed: %s runs on %s at epoch %d. Moving it back is a new "+ "`hull migrate` run on %s", rec.TxID, name, rec.Peer, rec.Epoch, rec.Peer) case migAborted: - _, _ = fmt.Fprintf(w, "Move %s was already aborted\n", rec.TxID) - return nil + return finishAbort(ctx, s, name, rec, path, w, peer) } state, err := s.GetInstance(name) if err != nil { return err } + // Fenced by the record, by the instance record, or by hvi itself: a + // fence whose reply was lost before hull recorded it. fenced := migrateStateOrder(roleSource, rec.State) >= migrateStateOrder(roleSource, srcFenced) || fencedFor(state, rec.TxID) + if !fenced && hviVMState(state) == store.MigrationFenced { + // Adopted before anything changes: the abort must have the bundle + // to restore the instance from, or it has nothing to bring back. + if err := adoptFenceForAbort(s, name, state, rec, w); err != nil { + return err + } + fenced = true + } + // From committing on the commit may have been delivered, so only the + // destination's record of this very move, aborted, lets the source go. + committing := rec.State == srcCommitting p := peerFor(rec, peer) reply, err := p.call(ctx, "migrate-abort", []string{"--txid", rec.TxID}, nil) + released := err == nil && abortedThere(reply, rec) switch { case err != nil && refusalCode(err) == codeRunning: - return fmt.Errorf("the destination %s says move %s committed there and runs %s: it cannot be "+ - "aborted. `hull migrate %s --txid %s` records the commit here", rec.Peer, rec.TxID, name, name, rec.TxID) + return fmt.Errorf("the destination %s says move %s committed there (%v): it cannot be aborted. "+ + "`hull migrate %s --txid %s` records the commit here", rec.Peer, rec.TxID, refusalDetail(err), name, + rec.TxID) + case err != nil && refusalCode(err) == codeUnknownTxID && committing: + return rec.fail(path, fmt.Errorf("cannot abort move %s: %s has no record of it, but this host has "+ + "sent it the commit, which it may have run before its record was lost. The source stays fenced: "+ + "running %s here could put it on two hosts. Find out where it runs (docs/migration.md, "+ + "\"Operator recovery\")", rec.TxID, rec.Peer, name)) + case err != nil && refusalCode(err) == codeUnknownTxID: + // No commit was ever sent, so a destination that never heard of + // the move, or lost its reservation, cannot be running it. + err, released = nil, true case err != nil && fenced: return rec.fail(path, fmt.Errorf("cannot abort move %s: %w\nWithout the destination's word that it "+ "is not running %s and will refuse the commit, the source stays fenced: running it here could "+ "put it on two hosts. Retry when %s answers", rec.TxID, err, name, rec.Peer)) case err != nil: _, _ = fmt.Fprintf(w, "warning: %s did not confirm the abort (%v); it may still hold a reservation for %s, "+ - "which `hull migrate --abort %s` again releases\n", rec.Peer, err, name, rec.TxID) + "which `hull migrate --abort %s` run again releases\n", rec.Peer, err, name, rec.TxID) } - if reply != nil && reply.Error != "" { - _, _ = fmt.Fprintf(w, "warning: %s: %s\n", rec.Peer, reply.Error) + if err == nil && fenced && reply.OK && !abortedThere(reply, rec) { + return rec.fail(path, fmt.Errorf("cannot abort move %s: the reply of %s does not say it aborted "+ + "this move at epoch %d, so the source stays fenced. Retry", rec.TxID, rec.Peer, rec.Epoch)) } + rec.DestReleased = released if err := rec.advance(path, migAborted); err != nil { return err } @@ -913,12 +982,78 @@ func sourceAbort(ctx context.Context, s *store.Store, name string, w io.Writer, _, _ = fmt.Fprintf(w, "Aborted move %s; %s never stopped running here\n", rec.TxID, name) return nil } - // The destination is out of it. The instance is the fenced checkpoint - // again, and is restored from it here. - if state.Migration != nil && state.Migration.TxID == rec.TxID { + return abortLocally(s, name, rec, w) +} + +// finishAbort is `hull migrate --abort` of a move already aborted here. It +// asks the destination again if it never confirmed the release, and it +// restores the instance if the first run recorded the abort but did not +// live to hand the fenced checkpoint back. +func finishAbort(ctx context.Context, s *store.Store, name string, rec *migrationRecord, path string, w io.Writer, + peer movePeer) error { + if !rec.DestReleased && rec.Remote != nil { + reply, err := peerFor(rec, peer).call(ctx, "migrate-abort", []string{"--txid", rec.TxID}, nil) + if (err == nil && abortedThere(reply, rec)) || refusalCode(err) == codeUnknownTxID { + rec.DestReleased = true + if err := writeMigrationRecord(path, rec); err != nil { + return err + } + _, _ = fmt.Fprintf(w, "%s has released move %s\n", rec.Peer, rec.TxID) + } else { + if err == nil { + err = errors.New("its reply does not say it aborted the move") + } + _, _ = fmt.Fprintf(w, "warning: %s still did not confirm the abort (%v); `hull migrate --abort %s` "+ + "run again asks it again\n", rec.Peer, err, rec.TxID) + } + } + state, err := s.GetInstance(name) + if err != nil { + return err + } + if m := state.Migration; m != nil && m.TxID == rec.TxID && + (m.State == store.MigrationMigrating || m.State == store.MigrationFenced) { + return abortLocally(s, name, rec, w) + } + _, _ = fmt.Fprintf(w, "Move %s was already aborted\n", rec.TxID) + return nil +} + +// adoptFenceForAbort records the bundle hvi fenced the VM into for move rec +// when hull did not live to record it, as a re-run of the move would. +func adoptFenceForAbort(s *store.Store, name string, state *store.InstanceState, rec *migrationRecord, + w io.Writer) error { + unlock, err := lockInstanceMigration(s, name) + if err != nil { + return err + } + defer unlock() + root, ok := flagValue(state.CmdLine, "--checkpoint-root") + if !ok { + return errors.New("the instance is fenced and has no checkpoint root") + } + dir := filepath.Join(root, rec.TxID) + if _, err := os.Stat(filepath.Join(dir, "manifest.json")); err != nil { + return fmt.Errorf("cannot abort move %s: the VM of %s is fenced, but no bundle of this move is at %s "+ + "to restore it from. Nothing was changed", rec.TxID, name, dir) + } + m := &moveSource{s: s, name: name, rec: rec, out: w} + return m.adoptFence(state, dir) +} + +// abortLocally is the source's half of an abort, once the move is recorded +// aborted: the instance is the fenced checkpoint again, the fenced VMM is +// stopped, and the instance is restored from the checkpoint here. +func abortLocally(s *store.Store, name string, rec *migrationRecord, w io.Writer) error { + state, err := s.GetInstance(name) + if err != nil { + return err + } + if state.Migration != nil && state.Migration.TxID == rec.TxID && state.Migration.State != store.MigrationFenced { state.Migration.State = store.MigrationFenced if err := s.SaveInstance(state); err != nil { - return err + return fmt.Errorf("aborted move %s, but recording its checkpoint as %s's again failed: %w; "+ + "`hull migrate --abort %s` run again finishes", rec.TxID, name, err, rec.TxID) } } if state.Status == "running" && state.PID > 0 && processIsAVMM(state.PID, state) { @@ -935,6 +1070,13 @@ func sourceAbort(ctx context.Context, s *store.Store, name string, w io.Writer, return nil } +// abortedThere reports whether reply is the destination's record of move rec, +// aborted. +func abortedThere(reply *migrateReply, rec *migrationRecord) bool { + r := reply.Record + return r != nil && r.State == migAborted && r.TxID == rec.TxID && r.Epoch == rec.Epoch +} + // localRestore runs `hull restore --detach NAME` here, as a child for the // same reason as on the destination. var localRestore = func(s *store.Store, name string) error { diff --git a/cmd/hull/migrate_ssh.go b/cmd/hull/migrate_ssh.go index d9eaf5ba..4a4f74d3 100644 --- a/cmd/hull/migrate_ssh.go +++ b/cmd/hull/migrate_ssh.go @@ -118,6 +118,16 @@ func refusalCode(err error) string { return "" } +// refusalDetail returns what the destination said in a migrateRefusal in +// err's chain, or err itself. +func refusalDetail(err error) string { + var r *migrateRefusal + if errors.As(err, &r) { + return r.Reply.Error + } + return err.Error() +} + // movePeer runs one remote verb on the destination. stdin, when not nil, is // copied to the verb's standard input. The source drives every step through // this, so the tests can put the destination in the same process, drop a diff --git a/cmd/hull/migrate_test.go b/cmd/hull/migrate_test.go index a575aade..d15bf8b4 100644 --- a/cmd/hull/migrate_test.go +++ b/cmd/hull/migrate_test.go @@ -17,6 +17,7 @@ package main import ( "bytes" "context" + "errors" "io" "os" "path/filepath" @@ -571,3 +572,452 @@ func TestMoveThereAndBack(t *testing.T) { t.Errorf("the second move's source record: %+v %v", away, err) } } + +// loseCommitAck runs a move whose commit reply is lost: the destination +// runs the guest, the source stays fenced in committing. +func loseCommitAck(t *testing.T, f *moveFixture) *localPeer { + t.Helper() + peer := newLocalPeer(f.dst) + peer.drop["migrate-commit"] = 1 + _ = sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer) + if got := f.srcRecord(t).State; got != srcCommitting { + t.Fatalf("source is %s, want committing", got) + } + return peer +} + +// abortRefusedThenCommitted checks that the source's abort is refused +// because the destination committed, and that a re-run records the commit. +func abortRefusedThenCommitted(t *testing.T, f *moveFixture, peer movePeer) { + t.Helper() + restored := 0 + localRestore = func(*store.Store, string) error { restored++; return nil } + err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer) + if err == nil || !strings.Contains(err.Error(), "cannot be aborted") { + t.Errorf("abort of a move the destination committed: %v", err) + } + if restored != 0 || f.srcRecord(t).State != srcCommitting { + t.Fatalf("the source was brought back (restored %d, record %s)", restored, f.srcRecord(t).State) + } + opt := f.options(false) + opt.txid = f.srcRecord(t).TxID + if err := sourceMigrate(context.Background(), f.src, f.name, opt, peer); err != nil { + t.Fatal(err) + } + if got := f.srcRecord(t).State; got != srcCommitted { + t.Errorf("after the re-run the source is %s", got) + } +} + +// The commit's acknowledgement is lost and the destination then moves the +// instance on to a third host, which replaces its migration.json. Its +// ledger still says the first move committed there, so the source's abort +// is refused and a re-run records the commit: the guest never runs on two +// hosts. +func TestMoveLedgerAnswersAfterTheInstanceMovedOn(t *testing.T) { + f := newMoveFixture(t, true) + peerB := loseCommitAck(t, f) + txid := f.srcRecord(t).TxID + c := shortStore(t) + on := f.options(false) + on.to, on.remote.Target = "pm@c", "pm@c" + if err := sourceMigrate(context.Background(), f.dst, f.name, on, newLocalPeer(c)); err != nil { + t.Fatalf("moving on to c: %v", err) + } + if mustInstance(t, c, f.name).Status != "running" { + t.Fatal("c does not run the guest") + } + r := destStatus(f.dst, txid) + if !r.OK || r.Record.State != dstRunning || r.VM != "" { + t.Errorf("status of the first move after the move on: %+v", r) + } + abortRefusedThenCommitted(t, f, peerB) +} + +// The same with the destination's operator removing the instance after the +// commit: `hull rm` leaves the ledger, so the source cannot abort a move +// that ran on the destination and resume the guest from before it. +func TestMoveLedgerAnswersAfterRm(t *testing.T) { + f := newMoveFixture(t, true) + peer := loseCommitAck(t, f) + if err := removeInstanceIn(f.dst, f.name, true); err != nil { + t.Fatalf("rm on the destination: %v", err) + } + abortRefusedThenCommitted(t, f, peer) +} + +// Once the source has sent the commit, a destination with no record of the +// move, or a reply that is not its record of this move aborted, leaves the +// source fenced: the commit may have run there. +func TestMoveAbortWhileCommittingNeedsTheAbortedRecord(t *testing.T) { + f := newMoveFixture(t, true) + peer := loseCommitAck(t, f) + rec := f.srcRecord(t) + restored := 0 + localRestore = func(*store.Store, string) error { restored++; return nil } + for _, answer := range []*migrateReply{ + {OK: false, Code: codeUnknownTxID, Error: "no move is recorded here"}, + {OK: true}, + {OK: true, Record: &migrationRecord{TxID: rec.TxID, Epoch: rec.Epoch + 1, State: migAborted}}, + {OK: true, Record: &migrationRecord{TxID: rec.TxID, Epoch: rec.Epoch, State: dstRestoredPaused}}, + } { + peer.answer["migrate-abort"] = answer + err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer) + if err == nil || !strings.Contains(err.Error(), "stays fenced") { + t.Errorf("abort answered %+v: %v", answer, err) + } + if restored != 0 || f.vm.stopped || f.srcRecord(t).State != srcCommitting { + t.Fatalf("abort answered %+v brought the source back", answer) + } + } + // The destination's own answer, from its ledger, is "running". + delete(peer.answer, "migrate-abort") + if err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer); err == nil || + !strings.Contains(err.Error(), "cannot be aborted") { + t.Errorf("abort: %v", err) + } +} + +// Before the commit was sent, a destination that has no record of the move +// cannot run it, and the abort goes ahead: here after the fence, with the +// destination's store lost. +func TestMoveAbortBeforeTheCommitAcceptsAnUnknownMove(t *testing.T) { + f := newMoveFixture(t, false) + peer := newLocalPeer(f.dst) + _ = sourceMigrate(context.Background(), f.src, f.name, f.options(true), peer) + if got := f.srcRecord(t).State; got != srcTransferred { + t.Fatalf("source is %s", got) + } + if err := removeInstanceIn(f.dst, f.name, true); err == nil { + t.Fatal("rm of an arriving instance was allowed") + } + if err := f.dst.DeleteInstance(f.name); err != nil { + t.Fatal(err) + } + if r := destAbort(f.dst, f.srcRecord(t).TxID); r.OK || r.Code != codeUnknownTxID { + t.Fatalf("destination abort of an unknown move: %+v", r) + } + restored := 0 + localRestore = func(*store.Store, string) error { restored++; return nil } + if err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer); err != nil { + t.Fatal(err) + } + if restored != 1 || f.srcRecord(t).State != migAborted { + t.Errorf("restored %d, source %s", restored, f.srcRecord(t).State) + } +} + +// A destination abort whose cleanup fails is not an abort the source may +// act on: the source stays fenced, nothing on the destination may run what +// is left, and the abort run again finishes the job. +func TestMoveAbortedDestinationWithALeftoverStaysGuarded(t *testing.T) { + f := newMoveFixture(t, true) + peer := newLocalPeer(f.dst) + peer.answer["migrate-commit"] = &migrateReply{OK: false, Code: codeFailed, Error: "lost"} + _ = sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer) + delete(peer.answer, "migrate-commit") + txid := f.srcRecord(t).TxID + if got := f.dstRecord(t, txid).State; got != dstRestoredPaused { + t.Fatalf("destination is %s", got) + } + prev := destCleanup + t.Cleanup(func() { destCleanup = prev }) + destCleanup = func(*store.Store, string) error { return errors.New("pid 4242 is still alive after SIGKILL") } + restored := 0 + localRestore = func(*store.Store, string) error { restored++; return nil } + err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer) + if err == nil || !strings.Contains(err.Error(), "stays fenced") || !strings.Contains(err.Error(), "still alive") { + t.Fatalf("abort with a failed cleanup: %v", err) + } + if restored != 0 || f.srcRecord(t).State != srcCommitting { + t.Fatalf("the source was brought back (restored %d, %s)", restored, f.srcRecord(t).State) + } + if got := f.dstRecord(t, txid).State; got != migAborted { + t.Fatalf("destination is %s", got) + } + for _, verb := range []string{"resume", "restore", "checkpoint", "migrate"} { + if err := refuseDuringMove(f.dst, f.name, verb); err == nil || !strings.Contains(err.Error(), "aborted") { + t.Errorf("%s of the aborted copy: %v", verb, err) + } + } + st := mustInstance(t, f.dst, f.name) + if err := checkMoveRestore(f.dst, st, hviRestoreOptions{}); err == nil { + t.Error("a plain restore of the aborted copy was allowed") + } + back := f.options(false) + back.to, back.remote.Target = "pm@src", "pm@src" + if err := sourceMigrate(context.Background(), f.dst, f.name, back, newLocalPeer(shortStore(t))); err == nil { + t.Error("a move of the aborted copy was allowed") + } + // The commit stays refused, and a second abort finishes. + if r := destCommit(f.dst, txid, f.srcRecord(t).Epoch); r.OK || r.Code != codeAborted { + t.Errorf("late commit: %+v", r) + } + destCleanup = prev + if err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(f.dst.InstanceDir(f.name)); !os.IsNotExist(err) { + t.Errorf("the destination kept the instance: %v", err) + } + if restored != 1 || f.srcRecord(t).State != migAborted { + t.Errorf("restored %d, source %s", restored, f.srcRecord(t).State) + } + if r := destCommit(f.dst, txid, f.srcRecord(t).Epoch); r.OK || r.Code != codeAborted { + t.Errorf("late commit after the cleanup: %+v", r) + } +} + +// dropNthPeer runs a verb and loses its reply on the nth call of it. +type dropNthPeer struct { + *localPeer + verb string + n int + seen int +} + +func (p *dropNthPeer) call(ctx context.Context, verb string, args []string, stdin io.Reader) (*migrateReply, error) { + r, err := p.localPeer.call(ctx, verb, args, stdin) + if verb == p.verb { + p.seen++ + if p.seen == p.n { + return nil, errMoveNoAnswer + } + } + return r, err +} + +// loseReceiveReply runs a move whose stream arrives and verifies, but whose +// reply is lost: the source stays in transferring, the destination is +// verified. +func loseReceiveReply(t *testing.T, f *moveFixture) *localPeer { + t.Helper() + // The first migrate-receive is the reservation, the second the stream. + peer := &dropNthPeer{localPeer: newLocalPeer(f.dst), verb: "migrate-receive", n: 2} + _ = sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer) + src := f.srcRecord(t) + if src.State != srcTransferring { + t.Fatalf("source %s", src.State) + } + if got := f.dstRecord(t, src.TxID).State; got != dstVerified { + t.Fatalf("destination %s", got) + } + return peer.localPeer +} + +// A re-run that finds the destination verified checks the disk as a stream +// would have: a disk written after the fence is refused there too. +func TestMoveRetryAfterALostReceiveReplyChecksTheDisk(t *testing.T) { + f := newMoveFixture(t, true) + peer := loseReceiveReply(t, f) + if f.srcRecord(t).Disk.SHA256 == "" { + t.Error("the source did not record the hash of the disk it sent") + } + later := time.Now().Add(time.Hour) + if err := os.Chtimes(f.disk, later, later); err != nil { + t.Fatal(err) + } + opt := f.options(false) + opt.txid = f.srcRecord(t).TxID + err := sourceMigrate(context.Background(), f.src, f.name, opt, peer) + if err == nil || !strings.Contains(err.Error(), "changed after the fence") { + t.Errorf("err = %v", err) + } + if got := f.srcRecord(t).State; got != srcTransferring { + t.Errorf("source %s", got) + } +} + +// The source compares the destination's disk with its own hash, never takes +// the destination's: with none on record it hashes the disk again, and a +// record that disagrees with the destination is refused. +func TestMoveRetryAfterALostReceiveReplyComparesTheSourcesHash(t *testing.T) { + f := newMoveFixture(t, true) + peer := loseReceiveReply(t, f) + rec := f.srcRecord(t) + sent := rec.Disk.SHA256 + path := f.src.InstanceMigrationRecord(f.name) + rec.Disk.SHA256 = strings.Repeat("0", 64) + if err := writeMigrationRecord(path, rec); err != nil { + t.Fatal(err) + } + opt := f.options(false) + opt.txid = rec.TxID + err := sourceMigrate(context.Background(), f.src, f.name, opt, peer) + if err == nil || !strings.Contains(err.Error(), "not the one the source sent") { + t.Errorf("a destination disk that differs from the source's hash: %v", err) + } + // As if hull had died before it recorded the hash. + rec = f.srcRecord(t) + rec.Disk.SHA256, rec.Disk.DataBytes = "", 0 + if err := writeMigrationRecord(path, rec); err != nil { + t.Fatal(err) + } + if err := sourceMigrate(context.Background(), f.src, f.name, opt, peer); err != nil { + t.Fatal(err) + } + if got := f.srcRecord(t); got.State != srcCommitted || got.Disk.SHA256 != sent || got.Disk.DataBytes == 0 { + t.Errorf("source %s, disk %+v, sent %s", got.State, got.Disk, sent) + } +} + +// An abort before the fence that the destination does not confirm says a +// second abort releases the reservation, and the second abort does. +func TestMoveSecondAbortReleasesTheReservation(t *testing.T) { + f := newMoveFixture(t, true) + peer := newLocalPeer(f.dst) + f.vm.mu.Lock() + f.vm.failCheckpoint = true + f.vm.mu.Unlock() + _ = sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer) + peer.down = true + var w bytes.Buffer + if err := sourceAbort(context.Background(), f.src, f.name, &w, peer); err != nil { + t.Fatal(err) + } + if !strings.Contains(w.String(), "run again releases") || f.srcRecord(t).DestReleased { + t.Fatalf("first abort said %q, released %v", w.String(), f.srcRecord(t).DestReleased) + } + peer.down = false + w.Reset() + if err := sourceAbort(context.Background(), f.src, f.name, &w, peer); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(f.dst.InstanceDir(f.name)); !os.IsNotExist(err) { + t.Errorf("the reservation is still there after the second abort (%q): %v", w.String(), err) + } + if !f.srcRecord(t).DestReleased { + t.Error("the release is not on record") + } + // Once released, an abort asks nothing more. + before := peer.called("migrate-abort") + if err := sourceAbort(context.Background(), f.src, f.name, &w, peer); err != nil { + t.Fatal(err) + } + if peer.called("migrate-abort") != before || f.vm.stopped { + t.Error("a third abort asked the destination again, or stopped the source") + } +} + +// An abort that recorded aborted but did not live to hand the fenced +// checkpoint back is finished by running the abort again. +func TestMoveAbortRunAgainRestoresTheSource(t *testing.T) { + f := newMoveFixture(t, false) + peer := newLocalPeer(f.dst) + _ = sourceMigrate(context.Background(), f.src, f.name, f.options(true), peer) + rec := f.srcRecord(t) + if r := destAbort(f.dst, rec.TxID); !r.OK { + t.Fatalf("destination abort: %+v", r) + } + // Recorded aborted here; the instance record still says migrating. + if err := rec.advance(f.src.InstanceMigrationRecord(f.name), migAborted); err != nil { + t.Fatal(err) + } + if got := mustInstance(t, f.src, f.name).Migration.State; got != store.MigrationMigrating { + t.Fatalf("instance migration %s", got) + } + restored := 0 + localRestore = func(s *store.Store, name string) error { + restored++ + st := mustInstance(t, s, name) + if err := restorableMigration(st.Migration); err != nil { + return err + } + st.Migration.State = store.MigrationRestored + return s.SaveInstance(st) + } + if err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer); err != nil { + t.Fatal(err) + } + if restored != 1 || !f.vm.stopped { + t.Errorf("restored %d, fenced VMM stopped %v", restored, f.vm.stopped) + } + var w bytes.Buffer + if err := sourceAbort(context.Background(), f.src, f.name, &w, peer); err != nil || + !strings.Contains(w.String(), "already aborted") || restored != 1 { + t.Errorf("abort of a finished abort: %v, %q, restored %d", err, w.String(), restored) + } +} + +// A fence whose reply was lost before hull recorded it: the VM is fenced, +// the record still says reserved. The abort counts it as fenced, adopts the +// bundle, needs the destination's word, and restores the instance here. +func TestMoveAbortAfterALostFenceReply(t *testing.T) { + f := newMoveFixture(t, true) + peer := newLocalPeer(f.dst) + f.vm.mu.Lock() + f.vm.failCheckpoint = true + f.vm.mu.Unlock() + _ = sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer) + rec := f.srcRecord(t) + if rec.State != srcReserved { + t.Fatalf("source %s", rec.State) + } + f.vm.mu.Lock() + f.vm.fenced = true + f.vm.mu.Unlock() + root, _ := flagValue(mustInstance(t, f.src, f.name).CmdLine, "--checkpoint-root") + restored := 0 + localRestore = func(*store.Store, string) error { restored++; return nil } + + // No bundle of this move: nothing to bring back, and nothing changes. + err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer) + if err == nil || !strings.Contains(err.Error(), "no bundle of this move") { + t.Fatalf("abort with no bundle: %v", err) + } + if f.srcRecord(t).State != srcReserved || peer.called("migrate-abort") != 0 { + t.Fatal("the abort went ahead with nothing to restore") + } + dir := filepath.Join(root, rec.TxID) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + writeFakeBundle(t, dir, fixtureDiskSize) + // Fenced, so the destination's word is needed. + peer.down = true + if err := sourceAbort(context.Background(), f.src, f.name, &bytes.Buffer{}, peer); err == nil || + !strings.Contains(err.Error(), "stays fenced") { + t.Fatalf("abort with no answer: %v", err) + } + peer.down = false + var w bytes.Buffer + if err := sourceAbort(context.Background(), f.src, f.name, &w, peer); err != nil { + t.Fatal(err) + } + if strings.Contains(w.String(), "never stopped") || restored != 1 || !f.vm.stopped { + t.Errorf("abort said %q; restored %d, stopped %v", w.String(), restored, f.vm.stopped) + } + if m := mustInstance(t, f.src, f.name).Migration; m == nil || m.TxID != rec.TxID || m.BundleDir != dir { + t.Errorf("instance migration %+v", m) + } +} + +// A txid given to --abort or --status names that move, not whatever move the +// instance holds now: after a move on, the first move's txid is only in the +// ledger, and resolving it to the instance would act on the second move. +func TestMoveTxIDArgumentNamesThatMove(t *testing.T) { + f := newMoveFixture(t, true) + if err := sourceMigrate(context.Background(), f.src, f.name, f.options(false), newLocalPeer(f.dst)); err != nil { + t.Fatal(err) + } + first := f.srcRecord(t).TxID + // The move on stops before the commit: c cannot restore. + c := newLocalPeer(shortStore(t)) + c.answer["migrate-prepare"] = &migrateReply{OK: false, Code: codeRestoreRefused, Error: "no restore"} + on := f.options(false) + on.to, on.remote.Target = "pm@c", "pm@c" + _ = sourceMigrate(context.Background(), f.dst, f.name, on, c) + second, err := readMigrationRecord(f.dst.InstanceMigrationRecord(f.name)) + if err != nil || second.State != srcTransferred { + t.Fatalf("the move on: %+v %v", second, err) + } + if _, err := moveInstanceName(f.dst, first); err == nil || !strings.Contains(err.Error(), "migrate-status") { + t.Errorf("the first move's txid resolved to the instance: %v", err) + } + if name, err := moveInstanceName(f.dst, second.TxID); err != nil || name != f.name { + t.Errorf("the second move's txid: %q %v", name, err) + } + if name, err := moveInstanceName(f.dst, f.name); err != nil || name != f.name { + t.Errorf("the name: %q %v", name, err) + } +} diff --git a/docs/checkpoint-restore.md b/docs/checkpoint-restore.md index f2ca2432..a80b5f52 100644 --- a/docs/checkpoint-restore.md +++ b/docs/checkpoint-restore.md @@ -7,16 +7,19 @@ hull picks the path from the instance's record: | | `vz` | `hvi` | |---|---|---| | How | vz-runner, on a signal | hvi's control socket | -| Captures | Virtualization.framework's machine state, plus an APFS clone of the disk | today a [partial bundle](#what-a-bundle-holds): RAM and device state, not the vCPU, interrupt controller or timer state a resume needs; not the disk | +| Captures | Virtualization.framework's machine state, plus an APFS clone of the disk | a [bundle](#what-a-bundle-holds) of RAM and device state, and on macOS (HVF, `hvi` from 4ea836f) the vCPU, interrupt controller and timer state a resume needs; on KVM a partial bundle without those; not the disk | | After `checkpoint` | the guest keeps running | `--then continue` (the default) or `--then fence` | -| Restore from | the latest checkpoint, any number of times | **not yet**: needs an `hvi` with a `restore` subcommand, which no `hvi` has. hull's side takes a fenced checkpoint, onto the disk the guest stopped with | +| Restore from | the latest checkpoint, any number of times | a fenced checkpoint, onto the disk the guest stopped with, with an `hvi` that has a `restore` subcommand: on macOS (HVF) from 4ea836f on `hvi`'s `feat/teleport`, not yet pushed and not the submodule's pin; a KVM bundle cannot be restored yet | | Also | | `hull pause` and `hull resume` | | Hosts | macOS 14+, Apple silicon | macOS (HVF) and Linux (KVM) | Both need a block rootfs (`run --rootfs-type block`) and no shared host -directory. For `hvi`, pause, resume and checkpoint work today; restore is -built in hull and refuses until `hvi` gains `restore`, because no bundle `hvi` -writes yet can be resumed. +directory. For `hvi`, pause, resume and checkpoint work today. Restore works +on macOS (HVF) with an `hvi` that has `restore` (4ea836f on `hvi`'s +`feat/teleport`, the one the moves in [migration.md](migration.md#verified) +ran with). The `hvi` this branch pins (890f9e0) has none, and hull refuses a +restore with it. On Linux (KVM) `hvi` writes a partial bundle without the +vCPU, interrupt controller and timer state, which no restore can resume yet. An `hvi` checkpoint taken with `--then fence` is also what moves an instance to another host: `hull migrate` takes one, sends it and the disk over ssh, and @@ -278,12 +281,13 @@ and `ram.bin` (the guest's RAM, sparse: all-zero pages take no space), devices and the UART), `disk.json` and `session.json`, which carries hull's txid. `hvi checkpoint-verify DIR` checks one. -**Today the bundle is partial.** The hvi that serves the op today +**On KVM, and with the pinned hvi, the bundle is partial.** That hvi records only part of each vCPU's registers and none of the interrupt controller or timer state, and `checkpoint-verify` says so: "vCPU registers, interrupt controller and timer not captured, so no restore can resume this -bundle". The full vCPU, GIC and timer capture comes with hvi's `restore`, and until -then no bundle can be restored. The disk is not in the +bundle". On macOS the hvi with `restore` (4ea836f on its `feat/teleport`) +captures them whole, and its `checkpoint-verify` answers "complete: every +part a restore needs is captured". The disk is not in the bundle: hvi syncs it and records its size and write count, and the restore reopens the same file. hvi's own [docs/checkpoint.md](../hvi-vmm/docs/checkpoint.md) describes the format and diff --git a/docs/cli.md b/docs/cli.md index c936976d..ba6a7e40 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -43,7 +43,7 @@ hidden flag is `restore --move`, which only a move's own restore passes. | `pause ` | pause a running `hvi` instance: its I/O threads and every vCPU stop | | `resume ` | resume an `hvi` instance paused by `pause` | | `checkpoint ` | `vz`: pause, save VM and disk state, resume. `hvi`: write a checkpoint bundle, then continue or fence | -| `restore ` | restore a stopped instance: `vz` from its saved state; `hvi` from a fenced bundle, refused until `hvi` has a `restore` subcommand | +| `restore ` | restore a stopped instance: `vz` from its saved state; `hvi` from a fenced bundle, with an `hvi` that has a `restore` subcommand (on macOS, from 4ea836f on `hvi`'s `feat/teleport`; a KVM bundle cannot be restored yet) | | `migrate --to USER@HOST` | move a running `hvi` instance to another host over ssh; see [migration.md](migration.md) | | `rm ` | remove a stopped instance | | `logs ` | show an instance's log | @@ -186,7 +186,8 @@ the guest here, streams the checkpoint and the disk over ssh, has the destination restore it paused, and only then commits, after which the destination resumes it and the copy here is stale. While a move holds an instance, `checkpoint`, `resume`, `restore` and `rm` of it are refused on -both hosts. The protocol, its states and its recovery are in +both hosts, and what an aborted move left on the destination can only be +removed. The protocol, its states and its recovery are in [migration.md](migration.md). The destination's verbs each print one JSON object on stdout: diff --git a/docs/migration.md b/docs/migration.md index 6eef4ea0..7dbf6bb0 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -10,7 +10,11 @@ what can go wrong, and how to recover. > **Status, 2026-10-14.** On macOS (HVF) a move runs end to end with the > `hvi` on `feat/teleport` at 4ea836f, the first with a `restore` -> subcommand. On Linux (KVM) `hvi` has no restore yet (H5), so a move stops +> subcommand. That commit is not pushed yet, and the `hvi-vmm` submodule of +> this branch is still pinned at 890f9e0, which has neither the control +> flags nor `restore`: a hull built with its pinned `hvi` refuses +> `hull migrate` at preflight. The pin moves once `feat/teleport` is pushed. +> On Linux (KVM) `hvi` cannot restore a bundle yet (H5), so a move stops > before the commit with the destination's answer "restore refused" (see > [Verified](#verified)). Without `--allow-no-restore`, `hull migrate` > refuses at preflight on an `hvi` with no restore, before anything is @@ -107,19 +111,33 @@ the next epoch again. sends the commit again; the destination answers it the same way. 6. `hull migrate --abort` works only before the commit. After the fence it first asks the destination, which records `aborted` (so a commit that - arrives later is refused) and stops any paused VM it restored. Only on that - answer does the source restore the instance locally. With no answer, or the - answer `running`, the source stays fenced and says why. + arrives later is refused), keeps that in its ledger, stops any paused VM + it restored and removes the instance. It answers ok only when all of that + is done. Only on that answer does the source restore the instance locally. + With no answer, a failure, or the answer `running`, the source stays + fenced and says why. A destination that has no record of the move answers + `unknown_txid`: before `committing` no commit was sent, so the abort goes + ahead; in `committing` the source stays fenced, since the commit may have + run there. What an aborted move leaves on the destination, until the + abort removes it, refuses every verb but `hull rm`. 7. After the commit the source's checkpoint is `migrated` (stale): it cannot be restored, resumed, checkpointed or moved. `hull migrate --status` explains. +8. The destination writes the move to its ledger, + `/migrations/.json`, before its instance record says + `running`. `hull rm` and a later move of the instance leave the ledger, + so the destination answers for the txid of every move that committed or + aborted on it, whatever became of the instance. **Every step can be run again.** A re-run of `hull migrate NAME --txid T` starts from the recorded state. A step that finds its effect already there succeeds: a reserve of a name this move reserved, a fence the record already holds (or one hvi took but hull did not live to record, which is adopted after `hvi checkpoint-verify`), a transfer the destination has verified, a -restore that is up and paused, a commit that is running. +restore that is up and paused, a commit that is running. An abort run +again on an aborted move asks the destination again if it never confirmed +the release, and restores the instance if the first run recorded the abort +but did not live to restore it. **Locks.** On each host a move takes `/migrate.lock` for each step. The source holds it for the whole run of `hull migrate`, and the fence also @@ -144,12 +162,16 @@ which takes `migration.lock` itself, so the two locks nest in one order only: - **The records are the authority on each host.** `migration.json` and `state.json` live in the 0700 instance directory, outside the checkpoint root hvi may write to. -- **A partition never leaves two hosts running.** If the network fails during - the commit, the source stays fenced in `committing` and the destination is - either `restored-paused` (paused, not running) or `running`. Both are - visible with `hull migrate --status` and the hidden `migrate-status` verb. - Either the commit is sent again, or the abort is confirmed by the - destination. Neither side guesses. +- **A partition does not leave two hosts running.** If the network fails + during the commit, the source stays fenced in `committing` and the + destination is either `restored-paused` (paused, not running) or `running`. + Both are visible with `hull migrate --status` and the hidden + `migrate-status` verb. Either the commit is sent again, or the destination + confirms the abort with its record of this move, aborted. That stays true + after the destination moved the instance on or removed it, because its + ledger keeps the move. A destination with no record at all (its store lost + or replaced) leaves the source fenced; the operator decides, see + [Operator recovery](#operator-recovery). - **Operators can still break it.** Deleting a `migration.json` by hand, or running hvi directly on a bundle, is outside these rules. @@ -200,8 +222,13 @@ it is refused. The source checks the disk's size and modification time against the ones it recorded at the fence before and after it sends, and refuses a disk that -changed: something wrote it while the guest was fenced. It also checks each -bundle file against the manifest as it reads it. +changed: something wrote it while the guest was fenced. It records the +disk's sha256 as sent before it reads the destination's reply. A re-run that +finds the destination already verified (the reply was lost) checks the size +and modification time again and compares the destination's sha256 with the +source's own, hashing the disk again if hull did not live to record it. It +never takes the destination's sha256 as its own. The source also checks +each bundle file against the manifest as it reads it. `migration.json` records the stream's bytes, its duration and its rate. @@ -214,6 +241,7 @@ Both hosts keep `/instances//migration.json`, 0600: | `version` | 1 | | `txid`, `role` (`source` or `destination`), `name`, `epoch`, `state`, `peer` | the move | | `remote` | source only: `target`, `store`, `hull`, `hvi`, `gatewaySock`, `allowNoRestore` | +| `destReleased` | source only, once aborted: the destination confirmed it keeps nothing of the move | | `bundleDir`, `manifestSha256` | the fenced checkpoint, on either host | | `disk` | `path`, `size`, `modNs` (at the fence), `sha256`, `dataBytes` | | `transfer` | `bytes`, `seconds`, `bytesPerSec`, `startedAt` | @@ -223,8 +251,12 @@ Both hosts keep `/instances//migration.json`, 0600: | `error` | why the last step stopped | | `history` | every state, with its time | -A destination that aborts a move removes the instance directory, so the name -is free again, and keeps the record at `/migrations/.json`. +A destination keeps a ledger of the moves that ended on it, committed or +aborted, at `/migrations/.json`. A commit writes its entry +before the instance record says `running`; an abort writes it before it +removes the instance directory, so the name is free again. Lookups by txid +read the instance's `migration.json` first and the ledger after it. Nothing +removes a ledger entry. The instance record's `migration` field (see [storage.md](storage.md)) gains three states: `migrating` (the source's fenced checkpoint, held by a move), @@ -246,8 +278,12 @@ destination before the commit), and two fields, `moveTxid` and `peer`. | the commit's reply is lost | fenced, `committing` | `restored-paused` or `running` | run it again: it learns the commit | | commit refused (wrong epoch, aborted) | fenced, `committing` | `aborted` or other | `--abort` | | the resume fails after the commit | `committed` (the destination owns it) | `running` with the error | `hull resume NAME` on the destination | -| `--abort` gets no answer | fenced, unchanged | unknown | retry when the destination answers | -| `--abort` answered `running` | fenced, unchanged | `running` | `hull migrate NAME --txid T` records the commit | +| `--abort` before the fence gets no answer | running, `aborted` | maybe `reserved` | `--abort` again releases the reservation | +| `--abort` after the fence gets no answer | fenced, unchanged | unknown | retry when the destination answers | +| `--abort` answered `running` (the destination committed, even if it has since moved the instance on or removed it) | fenced, unchanged | `running`, or only in the ledger | `hull migrate NAME --txid T` records the commit | +| `--abort` in `committing` answered `unknown_txid` | fenced, unchanged | no record of the move | find where the guest runs; see [Operator recovery](#operator-recovery) | +| the destination's cleanup after an abort fails | fenced, unchanged | `aborted`, its copy refused every verb but `rm` | `--abort` again | +| hull dies after recording the abort | `aborted`, checkpoint still `migrating` | `aborted` | `--abort` again restores it | | the local restore after an abort fails | `aborted`, stopped, `fenced` checkpoint | `aborted` | `hull restore NAME` once hvi can | ## Operator recovery @@ -260,8 +296,12 @@ destination before the commit), and two fields, `moveTxid` and `peer`. never runs the VM on its own. Abort it there with `hull migrate-abort --txid T`, which removes the received copy. The fenced checkpoint on the dead source was the only runnable copy. -- **The destination is gone for good, after the fence.** `--abort` cannot be - confirmed, so it refuses. If you know the destination will never come back, +- **The destination is gone for good, after the fence**, or in `committing` + it answers that it has no record of the move (its store was lost or + replaced). `--abort` cannot be confirmed, so it refuses. In `committing`, + first make sure no host runs the guest: the commit may have run on the + destination before its record was lost, and the guest may have moved on + from there. If you know the destination will never come back, stop the source's fenced VMM, set the source `migration.json` state to `aborted` and the instance record's `migration.state` to `fenced` by hand, and `hull restore NAME`. This is the one recovery that relies on your word @@ -288,9 +328,12 @@ exec sh -c "$last" The unit tests (`cmd/hull/migrate_test.go`) run every step in one process with the destination behind an interface that can drop a reply or answer for -it: a commit whose acknowledgement is lost, an abort before and after the -commit, a destination that answers `running` to an abort, a transfer cut off -midway, a disk changed after the fence. `TestMoveOverFakeSSH` runs the +it: a commit whose acknowledgement is lost, then the destination moving the +instance on or removing it before the source aborts, an abort before and +after the commit, a destination that answers `running` or `unknown_txid` to +an abort, a destination whose cleanup fails, an abort run again, a transfer +cut off midway, a stream whose reply is lost, a disk changed after the +fence. `TestMoveOverFakeSSH` runs the destination as a separate hull process through that transport, with a fake hvi that has no restore, and ends in "restore refused" and an abort. @@ -336,6 +379,33 @@ every second. This Mac has no sshd, so the Mac runs used the | Plain `hull restore` of either side | refused by the move | refused by the move | | `--abort` | destination removed the instance, kept the tombstone; source stopped its fenced VMM and tried `hull restore`, refused for the same reason | the same | -Not run live: a lost commit acknowledgement, an abort answered `running`, a -transfer cut midway and a disk changed after the fence (the unit tests cover -each), any gateway case, and a move on KVM past the restore. +**A lost commit acknowledgement, Mac (HVF), hvi 4ea836f.** A stand-in for +ssh ran `migrate-commit` on the destination and then failed as a cut link +would, so the source never read the reply: + +- The source stopped in `committing`, fenced (its VM `fenced`, its + checkpoint `migrating`); the destination recorded `running` and the guest + counted on there (TICK 12, 14). +- The destination then moved the guest on to a third store, which committed + at epoch 2 and replaced the destination's `migration.json`. Its ledger + still held the first move as `running`, and `migrate-status` answered + from it. In a second run the destination's operator ran + `hull rm --force` instead, with the same ledger answer. +- In both runs the source's `hull migrate --abort` was refused, saying the + move committed there, and the source stayed fenced. `hull migrate NAME + --txid T` then recorded `committed` and stopped the fenced VMM. After the + move on, the third store went on counting (TICK 18, 21) with no boot line. + +**An abort after the transfer, Mac (HVF), hvi 4ea836f.** The stand-in +refused `migrate-prepare` once, so the move stopped in `transferred`. On +the destination a plain `hull restore` was refused by the move. The abort +removed the destination's instance, left its ledger entry `aborted` and +recorded `destReleased` on the source. The source then restored the guest +from its fenced checkpoint, and it counted on (TICK 9 at the fence, then 13, +15) with no boot line. A late `migrate-commit` on the destination was +refused with `aborted`, and a second abort said "already aborted". + +Not run live: an abort answered `unknown_txid`, a destination whose cleanup +fails, a transfer cut midway, a stream whose reply is lost and a disk +changed after the fence (the unit tests cover each), any gateway case, and +a move on KVM past the restore. diff --git a/docs/storage.md b/docs/storage.md index ea6def8b..9ccdb653 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -189,7 +189,7 @@ in the name only keeps two pulls' directories apart. │ └── provenance.json ├── volumes/_/ compose named volumes ├── compose/.json compose project state, plus its gateway sockets and log -├── migrations/.json the records of moves this store aborted as their destination +├── migrations/.json the ledger of moves that committed or aborted here as their destination └── instances// ├── state.json 0600 ├── bundle/ config.json plus the rootfs symlink or image @@ -543,6 +543,7 @@ states. It is written through a temp file, an fsync, a rename and an fsync of the directory, because each state is a promise to the other host. The instance record's `migration` field gains `moveTxid` and `peer`, and three states (`migrating`, `migrated`, `incoming`); `state.json` gains -`ownerEpoch`. A destination that aborts a move removes the instance directory -and keeps the record as `/migrations/.json`. See +`ownerEpoch`. A destination keeps the record of every move that committed +or aborted on it as `/migrations/.json`, which neither `hull rm` +nor a later move removes; an abort removes the instance directory. See [migration.md](migration.md#records). diff --git a/pkg/store/store.go b/pkg/store/store.go index ce052f46..e3ef4f8b 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -747,10 +747,12 @@ func (s *Store) InstanceMigrationRecord(id string) string { return s.instancePath(id, "migration.json") } -// MigrationTombstoneDir is where a destination keeps the records of moves it -// aborted. Aborting removes the instance directory, so the name is free for -// another run, and the record moves here so a late commit or a status query -// for its txid still gets an answer. +// MigrationTombstoneDir is the ledger where a destination keeps the records +// of moves that ended on it, committed or aborted, as .json. Aborting +// removes the instance directory, so the name is free for another run, and +// `hull rm` or a later move can remove or replace a committed one; the +// record here outlives both, so a late commit, an abort or a status query +// for its txid still gets the true answer. func (s *Store) MigrationTombstoneDir() string { return filepath.Join(s.rootDir, "migrations") } From 7eceff3dae2c0324b3b576bc3a5c276f1f971a8f Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Wed, 14 Oct 2026 07:53:58 +0300 Subject: [PATCH 12/33] feat(hvi): run and move instances under the portable machine profile hvi's portable-arm64-v1 profile shows the guest the same machine on an Apple host and on a Linux arm64 host, and its guest keeps time through the hvi-time device, so it needs a kernel built with that driver. `hull assets pull --profile portable-arm64-v1` installs that kernel as a variant of the boot bundle with its own provenance, from an OCI artifact or a local file, checks the artifact holds a kernel before it writes anything, and names the right pull when the kernel fails its check. `hull run --hypervisor hvi --profile portable-arm64-v1`, or the com.nofire.hvi.profile annotation, passes --profile to hvi and records the profile in state.json. The run boots only the profile's kernel, checked by its contents whatever names it, and refuses other backends, non-block roots, host shares, an hvi without --profile and an hvi without the control socket. A move carries the profile. The probe refuses a portable instance when the destination's hvi takes no --profile, relaxes the same-OS check to the architecture for a portable guest, and the destination checks the received bundle's profile against the move. The docs describe profiles, the kernel, what require refuses, and the moves run on the Linux pair and on this Mac. Signed-off-by: Panagiotis Moustafellos --- cmd/hull/assets.go | 137 +++++++++- cmd/hull/hviflags.go | 22 +- cmd/hull/hviflags_test.go | 29 +++ cmd/hull/hviprofile.go | 146 +++++++++++ cmd/hull/hviprofile_test.go | 124 +++++++++ cmd/hull/hvirestore.go | 18 +- cmd/hull/hvirestore_test.go | 11 + cmd/hull/migrate_dest.go | 62 +++++ cmd/hull/migrate_fixture_test.go | 18 +- cmd/hull/migrate_profile_test.go | 174 +++++++++++++ cmd/hull/migrate_record.go | 3 + cmd/hull/migrate_source.go | 32 ++- cmd/hull/migrate_ssh.go | 3 + cmd/hull/run.go | 65 ++++- cmd/hull/run_profile_test.go | 345 +++++++++++++++++++++++++ docs/backends.md | 56 ++++ docs/cli.md | 22 +- docs/images.md | 45 ++++ docs/migration.md | 117 ++++++++- internal/bootassets/bootassets.go | 21 ++ internal/bootassets/profile.go | 382 +++++++++++++++++++++++++++ internal/bootassets/profile_test.go | 385 ++++++++++++++++++++++++++++ internal/bootassets/provenance.go | 9 +- pkg/store/store.go | 7 + 24 files changed, 2198 insertions(+), 35 deletions(-) create mode 100644 cmd/hull/hviprofile.go create mode 100644 cmd/hull/hviprofile_test.go create mode 100644 cmd/hull/migrate_profile_test.go create mode 100644 cmd/hull/run_profile_test.go create mode 100644 internal/bootassets/profile.go create mode 100644 internal/bootassets/profile_test.go diff --git a/cmd/hull/assets.go b/cmd/hull/assets.go index d6d14b79..b796fc0f 100644 --- a/cmd/hull/assets.go +++ b/cmd/hull/assets.go @@ -40,7 +40,15 @@ func assetsCommand() *cli.Command { { Name: "show", Usage: "show where the boot assets are and whether they are present", + Flags: []cli.Flag{assetsProfileFlag()}, Action: func(_ context.Context, cmd *cli.Command) error { + profile, err := bootassets.NormalizeProfile(cmd.String("profile")) + if err != nil { + return err + } + if profile != "" { + return showProfileAssets(cmd.String("store-dir"), profile) + } return showBootAssets(cmd.String("store-dir")) }, }, @@ -57,8 +65,12 @@ func assetsCommand() *cli.Command { // // One bare line, no decoration, so a shell can consume it: // assets=$(hull assets dir) + // + // With --profile it is that profile's kernel directory + // instead; the initrd a profile boots stays in the plain one. + Flags: []cli.Flag{assetsProfileFlag()}, Action: func(_ context.Context, cmd *cli.Command) error { - dir, err := bootassets.Dir(cmd.String("store-dir")) + dir, err := assetsDir(cmd.String("store-dir"), cmd.String("profile")) if err != nil { return err } @@ -68,15 +80,23 @@ func assetsCommand() *cli.Command { }, { Name: "pull", - Usage: "download the boot assets for this host", + Usage: "download the boot assets for this host, or install a profile's kernel", ArgsUsage: "[REF]", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "force", Usage: "download even when the assets are already present", }, + assetsProfileFlag(), }, Action: func(ctx context.Context, cmd *cli.Command) error { + profile, err := bootassets.NormalizeProfile(cmd.String("profile")) + if err != nil { + return err + } + if profile != "" { + return pullProfileAssets(ctx, cmd, profile) + } ref := bootassets.Ref() if cmd.Args().Len() > 0 { ref = cmd.Args().First() @@ -113,6 +133,119 @@ func assetsCommand() *cli.Command { } } +// assetsProfileFlag selects a machine profile whose kernel is a variant of +// the bundle (see internal/bootassets/profile.go). +func assetsProfileFlag() cli.Flag { + return &cli.StringFlag{ + Name: "profile", + Usage: "the machine profile whose kernel to act on: " + bootassets.PortableArm64V1 + + " (default: native, the bundle itself)", + } +} + +// assetsDir is the directory `hull assets dir` prints. +func assetsDir(storeDir, name string) (string, error) { + profile, err := bootassets.NormalizeProfile(name) + if err != nil { + return "", err + } + if profile == "" { + return bootassets.Dir(storeDir) + } + return bootassets.ProfileDir(storeDir, profile) +} + +// pullProfileAssets installs a profile's kernel: from REF when one is given, +// otherwise from HULL_PORTABLE_KERNEL or HULL_PORTABLE_KERNEL_REF. The +// default bundle is fetched too when it is missing, since the profile boots +// its initrd. +func pullProfileAssets(ctx context.Context, cmd *cli.Command, profile string) error { + storeDir := cmd.String("store-dir") + if _, err := bootassets.ProfileDir(storeDir, profile); err != nil { + return err + } + if err := ensureStore(storeDir, defaultStoreDir()); err != nil { + return err + } + if !bootassets.Present(storeDir) { + if err := bootassets.Fetch(ctx, bootassets.Ref(), storeDir); err != nil { + return err + } + } + if !cmd.Bool("force") && bootassets.ProfileKernelPresent(storeDir, profile) { + dir, err := bootassets.ProfileDir(storeDir, profile) + if err != nil { + return err + } + // Present is not the same as intact: a kernel that no longer + // matches its record would otherwise be reported as installed, + // and the run that boots it refused. + kernel, _, err := bootassets.ProfilePaths(storeDir, profile) + if err != nil { + return err + } + if verr := bootassets.VerifyFiles(kernel); verr != nil && !errors.Is(verr, bootassets.ErrNoProvenance) { + return verr + } + fmt.Printf("the %s kernel is already present in %s (use --force to replace it)\n", profile, dir) + return nil + } + var err error + if cmd.Args().Len() > 0 { + err = bootassets.FetchProfile(ctx, cmd.Args().First(), storeDir, profile) + } else { + err = bootassets.InstallProfile(ctx, storeDir, profile) + } + if err != nil { + return err + } + return showProfileAssets(storeDir, profile) +} + +// showProfileAssets is `hull assets show --profile`: the profile's kernel, +// where it came from, and the default initrd it boots with. +func showProfileAssets(storeDir, profile string) error { + dir, err := bootassets.ProfileDir(storeDir, profile) + if err != nil { + return err + } + kernel, initrd, err := bootassets.ProfilePaths(storeDir, profile) + if err != nil { + return err + } + fmt.Printf("profile: %s\ndirectory: %s\n", profile, dir) + switch record, err := bootassets.ProfileRecord(storeDir, profile); { + case errors.Is(err, bootassets.ErrNoProvenance): + fmt.Printf("provenance: none\n") + case err != nil: + fmt.Printf("provenance: unreadable: %v\n", err) + case record.VerifiedDigest != "": + fmt.Printf("installed from: %s (%s)\nsignature: verified against %s\n", record.Ref, + record.Digest, record.VerifiedDigest) + default: + fmt.Printf("installed from: %s (%s)\nsignature: none (a local file, or nothing was checked)\n", + record.Ref, record.Digest) + } + fmt.Println() + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + row := func(format string, args ...any) { + _, _ = fmt.Fprintf(w, format, args...) + } + row("FILE\tSIZE\tSTATUS\n") + for _, path := range []string{kernel, initrd} { + info, statErr := os.Stat(path) + switch { + case statErr != nil: + row("%s\t-\tmissing\n", path) + case info.Size() == 0: + row("%s\t0\tempty\n", path) + default: + row("%s\t%d\tok\n", path, info.Size()) + } + } + return w.Flush() +} + func showBootAssets(storeDir string) error { dir, err := bootassets.Dir(storeDir) if err != nil { diff --git a/cmd/hull/hviflags.go b/cmd/hull/hviflags.go index 40998939..daa95967 100644 --- a/cmd/hull/hviflags.go +++ b/cmd/hull/hviflags.go @@ -41,10 +41,28 @@ import ( // answer is taken as "supported": the boot itself then fails with hvi's own // reason, which says more than a guess here would. func hviBootTakesControlFlags(binary string) (bool, error) { + return hviBootKnowsFlag(binary, "--checkpoint-root") +} + +// hviBootTakesProfile reports whether the hvi at binary accepts --profile +// on `boot`. Asked the same way: `boot --profile` with no name fails while +// parsing, and an hvi from before machine profiles answers `unknown boot arg +// "--profile"` (its --profile-out is a different argument, matched whole). +// +// Unlike the control flags, a run that asked for a profile does not go on +// without it: an hvi that cannot take --profile would boot the native +// machine, whose checkpoints restore only on the same kind of host. +func hviBootTakesProfile(binary string) (bool, error) { + return hviBootKnowsFlag(binary, "--profile") +} + +// hviBootKnowsFlag runs `binary boot FLAG` with no value and reports +// whether hvi named the flag unknown. +func hviBootKnowsFlag(binary, flag string) (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() var out bytes.Buffer - probe := exec.CommandContext(ctx, binary, "boot", "--checkpoint-root") + probe := exec.CommandContext(ctx, binary, "boot", flag) probe.Stdout = &out probe.Stderr = &out err := probe.Run() @@ -55,5 +73,5 @@ func hviBootTakesControlFlags(binary string) (bool, error) { if err != nil && !errors.As(err, &exitErr) { return false, fmt.Errorf("run %s: %w", binary, err) } - return !strings.Contains(out.String(), `unknown boot arg "--checkpoint-root"`), nil + return !strings.Contains(out.String(), fmt.Sprintf("unknown boot arg %q", flag)), nil } diff --git a/cmd/hull/hviflags_test.go b/cmd/hull/hviflags_test.go index d5273581..2b4a8f27 100644 --- a/cmd/hull/hviflags_test.go +++ b/cmd/hull/hviflags_test.go @@ -53,3 +53,32 @@ echo 'hvi: --checkpoint-root needs a directory' >&2; exit 1`) t.Error("a missing binary was not reported") } } + +// An hvi from before machine profiles (feat/teleport at 5aafb49) names +// --profile unknown; one with them (the P1..P3 series) says it needs a name. +// --profile-out, which both know, must not read as --profile. +func TestHviBootTakesProfile(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil { + t.Fatal(err) + } + return p + } + old := write("old", `[ "$1 $2 $#" = "boot --profile 2" ] || exit 9 +echo 'hvi: unknown boot arg "--profile"' >&2; exit 1`) + current := write("current", `[ "$1 $2 $#" = "boot --profile 2" ] || { echo 'hvi: unknown boot arg "--profile"' >&2; exit 1; } +echo 'hvi: --profile needs a name' >&2; exit 1`) + outOnly := write("out-only", `echo 'hvi: unknown boot arg "--profile-out-x"' >&2; exit 1`) + + if takes, err := hviBootTakesProfile(old); err != nil || takes { + t.Errorf("an hvi without profiles: takes=%v err=%v", takes, err) + } + if takes, err := hviBootTakesProfile(current); err != nil || !takes { + t.Errorf("an hvi with profiles: takes=%v err=%v", takes, err) + } + if takes, err := hviBootTakesProfile(outOnly); err != nil || !takes { + t.Errorf("another unknown flag read as --profile: takes=%v err=%v", takes, err) + } +} diff --git a/cmd/hull/hviprofile.go b/cmd/hull/hviprofile.go new file mode 100644 index 00000000..67be8133 --- /dev/null +++ b/cmd/hull/hviprofile.go @@ -0,0 +1,146 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + + "github.com/brig-sh/hull/internal/bootassets" + "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" +) + +// hviProfileAnnotation selects the hvi machine profile, as --profile does. +// +// It is read from this invocation's --annotation flags only, never from the +// image's metadata or urunc.json, for the same reason as bootKernel: the +// profile decides which host kernel boots and what CPU the guest is shown, +// and an image must not choose either. That is also how brig passes it. +const hviProfileAnnotation = "com.nofire.hvi.profile" + +// resolveRunProfile combines --profile and the profile annotation into the +// profile hull records: "" for hvi's native profile. The two may both be +// given, but only for the same profile. +func resolveRunProfile(flag string, invocationAnnotations map[string]string) (string, error) { + fromFlag, err := bootassets.NormalizeProfile(flag) + if err != nil { + return "", fmt.Errorf("--profile: %w", err) + } + annotation, set := invocationAnnotations[hviProfileAnnotation] + if !set { + return fromFlag, nil + } + fromAnnotation, err := bootassets.NormalizeProfile(annotation) + if err != nil { + return "", fmt.Errorf("annotation %s: %w", hviProfileAnnotation, err) + } + if flag != "" && fromFlag != fromAnnotation { + return "", fmt.Errorf("--profile %s and annotation %s=%s name different profiles", + flag, hviProfileAnnotation, annotation) + } + return fromAnnotation, nil +} + +// checkProfileRun refuses a run the profile cannot describe, before the +// image is pulled. hypervisor and rootfsType are the flags as the platform +// preflight left them, so on Linux an unset --rootfs-type is already block. +// +// The portable profile is an hvi machine with one rootfs disk: its guest is +// the same machine on HVF and on KVM because nothing in it lives on the +// host. A virtio-fs rootfs or share is a host directory, which neither a +// checkpoint nor the other host carries, so they are refused rather than +// left to fail the first checkpoint. An unset --rootfs-type on macOS is +// virtiofs for hvi, so it is refused too, not switched to block behind the +// caller's back. +func checkProfileRun(profile string, r runRequest) error { + if profile == "" { + return nil + } + if r.hypervisor != "" && r.hypervisor != string(hypervisors.HviVmm) { + return fmt.Errorf("--profile %s is an hvi machine profile, and --hypervisor is %s", + profile, r.hypervisor) + } + if r.rootfsType != "block" { + rootfs := r.rootfsType + if rootfs == "" { + rootfs = "virtiofs, the default" + } + return fmt.Errorf("--profile %s needs --rootfs-type block (got %s): the profile's guest "+ + "keeps its root on a disk a checkpoint carries, not in a host directory", profile, rootfs) + } + if r.sharedDirs > 0 || r.sharedDirFds > 0 { + return fmt.Errorf("--profile %s cannot share host directories (--shared-dir, "+ + "--shared-dir-fd): a share is a host directory no checkpoint carries", profile) + } + return nil +} + +// checkProfileBackend is the check that waits for the image: an image's own +// annotation can choose the backend when --hypervisor is not given. +func checkProfileBackend(profile, vmmName, source string) error { + if profile == "" || vmmName == string(hypervisors.HviVmm) { + return nil + } + return fmt.Errorf("--profile %s is an hvi machine profile, and this run uses %s (from %s); "+ + "pass --hypervisor hvi", profile, vmmName, source) +} + +// checkProfileKernel refuses a profile run whose kernel is not the +// profile's. hull resolves the profile's kernel itself only when no +// bootKernel annotation is given, --no-boot-assets is not set and the image +// has no kernel of its own; any other way of arriving at a kernel would boot +// it under the profile's name and record a portable machine whose guest +// cannot keep time on it. So the kernel is checked by its bytes, wherever it +// came from: the staged copy must hash to the kernel the profile's record +// in the asset directory describes. That keeps brig's path, a bootKernel +// annotation naming the installed Image or a copy of it, and refuses the +// rest. +// +// bootKernel is the annotation as the run resolved it ("" when the kernel +// came from the image), staged the copy the VMM will boot ("" when none was +// found), and why says where a kernel not from the annotation came from. +func checkProfileKernel(storeDir, profile, bootKernel, staged, why string) error { + if profile == "" { + return nil + } + if bootKernel == "" || staged == "" { + return fmt.Errorf("--profile %s boots the profile's own kernel, and this run %s; drop it, "+ + "or name the profile's kernel (`hull assets dir --profile %s`) in "+ + "com.urunc.unikernel.bootKernel", profile, why, profile) + } + what := "the kernel in com.urunc.unikernel.bootKernel (" + bootKernel + ")" + if err := bootassets.VerifyProfileKernel(storeDir, profile, staged, what); err != nil { + return fmt.Errorf("--profile %s: %w", profile, err) + } + return nil +} + +// hviProfileArgs is what a run passes hvi for its profile: nothing for the +// native one, so its command line stays the one an hvi from before profiles +// accepts, and --profile NAME otherwise, once the hvi that will run has said +// it takes the flag. +func hviProfileArgs(binary, profile string) ([]string, error) { + if profile == "" { + return nil, nil + } + takes, err := hviBootTakesProfile(binary) + if err != nil { + return nil, err + } + if !takes { + return nil, fmt.Errorf("--profile %s needs an hvi that takes --profile, and %s does not "+ + "(it would boot hvi's native machine)", profile, binary) + } + return []string{"--profile", profile}, nil +} diff --git a/cmd/hull/hviprofile_test.go b/cmd/hull/hviprofile_test.go new file mode 100644 index 00000000..4e4e9a47 --- /dev/null +++ b/cmd/hull/hviprofile_test.go @@ -0,0 +1,124 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +const portable = "portable-arm64-v1" + +func TestResolveRunProfile(t *testing.T) { + cases := []struct { + flag string + annotations map[string]string + want string + refusal string + }{ + {flag: "", want: ""}, + {flag: "native", want: ""}, + {flag: portable, want: portable}, + {annotations: map[string]string{hviProfileAnnotation: portable}, want: portable}, + {flag: portable, annotations: map[string]string{hviProfileAnnotation: portable}, want: portable}, + {flag: "native", annotations: map[string]string{hviProfileAnnotation: portable}, + refusal: "different profiles"}, + {flag: "portable", refusal: "unknown machine profile"}, + {annotations: map[string]string{hviProfileAnnotation: "x"}, refusal: hviProfileAnnotation}, + } + for _, c := range cases { + got, err := resolveRunProfile(c.flag, c.annotations) + if c.refusal != "" { + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%q %v: err = %v, want %q", c.flag, c.annotations, err, c.refusal) + } + continue + } + if err != nil || got != c.want { + t.Errorf("%q %v = %q, %v; want %q", c.flag, c.annotations, got, err, c.want) + } + } +} + +func TestCheckProfileRun(t *testing.T) { + block := runRequest{hypervisor: "hvi", rootfsType: "block"} + if err := checkProfileRun(portable, block); err != nil { + t.Fatalf("hvi with a block rootfs was refused: %v", err) + } + // No --hypervisor: the image annotation or the platform picks the + // backend, and checkProfileBackend checks it once known. + if err := checkProfileRun(portable, runRequest{rootfsType: "block"}); err != nil { + t.Fatalf("an unset --hypervisor was refused before the image was read: %v", err) + } + for _, c := range []struct { + r runRequest + refusal string + }{ + {runRequest{hypervisor: "vz", rootfsType: "block"}, "--hypervisor is vz"}, + {runRequest{hypervisor: "qemu", rootfsType: "block"}, "--hypervisor is qemu"}, + {runRequest{hypervisor: "hvi", rootfsType: "virtiofs"}, "got virtiofs"}, + {runRequest{hypervisor: "hvi"}, "virtiofs, the default"}, + {runRequest{hypervisor: "hvi", rootfsType: "block", sharedDirs: 1}, "cannot share"}, + {runRequest{hypervisor: "hvi", rootfsType: "block", sharedDirFds: 1}, "cannot share"}, + } { + err := checkProfileRun(portable, c.r) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%+v: err = %v, want %q", c.r, err, c.refusal) + } + if err := checkProfileRun("", c.r); err != nil { + t.Errorf("a native run was refused by the profile check: %v", err) + } + } + if err := checkProfileBackend(portable, "vz", "image annotation"); err == nil || + !strings.Contains(err.Error(), "image annotation") { + t.Errorf("an image that picks vz: err = %v", err) + } + if err := checkProfileBackend(portable, "hvi", "flag"); err != nil { + t.Errorf("hvi refused: %v", err) + } + if err := checkProfileBackend("", "vz", "flag"); err != nil { + t.Errorf("native vz refused: %v", err) + } +} + +// A native run's hvi command line is the one an hvi from before profiles +// accepts; a portable run adds --profile only when the hvi takes it, and +// otherwise refuses rather than boot the native machine. +func TestHviProfileArgs(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil { + t.Fatal(err) + } + return p + } + old := write("old", `echo 'hvi: unknown boot arg "--profile"' >&2; exit 1`) + current := write("current", `echo 'hvi: --profile needs a name' >&2; exit 1`) + + if args, err := hviProfileArgs(old, ""); err != nil || args != nil { + t.Fatalf("native: %v %v", args, err) + } + if args, err := hviProfileArgs(current, portable); err != nil || + !slices.Equal(args, []string{"--profile", portable}) { + t.Fatalf("portable: %v %v", args, err) + } + if _, err := hviProfileArgs(old, portable); err == nil || !strings.Contains(err.Error(), old) { + t.Fatalf("an hvi without profiles: err = %v, want a refusal naming it", err) + } +} diff --git a/cmd/hull/hvirestore.go b/cmd/hull/hvirestore.go index 2b55a2de..3fc1d4d7 100644 --- a/cmd/hull/hvirestore.go +++ b/cmd/hull/hvirestore.go @@ -132,13 +132,14 @@ type hviRestoreOptions struct { // the `hvi restore` command line that brings it back from the bundle in from. // // The bundle holds the guest's RAM, so it already has the kernel, initramfs -// and command line in it, and its profile fixes the memory size and vCPU -// count: those flags are dropped. What the bundle cannot hold are the host -// resources the devices were attached to, and those are kept: the disk, the -// gateway socket (so a restore on the same host re-joins the same gateway), -// the MAC the gateway's leases are keyed by, the agent socket, the sandbox id, -// the event ledger, and the checkpoint root, without which the restored guest -// could never be checkpointed again. A bare --net is the deprecated spelling of +// and command line in it, and its profile fixes the memory size, the vCPU +// count and the machine profile (--profile): those flags are dropped. What +// the bundle cannot hold are the host resources the devices were attached +// to, and those are kept: the disk, the gateway socket (so a restore on the +// same host re-joins the same gateway), the MAC the gateway's leases are +// keyed by, the agent socket, the sandbox id, the event ledger, and the +// checkpoint root, without which the restored guest could never be +// checkpointed again. A bare --net is the deprecated spelling of // --net-stub and is passed under its new name, since restore takes only that. // // An allowlist, not a list of flags to remove: a flag this function has never @@ -162,6 +163,9 @@ func hviRestoreArgv(cmdLine []string, from, controlSock, profilePath string) ([] "--kernel": true, "--initramfs": true, "--cmdline": true, "--mem-mib": true, "--cpus": true, "--fs-uid": true, "--fs-gid": true, "--from": true, "--control-sock": true, "--profile-out": true, + // The machine profile is in the bundle's own profile.json, which + // hvi restore checks against the host; it takes no --profile. + "--profile": true, } out := []string{cmdLine[0], "restore", "--from", from} sawDisk := false diff --git a/cmd/hull/hvirestore_test.go b/cmd/hull/hvirestore_test.go index b580ec46..2e167769 100644 --- a/cmd/hull/hvirestore_test.go +++ b/cmd/hull/hvirestore_test.go @@ -59,6 +59,17 @@ func TestHviRestoreArgv(t *testing.T) { "--net-gateway", "/g.sock", "--net-mac", "m", "--events", "/e.ndjson", "--checkpoint-root", "/s/i/checkpoint"}, tail...), }, + { + // The bundle's profile.json names the machine; hvi restore + // takes no --profile, so a portable run maps like any other. + name: "linux run under the portable profile", + cmdLine: []string{"/h/hvi", "boot", "--kernel", "/k", "--mem-mib", "512", "--cpus", "1", + "--disk", "/r", "--agent-sock", "/s/i/agent.sock", "--checkpoint-root", "/s/i/checkpoint", + "--control-sock", "/s/i/control.sock", "--profile-out", "/s/i/profile.json", + "--profile", "portable-arm64-v1"}, + want: append([]string{"/h/hvi", "restore", "--from", from, "--disk", "/r", + "--agent-sock", "/s/i/agent.sock", "--checkpoint-root", "/s/i/checkpoint"}, tail...), + }, { name: "a share", cmdLine: []string{"hvi", "boot", "--disk", "/r", "--share-ro", "/src", "t"}, diff --git a/cmd/hull/migrate_dest.go b/cmd/hull/migrate_dest.go index e6ec37e4..dd53c30e 100644 --- a/cmd/hull/migrate_dest.go +++ b/cmd/hull/migrate_dest.go @@ -37,6 +37,7 @@ import ( "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" "golang.org/x/sys/unix" + "github.com/brig-sh/hull/internal/bootassets" "github.com/brig-sh/hull/pkg/store" ) @@ -136,6 +137,11 @@ func destProbe(s *store.Store, name, hviOverride, gatewaySock string) *migrateRe } else { p.HviRestore = ok } + if ok, err := hviBootTakesProfile(hvi); err != nil { + p.HviError = err.Error() + } else { + p.HviProfile = ok + } } if name != "" { if err := store.ValidateInstanceID(name); err != nil { @@ -195,6 +201,12 @@ func destReserve(s *store.Store, txid, name string, req migrateReserveRequest) * if _, err := destHviPath(req.Spec.Hvi); err != nil { return refuse(codeFailed, err) } + // The profile is the source's word for now; the bundle's own + // profile.json is checked against it on receive. + profile, err := bootassets.NormalizeProfile(req.Spec.Profile) + if err != nil { + return refuse(codeFailed, fmt.Errorf("the move's machine profile: %w", err)) + } for _, sock := range []struct{ what, path string }{ {"agent", s.InstanceAgentSocket(name)}, {"control", s.InstanceControlSocket(name)}, } { @@ -210,6 +222,7 @@ func destReserve(s *store.Store, txid, name string, req migrateReserveRequest) * return refuse(codeFailed, err) } spec := req.Spec + spec.Profile = profile rec := &migrationRecord{ TxID: txid, Role: roleDestination, Name: name, Epoch: req.Epoch, State: dstReserved, Peer: spec.SourceHost, Spec: &spec, @@ -296,6 +309,9 @@ func destReceive(s *store.Store, txid, name string, in io.Reader) *migrateReply } took := time.Since(started) disk, err := checkReceivedBundle(filepath.Join(staging, "bundle"), res) + if err == nil { + err = checkBundleProfile(filepath.Join(staging, "bundle"), rec.Spec.Profile) + } if err != nil { _ = store.RemoveTree(staging) return refuseWith(codeVerifyFailed, rec.fail(path, err), rec) @@ -338,6 +354,51 @@ func destReceive(s *store.Store, txid, name string, in io.Reader) *migrateReply return &migrateReply{OK: true, Record: rec} } +// checkBundleProfile refuses a bundle whose machine profile is not the one +// the move's spec names. The spec's profile is what the destination records +// and what relaxes the OS check on the instance's next move, so it is taken +// only when the machine hvi captured agrees: a native bundle labelled +// portable would be let through to a host it cannot restore on. +func checkBundleProfile(dir, want string) error { + got, err := bundleProfile(dir) + if err != nil { + return err + } + if got != want { + return fmt.Errorf("the move says the instance runs the %s machine profile, and the bundle's "+ + "profile.json records %s", profileName(want), profileName(got)) + } + return nil +} + +// bundleProfile is the machine profile a bundle's profile.json records, +// as hull names it: "" for native, which hvi leaves out of the file. +func bundleProfile(dir string) (string, error) { + data, err := os.ReadFile(filepath.Join(dir, "profile.json")) + if err != nil { + return "", fmt.Errorf("read the bundle's machine profile: %w", err) + } + var p struct { + Profile string `json:"profile"` + } + if err := json.Unmarshal(data, &p); err != nil { + return "", fmt.Errorf("the bundle's profile.json: %w", err) + } + profile, err := bootassets.NormalizeProfile(p.Profile) + if err != nil { + return "", fmt.Errorf("the bundle's profile.json: %w", err) + } + return profile, nil +} + +// profileName is a profile as a message names it. +func profileName(profile string) string { + if profile == "" { + return "native" + } + return profile +} + // checkReceivedBundle checks the received bundle against its manifest: // manifest.json hashes to what the stream header says hvi reported at the // fence, every component the manifest lists arrived with its size and @@ -527,6 +588,7 @@ func writeIncomingInstance(s *store.Store, rec *migrationRecord, hvi, root strin MAC: spec.MAC, IP: spec.IP, Backend: string(hypervisors.HviVmm), + Profile: spec.Profile, StartTime: time.Now(), Migration: &store.MigrationState{State: store.MigrationIncoming, BundleDir: rec.BundleDir, TxID: rec.TxID, ManifestSHA256: rec.ManifestSHA256, MoveTxID: rec.TxID, Peer: rec.Peer}, diff --git a/cmd/hull/migrate_fixture_test.go b/cmd/hull/migrate_fixture_test.go index 0f177df5..c25d750e 100644 --- a/cmd/hull/migrate_fixture_test.go +++ b/cmd/hull/migrate_fixture_test.go @@ -45,8 +45,20 @@ import ( // It returns the manifest's sha256. func writeFakeBundle(t *testing.T, dir string, diskSize int64) string { t.Helper() + return writeFakeBundleWithProfile(t, dir, diskSize, "") +} + +// writeFakeBundleWithProfile is writeFakeBundle for a guest of the given +// machine profile, which profile.json names unless it is native, as hvi +// writes it. +func writeFakeBundleWithProfile(t *testing.T, dir string, diskSize int64, profile string) string { + t.Helper() + profileJSON := `{"profile_version":1}` + if profile != "" { + profileJSON = `{"profile_version":1,"profile":"` + profile + `"}` + } parts := map[string][]byte{ - "profile.json": []byte(`{"profile_version":1}`), + "profile.json": []byte(profileJSON), "ram.bin": bytes.Repeat([]byte("ram!"), 4096), "disk.json": []byte(fmt.Sprintf(`{"version":1,"path_hint":"/d","size_bytes":%d,"sha256_sparse":null,`+ `"generation":3,"synced":true}`, diskSize)), @@ -80,6 +92,8 @@ type moveVM struct { fenced bool stopped bool failCheckpoint bool + // profile is the machine profile the bundles it writes record. + profile string } func (v *moveVM) ops() []string { @@ -136,7 +150,7 @@ func newMoveFixture(t *testing.T, hasRestore bool) *moveFixture { if vm.failCheckpoint { return stubAnswer{reply: `{"v":1,"ok":false,"code":"capture_failed","error":"disk sync failed"}`} } - sum := writeFakeBundle(t, req["dir"].(string), fixtureDiskSize) + sum := writeFakeBundleWithProfile(t, req["dir"].(string), fixtureDiskSize, vm.profile) if req["then"] == "fence" { vm.fenced = true } diff --git a/cmd/hull/migrate_profile_test.go b/cmd/hull/migrate_profile_test.go new file mode 100644 index 00000000..86b357ad --- /dev/null +++ b/cmd/hull/migrate_profile_test.go @@ -0,0 +1,174 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + + "github.com/brig-sh/hull/pkg/store" +) + +// A portable instance may move between an Apple host and a Linux arm64 +// host; a native one only to the same kind of host, and neither across +// architectures. +func TestHostKindMismatch(t *testing.T) { + other := "linux" + if runtime.GOOS == "linux" { + other = "darwin" + } + same := &migrateProbe{OS: runtime.GOOS, Arch: runtime.GOARCH} + otherOS := &migrateProbe{OS: other, Arch: runtime.GOARCH} + otherArch := &migrateProbe{OS: runtime.GOOS, Arch: "riscv64"} + if why := hostKindMismatch(same, ""); len(why) != 0 { + t.Errorf("the same host kind: %v", why) + } + if why := hostKindMismatch(otherOS, ""); len(why) != 1 || !strings.Contains(why[0], portable) { + t.Errorf("native to another OS: %v", why) + } + if why := hostKindMismatch(otherOS, portable); len(why) != 0 { + t.Errorf("portable to another OS: %v", why) + } + if why := hostKindMismatch(otherArch, portable); len(why) != 1 { + t.Errorf("portable to another architecture: %v", why) + } +} + +// The destination records the profile the source ran under, so its ps, +// inspect and next move see the same machine. +func TestIncomingInstanceKeepsTheProfile(t *testing.T) { + s := newCmdTestStore(t) + rec := &migrationRecord{TxID: "tx-profile", Role: roleDestination, Name: "vm-p", + BundleDir: filepath.Join(s.InstanceDir("vm-p"), "checkpoint", "tx-profile"), + Disk: &migrateDisk{Path: filepath.Join(s.InstanceDir("vm-p"), "bundle", "rootfs.ext4")}, + Spec: &migrateSpec{SourceHost: "src", Net: "none", Profile: portable}} + if _, err := s.CreateInstance("vm-p"); err != nil { + t.Fatal(err) + } + if err := writeIncomingInstance(s, rec, "/bin/hvi", "/root"); err != nil { + t.Fatal(err) + } + state, err := s.GetInstance("vm-p") + if err != nil { + t.Fatal(err) + } + if state.Profile != portable || state.Migration == nil || + state.Migration.State != store.MigrationIncoming { + t.Fatalf("incoming instance = profile %q, migration %+v", state.Profile, state.Migration) + } + if slices.Contains(state.CmdLine, "--profile") { + t.Fatalf("the restore command line carries --profile: %v", state.CmdLine) + } +} + +// A portable instance is not fenced for a destination whose hvi has no +// machine profiles: the probe refuses it first. A native instance still +// moves to that hvi. +func TestMovePortableNeedsAnHviWithProfiles(t *testing.T) { + f := newMoveFixture(t, true) + noProfiles := filepath.Join(t.TempDir(), "hvi") + script := "#!/bin/sh\n" + + `[ "$1 $2" = "boot --profile" ] && { echo 'hvi: unknown boot arg "--profile"' >&2; exit 1; }` + "\n" + + `exec "` + f.hvi + `" "$@"` + "\n" + if err := os.WriteFile(noProfiles, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + state := mustInstance(t, f.src, f.name) + state.Profile = portable + if err := f.src.SaveInstance(state); err != nil { + t.Fatal(err) + } + peer := newLocalPeer(f.dst) + opt := f.options(false) + opt.remote.Hvi = noProfiles + err := sourceMigrate(context.Background(), f.src, f.name, opt, peer) + if err == nil || !strings.Contains(err.Error(), "has no machine profiles") { + t.Fatalf("err = %v, want the probe's refusal", err) + } + if slices.Contains(f.vm.ops(), "checkpoint") { + t.Fatal("the source was fenced for a destination that cannot restore it") + } + if _, err := os.Stat(f.src.InstanceMigrationRecord(f.name)); !os.IsNotExist(err) { + t.Fatal("a refused probe left a move record") + } + + state.Profile = "" + if err := f.src.SaveInstance(state); err != nil { + t.Fatal(err) + } + opt.dryRun = true + if err := sourceMigrate(context.Background(), f.src, f.name, opt, peer); err != nil { + t.Fatalf("a native instance to the same hvi: %v", err) + } +} + +// The destination records the profile the move names only when the bundle +// hvi wrote agrees: a native guest labelled portable is refused on receive, +// and nothing is recorded that would relax the OS check on its next move. +// The bundle of a portable guest moves through the commit. +func TestReceiveChecksTheBundleProfile(t *testing.T) { + f := newMoveFixture(t, true) + state := mustInstance(t, f.src, f.name) + state.Profile = portable + if err := f.src.SaveInstance(state); err != nil { + t.Fatal(err) + } + peer := newLocalPeer(f.dst) + err := sourceMigrate(context.Background(), f.src, f.name, f.options(false), peer) + if err == nil || !strings.Contains(err.Error(), "profile.json records native") { + t.Fatalf("err = %v, want the destination's refusal of a native bundle labelled %s", err, portable) + } + if st, err := f.dst.GetInstance(f.name); err == nil && st.Profile != "" { + t.Fatalf("the destination recorded profile %q for the refused bundle", st.Profile) + } + + g := newMoveFixture(t, true) + g.vm.profile = portable + state = mustInstance(t, g.src, g.name) + state.Profile = portable + if err := g.src.SaveInstance(state); err != nil { + t.Fatal(err) + } + if err := sourceMigrate(context.Background(), g.src, g.name, g.options(false), newLocalPeer(g.dst)); err != nil { + t.Fatalf("a portable bundle labelled portable: %v", err) + } + if st := mustInstance(t, g.dst, g.name); st.Profile != portable { + t.Fatalf("the destination recorded profile %q", st.Profile) + } +} + +// A reserve takes only a profile this hull knows, and records native as "". +func TestReserveNormalizesTheProfile(t *testing.T) { + s := shortStore(t) + hvi := fakeHviProgram(t, true) + req := migrateReserveRequest{Epoch: 1, Spec: migrateSpec{SourceHost: "src", Net: "none", Hvi: hvi, + Profile: "portable-arm64-v2"}} + if r := destReserve(s, strings.Repeat("ab", 16), "vm1", req); r.OK || !strings.Contains(r.Error, "portable-arm64-v2") { + t.Fatalf("reserve of an unknown profile: %+v", r) + } + if _, err := os.Stat(s.InstanceDir("vm1")); !os.IsNotExist(err) { + t.Fatal("a refused reserve left the instance directory") + } + req.Spec.Profile = "native" + r := destReserve(s, strings.Repeat("ab", 16), "vm1", req) + if !r.OK || r.Record.Spec.Profile != "" { + t.Fatalf("reserve of native: %+v", r) + } +} diff --git a/cmd/hull/migrate_record.go b/cmd/hull/migrate_record.go index 52c02da2..3f0f8d46 100644 --- a/cmd/hull/migrate_record.go +++ b/cmd/hull/migrate_record.go @@ -181,6 +181,9 @@ type migrateSpec struct { GatewaySock string `json:"gatewaySock,omitempty"` // Hvi is the destination's hvi, when the operator named one. Hvi string `json:"hvi,omitempty"` + // Profile is the instance's hvi machine profile ("" for native), which + // the destination records on the instance it takes in. + Profile string `json:"profile,omitempty"` } // migrateReserveRequest is migrate-receive --reserve's standard input. diff --git a/cmd/hull/migrate_source.go b/cmd/hull/migrate_source.go index 050c89f5..35bc5b2c 100644 --- a/cmd/hull/migrate_source.go +++ b/cmd/hull/migrate_source.go @@ -32,6 +32,7 @@ import ( "github.com/urfave/cli/v3" + "github.com/brig-sh/hull/internal/bootassets" "github.com/brig-sh/hull/pkg/store" ) @@ -293,7 +294,7 @@ func sourcePreflight(s *store.Store, state *store.InstanceState, opt migrateOpti } host, _ := os.Hostname() spec := &migrateSpec{SourceHost: host, ImageDigest: state.ImageDigest, MAC: state.MAC, IP: state.IP, - Hvi: opt.remote.Hvi, Net: "none"} + Hvi: opt.remote.Hvi, Net: "none", Profile: state.Profile} argv := state.CmdLine if v, ok := flagValue(argv, "--net-mac"); ok { spec.MAC = v @@ -337,11 +338,7 @@ func probeDestination(ctx context.Context, peer movePeer, rec *migrationRecord, if p == nil { return nil, fmt.Errorf("probe %s: the reply has no probe", rec.Peer) } - var why []string - if p.OS != runtime.GOOS || p.Arch != runtime.GOARCH { - why = append(why, fmt.Sprintf("it is %s/%s and this host %s/%s: a checkpoint restores only on "+ - "the same kind of host", p.OS, p.Arch, runtime.GOOS, runtime.GOARCH)) - } + why := hostKindMismatch(p, rec.Spec.Profile) if !p.Hypervisor { why = append(why, "it cannot run a VM: "+p.HypervisorDetail) } @@ -352,6 +349,9 @@ func probeDestination(ctx context.Context, peer movePeer, rec *migrationRecord, why = append(why, fmt.Sprintf("its hvi (%s) predates the control socket", p.Hvi)) case !p.HviRestore && !rec.Remote.AllowNoRestore: why = append(why, fmt.Sprintf("its hvi (%s) has no restore subcommand", p.Hvi)) + case rec.Spec.Profile != "" && !p.HviProfile: + why = append(why, fmt.Sprintf("its hvi (%s) has no machine profiles, so it cannot restore a "+ + "%s instance", p.Hvi, rec.Spec.Profile)) } switch { case p.NameTaken && p.NameTxID != rec.TxID && p.NameMove == roleSource+" "+srcCommitted: @@ -375,6 +375,26 @@ func probeDestination(ctx context.Context, peer movePeer, rec *migrationRecord, return p, nil } +// hostKindMismatch says why a checkpoint taken here cannot restore on the +// probed host's kind of machine, or nothing. A native checkpoint holds the +// host's own interrupt controller, timer and CPU, so it restores only on the +// same OS and architecture. A portable-arm64-v1 one holds the profile's +// machine, which hvi presents on an Apple host and on a Linux arm64 host +// alike, so only the architecture has to match; the destination hvi still +// compares the bundle's profile with its own host before it restores. +func hostKindMismatch(p *migrateProbe, profile string) []string { + switch { + case p.Arch != runtime.GOARCH: + return []string{fmt.Sprintf("it is %s/%s and this host %s/%s: a checkpoint restores only on "+ + "the same architecture", p.OS, p.Arch, runtime.GOOS, runtime.GOARCH)} + case p.OS != runtime.GOOS && profile != bootassets.PortableArm64V1: + return []string{fmt.Sprintf("it is %s/%s and this host %s/%s: a checkpoint restores only on "+ + "the same kind of host, unless the instance runs --profile %s", p.OS, p.Arch, + runtime.GOOS, runtime.GOARCH, bootassets.PortableArm64V1)} + } + return nil +} + // moveSpaceNeeded estimates what a move writes on the destination: the // disk's allocated bytes and the guest's whole RAM (the bundle holds at most // that), plus a tenth. diff --git a/cmd/hull/migrate_ssh.go b/cmd/hull/migrate_ssh.go index 4a4f74d3..df199598 100644 --- a/cmd/hull/migrate_ssh.go +++ b/cmd/hull/migrate_ssh.go @@ -77,6 +77,9 @@ type migrateProbe struct { NameMove string `json:"nameMove,omitempty"` GatewayOK bool `json:"gatewayOk,omitempty"` GatewayDetail string `json:"gatewayDetail,omitempty"` + // HviProfile is whether the destination's hvi takes --profile: one + // that does not cannot restore a portable instance. + HviProfile bool `json:"hviProfile,omitempty"` } // Reply codes a source branches on. diff --git a/cmd/hull/run.go b/cmd/hull/run.go index f1ca9d7b..1365fa2c 100644 --- a/cmd/hull/run.go +++ b/cmd/hull/run.go @@ -104,6 +104,13 @@ func runCommand() *cli.Command { Value: "", Usage: "path to qemu-system-aarch64 binary (default: auto-detect, prefers signed copy)", }, + &cli.StringFlag{ + Name: "profile", + Usage: "hvi machine profile: 'native' (default) or 'portable-arm64-v1', a machine whose " + + "checkpoints restore on an Apple host and a Linux arm64 host alike (hvi only; needs " + + "--rootfs-type block and the profile's kernel, see `hull assets pull --profile`). " + + "Also the annotation com.nofire.hvi.profile", + }, &cli.StringFlag{ Name: "rootfs-type", Value: "", @@ -262,6 +269,13 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { return err } hypervisorOverride, rootfsTypeOverride = preflight.hypervisor, preflight.rootfsType + profile, err := resolveRunProfile(cmd.String("profile"), annotationOverrides) + if err != nil { + return err + } + if err := checkProfileRun(profile, preflight); err != nil { + return err + } // Generate instance ID if not provided if instanceName == "" { @@ -534,6 +548,9 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { if err := platformCheckHypervisor(vmmName, vmmSource); err != nil { return err } + if err := checkProfileBackend(profile, vmmName, vmmSource); err != nil { + return err + } vmmType := hypervisors.VmmType(vmmName) telemetryBackendSource = vmmSource @@ -569,7 +586,14 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { annotationOverrides["com.urunc.unikernel.bootInitrd"] == "" && (vmmType == vzVmm || vmmType == hypervisors.HviVmm) && !imageCarriesKernel(ociSpec.Annotations) { - kernel, initrdFile, assetErr := bootassets.Ensure(ctx, cmd.String("store-dir")) + ensure := bootassets.Ensure + if profile != "" { + // The profile boots its own kernel with the default initrd. + ensure = func(ctx context.Context, storeDir string) (string, string, error) { + return bootassets.EnsureProfile(ctx, storeDir, profile) + } + } + kernel, initrdFile, assetErr := ensure(ctx, cmd.String("store-dir")) if assetErr != nil { return fmt.Errorf("this image carries no kernel, so it needs the generic boot assets: %w", assetErr) } @@ -761,6 +785,17 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { } } + // Before the disk is built: a profile boots its own kernel, whichever + // way the run arrived at one. + profileKernelWhy := "takes its kernel from the image" + if cmd.Bool("no-boot-assets") { + profileKernelWhy = "has --no-boot-assets, so its kernel comes from the image" + } + if err := checkProfileKernel(cmd.String("store-dir"), profile, bootKernel, kernelPath, + profileKernelWhy); err != nil { + return err + } + // Determine rootfs mode: initrd file vs directory (virtiofs/9pfs/block) initrdPath := "" rootfsDir := "" @@ -1512,6 +1547,11 @@ exec %s "$@" } cmdArgs = append(cmdArgs, "--control-sock", controlSock, "--profile-out", profilePath, "--checkpoint-root", root) + } else if profile != "" { + // A profile exists to be checkpointed and moved, and without + // the control socket the instance never can be. + return fmt.Errorf("--profile %s needs an hvi with a control socket (--control-sock), "+ + "and %s takes none: the instance could never be checkpointed or moved", profile, cmdArgs[0]) } else { // Debug, not a warning: this is every run on the hvi hull // ships with until that one has the socket, and nobody asked @@ -1520,6 +1560,19 @@ exec %s "$@" cmdArgs[0], instanceName) } } + if profile != "" { + // The image may have booted from its own initrd rather than the + // block rootfs the flags asked for; the profile needs the disk. + if rootfsDiskImage == "" { + return fmt.Errorf("--profile %s needs a block rootfs, and this image boots from its own "+ + "initrd", profile) + } + profileArgs, err := hviProfileArgs(cmdArgs[0], profile) + if err != nil { + return err + } + cmdArgs = append(cmdArgs, profileArgs...) + } state := &store.InstanceState{ ID: instanceName, @@ -1531,6 +1584,7 @@ exec %s "$@" Backend: string(vmmType), ControlSock: controlSock, ProfilePath: profilePath, + Profile: profile, } started, err := launchVMM(ctx, cmd, s, state, cmdArgs, append(gatewayFiles, shareFiles...), vmmType, detach, netMode, gatewayIP) instanceStarted = started @@ -1992,14 +2046,15 @@ func verifyStagedBootAsset(storeDir, original, staged string) error { if bootassets.VerifyModeFromEnv() == bootassets.VerifyRequire { return fmt.Errorf("refusing to boot %s: %s=require and it has no provenance "+ "record beside it, so nothing vouches for its contents. Run "+ - "`hull assets pull --force` to fetch and record them again", - original, bootassets.VerifyModeEnv) + "%s to fetch and record them again", + original, bootassets.VerifyModeEnv, bootassets.ReinstallCommand(filepath.Dir(original))) } if dirErr == nil && inOurs { log.Warnf("%s is in hull's own boot-asset directory but has no provenance "+ "record beside it, so nothing vouches for its contents. Fetch writes one; "+ - "`hull assets pull --force` will record them. Booting as given "+ - "(%s=require refuses instead)", original, bootassets.VerifyModeEnv) + "%s will record them. Booting as given "+ + "(%s=require refuses instead)", original, + bootassets.ReinstallCommand(filepath.Dir(original)), bootassets.VerifyModeEnv) return nil } log.Warnf("%s has no provenance record beside it, so nothing vouches for its "+ diff --git a/cmd/hull/run_profile_test.go b/cmd/hull/run_profile_test.go new file mode 100644 index 00000000..3601b963 --- /dev/null +++ b/cmd/hull/run_profile_test.go @@ -0,0 +1,345 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + "time" + + "github.com/brig-sh/hull/internal/bootassets" + "github.com/brig-sh/hull/pkg/store" + "github.com/urfave/cli/v3" +) + +// These tests run `hull run` itself, up to the exec of a fake hvi that +// records its command line, so they check the call site that puts the +// profile on the hvi argv, in state.json and in the kernel staged, not only +// the helpers it calls. + +const ( + runNativeKernel = "the default bundle's kernel" + runPortableKernel = "the portable profile's kernel" +) + +// runProfileEnv is a store whose asset directory holds a default bundle and +// a portable kernel, each with its record, so nothing is fetched; a local +// OCI bundle to run; and a fake hvi first on PATH. controlFlags says whether +// that hvi takes the control flags. It returns the store directory, the +// bundle and the file the fake hvi writes its argv to. +func runProfileEnv(t *testing.T, controlFlags bool) (storeDir, bundle, argvFile string) { + t.Helper() + if _, err := os.Stat(mke2fsBin); err != nil { + t.Skipf("the block rootfs needs e2fsprogs: %v", err) + } + for _, k := range []string{"HULL_BOOT_ASSETS", "BRIG_BOOT_ASSETS", bootassets.PortableKernelEnv, + bootassets.PortableKernelRefEnv, bootassets.VerifyModeEnv} { + t.Setenv(k, "") + } + t.Setenv("DO_NOT_TRACK", "1") + storeDir = shortSocketDir(t) + // The macOS store marker, so the run takes /tmp for a store hull + // mounted rather than creating a disk image; Linux ignores the file. + if err := os.WriteFile(filepath.Join(storeDir, ".case-sensitive-apfs"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := store.New(storeDir); err != nil { + t.Fatal(err) + } + assets := filepath.Join(storeDir, "assets") + seedAssetDir(t, assets, map[string]string{ + bootassets.KernelName(): runNativeKernel, + bootassets.InitrdName: "initrd", + }) + seedAssetDir(t, filepath.Join(assets, bootassets.PortableArm64V1), map[string]string{ + bootassets.ProfileKernelName: runPortableKernel, + }) + + bundle = filepath.Join(t.TempDir(), "bundle") + if err := os.MkdirAll(filepath.Join(bundle, "rootfs", "bin"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bundle, "rootfs", "bin", "sh"), []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + spec := `{"ociVersion":"1.0.2","process":{"args":["/bin/sh"],"env":["PATH=/bin"],"cwd":"/"},` + + `"root":{"path":"rootfs"},"annotations":{}}` + if err := os.WriteFile(filepath.Join(bundle, "config.json"), []byte(spec), 0o644); err != nil { + t.Fatal(err) + } + + bin := t.TempDir() + argvFile = filepath.Join(bin, "argv") + checkpointRoot := `echo "error: --checkpoint-root needs a directory" >&2; exit 2` + if !controlFlags { + checkpointRoot = `echo 'error: unknown boot arg "--checkpoint-root"' >&2; exit 2` + } + script := "#!/bin/sh\n" + + "case \"$1 $2\" in\n" + + "\"boot --checkpoint-root\") " + checkpointRoot + " ;;\n" + + "\"boot --profile\") echo 'error: --profile needs a name' >&2; exit 2 ;;\n" + + "esac\n" + + "printf '%s\\n' \"$@\" > '" + argvFile + ".tmp' && mv '" + argvFile + ".tmp' '" + argvFile + "'\n" + if err := os.WriteFile(filepath.Join(bin, "hvi"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + return storeDir, bundle, argvFile +} + +// seedAssetDir writes files and the provenance record a fetch would write +// beside them. +func seedAssetDir(t *testing.T, dir string, files map[string]string) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + rec := map[string]any{"ref": "file:///test", "digest": "sha256:test"} + entries := map[string]any{} + for name, body := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256([]byte(body)) + entries[name] = map[string]any{"sha256": hex.EncodeToString(sum[:]), "size": len(body)} + } + rec["files"] = entries + b, err := json.Marshal(rec) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "provenance.json"), b, 0o644); err != nil { + t.Fatal(err) + } +} + +// hullRun runs `hull --store-dir STORE run --detach ...` in this process. +func hullRun(t *testing.T, storeDir string, args ...string) error { + t.Helper() + app := &cli.Command{ + Name: "hull", + Flags: []cli.Flag{&cli.StringFlag{Name: "store-dir", Value: storeDir}}, + Commands: []*cli.Command{runCommand()}, + } + argv := append([]string{"hull", "run", "--detach", "--hypervisor", "hvi", "--rootfs-type", "block", + "--net", "none"}, args...) + return app.Run(context.Background(), argv) +} + +// recordedArgv waits for the fake hvi to write its command line, which it +// renames into place whole. +func recordedArgv(t *testing.T, path string) []string { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + b, err := os.ReadFile(path) + if err == nil { + return strings.Split(strings.TrimSuffix(string(b), "\n"), "\n") + } + if time.Now().After(deadline) { + t.Fatalf("the fake hvi never ran: %v", err) + } + time.Sleep(20 * time.Millisecond) + } +} + +func stagedKernel(t *testing.T, storeDir, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(storeDir, "instances", name, stagedHostKernelName)) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func instanceState(t *testing.T, storeDir, name string) *store.InstanceState { + t.Helper() + s, err := store.New(storeDir) + if err != nil { + t.Fatal(err) + } + return mustInstance(t, s, name) +} + +func requireArm64(t *testing.T) { + t.Helper() + if runtime.GOARCH != "arm64" { + t.Skip("portable-arm64-v1 runs on arm64 hosts only") + } +} + +// A portable run passes --profile to hvi after the control flags, records +// the profile, and boots the profile's kernel; a native run does none of +// it, so its command line is the one an hvi from before profiles takes. +func TestRunWiresTheProfile(t *testing.T) { + requireArm64(t) + storeDir, bundle, argvFile := runProfileEnv(t, true) + + if err := hullRun(t, storeDir, "--name", "port", "--profile", bootassets.PortableArm64V1, bundle); err != nil { + t.Fatal(err) + } + argv := recordedArgv(t, argvFile) + i := slices.Index(argv, "--profile") + if i < 0 || i+1 >= len(argv) || argv[i+1] != bootassets.PortableArm64V1 { + t.Fatalf("the portable run's hvi argv has no --profile %s: %v", bootassets.PortableArm64V1, argv) + } + if !slices.Contains(argv, "--control-sock") { + t.Fatalf("the portable run's hvi argv has no --control-sock: %v", argv) + } + state := instanceState(t, storeDir, "port") + if state.Profile != bootassets.PortableArm64V1 { + t.Fatalf("state.json profile = %q", state.Profile) + } + if got := stagedKernel(t, storeDir, "port"); got != runPortableKernel { + t.Fatalf("the portable run staged %q, not the profile's kernel", got) + } + + if err := os.Remove(argvFile); err != nil { + t.Fatal(err) + } + if err := hullRun(t, storeDir, "--name", "nat", bundle); err != nil { + t.Fatal(err) + } + argv = recordedArgv(t, argvFile) + if slices.Contains(argv, "--profile") { + t.Fatalf("the native run's hvi argv has --profile: %v", argv) + } + state = instanceState(t, storeDir, "nat") + if state.Profile != "" { + t.Fatalf("the native run recorded profile %q", state.Profile) + } + raw, err := os.ReadFile(filepath.Join(storeDir, "instances", "nat", "state.json")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), `"profile"`) { + t.Fatalf("the native run's state.json names a profile: %s", raw) + } + if got := stagedKernel(t, storeDir, "nat"); got != runNativeKernel { + t.Fatalf("the native run staged %q, not the default kernel", got) + } +} + +// A profile boots its own kernel however the run arrives at one. A +// bootKernel annotation is checked by its bytes: the installed kernel, or a +// copy of it elsewhere (brig's BRIG_PORTABLE_KERNEL), boots; the default +// bundle's kernel is refused, as is a run whose kernel comes from the image. +// Each refusal comes before the disk is built and leaves no instance. +func TestRunRefusesAKernelThatIsNotTheProfiles(t *testing.T) { + requireArm64(t) + storeDir, bundle, argvFile := runProfileEnv(t, true) + assets := filepath.Join(storeDir, "assets") + initrd := "com.urunc.unikernel.bootInitrd=" + filepath.Join(assets, bootassets.InitrdName) + copyOf := filepath.Join(t.TempDir(), "Image") + if err := os.WriteFile(copyOf, []byte(runPortableKernel), 0o644); err != nil { + t.Fatal(err) + } + profile := []string{"--profile", bootassets.PortableArm64V1} + + for _, tc := range []struct { + name string + args []string + want string + }{ + {"default", []string{"--annotation", "com.urunc.unikernel.bootKernel=" + + filepath.Join(assets, bootassets.KernelName()), "--annotation", initrd}, + "is not the " + bootassets.PortableArm64V1 + " kernel"}, + {"noassets", []string{"--no-boot-assets"}, "--no-boot-assets"}, + } { + err := hullRun(t, storeDir, append(append([]string{"--name", tc.name}, profile...), + append(tc.args, bundle)...)...) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: err = %v, want a refusal naming %q", tc.name, err, tc.want) + } + if _, statErr := os.Stat(filepath.Join(storeDir, "instances", tc.name)); !os.IsNotExist(statErr) { + t.Errorf("%s: the refused run left its instance directory: %v", tc.name, statErr) + } + } + if _, err := os.Stat(argvFile); !os.IsNotExist(err) { + t.Fatalf("a refused run started hvi: %v", err) + } + + for _, tc := range []struct{ name, kernel string }{ + {"installed", filepath.Join(assets, bootassets.PortableArm64V1, bootassets.ProfileKernelName)}, + {"copy", copyOf}, + } { + _ = os.Remove(argvFile) + err := hullRun(t, storeDir, append(append([]string{"--name", tc.name}, profile...), + "--annotation", "com.urunc.unikernel.bootKernel="+tc.kernel, "--annotation", initrd, bundle)...) + if err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + if argv := recordedArgv(t, argvFile); !slices.Contains(argv, bootassets.PortableArm64V1) { + t.Fatalf("%s: the hvi argv has no profile: %v", tc.name, argv) + } + if got := stagedKernel(t, storeDir, tc.name); got != runPortableKernel { + t.Fatalf("%s: staged %q", tc.name, got) + } + } +} + +// A portable run on an hvi that takes --profile but not the control flags +// is refused: it would be a portable instance that can never be +// checkpointed or moved. +func TestRunRefusesAProfileWithoutTheControlSocket(t *testing.T) { + requireArm64(t) + storeDir, bundle, argvFile := runProfileEnv(t, false) + err := hullRun(t, storeDir, "--name", "nosock", "--profile", bootassets.PortableArm64V1, bundle) + if err == nil || !strings.Contains(err.Error(), "--control-sock") { + t.Fatalf("err = %v, want the refusal naming the control socket", err) + } + if _, statErr := os.Stat(filepath.Join(storeDir, "instances", "nosock")); !os.IsNotExist(statErr) { + t.Fatalf("the refused run left its instance directory: %v", statErr) + } + if _, statErr := os.Stat(argvFile); !os.IsNotExist(statErr) { + t.Fatal("the refused run started hvi") + } + // A native run on the same hvi still boots, without the socket. + if err := hullRun(t, storeDir, "--name", "nat", bundle); err != nil { + t.Fatal(err) + } + if argv := recordedArgv(t, argvFile); slices.Contains(argv, "--control-sock") { + t.Fatalf("the native run passed --control-sock to an hvi that takes none: %v", argv) + } +} + +// `assets pull --profile` without --force does not call a kernel that fails +// its record "already present": it says what is wrong and names the pull +// that replaces it. +func TestAssetsPullProfileReportsATamperedKernel(t *testing.T) { + requireArm64(t) + storeDir, _, _ := runProfileEnv(t, true) + kernel := filepath.Join(storeDir, "assets", bootassets.PortableArm64V1, bootassets.ProfileKernelName) + if err := os.WriteFile(kernel, []byte("something else"), 0o644); err != nil { + t.Fatal(err) + } + app := &cli.Command{ + Name: "hull", + Flags: []cli.Flag{&cli.StringFlag{Name: "store-dir", Value: storeDir}}, + Commands: []*cli.Command{assetsCommand()}, + } + err := app.Run(context.Background(), []string{"hull", "assets", "pull", "--profile", bootassets.PortableArm64V1}) + if err == nil || !strings.Contains(err.Error(), "`hull assets pull --profile "+bootassets.PortableArm64V1+" --force`") { + t.Fatalf("err = %v, want the mismatch and the profile's pull", err) + } +} diff --git a/docs/backends.md b/docs/backends.md index 3a6b160f..1c6dee7a 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -52,6 +52,7 @@ Start with `vz`. Move to another backend only for a reason on this list. | `exec` | yes | yes | yes | yes | | Checkpoint and restore | yes, with a block rootfs | checkpoint with a block rootfs and no shares, into a [partial bundle](checkpoint-restore.md#what-a-bundle-holds); **no restore yet** | **no** | as on macOS | | Pause and resume | **no** | yes | **no** | yes | +| Machine profile (`--profile`) | **refused** | `native`, or `portable-arm64-v1` with a block rootfs, no shares and the [profile's kernel](images.md#a-profiles-kernel); needs an `hvi` that takes `--profile` | **refused** | as on macOS | | `--gui`, `--gui-title` | yes | no | no | no | | `--rosetta` | yes | no | no | no | @@ -220,6 +221,61 @@ pin predates the socket; hull asks each `hvi` before a run and leaves the flags off one that does not take them, so it boots without pause or checkpoint. See [checkpoint-restore.md](checkpoint-restore.md#hvi). +### Machine profiles + +`hvi` boots one of two machines. `native`, the default, shows the guest the +host's own interrupt controller, timer and CPU, so a checkpoint of it +restores only on the same kind of host. `portable-arm64-v1` shows the guest +the same machine on an Apple host (Hypervisor.framework) and on a Linux +arm64 host (KVM): a GICv2, a Cortex-A76 identity with a reduced feature +set, and the `hvi-time` device for its clocks and timers in place of the +architected timer, whose rate differs between the two (24 MHz on Apple, +54 MHz on a Raspberry Pi 5). `hvi`'s `docs/portable-profile.md` is the +contract. + +```bash +HULL_PORTABLE_KERNEL=/path/to/Image hull assets pull --profile portable-arm64-v1 +hull run --hypervisor hvi --profile portable-arm64-v1 --rootfs-type block IMAGE +``` + +hull records the profile in `state.json` (`profile`), passes `--profile` to +`hvi`, and boots the profile's kernel with the default initrd; `hvi` adds +the kernel arguments the profile needs. A native run passes nothing new, so +an `hvi` from before profiles still boots it. + +The kernel is checked by its contents, whatever names it. When hull finds +the kernel itself, it takes the profile's. When a +`com.urunc.unikernel.bootKernel` annotation names it (as brig does), that +file must hash to the kernel recorded in `hull assets dir --profile +portable-arm64-v1`, so a copy of that kernel at another path boots and any +other kernel is refused. The check runs before the disk is built. + +hull refuses the profile: + +- on `vz` and `qemu`; +- without `--rootfs-type block` (on macOS an unset `--rootfs-type` is + virtiofs for `hvi`, and is refused rather than switched); +- with `--shared-dir` or `--shared-dir-fd`, and for an image that boots from + its own initrd: none of these lives on the disk a checkpoint carries; +- with an `hvi` that does not take `--profile` (hull asks it, as it asks + about the control flags). That `hvi` would boot the native machine; +- with an `hvi` that takes `--profile` but has no control socket + (`--control-sock`): that instance could never be checkpointed or moved; +- with a kernel that is not the profile's: a bootKernel annotation that + hashes to another kernel, a kernel from the image, or `--no-boot-assets`, + which leaves the image to supply one. + +The annotation `com.nofire.hvi.profile` selects the profile too, but only +from `--annotation`, never from the image: like the boot kernel, the +profile decides what host kernel boots. `hull restore` and a move take the +machine from the checkpoint's own profile, so `hvi restore` gets no +`--profile`. A move carries the profile to the destination's record, and a +portable instance may move between macOS and Linux (see +[migration.md](migration.md), and its runs in [The portable profile on this +Mac](migration.md#the-portable-profile-on-this-mac) and [on the Linux +pair](migration.md#the-portable-profile-on-the-linux-pair)); a native one +only to the same OS. + **Other limits `hvi` states about itself.** Eight vCPUs on a GICv2 arm64 Linux host, which does not apply on macOS because Apple's in-kernel GIC is a GICv3. One disk and one NIC, no hotplug, no PCI. virtio-fs on macOS only. It has had diff --git a/docs/cli.md b/docs/cli.md index ba6a7e40..15d99fc2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -65,7 +65,7 @@ The image reference is the first positional argument. Everything after it replaces the image's `Cmd`, the way `docker run` does. The image entrypoint is kept when that entrypoint is `urunit`. -Twenty-three flags: +Twenty-four flags: | Flag | Default | What it does | |---|---|---| @@ -80,6 +80,7 @@ Twenty-three flags: | `--hypervisor ` | from image annotation, else `qemu` | `vz`, `hvi`, `qemu`, or `qemu-hvf` | | `--qemu-path ` | auto-detect, prefers a signed copy | path to `qemu-system-aarch64` | | `--rootfs-type ` | `virtiofs` on `vz` and `hvi`, `9pfs` on `qemu` | `block`, `virtiofs` or `9pfs` | +| `--profile ` | `native` | `hvi` machine profile: `native` or `portable-arm64-v1`, whose checkpoints restore on an Apple host and a Linux arm64 host alike. Also the annotation `com.nofire.hvi.profile`, read from `--annotation` only. Recorded in `state.json` as `profile` and passed to `hvi` as `--profile`. See [backends.md](backends.md#machine-profiles) | | `--annotation ` | | set an OCI runtime annotation. Repeatable | | `--no-boot-assets` | off | do not fall back to the published boot assets when the image carries no kernel | | `--env`, `-e ` | | `KEY=VALUE`, or a bare `KEY` to inherit it from the host without putting the value in argv. Repeatable | @@ -104,6 +105,15 @@ These fail before anything starts, rather than being documented and ignored: because hvi's built-in stack forwards no traffic. The error names the gateway - `--gui` or `--rosetta` on any backend but `vz` - `--gui-title` without `--gui` +- `--profile` other than `native` with a backend other than `hvi`, without + `--rootfs-type block` (on macOS an unset `--rootfs-type` is virtiofs, and is + refused, not switched), with `--shared-dir` or `--shared-dir-fd`, with an + image that boots from its own initrd, with an `hvi` that does not take + `--profile` or has no control socket, with `--profile` and the annotation + naming different profiles, or with a kernel that is not the profile's: a + `com.urunc.unikernel.bootKernel` that does not hash to the kernel in + `hull assets dir --profile NAME`, a kernel from the image, or + `--no-boot-assets` - `--rosetta` without the virtiofs rootfs mode, a `urunit` init, and a static arm64 `busybox` at `/.rosetta/busybox` in the image @@ -155,9 +165,9 @@ as a default that an explicit `--env TERM` beats. | `checkpoint ` | `--timeout `, default 60; `hvi` only: `--then continue\|fence`, default `continue`; `--dir `, an empty directory for the bundle, below `/checkpoint` | | `restore ` | `--detach`, `-d`; `--stop-grace `, default 10; `--wait-ip`; `--gateway-sock ` (`vz` only); `hvi` only: `--paused`, `--timeout `, default 120, `--adopt-latest` | | `migrate ` | `--to USER@HOST`; `--remote-store `; `--remote-hull `; `--remote-hvi `; `--remote-gateway-sock `; `--txid ` resume a move; `--dry-run`; `--abort` and `--status` (with a name or a txid); `--timeout `, default 120, for the fence; `--allow-no-restore`, testing only | -| `assets show` | none | -| `assets dir` | none. Prints the directory and nothing else, for scripts | -| `assets pull [REF]` | `--force` download even when the assets are present | +| `assets show` | `--profile NAME` shows that profile's kernel and the initrd it boots with | +| `assets dir` | `--profile NAME` prints the profile's kernel directory instead. Prints the directory and nothing else, for scripts | +| `assets pull [REF]` | `--force` download even when the assets are present. `--profile NAME` installs the profile's kernel from REF, `HULL_PORTABLE_KERNEL` or `HULL_PORTABLE_KERNEL_REF` (see [images.md](images.md#a-profiles-kernel)); without `--force` it checks the installed one against its record. Plain `--force` leaves the profile's kernel alone: replace it with `--profile NAME --force` | | `store detach` | `--force` detach even with files still open | | `store compact` | `--force` detach even with files still open, before compacting | | `telemetry on\|off\|status` | none | @@ -195,7 +205,7 @@ The destination's verbs each print one JSON object on stdout: | Verb | Flags | What it does | |---|---|---| -| `migrate-probe` | `--name`, `--hvi`, `--gateway-sock` | reports hull and hvi versions, whether hvi takes the control flags and has `restore`, the hypervisor, free space, whether the name is taken, and whether the gateway answers | +| `migrate-probe` | `--name`, `--hvi`, `--gateway-sock` | reports hull and hvi versions, whether hvi takes the control flags and `--profile` (a stand-in for "can restore a portable bundle"; see [migration.md](migration.md#usage)) and has `restore`, the hypervisor, free space, whether the name is taken, and whether the gateway answers | | `migrate-receive` | `--txid`, `--name`, `--reserve` | with `--reserve`, claims the name (a JSON request on stdin); without, reads the move stream on stdin and ends `verified` | | `migrate-prepare` | `--txid` | restores the received checkpoint paused | | `migrate-commit` | `--txid`, `--epoch` | records `running`, then resumes the guest | @@ -312,6 +322,8 @@ whole kernel command line. | `HULL_VERIFY` | how the bundle's cosign signature is checked: `warn` (default), `require`, `strict`, or `off`/`none`/`0`. Anything unrecognized is `warn` | | `HULL_BOOT_ASSETS_ALLOW_FOREIGN` | `1`, or the repository being allowed, to fetch from a repository other than the published one | | `HULL_BOOT_ASSETS_INSECURE` | `1` to fetch from a registry that resolves to a scheme other than https | +| `HULL_PORTABLE_KERNEL` | a local kernel `Image` for `--profile portable-arm64-v1`, installed by `hull assets pull --profile portable-arm64-v1` or the first run that needs it | +| `HULL_PORTABLE_KERNEL_REF` | an OCI artifact carrying that kernel instead. Setting both is refused | | `XDG_DATA_HOME` | consulted only with no store and off macOS; assets then live under `$XDG_DATA_HOME/brig/assets` | With no store, the fallback asset directory is `~/.hull/assets` on macOS. diff --git a/docs/images.md b/docs/images.md index 22aadbdc..f92a2c95 100644 --- a/docs/images.md +++ b/docs/images.md @@ -177,6 +177,51 @@ kernel, and an `assets pull` against one store cannot change what a run using another store boots. That was a real defect once. An unrelated command in another shell could change what a CI job booted. +### A profile's kernel + +`hull run --hypervisor hvi --profile portable-arm64-v1` boots a different +kernel: one built with the `hvi-time` clock driver, which keeps time on +hvi's portable machine (see [backends.md](backends.md#machine-profiles)). +It is a variant of the bundle. It lives in a directory named after the +profile inside the asset directory, `/assets/portable-arm64-v1/Image`, +with its own `provenance.json`, and the profile boots it with the default +bundle's `container-initrd`. Each file is checked against the record in its +own directory before a boot. A run with the profile boots no other kernel: +one named in a `com.urunc.unikernel.bootKernel` annotation must hash to the +`Image` this record describes, and a kernel from the image is refused (see +[backends.md](backends.md#machine-profiles)). + +No artifact carries that kernel yet, so it comes from a local file for now: + +```bash +HULL_PORTABLE_KERNEL=/path/to/Image hull assets pull --profile portable-arm64-v1 +hull assets show --profile portable-arm64-v1 # where it came from, and its sha256 +hull assets dir --profile portable-arm64-v1 # the variant directory, for scripts +hull assets pull --profile portable-arm64-v1 REF # or from an OCI artifact +``` + +| Source | How | Record | +|---|---|---| +| `HULL_PORTABLE_KERNEL=PATH` | copied into the variant directory and hashed on the way in | `ref` is `file://PATH`, `digest` the file's sha256, no verified digest | +| `HULL_PORTABLE_KERNEL_REF=REF`, or `REF` on the command line | fetched like the bundle: the same reference check, signature check and staged writes; the artifact must carry a layer titled `Image`, and one without is refused before anything in the variant directory changes | as for the bundle | + +Setting both variables is refused. `HULL_VERIFY=require` refuses to +install a local file, since no signature covers it. It does not refuse to +boot one already installed: a kernel installed from `HULL_PORTABLE_KERNEL` +before `require` was set still boots under it, as an unverified default +bundle does. `hull assets show --profile portable-arm64-v1` says which kind +is installed (`signature: none`). The first `hull run` with the profile +installs the kernel from whichever variable is set if none is installed; an +installed kernel is never replaced unasked, so after rebuilding the file run +`hull assets pull --profile portable-arm64-v1 --force`. A plain `hull assets +pull --force` re-fetches the default bundle and leaves the profile's kernel +as it is. A kernel that no longer matches its record stops the run, and the +refusal names the `--profile` pull and the source it would take the kernel +from. A run whose configured source differs from the recorded one boots the +installed kernel and says so in a warning. The variant follows `HULL_BOOT_ASSETS` and +`BRIG_BOOT_ASSETS` like the rest of the asset directory. The profile is +arm64 only. + ### Offline preparation ```bash diff --git a/docs/migration.md b/docs/migration.md index 7dbf6bb0..a7a6fad3 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -19,6 +19,16 @@ what can go wrong, and how to recover. > [Verified](#verified)). Without `--allow-no-restore`, `hull migrate` > refuses at preflight on an `hvi` with no restore, before anything is > fenced. +> +> A `--profile portable-arm64-v1` instance moves through the commit on +> this Mac with the `hvi` at c580223, which checkpoints and restores the +> portable machine (see [The portable profile on this +> Mac](#the-portable-profile-on-this-mac)). That move ran between two +> stores through the `HULL_MIGRATE_SSH` stand-in, not real ssh. With the +> `hvi` at f4594f0 a portable move stops at the fence on the Linux pair (see +> [The portable profile on the Linux +> pair](#the-portable-profile-on-the-linux-pair)). A portable move between +> this Mac and a Pi has not run. ## Usage @@ -48,7 +58,19 @@ hull migrate --abort 3f...e1 # end a move that has not com The instance must be an `hvi` instance on a block rootfs, with no shared host directory, running or paused, started with a control socket. Both hosts -must be the same OS and architecture. Everything else about the host is left +must be the same architecture, and for a native instance the same OS. An +instance started with `--profile portable-arm64-v1` (see +[backends.md](backends.md#machine-profiles)) may move between macOS and +Linux arm64, since its checkpoint holds the profile's machine rather than +the host's; the probe then also needs the destination's `hvi` to take +`--profile`, and refuses before the fence when it does not. That is a +proxy: `hvi` has no answer that says it can restore a portable bundle, and +one that takes `--profile` but cannot checkpoint or restore the portable +machine (f4594f0) passes the probe, so the move stops later, at the fence +or at the restore. On receive, the destination reads the bundle's own +`profile.json` and refuses the bundle, as a failed verification, when its +profile is not the one the source named; the profile it records is the +one `hvi` captured. Everything else about the host is left to `hvi restore` on the destination, which by its specification (SPEC H4) compares the bundle's machine profile with the host's (CPU, counter frequency, interrupt controller, hvi build) and refuses before it creates @@ -409,3 +431,96 @@ Not run live: an abort answered `unknown_txid`, a destination whose cleanup fails, a transfer cut midway, a stream whose reply is lost and a disk changed after the fence (the unit tests cover each), any gateway case, and a move on KVM past the restore. + +### The portable profile on this Mac + +2026-10-14, hull at this branch, `hvi` at `feat/teleport` c580223 (built +from `git archive` with `cargo build --release`, signed ad hoc with +`hvi.entitlements`), HVF. The store held the default boot assets and the +portable kernel (sha256 `32725b9b...`) installed from +`HULL_PORTABLE_KERNEL`. Guest: alpine on a block rootfs, 512 MiB, 2 +vCPUs, `--net none`, printing a counter every second. Each run below was +done twice, in the review of the profile commits and again after the +fixes that followed it; the numbers are the range of the two. + +- **Run.** `hull run --hypervisor hvi --profile portable-arm64-v1 + --rootfs-type block --net none` passed `--control-sock ... --profile + portable-arm64-v1` and recorded `"profile": "portable-arm64-v1"`. In the + guest: clocksource `hvi-time`, kernel 6.12.95-00002, CPU part 0xd0b. +- **Same host, fence and restore.** `hull checkpoint --then fence` took + 426 to 433 ms (capture 398 to 408 ms, 80 to 81 of 512 MiB RAM stored). + The counter held at TICK 10 for 3 s, and exec and resume were refused. + After `hull stop`, `hull restore --detach` took 0.55 to 0.57 s, with no + `--profile` on the restore argv. The guest went on (counter 19, then + 21) on `hvi-time`, a file written before the fence was there, and it + printed no boot line. +- **A move between two stores.** Through the `HULL_MIGRATE_SSH` stand-in + (see [Testing a move](#testing-a-move)), not real ssh. The destination's + probe answered `hviProfile` true. A dry run with `--remote-hvi` naming a + wrapper that rejects `--profile` was refused before the fence, and no + record was written. `hull migrate` then ran through the commit in 2.2 s: + fence 472 ms, transfer 639 to 643 ms (95.9 to 96.6 MiB), restore paused + 585 to 586 ms, commit 88 to 93 ms. On the destination the counter went + on from 11 to 15 to 17 on `hvi-time`, the file was intact, and there + was no boot line. The destination recorded that profile and owner + epoch 1. After the fixes, the bundle's `profile.json` was also read on + both stores: it named `portable-arm64-v1`. The source was stopped and `migrated`, and + its `hull restore` was refused as stale. +- **The kernel rule** (after the fixes only). With `--profile`, a + bootKernel annotation naming the default bundle's kernel was refused + before the disk was built, with no instance left. The annotation naming + the installed portable kernel, and naming a copy of it at another path, + booted on `hvi-time`. `--no-boot-assets` was refused, and so was an + `hvi` wrapper that takes `--profile` but not `--checkpoint-root`. A + portable kernel changed by one byte was refused, and the refusal named + `hull assets pull --profile portable-arm64-v1 --force`; after that + command the next run booted. + +Not run: real ssh, KVM, and a move between this Mac and a Pi. + +### The portable profile on the Linux pair + +2026-10-14, hull at this branch (f2a7a5c), between two Raspberry Pi 5 +hosts (rp1, rp2: Linux 6.12.25, KVM with a vGICv2) over real ssh with a +key from each Pi to the other. Each store held the default boot assets +and the portable kernel (Linux 6.12.95 with `hvi-time`, sha256 +`32725b9b...`) installed by `hull assets pull --profile +portable-arm64-v1` from `HULL_PORTABLE_KERNEL`. Guest: alpine on a block +rootfs, 512 MiB, 1 vCPU, `--net none`. Its init only sleeps; `hull exec` +starts a detached shell that counts in a variable and writes the count +to `/dev/shm/counter` every second, and writes a 5.5 KiB random file to +`/root/marker` with `sync`. + +- **Run.** With `hvi` at `feat/teleport` f4594f0 (the first with + `--profile` and the control flags), `hull run --hypervisor hvi + --profile portable-arm64-v1 --rootfs-type block --net none` booted in + 0.9 s on hvi-time (`Switched to clocksource hvi-time`, the A76 CPU + baseline, PSCI 1.1), recorded `"profile": "portable-arm64-v1"` and + passed `--control-sock ... --profile portable-arm64-v1`. `hull pause` + and `hull resume` worked. The same run on this Mac (HVF, `hvi` from the + P1 to P3 series) booted on the software GICv2 with hvi-time at 24 MHz. +- **Probe.** A dry run to rp2 passed. With `--remote-hvi` naming an + `hvi` without profiles (5aafb49), the probe refused before the fence: + "its hvi ... has no machine profiles, so it cannot restore a + portable-arm64-v1 instance". +- **Fence.** `hvi` f4594f0 refuses to checkpoint a portable guest ("no + backend checkpoints yet"). `hull migrate` stopped at `reserved` in + 0.6 s with that reason, the guest kept running on rp1, and `--abort` + released rp2's reservation. +- **Native, up to the restore** (`hvi` 5aafb49): fence 655 ms (capture + 649 ms, 55.5 of 512 MiB RAM stored), 70.8 MiB in 1.44 s (49 MiB/s) + with 15.2 MiB of the 15 GiB disk, bundle sha256 identical on both + hosts, rp2 `verified`. rp2's `hvi restore` refused ("the KVM restore is + not built yet"); the move stopped at `transferred`, and the abort + removed rp2's copy and tried the source's restore, refused the same way. + Before f2a7a5c each of those two refusals took the full two-minute + restore deadline (`hull migrate` 123 s); after it, 3.4 s, with hvi's + reason in the error. + +Not run yet: a portable move past the fence on the Linux pair, and a +portable move between this Mac and a Pi. The `hvi` branch at c580223 +is meant to checkpoint and restore the portable profile on HVF and on +KVM; only HVF was run here, and its move on this Mac is in [The portable profile on this +Mac](#the-portable-profile-on-this-mac). This Mac's same-host move over +real ssh was not run either, as enabling Remote Login needs a password +for sudo. diff --git a/internal/bootassets/bootassets.go b/internal/bootassets/bootassets.go index dcd05ace..795d099d 100644 --- a/internal/bootassets/bootassets.go +++ b/internal/bootassets/bootassets.go @@ -202,6 +202,18 @@ func Fetch(ctx context.Context, ref, storeDir string) error { if err != nil { return err } + return fetchInto(ctx, ref, dir, nil) +} + +// fetchInto downloads a bundle into dir and records it there. The default +// bundle and a profile's kernel (FetchProfile) come in through the same +// checks: the reference, the signature, the staged writes and the record. +// +// required names layer titles the artifact must carry. They are checked in +// the manifest, before anything is written: an artifact without them would +// otherwise replace the files and the record already in dir and only then +// be refused, leaving a record that no longer covers what boots. +func fetchInto(ctx context.Context, ref, dir string, required []string) error { // Decide whether this is a reference we are willing to take a kernel from // before any bytes move, rather than after they are on disk. warning, err := checkRef(ref) @@ -234,6 +246,15 @@ func Fetch(ctx context.Context, ref, storeDir string) error { if err != nil { return fmt.Errorf("read boot asset manifest from %s: %w", ref, err) } + titles := make(map[string]bool, len(manifest.Layers)) + for _, desc := range manifest.Layers { + titles[desc.Annotations[titleAnnotation]] = true + } + for _, name := range required { + if !titles[name] { + return fmt.Errorf("%s carries no layer titled %s; nothing in %s was changed", ref, name, dir) + } + } // The manifest digest is what a signature would be made over, so it is // recorded now while the bundle is resolved, rather than left to a later diff --git a/internal/bootassets/profile.go b/internal/bootassets/profile.go new file mode 100644 index 00000000..6fbfd298 --- /dev/null +++ b/internal/bootassets/profile.go @@ -0,0 +1,382 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bootassets + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" +) + +// A machine profile other than hvi's native one can need its own kernel. +// portable-arm64-v1 does: its guest keeps time through the hvi-time device, +// and only a kernel built with that driver ticks on it. Such a kernel is a +// variant of the bundle: it lives in a directory of its own under the asset +// directory, named after the profile, with its own provenance record, and +// the initrd stays the default bundle's. The record is per directory +// (VerifyFiles groups by directory), so the variant kernel and the default +// initrd are each checked against the record that describes them. + +// PortableArm64V1 is the machine profile whose guest is the same machine on +// an Apple host (hvi on Hypervisor.framework) and on a Linux arm64 host (hvi +// on KVM), so a checkpoint of it restores on either. +const PortableArm64V1 = "portable-arm64-v1" + +// PortableKernelEnv names a local kernel Image for portable-arm64-v1. It is +// how the variant is installed until a published artifact exists: `hull +// assets pull --profile portable-arm64-v1` (or the first run that needs it) +// copies the file into the variant directory and records its sha256. +const PortableKernelEnv = "HULL_PORTABLE_KERNEL" + +// PortableKernelRefEnv names an OCI artifact that carries the +// portable-arm64-v1 kernel, fetched with the same checks as the default +// bundle. +const PortableKernelRefEnv = "HULL_PORTABLE_KERNEL_REF" + +// ProfileKernelName is the kernel file of a profile variant. The only +// profile with one is arm64, whose kernel is a flat Image. +const ProfileKernelName = "Image" + +// hostArch is runtime.GOARCH, a variable so a test on an amd64 runner can +// exercise the arm64 paths. +var hostArch = runtime.GOARCH + +// ErrUnknownProfile is a profile name hull has no boot assets for. +var ErrUnknownProfile = errors.New("unknown machine profile") + +// NormalizeProfile returns the profile name hull records: "" for hvi's +// native profile, which needs no variant and no flag, and the name itself +// for one it knows. +func NormalizeProfile(name string) (string, error) { + switch name { + case "", "native": + return "", nil + case PortableArm64V1: + return name, nil + } + return "", fmt.Errorf("%w %q: expected native or %s", ErrUnknownProfile, name, PortableArm64V1) +} + +// checkProfile refuses a profile this host has no kernel variant for. +func checkProfile(profile string) error { + name, err := NormalizeProfile(profile) + if err != nil { + return err + } + if name == "" { + return errors.New("the native profile boots the default boot assets; it has no variant") + } + if hostArch != "arm64" { + return fmt.Errorf("%s is an arm64 machine, and this host is %s", name, hostArch) + } + return nil +} + +// ProfileDir is the directory a profile's kernel lives in: a directory named +// after the profile inside the asset directory (Dir), so HULL_BOOT_ASSETS +// and BRIG_BOOT_ASSETS move it along with the default bundle. +func ProfileDir(storeDir, profile string) (string, error) { + if err := checkProfile(profile); err != nil { + return "", err + } + dir, err := Dir(storeDir) + if err != nil { + return "", err + } + return filepath.Join(dir, profile), nil +} + +// ProfilePaths returns the kernel and initrd a profile boots: the profile's +// own kernel and the default bundle's initrd. +func ProfilePaths(storeDir, profile string) (kernel, initrd string, err error) { + dir, err := ProfileDir(storeDir, profile) + if err != nil { + return "", "", err + } + if _, initrd, err = Paths(storeDir); err != nil { + return "", "", err + } + return filepath.Join(dir, ProfileKernelName), initrd, nil +} + +// ProfileKernelPresent reports whether a profile's kernel is on disk. +func ProfileKernelPresent(storeDir, profile string) bool { + kernel, _, err := ProfilePaths(storeDir, profile) + if err != nil { + return false + } + info, err := os.Stat(kernel) + return err == nil && !info.IsDir() && info.Size() > 0 +} + +// ProfileSource is where a profile's kernel comes from when hull has to +// install it: a local file (PortableKernelEnv) or an OCI reference +// (PortableKernelRefEnv). Both empty means nothing is configured; both set +// is refused, since the two would install different kernels. +func ProfileSource(profile string) (file, ref string, err error) { + if err := checkProfile(profile); err != nil { + return "", "", err + } + file, ref = os.Getenv(PortableKernelEnv), os.Getenv(PortableKernelRefEnv) + if file != "" && ref != "" { + return "", "", fmt.Errorf("both %s and %s are set; set one, so it is clear which kernel "+ + "%s boots", PortableKernelEnv, PortableKernelRefEnv, profile) + } + return file, ref, nil +} + +// errNoProfileSource is the remedy when nothing says where the kernel is. +func errNoProfileSource(profile string) error { + return fmt.Errorf("no %s kernel is installed, and none is published yet: set %s to a kernel "+ + "Image built with the hvi-time driver (or %s to an artifact that carries one) and run "+ + "`hull assets pull --profile %s`", profile, PortableKernelEnv, PortableKernelRefEnv, profile) +} + +// InstallProfile installs a profile's kernel from its configured source +// (ProfileSource), replacing the one there. +func InstallProfile(ctx context.Context, storeDir, profile string) error { + file, ref, err := ProfileSource(profile) + if err != nil { + return err + } + switch { + case file != "": + return ImportProfileKernel(storeDir, profile, file) + case ref != "": + return FetchProfile(ctx, ref, storeDir, profile) + } + return errNoProfileSource(profile) +} + +// FetchProfile downloads an OCI artifact carrying a profile's kernel into +// the profile's directory. It goes through the same reference check, +// signature check and record as the default bundle (Fetch), and the artifact +// must carry a layer titled ProfileKernelName: one that does not is refused +// before anything in the directory changes. +func FetchProfile(ctx context.Context, ref, storeDir, profile string) error { + dir, err := ProfileDir(storeDir, profile) + if err != nil { + return err + } + if err := fetchInto(ctx, ref, dir, []string{ProfileKernelName}); err != nil { + return fmt.Errorf("install the %s kernel: %w", profile, err) + } + return nil +} + +// ImportProfileKernel copies a local kernel into a profile's directory and +// records it. The record's Ref is the file's absolute path as a file: URL and +// its Digest the file's sha256; VerifiedDigest stays empty, because no +// signature covers a local build. That is why HULL_VERIFY=require refuses an +// import: it accepts only what a signature proved. +// +// The kernel is hashed as it is copied, the way Fetch hashes a layer on its +// way to disk, so the record describes the bytes that landed. +func ImportProfileKernel(storeDir, profile, path string) error { + dir, err := ProfileDir(storeDir, profile) + if err != nil { + return err + } + if VerifyModeFromEnv() == VerifyRequire { + return fmt.Errorf("%s=require, and a local kernel (%s=%s) carries no signature to verify", + VerifyModeEnv, PortableKernelEnv, path) + } + abs, err := filepath.Abs(path) + if err != nil { + return err + } + src, err := os.Open(abs) + if err != nil { + return fmt.Errorf("open the %s kernel: %w", profile, err) + } + defer func() { _ = src.Close() }() + info, err := src.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("%s is not a kernel Image: it is empty or not a regular file", abs) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create %s: %w", dir, err) + } + var rec File + if err := writeFile(filepath.Join(dir, ProfileKernelName), src, bundleFileMode, &rec); err != nil { + return err + } + log.Warnf("Installed %s as the %s kernel; it is a local file, so no signature vouches for it "+ + "(sha256:%s)", abs, profile, rec.SHA256) + return writeProvenance(dir, "file://"+abs, "sha256:"+rec.SHA256, "", + map[string]File{ProfileKernelName: rec}) +} + +// EnsureProfile returns the kernel and initrd a profile boots, installing +// the kernel from its configured source first if it is not on disk. +// +// The default initrd comes from Ensure, so it is fetched and checked exactly +// as for a default boot. The kernel already on disk is checked against its +// record; like Ensure it is never replaced unasked, so a changed +// HULL_PORTABLE_KERNEL file is picked up by `hull assets pull --profile +// ... --force`, and a run whose configured source differs from the one +// recorded says so. +func EnsureProfile(ctx context.Context, storeDir, profile string) (kernel, initrd string, err error) { + kernel, initrd, err = ProfilePaths(storeDir, profile) + if err != nil { + return "", "", err + } + if _, _, err := Ensure(ctx, storeDir); err != nil { + return "", "", err + } + if ProfileKernelPresent(storeDir, profile) { + err := VerifyFiles(kernel) + switch { + case err == nil: + warnIfSourceMoved(storeDir, profile) + return kernel, initrd, nil + case !errors.Is(err, ErrNoProvenance): + return "", "", err + } + log.Warnf("The %s kernel in %s has no provenance record; installing it again from its "+ + "configured source", profile, filepath.Dir(kernel)) + } + if err := InstallProfile(ctx, storeDir, profile); err != nil { + return "", "", err + } + if !ProfileKernelPresent(storeDir, profile) { + return "", "", fmt.Errorf("the %s kernel is still missing after installing it: expected %s", + profile, kernel) + } + return kernel, initrd, nil +} + +// warnIfSourceMoved says when the kernel on disk came from somewhere other +// than the source configured now: the run boots the recorded one. +func warnIfSourceMoved(storeDir, profile string) { + file, ref, err := ProfileSource(profile) + if err != nil || (file == "" && ref == "") { + return + } + dir, err := ProfileDir(storeDir, profile) + if err != nil { + return + } + rec, err := readProvenance(dir) + if err != nil { + return + } + want := ref + if file != "" { + if abs, err := filepath.Abs(file); err == nil { + want = "file://" + abs + } + } + if rec.Ref != want { + log.Warnf("The %s kernel installed in %s came from %s, not from %s; booting the installed "+ + "one (`hull assets pull --profile %s --force` replaces it)", profile, dir, rec.Ref, want, profile) + } +} + +// ReinstallCommand is the command that puts the files of an asset directory +// back: `hull assets pull --force` for the bundle, and `hull assets pull +// --profile NAME --force` for a profile's directory, which the plain one +// leaves alone. A profile's directory is recognised by its name, which +// ProfileDir makes the profile's. +func ReinstallCommand(dir string) string { + if profile, err := NormalizeProfile(filepath.Base(dir)); err == nil && profile != "" { + return fmt.Sprintf("`hull assets pull --profile %s --force`", profile) + } + return "`hull assets pull --force`" +} + +// refetchHint is the remedy a failed check of dir names: the command, and +// for a profile's kernel where that command takes it from now. +func refetchHint(dir string, rec Provenance) string { + profile, err := NormalizeProfile(filepath.Base(dir)) + if err != nil || profile == "" { + return "Re-fetch with `hull assets pull --force`" + } + return fmt.Sprintf("Install it again with %s, which takes it from %s,", ReinstallCommand(dir), + describeProfileSource(profile, rec.Ref)) +} + +// describeProfileSource names the source `hull assets pull --profile` +// would install from now, and says so when none is configured. +func describeProfileSource(profile, recorded string) string { + file, ref, err := ProfileSource(profile) + switch { + case err != nil: + return fmt.Sprintf("one of %s and %s (both are set now; unset one)", PortableKernelEnv, + PortableKernelRefEnv) + case file != "": + return fmt.Sprintf("%s=%s", PortableKernelEnv, file) + case ref != "": + return fmt.Sprintf("%s=%s", PortableKernelRefEnv, ref) + } + return fmt.Sprintf("%s or %s (neither is set now; the installed kernel came from %s)", + PortableKernelEnv, PortableKernelRefEnv, recorded) +} + +// VerifyProfileKernel checks that the file at path is the kernel installed +// for profile: that it hashes to the kernel entry of the record in the +// profile's directory. what names the kernel in the refusal, since path +// may be a staged copy the operator never named. It is the check for a kernel hull did not resolve +// itself, a bootKernel annotation (brig's path) or a copy of the kernel at +// another path: the bytes decide, not the path they were named by. +// +// The profile needs its own kernel, so this refuses rather than warns: a +// run with --profile that booted another kernel would record a portable +// machine whose guest cannot keep time on it, and a move would treat it +// as portable. +func VerifyProfileKernel(storeDir, profile, path, what string) error { + dir, err := ProfileDir(storeDir, profile) + if err != nil { + return err + } + rec, err := readProvenance(dir) + if errors.Is(err, ErrNoProvenance) { + return fmt.Errorf("no %s kernel is installed in %s, so nothing says which kernel the "+ + "profile boots: `hull assets pull --profile %s` installs one", profile, dir, profile) + } + if err != nil { + return err + } + want, ok := rec.Files[ProfileKernelName] + if !ok { + return fmt.Errorf("the record in %s does not cover %s. %s after working out what wrote to %s", + dir, ProfileKernelName, refetchHint(dir, rec), dir) + } + got, size, err := hashFile(path) + if err != nil { + return fmt.Errorf("hash the kernel for %s: %w", profile, err) + } + if got != want.SHA256 || size != want.Size { + return fmt.Errorf("%s is not the %s kernel: it hashes sha256:%s (%d bytes), and the kernel "+ + "installed in %s is sha256:%s (%d bytes, from %s)", + what, profile, got, size, dir, want.SHA256, want.Size, rec.Ref) + } + return nil +} + +// ProfileRecord returns a profile's provenance record. +func ProfileRecord(storeDir, profile string) (Provenance, error) { + dir, err := ProfileDir(storeDir, profile) + if err != nil { + return Provenance{}, err + } + return readProvenance(dir) +} diff --git a/internal/bootassets/profile_test.go b/internal/bootassets/profile_test.go new file mode 100644 index 00000000..a2b78272 --- /dev/null +++ b/internal/bootassets/profile_test.go @@ -0,0 +1,385 @@ +// Copyright (c) 2026, NOFire AI +// SPDX-License-Identifier: Apache-2.0 + +package bootassets + +import ( + "context" + "errors" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// profileEnv isolates a test from the host's asset and profile settings and +// pretends the host is arm64, the only architecture a profile variant has. +func profileEnv(t *testing.T) string { + t.Helper() + for _, k := range []string{"HULL_BOOT_ASSETS", "BRIG_BOOT_ASSETS", PortableKernelEnv, + PortableKernelRefEnv, VerifyModeEnv} { + t.Setenv(k, "") + } + old := hostArch + hostArch = "arm64" + t.Cleanup(func() { hostArch = old }) + return t.TempDir() +} + +// seedDefaultAssets writes a default kernel and initrd with their record, as +// a fetch would, so Ensure finds them and fetches nothing. +func seedDefaultAssets(t *testing.T, store string) string { + t.Helper() + dir := filepath.Join(store, "assets") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + files := map[string]File{} + for name, body := range map[string]string{KernelName(): "default kernel", InitrdName: "initrd"} { + var rec File + if err := writeFile(filepath.Join(dir, name), strings.NewReader(body), 0o644, &rec); err != nil { + t.Fatal(err) + } + files[name] = rec + } + if err := writeProvenance(dir, Ref(), "sha256:feed", "", files); err != nil { + t.Fatal(err) + } + return dir +} + +func writeKernel(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "Image") + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func TestNormalizeProfile(t *testing.T) { + for in, want := range map[string]string{"": "", "native": "", PortableArm64V1: PortableArm64V1} { + got, err := NormalizeProfile(in) + if err != nil || got != want { + t.Fatalf("NormalizeProfile(%q) = %q, %v; want %q", in, got, err, want) + } + } + if _, err := NormalizeProfile("portable-arm64-v2"); !errors.Is(err, ErrUnknownProfile) { + t.Fatalf("an unknown profile gave %v, want ErrUnknownProfile", err) + } +} + +// The variant sits inside the asset directory, so every way of moving that +// directory (the store, HULL_BOOT_ASSETS) moves the variant with it. +func TestProfileDirFollowsTheAssetDirectory(t *testing.T) { + store := profileEnv(t) + got, err := ProfileDir(store, PortableArm64V1) + if err != nil { + t.Fatal(err) + } + if want := filepath.Join(store, "assets", PortableArm64V1); got != want { + t.Fatalf("ProfileDir = %q, want %q", got, want) + } + t.Setenv("HULL_BOOT_ASSETS", "/local/assets") + if got, _ := ProfileDir(store, PortableArm64V1); got != "/local/assets/"+PortableArm64V1 { + t.Fatalf("ProfileDir with HULL_BOOT_ASSETS = %q", got) + } + kernel, initrd, err := ProfilePaths(store, PortableArm64V1) + if err != nil { + t.Fatal(err) + } + if kernel != "/local/assets/"+PortableArm64V1+"/Image" || initrd != "/local/assets/"+InitrdName { + t.Fatalf("ProfilePaths = %q, %q: want the variant kernel and the default initrd", kernel, initrd) + } +} + +func TestProfileVariantRefusals(t *testing.T) { + store := profileEnv(t) + if _, err := ProfileDir(store, "native"); err == nil { + t.Fatal("the native profile has no variant directory") + } + hostArch = "amd64" + if _, err := ProfileDir(store, PortableArm64V1); err == nil || !strings.Contains(err.Error(), "amd64") { + t.Fatalf("an amd64 host gave %v, want a refusal naming the architecture", err) + } + hostArch = "arm64" + t.Setenv(PortableKernelEnv, "/a/Image") + t.Setenv(PortableKernelRefEnv, "ghcr.io/nofireai/hull-assets:portable") + if _, _, err := ProfileSource(PortableArm64V1); err == nil { + t.Fatal("both sources set must be refused") + } +} + +// A local kernel is copied, hashed on the way and recorded, so a later boot +// checks it like a fetched one; changing it afterwards is caught. +func TestImportProfileKernelRecordsTheFile(t *testing.T) { + store := profileEnv(t) + src := writeKernel(t, "hvi-time kernel") + if err := ImportProfileKernel(store, PortableArm64V1, src); err != nil { + t.Fatal(err) + } + rec, err := ProfileRecord(store, PortableArm64V1) + if err != nil { + t.Fatal(err) + } + sum, size, err := hashFile(src) + if err != nil { + t.Fatal(err) + } + if rec.Ref != "file://"+src || rec.Digest != "sha256:"+sum || rec.VerifiedDigest != "" { + t.Fatalf("record = %+v, want ref file://%s, digest sha256:%s and no verified digest", rec, src, sum) + } + if f := rec.Files["Image"]; f.SHA256 != sum || f.Size != size { + t.Fatalf("record file = %+v, want %s/%d", f, sum, size) + } + kernel, _, _ := ProfilePaths(store, PortableArm64V1) + if err := VerifyFiles(kernel); err != nil { + t.Fatalf("the imported kernel does not verify: %v", err) + } + if err := os.WriteFile(kernel, []byte("something else"), 0o644); err != nil { + t.Fatal(err) + } + if err := VerifyFiles(kernel); err == nil { + t.Fatal("a kernel changed after the import verified clean") + } +} + +func TestImportProfileKernelRefusals(t *testing.T) { + store := profileEnv(t) + if err := ImportProfileKernel(store, PortableArm64V1, writeKernel(t, "")); err == nil { + t.Fatal("an empty kernel was imported") + } + if err := ImportProfileKernel(store, PortableArm64V1, t.TempDir()); err == nil { + t.Fatal("a directory was imported as a kernel") + } + t.Setenv(VerifyModeEnv, "require") + err := ImportProfileKernel(store, PortableArm64V1, writeKernel(t, "k")) + if err == nil || !strings.Contains(err.Error(), "require") { + t.Fatalf("HULL_VERIFY=require gave %v, want a refusal", err) + } + if ProfileKernelPresent(store, PortableArm64V1) { + t.Fatal("a refused import left a kernel behind") + } +} + +// The first run of a profile installs its kernel from the configured file, +// takes the initrd from the default bundle, and a later run reuses the +// installed kernel rather than copying the file again. +func TestEnsureProfileInstallsFromTheConfiguredFile(t *testing.T) { + store := profileEnv(t) + defaultDir := seedDefaultAssets(t, store) + src := writeKernel(t, "portable kernel") + t.Setenv(PortableKernelEnv, src) + kernel, initrd, err := EnsureProfile(context.Background(), store, PortableArm64V1) + if err != nil { + t.Fatal(err) + } + if kernel != filepath.Join(defaultDir, PortableArm64V1, "Image") || initrd != filepath.Join(defaultDir, InitrdName) { + t.Fatalf("EnsureProfile = %q, %q", kernel, initrd) + } + if b, _ := os.ReadFile(kernel); string(b) != "portable kernel" { + t.Fatalf("installed kernel holds %q", b) + } + if err := VerifyFiles(kernel, initrd); err != nil { + t.Fatalf("the pair a profile boots does not verify: %v", err) + } + // The source changes; the installed kernel stays until asked. + if err := os.WriteFile(src, []byte("rebuilt kernel"), 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := EnsureProfile(context.Background(), store, PortableArm64V1); err != nil { + t.Fatal(err) + } + if b, _ := os.ReadFile(kernel); string(b) != "portable kernel" { + t.Fatalf("a second run replaced the installed kernel with %q", b) + } + if err := InstallProfile(context.Background(), store, PortableArm64V1); err != nil { + t.Fatal(err) + } + if b, _ := os.ReadFile(kernel); string(b) != "rebuilt kernel" { + t.Fatalf("InstallProfile left %q", b) + } +} + +func TestEnsureProfileWithoutASourceNamesTheRemedy(t *testing.T) { + store := profileEnv(t) + seedDefaultAssets(t, store) + _, _, err := EnsureProfile(context.Background(), store, PortableArm64V1) + if err == nil || !strings.Contains(err.Error(), PortableKernelEnv) { + t.Fatalf("EnsureProfile with nothing configured gave %v, want the %s remedy", err, PortableKernelEnv) + } +} + +// A tampered variant kernel stops the run; it is not quietly reinstalled. +func TestEnsureProfileRefusesATamperedKernel(t *testing.T) { + store := profileEnv(t) + seedDefaultAssets(t, store) + t.Setenv(PortableKernelEnv, writeKernel(t, "good")) + kernel, _, err := EnsureProfile(context.Background(), store, PortableArm64V1) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(kernel, []byte("bad"), 0o644); err != nil { + t.Fatal(err) + } + _, _, err = EnsureProfile(context.Background(), store, PortableArm64V1) + if err == nil { + t.Fatal("a tampered profile kernel was accepted") + } + // The remedy is the one that replaces the variant, and it names where + // that command takes the kernel from. The plain `assets pull --force` + // re-fetches the default bundle and leaves the variant as it is. + msg := err.Error() + if !strings.Contains(msg, "`hull assets pull --profile "+PortableArm64V1+" --force`") || + !strings.Contains(msg, PortableKernelEnv+"=") { + t.Fatalf("the tampered kernel's refusal names the wrong remedy: %v", err) + } + t.Setenv(PortableKernelEnv, "") + _, _, err = EnsureProfile(context.Background(), store, PortableArm64V1) + if err == nil || !strings.Contains(err.Error(), "neither is set now") { + t.Fatalf("with no source configured the remedy should say so: %v", err) + } +} + +// ReinstallCommand names the command that puts a directory's files back. +func TestReinstallCommand(t *testing.T) { + if got := ReinstallCommand("/s/assets"); got != "`hull assets pull --force`" { + t.Errorf("the bundle's directory: %s", got) + } + if got := ReinstallCommand("/s/assets/" + PortableArm64V1); !strings.Contains(got, "--profile "+PortableArm64V1) { + t.Errorf("the profile's directory: %s", got) + } +} + +// VerifyProfileKernel goes by the bytes: the installed kernel and a copy of +// it at another path pass, another kernel does not, and with no kernel +// installed there is nothing to compare against. +func TestVerifyProfileKernel(t *testing.T) { + store := profileEnv(t) + defaultDir := seedDefaultAssets(t, store) + copyOf := writeKernel(t, "portable kernel") + err := VerifyProfileKernel(store, PortableArm64V1, copyOf, "the copy") + if err == nil || !strings.Contains(err.Error(), "hull assets pull --profile") { + t.Fatalf("with no profile kernel installed: %v", err) + } + if err := ImportProfileKernel(store, PortableArm64V1, copyOf); err != nil { + t.Fatal(err) + } + installed := filepath.Join(defaultDir, PortableArm64V1, ProfileKernelName) + for _, path := range []string{installed, copyOf} { + if err := VerifyProfileKernel(store, PortableArm64V1, path, path); err != nil { + t.Errorf("%s: %v", path, err) + } + } + native := filepath.Join(defaultDir, KernelName()) + err = VerifyProfileKernel(store, PortableArm64V1, native, "the default kernel") + if err == nil || !strings.Contains(err.Error(), "the default kernel is not the "+PortableArm64V1+" kernel") { + t.Fatalf("the default bundle's kernel: %v", err) + } +} + +// fakeRegistry serves an OCI registry over plain http on loopback, and lets +// the fetch take a kernel from it as a mirror the operator allowed. +func fakeRegistry(t *testing.T) string { + t.Helper() + srv := httptest.NewServer(registry.New()) + t.Cleanup(srv.Close) + t.Setenv("HULL_BOOT_ASSETS_INSECURE", "1") + t.Setenv(ForeignRepoEnv, "1") + return strings.TrimPrefix(srv.URL, "http://") +} + +// pushArtifact pushes an artifact whose layers are the given titled files. +func pushArtifact(t *testing.T, ref string, files map[string]string) { + t.Helper() + img := empty.Image + for title, body := range files { + var err error + img, err = mutate.Append(img, mutate.Addendum{ + Layer: static.NewLayer([]byte(body), types.MediaType("application/octet-stream")), + Annotations: map[string]string{titleAnnotation: title}, + }) + if err != nil { + t.Fatal(err) + } + } + parsed, err := name.ParseReference(ref, name.Insecure) + if err != nil { + t.Fatal(err) + } + if err := remote.Write(parsed, img); err != nil { + t.Fatal(err) + } +} + +// An artifact without an Image layer is refused before it writes anything: +// the installed kernel and its record are as they were, and the next run +// still boots it. +func TestFetchProfileWithoutAKernelChangesNothing(t *testing.T) { + store := profileEnv(t) + seedDefaultAssets(t, store) + if err := ImportProfileKernel(store, PortableArm64V1, writeKernel(t, "portable kernel")); err != nil { + t.Fatal(err) + } + dir, err := ProfileDir(store, PortableArm64V1) + if err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(filepath.Join(dir, provenanceName)) + if err != nil { + t.Fatal(err) + } + host := fakeRegistry(t) + ref := host + "/kernels/portable:no-image" + pushArtifact(t, ref, map[string]string{"vmlinux": "a kernel by another name"}) + err = FetchProfile(context.Background(), ref, store, PortableArm64V1) + if err == nil || !strings.Contains(err.Error(), "no layer titled "+ProfileKernelName) { + t.Fatalf("FetchProfile of an artifact with no Image layer: %v", err) + } + after, err := os.ReadFile(filepath.Join(dir, provenanceName)) + if err != nil { + t.Fatal(err) + } + if string(after) != string(before) { + t.Fatalf("the refused fetch rewrote the record:\n%s\nwas\n%s", after, before) + } + if _, err := os.Stat(filepath.Join(dir, "vmlinux")); !os.IsNotExist(err) { + t.Fatalf("the refused fetch wrote its layer: %v", err) + } + if _, _, err := EnsureProfile(context.Background(), store, PortableArm64V1); err != nil { + t.Fatalf("the installed kernel no longer boots after the refused fetch: %v", err) + } + + // The same registry with an Image layer installs it, recorded. + ref = host + "/kernels/portable:good" + pushArtifact(t, ref, map[string]string{ProfileKernelName: "fetched kernel"}) + if err := FetchProfile(context.Background(), ref, store, PortableArm64V1); err != nil { + t.Fatal(err) + } + if b, _ := os.ReadFile(filepath.Join(dir, ProfileKernelName)); string(b) != "fetched kernel" { + t.Fatalf("the fetched kernel holds %q", b) + } + rec, err := ProfileRecord(store, PortableArm64V1) + if err != nil || rec.Ref != ref { + t.Fatalf("record after the fetch = %+v, %v", rec, err) + } +} + +// An artifact reference goes through the same reference check as the +// default bundle, before any bytes move. +func TestFetchProfileChecksTheReference(t *testing.T) { + store := profileEnv(t) + err := FetchProfile(context.Background(), "ghcr.io/someone/else:portable", store, PortableArm64V1) + if err == nil || !strings.Contains(err.Error(), ForeignRepoEnv) { + t.Fatalf("a foreign repository gave %v, want the %s refusal", err, ForeignRepoEnv) + } +} diff --git a/internal/bootassets/provenance.go b/internal/bootassets/provenance.go index 5b7db9aa..9a339993 100644 --- a/internal/bootassets/provenance.go +++ b/internal/bootassets/provenance.go @@ -303,9 +303,8 @@ func verifyAgainstRecord(dir string, p Provenance, required []string) error { if got != want.SHA256 || size != want.Size { return fmt.Errorf("cached boot asset %s does not match what was fetched from %s "+ "(recorded sha256:%s, %d bytes; found sha256:%s, %d bytes). Something has "+ - "replaced it since. Re-fetch with `hull assets pull --force` after working out "+ - "what wrote to %s", - path, p.Ref, want.SHA256, want.Size, got, size, dir) + "replaced it since. %s after working out what wrote to %s", + path, p.Ref, want.SHA256, want.Size, got, size, refetchHint(dir, p), dir) } } @@ -313,8 +312,8 @@ func verifyAgainstRecord(dir string, p Provenance, required []string) error { if _, ok := p.Files[name]; !ok { return fmt.Errorf("the boot asset provenance in %s does not cover %s, so nothing "+ "vouches for the file about to be booted -- a record that omits an entry is "+ - "not a record that passed. Re-fetch with `hull assets pull --force` after "+ - "working out what wrote to %s", dir, name, dir) + "not a record that passed. %s after working out what wrote to %s", + dir, name, refetchHint(dir, p), dir) } } return nil diff --git a/pkg/store/store.go b/pkg/store/store.go index e3ef4f8b..5d9cbec8 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -100,6 +100,13 @@ type InstanceState struct { // ProfilePath is where hvi wrote its machine profile (--profile-out): the // facts a destination host compares before it accepts a checkpoint. ProfilePath string `json:"profilePath,omitempty"` + // Profile is the hvi machine profile the instance boots under + // (`run --profile`), passed to hvi as --profile. Empty is hvi's native + // profile, which hull never names, so the record and the command line + // of a native run are what they were before profiles existed. + // portable-arm64-v1 is a machine a checkpoint of which restores on an + // Apple host and on a Linux arm64 host alike. + Profile string `json:"profile,omitempty"` // Migration is where this instance stands in a checkpoint, restore or // move. Nil until the first `hull checkpoint`. Stopping the instance // leaves it alone: a fenced instance that was stopped is exactly the one From 59ee5010fde5d732db231eda89ddb7d0423a3cb7 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Thu, 15 Oct 2026 01:26:06 +0300 Subject: [PATCH 13/33] feat(hvi): keep a run's disk base by content digest A move sent the guest's whole disk every time, although most of it is the image hull built and both hosts could hold already. To send only the blocks the guest changed, both hosts need that image and a name for it that does not depend on where it is stored. The disk digest hashes the size and every non-zero 64 KiB block with its index, so a hole and written zeros hash alike, and a clone, a sparse copy and a received image hash the same. An hvi run with a control socket on a block rootfs now keeps its image under /bases before launch, cloned where the filesystem can, records the digest and whether the VMM booted from it, and passes --disk-base-sha256 to an hvi that takes it. A restore clears the "from boot" mark, since hvi's record of dirty blocks restarts there. Signed-off-by: Panagiotis Moustafellos --- cmd/hull/clone_darwin.go | 23 +++ cmd/hull/clone_linux.go | 44 +++++ cmd/hull/diskbase.go | 316 +++++++++++++++++++++++++++++++++++ cmd/hull/diskbase_test.go | 178 ++++++++++++++++++++ cmd/hull/hvirestore.go | 8 + cmd/hull/run.go | 11 ++ cmd/hull/run_profile_test.go | 60 +++++++ pkg/store/store.go | 33 ++++ 8 files changed, 673 insertions(+) create mode 100644 cmd/hull/clone_darwin.go create mode 100644 cmd/hull/clone_linux.go create mode 100644 cmd/hull/diskbase.go create mode 100644 cmd/hull/diskbase_test.go diff --git a/cmd/hull/clone_darwin.go b/cmd/hull/clone_darwin.go new file mode 100644 index 00000000..76e58d5a --- /dev/null +++ b/cmd/hull/clone_darwin.go @@ -0,0 +1,23 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "golang.org/x/sys/unix" + +// cloneFile makes dst, which must not exist, an APFS clone of src: it shares +// src's blocks until either is written, so it costs no copy and no space. +func cloneFile(src, dst string) error { + return unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW) +} diff --git a/cmd/hull/clone_linux.go b/cmd/hull/clone_linux.go new file mode 100644 index 00000000..c013ea30 --- /dev/null +++ b/cmd/hull/clone_linux.go @@ -0,0 +1,44 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// cloneFile makes dst, which must not exist, a reflink of src (FICLONE). +// Filesystems that share extents (btrfs, XFS with reflink) take it; ext4 +// refuses, and the caller copies instead. +func cloneFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + err = unix.IoctlFileClone(int(out.Fd()), int(in.Fd())) + if cerr := out.Close(); err == nil { + err = cerr + } + if err != nil { + _ = os.Remove(dst) + } + return err +} diff --git a/cmd/hull/diskbase.go b/cmd/hull/diskbase.go new file mode 100644 index 00000000..d9d442cf --- /dev/null +++ b/cmd/hull/diskbase.go @@ -0,0 +1,316 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "os" + "path/filepath" + "regexp" + "time" + + "github.com/brig-sh/hull/pkg/store" +) + +// A base is the disk image exactly as `hull run` built it, before the guest +// wrote to it, kept read-only under /bases by its disk digest. A +// move whose destination holds the same base sends only the blocks the +// guest changed (docs/migration.md, "Bases"). + +// diskBlockSize is the grain of a disk digest and of the disk blocks a +// streamed move sends: 64 KiB, the block hvi's virtio-blk tracks writes at. +const diskBlockSize = 64 << 10 + +// diskDigestMagic opens every disk digest, so it can never equal the sha256 +// of some other byte string hull hashes. +const diskDigestMagic = "HULLDISK1\x00" + +// diskDigestPattern is a disk digest as hull writes it: 64 lowercase hex +// digits. It is also a base's file name, so nothing else is accepted. +var diskDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// checkDiskDigest refuses a string that is not a disk digest. +func checkDiskDigest(sum string) error { + if !diskDigestPattern.MatchString(sum) { + return fmt.Errorf("%q is not a disk digest (64 hex digits)", sum) + } + return nil +} + +// diskDigest hashes a disk image by its content alone: the magic, the size +// and the block size, then, for every 64 KiB block on an absolute boundary +// that is not all zeros, its index and its bytes (the last block may be +// short). A hole and a block of written zeros hash alike, and so do two +// files with the same bytes whatever their filesystems report as data: a +// clone, a sparse copy, an image received over a move. The sha256 of the +// plain file would say the same, but would have to read every hole. +type diskDigest struct { + h hash.Hash + size int64 + next int64 + blocks int64 + data int64 +} + +func newDiskDigest(size int64) *diskDigest { + d := &diskDigest{h: sha256.New(), size: size} + var hdr [16]byte + binary.BigEndian.PutUint64(hdr[:8], uint64(size)) + binary.BigEndian.PutUint64(hdr[8:], diskBlockSize) + _, _ = d.h.Write([]byte(diskDigestMagic)) + _, _ = d.h.Write(hdr[:]) + return d +} + +// add hashes block index, which must come after every block added before it +// and hold its full length. A zero block is left out, as a hole is. +func (d *diskDigest) add(index int64, data []byte) error { + if index < d.next { + return fmt.Errorf("disk block %d came after block %d", index, d.next-1) + } + if want := blockLen(d.size, index); want <= 0 || int64(len(data)) != want { + return fmt.Errorf("disk block %d has %d bytes, not %d", index, len(data), want) + } + d.next = index + 1 + if isZero(data) { + return nil + } + var idx [8]byte + binary.BigEndian.PutUint64(idx[:], uint64(index)) + _, _ = d.h.Write(idx[:]) + _, _ = d.h.Write(data) + d.blocks++ + d.data += int64(len(data)) + return nil +} + +// sum is the digest, as hex. +func (d *diskDigest) sum() string { return hex.EncodeToString(d.h.Sum(nil)) } + +// blockLen is the length of block index of a disk of size bytes: a whole +// block, or what is left of the disk for the last one. +func blockLen(size, index int64) int64 { + return min(diskBlockSize, size-index*diskBlockSize) +} + +// diskBlocks is the number of blocks of a disk of size bytes. +func diskBlocks(size int64) int64 { + return (size + diskBlockSize - 1) / diskBlockSize +} + +// forEachDataBlock calls fn, in index order, for every block of f that holds +// a non-zero byte. Only the extents SEEK_DATA reports are read, each rounded +// out to block boundaries; a block fn is handed is valid only during the +// call. +func forEachDataBlock(f *os.File, size int64, fn func(index int64, data []byte) error) error { + extents, err := dataExtents(f, size) + if err != nil { + return err + } + const span = 16 + buf := make([]byte, span*diskBlockSize) + next := int64(0) + for _, ext := range extents { + first := max(ext.off/diskBlockSize, next) + end := diskBlocks(ext.end) + for i := first; i < end; { + n := min(span, end-i) + length := min(n*diskBlockSize, size-i*diskBlockSize) + if _, err := f.ReadAt(buf[:length], i*diskBlockSize); err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("read %s at block %d: %w", f.Name(), i, err) + } + for j := int64(0); j < n; j++ { + lo := j * diskBlockSize + block := buf[lo:min(lo+diskBlockSize, length)] + if isZero(block) { + continue + } + if err := fn(i+j, block); err != nil { + return err + } + } + i += n + } + next = max(next, end) + } + return nil +} + +// diskDigestFile is the disk digest of the file at path, and the bytes of +// its non-zero blocks. +func diskDigestFile(path string) (string, int64, error) { + f, err := os.Open(path) + if err != nil { + return "", 0, err + } + defer func() { _ = f.Close() }() + info, err := f.Stat() + if err != nil { + return "", 0, err + } + if !info.Mode().IsRegular() { + return "", 0, fmt.Errorf("%s is not a regular file", path) + } + d := newDiskDigest(info.Size()) + if err := forEachDataBlock(f, info.Size(), d.add); err != nil { + return "", 0, err + } + return d.sum(), d.data, nil +} + +// copySparse copies src to dst, a new file, writing only the blocks of src +// that hold data, so the copy keeps the holes. +func copySparse(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + info, err := in.Stat() + if err != nil { + return err + } + out, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + err = out.Truncate(info.Size()) + if err == nil { + err = forEachDataBlock(in, info.Size(), func(index int64, data []byte) error { + _, err := out.WriteAt(data, index*diskBlockSize) + return err + }) + } + if err == nil { + err = out.Sync() + } + if cerr := out.Close(); err == nil { + err = cerr + } + if err != nil { + _ = os.Remove(dst) + } + return err +} + +// copyDisk copies the image src to dst, a path that must not exist: a clone +// where the filesystem has them (APFS, or a reflink on Linux), else a copy +// that keeps the holes. It reports which. +func copyDisk(src, dst string) (string, error) { + if err := cloneFile(src, dst); err == nil { + return "clone", nil + } + _ = os.Remove(dst) + if err := copySparse(src, dst); err != nil { + return "", err + } + return "copy", nil +} + +// keepBase puts an immutable copy of the image at disk under the store's +// bases, named by its disk digest, and returns the digest. It is taken from +// the image as hull built it, before the guest has run: the copy is made +// first, then hashed, so the digest names exactly the bytes kept. A base +// already there under the same digest is kept and the copy dropped. +func keepBase(s *store.Store, disk string) (string, error) { + started := time.Now() + if err := os.MkdirAll(s.BasesDir(), 0o700); err != nil { + return "", err + } + var rnd [8]byte + _, _ = rand.Read(rnd[:]) + tmp := filepath.Join(s.BasesDir(), ".new-"+hex.EncodeToString(rnd[:])) + how, err := copyDisk(disk, tmp) + if err != nil { + return "", fmt.Errorf("copy the base image: %w", err) + } + sum, _, err := diskDigestFile(tmp) + if err == nil { + err = sealBase(s, tmp, sum) + } + if err != nil { + _ = os.Remove(tmp) + return "", err + } + log.Debugf("Kept the base of %s as %s (%s, %s)", disk, s.BasePath(sum), how, time.Since(started)) + return sum, nil +} + +// sealBase makes the file at tmp, whose disk digest is sum, the base of +// that digest: read-only, synced, then renamed into place. A base already +// there is kept, and tmp removed. +func sealBase(s *store.Store, tmp, sum string) error { + if err := checkDiskDigest(sum); err != nil { + return err + } + dst := s.BasePath(sum) + if baseHeld(s, sum) { + return os.Remove(tmp) + } + f, err := os.Open(tmp) + if err != nil { + return err + } + err = f.Sync() + _ = f.Close() + if err == nil { + err = os.Chmod(tmp, 0o400) + } + if err == nil { + err = os.Rename(tmp, dst) + } + if err == nil { + err = syncDir(s.BasesDir()) + } + return err +} + +// baseHeld reports whether the store keeps the base of digest sum. +func baseHeld(s *store.Store, sum string) bool { + if checkDiskDigest(sum) != nil { + return false + } + info, err := os.Stat(s.BasePath(sum)) + return err == nil && info.Mode().IsRegular() +} + +// keepRunBase keeps the base of a run's freshly built disk image and returns +// the record of it, with the flag that hands its digest to hvi when the hvi +// at binary takes one (hvi records it in the checkpoint's disk.json). A base +// that cannot be kept costs the run nothing but the fast move: that move +// sends the whole disk instead. +func keepRunBase(s *store.Store, binary, disk string) (*store.BaseState, []string) { + sum, err := keepBase(s, disk) + if err != nil { + log.Warnf("Could not keep the base image of %s (%v); moving this instance will send its whole disk", + disk, err) + return nil, nil + } + base := &store.BaseState{SHA256: sum, FromBoot: true} + if knows, err := hviBootKnowsFlag(binary, hviDiskBaseFlag); err != nil || !knows { + return base, nil + } + return base, []string{hviDiskBaseFlag, sum} +} + +// hviDiskBaseFlag hands hvi the digest of the base the disk started from. +const hviDiskBaseFlag = "--disk-base-sha256" diff --git a/cmd/hull/diskbase_test.go b/cmd/hull/diskbase_test.go new file mode 100644 index 00000000..d4a3d2a7 --- /dev/null +++ b/cmd/hull/diskbase_test.go @@ -0,0 +1,178 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +// diskLayout writes size bytes at path whose content is data at each offset +// and zeros elsewhere. dense writes the zeros out; otherwise they are holes. +func diskLayout(t *testing.T, path string, size int64, data map[int64][]byte, dense bool) { + t.Helper() + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + if dense { + if _, err := f.Write(make([]byte, size)); err != nil { + t.Fatal(err) + } + } else if err := f.Truncate(size); err != nil { + t.Fatal(err) + } + for off, b := range data { + if _, err := f.WriteAt(b, off); err != nil { + t.Fatal(err) + } + } +} + +// The digest is of the bytes alone: the same content written densely, with +// holes, or with explicit zero runs inside an extent that starts off a block +// boundary hashes the same, and any change of one byte or of the size does +// not. +func TestDiskDigestIsOfTheContentOnly(t *testing.T) { + dir := t.TempDir() + const size = 3<<20 + 4096 + 17 // a short last block + content := map[int64][]byte{ + 0: bytes.Repeat([]byte{1}, 100), + 5*4096 + 3: bytes.Repeat([]byte{2}, 9000), + 1 << 20: bytes.Repeat([]byte{3}, diskBlockSize), + size - 5: {9, 9, 9, 9, 9}, + 2<<20 + diskBlockSize - 1: {7, 7}, + } + dense := filepath.Join(dir, "dense") + sparse := filepath.Join(dir, "sparse") + diskLayout(t, dense, size, content, true) + diskLayout(t, sparse, size, content, false) + // Zeros written over a hole make an extent of zeros, which hashes as + // the hole did. + odd := filepath.Join(dir, "odd") + diskLayout(t, odd, size, content, false) + f, err := os.OpenFile(odd, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt(make([]byte, 3*diskBlockSize), 2<<20+4*diskBlockSize+4096); err != nil { + t.Fatal(err) + } + _ = f.Close() + + want, data, err := diskDigestFile(dense) + if err != nil { + t.Fatal(err) + } + // Blocks 0, 16, 32 and 33 (the {7,7} pair straddles them), and the + // short last block, 48, of 4096+17 bytes. + if data != 4*diskBlockSize+4096+17 { + t.Errorf("data bytes %d", data) + } + for _, p := range []string{sparse, odd} { + got, _, err := diskDigestFile(p) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("%s hashes %s, the dense copy %s", filepath.Base(p), got, want) + } + } + + changed := filepath.Join(dir, "changed") + diskLayout(t, changed, size, content, false) + f, err = os.OpenFile(changed, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt([]byte{1}, size-1); err != nil { + t.Fatal(err) + } + _ = f.Close() + bigger := filepath.Join(dir, "bigger") + diskLayout(t, bigger, size+1, content, false) + for _, p := range []string{changed, bigger} { + if got, _, _ := diskDigestFile(p); got == want { + t.Errorf("%s hashes the same as the original", filepath.Base(p)) + } + } +} + +// A digest fed blocks out of order, or a block of the wrong length, refuses. +func TestDiskDigestRefusesBadBlocks(t *testing.T) { + d := newDiskDigest(3 * diskBlockSize) + if err := d.add(1, make([]byte, diskBlockSize)); err != nil { + t.Fatal(err) + } + if err := d.add(0, make([]byte, diskBlockSize)); err == nil { + t.Error("a block before the last one was taken") + } + if err := d.add(2, make([]byte, 10)); err == nil { + t.Error("a short middle block was taken") + } + if err := d.add(3, make([]byte, diskBlockSize)); err == nil { + t.Error("a block past the end was taken") + } +} + +// keepBase keeps one read-only copy per digest, with the image's bytes, and +// the copy keeps the holes. +func TestKeepBase(t *testing.T) { + s := shortStore(t) + disk := filepath.Join(t.TempDir(), "rootfs.ext4") + writeSparseDisk(t, disk, 64<<20) + sum, err := keepBase(s, disk) + if err != nil { + t.Fatal(err) + } + want, _, err := diskDigestFile(disk) + if err != nil || sum != want { + t.Fatalf("base %s, image %s (%v)", sum, want, err) + } + info, err := os.Stat(s.BasePath(sum)) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o400 { + t.Errorf("the base is %v, not read-only", info.Mode().Perm()) + } + if got := fileAllocated(s.BasePath(sum)); got > 4<<20 { + t.Errorf("the base of a 64 MiB sparse image occupies %d bytes", got) + } + assertSameContent(t, disk, s.BasePath(sum)) + // The same image again keeps the one base and leaves nothing behind. + if again, err := keepBase(s, disk); err != nil || again != sum { + t.Fatalf("again: %s %v", again, err) + } + entries, _ := os.ReadDir(s.BasesDir()) + if len(entries) != 1 { + t.Errorf("bases holds %d entries after two runs of one image", len(entries)) + } + // The guest's writes to its disk do not reach the base. + f, err := os.OpenFile(disk, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt([]byte("guest"), 0); err != nil { + t.Fatal(err) + } + _ = f.Close() + if got, _, _ := diskDigestFile(s.BasePath(sum)); got != sum { + t.Errorf("the base changed with the disk: %s", got) + } +} diff --git a/cmd/hull/hvirestore.go b/cmd/hull/hvirestore.go index 3fc1d4d7..13ddcba2 100644 --- a/cmd/hull/hvirestore.go +++ b/cmd/hull/hvirestore.go @@ -163,6 +163,9 @@ func hviRestoreArgv(cmdLine []string, from, controlSock, profilePath string) ([] "--kernel": true, "--initramfs": true, "--cmdline": true, "--mem-mib": true, "--cpus": true, "--fs-uid": true, "--fs-gid": true, "--from": true, "--control-sock": true, "--profile-out": true, + // The base digest describes the boot; a restored disk has moved on + // from it, and the bundle's disk.json carries what hvi recorded. + hviDiskBaseFlag: true, // The machine profile is in the bundle's own profile.json, which // hvi restore checks against the host; it takes no --profile. "--profile": true, @@ -449,6 +452,11 @@ func hviRestoreInstance(ctx context.Context, cmd *cli.Command, s *store.Store, s state.ControlSock = controlSock state.ProfilePath = profilePath state.Migration.State = store.MigrationRestoring + if state.Base != nil { + // The restored hvi's own record of dirty blocks starts here, not + // at the boot from the base. + state.Base.FromBoot = false + } txid := state.Migration.TxID type awaited struct { diff --git a/cmd/hull/run.go b/cmd/hull/run.go index 1365fa2c..f60d758b 100644 --- a/cmd/hull/run.go +++ b/cmd/hull/run.go @@ -1530,6 +1530,7 @@ exec %s "$@" // the flags would refuse to boot with them, so it runs without: it can // still be stopped, and pause and checkpoint say why they cannot. var controlSock, profilePath string + var instanceBase *store.BaseState if vmmType == hypervisors.HviVmm && len(cmdArgs) > 0 { takesFlags, err := hviBootTakesControlFlags(cmdArgs[0]) if err != nil { @@ -1547,6 +1548,15 @@ exec %s "$@" } cmdArgs = append(cmdArgs, "--control-sock", controlSock, "--profile-out", profilePath, "--checkpoint-root", root) + // A checkpointable guest on a block rootfs can be moved, and a + // move sends only what differs from the image as built when the + // destination holds that image: keep it now, before the guest + // writes to its copy. + if rootfsDiskImage != "" { + base, baseArgs := keepRunBase(s, cmdArgs[0], rootfsDiskImage) + instanceBase = base + cmdArgs = append(cmdArgs, baseArgs...) + } } else if profile != "" { // A profile exists to be checkpointed and moved, and without // the control socket the instance never can be. @@ -1585,6 +1595,7 @@ exec %s "$@" ControlSock: controlSock, ProfilePath: profilePath, Profile: profile, + Base: instanceBase, } started, err := launchVMM(ctx, cmd, s, state, cmdArgs, append(gatewayFiles, shareFiles...), vmmType, detach, netMode, gatewayIP) instanceStarted = started diff --git a/cmd/hull/run_profile_test.go b/cmd/hull/run_profile_test.go index 3601b963..3a02626d 100644 --- a/cmd/hull/run_profile_test.go +++ b/cmd/hull/run_profile_test.go @@ -98,6 +98,9 @@ func runProfileEnv(t *testing.T, controlFlags bool) (storeDir, bundle, argvFile "case \"$1 $2\" in\n" + "\"boot --checkpoint-root\") " + checkpointRoot + " ;;\n" + "\"boot --profile\") echo 'error: --profile needs a name' >&2; exit 2 ;;\n" + + "\"boot --disk-base-sha256\") if [ -f '" + filepath.Join(bin, "knows-base") + "' ]; then " + + "echo 'error: --disk-base-sha256 needs a digest' >&2; else " + + "echo 'error: unknown boot arg \"--disk-base-sha256\"' >&2; fi; exit 2 ;;\n" + "esac\n" + "printf '%s\\n' \"$@\" > '" + argvFile + ".tmp' && mv '" + argvFile + ".tmp' '" + argvFile + "'\n" if err := os.WriteFile(filepath.Join(bin, "hvi"), []byte(script), 0o755); err != nil { @@ -343,3 +346,60 @@ func TestAssetsPullProfileReportsATamperedKernel(t *testing.T) { t.Fatalf("err = %v, want the mismatch and the profile's pull", err) } } + +// A run on a block rootfs keeps the image as built under the store's bases +// by its digest before hvi starts, records it, and hands the digest to an +// hvi that takes --disk-base-sha256; an hvi that does not is not given it. +func TestRunKeepsTheBase(t *testing.T) { + storeDir, bundle, argvFile := runProfileEnv(t, true) + if err := hullRun(t, storeDir, "--name", "b1", bundle); err != nil { + t.Fatal(err) + } + argv := recordedArgv(t, argvFile) + if slices.Contains(argv, hviDiskBaseFlag) { + t.Fatalf("an hvi without %s was given it: %v", hviDiskBaseFlag, argv) + } + state := instanceState(t, storeDir, "b1") + if state.Base == nil || !state.Base.FromBoot { + t.Fatalf("state.json has no base from the boot: %+v", state.Base) + } + s, err := store.New(storeDir) + if err != nil { + t.Fatal(err) + } + disk := filepath.Join(storeDir, "instances", "b1", "bundle", "rootfs.ext4") + sum, _, err := diskDigestFile(disk) + if err != nil { + t.Fatal(err) + } + if sum != state.Base.SHA256 || !baseHeld(s, sum) { + t.Fatalf("base %s recorded, the disk hashes %s, held %v", state.Base.SHA256, sum, baseHeld(s, sum)) + } + + if err := os.WriteFile(filepath.Join(filepath.Dir(argvFile), "knows-base"), nil, 0o600); err != nil { + t.Fatal(err) + } + _ = os.Remove(argvFile) + if err := hullRun(t, storeDir, "--name", "b2", bundle); err != nil { + t.Fatal(err) + } + argv = recordedArgv(t, argvFile) + state = instanceState(t, storeDir, "b2") + i := slices.Index(argv, hviDiskBaseFlag) + if state.Base == nil || i < 0 || i+1 >= len(argv) || argv[i+1] != state.Base.SHA256 { + t.Fatalf("the hvi argv does not carry the recorded base %+v: %v", state.Base, argv) + } + // The restore command line drops the flag: a restored disk has moved + // on from the base. + if _, err := hviRestoreArgv(argv[:0:0], "", "", ""); err == nil { + t.Fatal("an empty command line mapped") + } + full := append([]string{"hvi"}, argv...) + restore, err := hviRestoreArgv(full, "/b", "/c", "/p") + if err != nil { + t.Fatal(err) + } + if slices.Contains(restore, hviDiskBaseFlag) { + t.Fatalf("the restore argv carries the base: %v", restore) + } +} diff --git a/pkg/store/store.go b/pkg/store/store.go index 5d9cbec8..b12a83c4 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -118,6 +118,24 @@ type InstanceState struct { // next epoch, and the destination accepts its commit only for that // epoch, so a commit meant for an earlier move is refused. OwnerEpoch uint64 `json:"ownerEpoch,omitempty"` + // Base names the immutable copy of the disk image this instance's disk + // started from, kept under /bases by its digest, so that a move + // sends only the blocks that differ from it. Nil for an instance with + // no block rootfs, one run by an hvi without a control socket, and one + // recorded before bases existed. + Base *BaseState `json:"base,omitempty"` +} + +// BaseState is the base image an instance's disk started from. +type BaseState struct { + // SHA256 is the base's disk digest (docs/migration.md, "Bases"), which + // is also its file name under /bases. + SHA256 string `json:"sha256"` + // FromBoot is set while the VMM that runs the instance is the one that + // booted from the fresh copy of the base. Only then do hvi's blocks + // dirtied since boot cover every difference from the base; a restore + // clears it, and a move then finds the difference by comparing. + FromBoot bool `json:"fromBoot,omitempty"` } // Migration states. A checkpointed instance kept running after the capture, @@ -764,6 +782,21 @@ func (s *Store) MigrationTombstoneDir() string { return filepath.Join(s.rootDir, "migrations") } +// BasesDir holds the base images of block rootfs disks, each an immutable +// file named .ext4 by its disk digest. A run puts the image it built +// there before the guest writes to its own copy, and a move's preparation +// puts the same file on the destination, so the move itself carries only +// the blocks the guest changed. +func (s *Store) BasesDir() string { + return filepath.Join(s.rootDir, "bases") +} + +// BasePath is where the base with disk digest sha256 is kept. The digest is +// not checked here; callers validate it before they read or write the file. +func (s *Store) BasePath(sha256 string) string { + return filepath.Join(s.BasesDir(), filepath.Base(sha256)+".ext4") +} + // DeleteInstance removes an instance directory func (s *Store) DeleteInstance(id string) error { // This one recurses, so it validates even though callers resolve the From 55a970dc93ffeac9c901a1d1ca551c7cf3c820d6 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Thu, 15 Oct 2026 01:31:26 +0300 Subject: [PATCH 14/33] feat(hvi): prepare a destination ahead of a move A move should not spend its time carrying what the destination could have held before it started: the image hull built the disk from, the portable kernel a later boot there needs, the OCI image. On the Pis the B4 demo had to install the kernel by hand, and every move sent the whole disk. `hull migrate-prepare NAME --to USER@HOST` now puts them there while the guest keeps running: the instance's base by its disk digest, sent as lz4-compressed non-zero blocks and kept on the destination only when what arrived hashes to that digest; the profile kernel this store has installed, checked against its sha256 and installed through the same import as a local kernel; and the image, pulled there by its reference unless the store has its digest. Each step finds its effect already there and sends nothing, and --json reports what was done and what the destination holds. The probe now lists the bases and profile kernels a store holds, whether its hvi restores from a stream, and a capability (stream-v1) that a hull from before this change does not list, so a source can tell an old destination apart without a protocol bump that would make the two refuse each other. The destination's hidden step of a move keeps its name: `migrate-prepare --txid T` still brings a received checkpoint up paused. Signed-off-by: Panagiotis Moustafellos --- cmd/hull/hvirestore.go | 31 ++ cmd/hull/migrate_dest.go | 14 +- cmd/hull/migrate_fixture_test.go | 8 + cmd/hull/migrate_prepare.go | 762 +++++++++++++++++++++++++++++++ cmd/hull/migrate_prepare_test.go | 239 ++++++++++ cmd/hull/migrate_ssh.go | 11 + go.mod | 2 +- 7 files changed, 1061 insertions(+), 6 deletions(-) create mode 100644 cmd/hull/migrate_prepare.go create mode 100644 cmd/hull/migrate_prepare_test.go diff --git a/cmd/hull/hvirestore.go b/cmd/hull/hvirestore.go index 13ddcba2..d116e9d6 100644 --- a/cmd/hull/hvirestore.go +++ b/cmd/hull/hvirestore.go @@ -235,6 +235,37 @@ func hviHasRestore(binary string) (bool, error) { return !strings.Contains(out.String(), `unknown subcommand "restore"`), nil } +// hviRestoreTakesStream reports whether the hvi at binary restores from a +// stream (`restore --from-stream`), which a streamed move hands it on stdin. +// Asked as the other probes are: hvi's restore parser answers an argument it +// does not know with `unknown restore arg "--from-stream"` (src/main.rs). +func hviRestoreTakesStream(binary string) (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var out bytes.Buffer + probe := exec.CommandContext(ctx, binary, "restore", hviFromStreamFlag) + probe.Stdout = &out + probe.Stderr = &out + err := probe.Run() + if ctx.Err() != nil { + return false, fmt.Errorf("%s restore did not answer a probe within 5s", binary) + } + var exitErr *exec.ExitError + if err != nil && !errors.As(err, &exitErr) { + return false, fmt.Errorf("run %s: %w", binary, err) + } + text := out.String() + return !strings.Contains(text, fmt.Sprintf("unknown restore arg %q", hviFromStreamFlag)) && + !strings.Contains(text, `unknown subcommand "restore"`), nil +} + +// hviFromStreamFlag is hvi restore's argument for a checkpoint read from a +// stream, and hviFromStreamStdin its value for standard input. +const ( + hviFromStreamFlag = "--from-stream" + hviFromStreamStdin = "-" +) + // reconcileLatestCheckpoint compares the record's checkpoint with the one // /checkpoint/latest.json names, and adopts the latter into the // record when asked and safe. diff --git a/cmd/hull/migrate_dest.go b/cmd/hull/migrate_dest.go index dd53c30e..c0a3ab4c 100644 --- a/cmd/hull/migrate_dest.go +++ b/cmd/hull/migrate_dest.go @@ -142,7 +142,13 @@ func destProbe(s *store.Store, name, hviOverride, gatewaySock string) *migrateRe } else { p.HviProfile = ok } + if ok, err := hviRestoreTakesStream(hvi); err != nil { + p.HviError = err.Error() + } else { + p.HviStream = ok + } } + destHolds(s, p) if name != "" { if err := store.ValidateInstanceID(name); err != nil { return refuse(codeFailed, err) @@ -876,7 +882,7 @@ func migrateRemoteVerb(name, usage string, flags []cli.Flag, func migrateRemoteCommands() []*cli.Command { txidFlag := &cli.StringFlag{Name: "txid", Required: true, Usage: "the move's txid"} nameFlag := &cli.StringFlag{Name: "name", Usage: "the instance's name on this host"} - return []*cli.Command{ + return append([]*cli.Command{ migrateRemoteVerb("migrate-probe", "report whether this host can take a moved instance", []cli.Flag{nameFlag, &cli.StringFlag{Name: "hvi"}, &cli.StringFlag{Name: "gateway-sock"}}, func(cmd *cli.Command, s *store.Store) *migrateReply { @@ -894,9 +900,7 @@ func migrateRemoteCommands() []*cli.Command { } return destReceive(s, cmd.String("txid"), cmd.String("name"), os.Stdin) }), - migrateRemoteVerb("migrate-prepare", "restore a received move paused", - []cli.Flag{txidFlag}, - func(cmd *cli.Command, s *store.Store) *migrateReply { return destPrepare(s, cmd.String("txid")) }), + migratePrepareCommand(), migrateRemoteVerb("migrate-commit", "take over a moved instance and run it", []cli.Flag{txidFlag, &cli.Uint64Flag{Name: "epoch", Required: true}}, func(cmd *cli.Command, s *store.Store) *migrateReply { @@ -908,7 +912,7 @@ func migrateRemoteCommands() []*cli.Command { migrateRemoteVerb("migrate-abort", "abort a move on this host unless it committed", []cli.Flag{txidFlag}, func(cmd *cli.Command, s *store.Store) *migrateReply { return destAbort(s, cmd.String("txid")) }), - } + }, migratePrepareRemoteCommands()...) } // fileSHA256 hashes the file at path. diff --git a/cmd/hull/migrate_fixture_test.go b/cmd/hull/migrate_fixture_test.go index c25d750e..b97339de 100644 --- a/cmd/hull/migrate_fixture_test.go +++ b/cmd/hull/migrate_fixture_test.go @@ -330,6 +330,14 @@ func (p *localPeer) call(_ context.Context, verb string, args []string, stdin io } case "migrate-prepare": reply = destPrepare(p.s, flags["--txid"]) + case "migrate-take-base": + reply = destTakeBase(p.s, flags["--sha256"], stdin) + _, _ = io.Copy(io.Discard, stdin) + case "migrate-take-kernel": + size, _ := strconv.ParseInt(flags["--size"], 10, 64) + reply = destTakeKernel(p.s, flags["--profile"], flags["--sha256"], size, stdin) + case "migrate-take-image": + reply = destTakeImage(p.s, flags["--ref"], flags["--digest"]) case "migrate-commit": epoch, _ := strconv.ParseUint(flags["--epoch"], 10, 64) reply = destCommit(p.s, flags["--txid"], epoch) diff --git a/cmd/hull/migrate_prepare.go b/cmd/hull/migrate_prepare.go new file mode 100644 index 00000000..7270bc24 --- /dev/null +++ b/cmd/hull/migrate_prepare.go @@ -0,0 +1,762 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "github.com/pierrec/lz4/v4" + "github.com/urfave/cli/v3" + + "github.com/brig-sh/hull/internal/bootassets" + "github.com/brig-sh/hull/pkg/store" +) + +// `hull migrate-prepare NAME --to USER@HOST` puts on the destination, ahead +// of a move and while the guest runs, what a streamed move then does not +// have to carry: the instance's base image (by its disk digest), the +// portable kernel of its machine profile, and its OCI image. Each step +// finds its effect already there and skips, so it can be run any number of +// times; the destination's probe lists what it holds. + +// moveCapStream is the capability a destination's probe lists when it takes +// a streamed move (`migrate-receive --stream`) and the prepare verbs. A hull +// from before them lists nothing and gets the whole-bundle move. +const moveCapStream = "stream-v1" + +// probeMaxBases bounds the bases a probe lists. +const probeMaxBases = 512 + +// destHolds is what the probe lists of the bases and profile kernels the +// destination store holds. +func destHolds(s *store.Store, p *migrateProbe) { + p.Capabilities = []string{moveCapStream} + if entries, err := os.ReadDir(s.BasesDir()); err == nil { + for _, e := range entries { + name, ok := strings.CutSuffix(e.Name(), ".ext4") + if ok && e.Type().IsRegular() && checkDiskDigest(name) == nil && len(p.Bases) < probeMaxBases { + p.Bases = append(p.Bases, name) + } + } + } + for _, profile := range []string{bootassets.PortableArm64V1} { + if sum := installedProfileKernel(s.RootDir(), profile); sum != "" { + if p.ProfileKernels == nil { + p.ProfileKernels = map[string]string{} + } + p.ProfileKernels[profile] = sum + } + } +} + +// installedProfileKernel is the sha256 a store's record gives its kernel of +// profile, once the kernel on disk matches it; "" when there is none. +func installedProfileKernel(storeDir, profile string) string { + rec, err := bootassets.ProfileRecord(storeDir, profile) + if err != nil { + return "" + } + f, ok := rec.Files[bootassets.ProfileKernelName] + if !ok { + return "" + } + kernel, _, err := bootassets.ProfilePaths(storeDir, profile) + if err != nil || bootassets.VerifyProfileKernel(storeDir, profile, kernel, "the installed kernel") != nil { + return "" + } + return f.SHA256 +} + +// hasCapability reports whether a probe lists capability c. +func (p *migrateProbe) hasCapability(c string) bool { + return p != nil && slices.Contains(p.Capabilities, c) +} + +// holdsBase reports whether a probe lists the base of digest sum. +func (p *migrateProbe) holdsBase(sum string) bool { + return p != nil && sum != "" && slices.Contains(p.Bases, sum) +} + +// The base stream is what `migrate-take-base` reads on stdin: +// +// "HULLBAS1" +// u32 len, JSON {size, sha256, blockSize} +// 'B' u64 index, u8 codec, u32 n, n bytes (one non-zero block) +// 'E' +// +// Blocks come in index order, each lz4-compressed (codec 1) unless that +// would not make it smaller (codec 0, raw). The receiver hashes what it +// writes with the disk digest and keeps the file only if the digest is the +// one the header and the request name. +const baseStreamMagic = "HULLBAS1" + +const ( + baseCodecRaw = 0 + baseCodecLZ4 = 1 +) + +// baseStreamHeader is the base stream's header. +type baseStreamHeader struct { + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + BlockSize int64 `json:"blockSize"` +} + +// baseSendStats is what sending a base carried. +type baseSendStats struct { + Blocks int64 + DataBytes int64 + WireBytes int64 +} + +// writeBaseStream writes the base at path, whose disk digest is sum, to w. +func writeBaseStream(w io.Writer, path, sum string) (baseSendStats, error) { + var st baseSendStats + f, err := os.Open(path) + if err != nil { + return st, err + } + defer func() { _ = f.Close() }() + info, err := f.Stat() + if err != nil { + return st, err + } + cw := &countingWriter{w: w} + bw := bufio.NewWriterSize(cw, 1<<20) + hdr, err := json.Marshal(baseStreamHeader{Size: info.Size(), SHA256: sum, BlockSize: diskBlockSize}) + if err != nil { + return st, err + } + var n4 [4]byte + binary.BigEndian.PutUint32(n4[:], uint32(len(hdr))) + _, _ = bw.WriteString(baseStreamMagic) + _, _ = bw.Write(n4[:]) + _, _ = bw.Write(hdr) + var c lz4.Compressor + comp := make([]byte, lz4.CompressBlockBound(diskBlockSize)) + err = forEachDataBlock(f, info.Size(), func(index int64, data []byte) error { + codec, payload := byte(baseCodecRaw), data + if n, err := c.CompressBlock(data, comp); err == nil && n > 0 && n < len(data) { + codec, payload = baseCodecLZ4, comp[:n] + } + var rec [14]byte + rec[0] = 'B' + binary.BigEndian.PutUint64(rec[1:9], uint64(index)) + rec[9] = codec + binary.BigEndian.PutUint32(rec[10:], uint32(len(payload))) + if _, err := bw.Write(rec[:]); err != nil { + return err + } + if _, err := bw.Write(payload); err != nil { + return err + } + st.Blocks++ + st.DataBytes += int64(len(data)) + return nil + }) + if err != nil { + return st, err + } + if err := bw.WriteByte('E'); err != nil { + return st, err + } + err = bw.Flush() + st.WireBytes = cw.n + return st, err +} + +// readBaseStream reads a base stream from r into out, a new empty file, and +// returns the disk digest of what it wrote. Every length and index is held +// to the header before anything is allocated or written, and a stream that +// ends without its end record is refused. +func readBaseStream(r io.Reader, out *os.File, want string) (string, error) { + br := bufio.NewReaderSize(r, 1<<20) + magic := make([]byte, len(baseStreamMagic)) + if _, err := io.ReadFull(br, magic); err != nil || string(magic) != baseStreamMagic { + return "", fmt.Errorf("the base stream does not start with %s", baseStreamMagic) + } + var n4 [4]byte + if _, err := io.ReadFull(br, n4[:]); err != nil { + return "", fmt.Errorf("the base stream ended in its header: %w", err) + } + n := binary.BigEndian.Uint32(n4[:]) + if n > migrateMaxHeader { + return "", fmt.Errorf("a base stream header of %d bytes is over the limit", n) + } + js := make([]byte, n) + if _, err := io.ReadFull(br, js); err != nil { + return "", fmt.Errorf("the base stream ended in its header: %w", err) + } + var hdr baseStreamHeader + if err := strictUnmarshal(js, &hdr); err != nil { + return "", fmt.Errorf("the base stream's header: %w", err) + } + switch { + case hdr.SHA256 != want: + return "", fmt.Errorf("the base stream carries base %s, not %s", hdr.SHA256, want) + case hdr.BlockSize != diskBlockSize: + return "", fmt.Errorf("the base stream's blocks are %d bytes, not %d", hdr.BlockSize, diskBlockSize) + case hdr.Size <= 0 || hdr.Size > migrateMaxDiskSize: + return "", fmt.Errorf("the base stream says the disk is %d bytes", hdr.Size) + } + if err := out.Truncate(hdr.Size); err != nil { + return "", err + } + d := newDiskDigest(hdr.Size) + comp := make([]byte, lz4.CompressBlockBound(diskBlockSize)) + block := make([]byte, diskBlockSize) + blocks := diskBlocks(hdr.Size) + for { + kind, err := br.ReadByte() + if err != nil { + return "", fmt.Errorf("the base stream ended before its end: %w", err) + } + if kind == 'E' { + break + } + if kind != 'B' { + return "", fmt.Errorf("the base stream has a record of unknown kind %q", kind) + } + var rec [13]byte + if _, err := io.ReadFull(br, rec[:]); err != nil { + return "", fmt.Errorf("the base stream ended in a block: %w", err) + } + index := int64(binary.BigEndian.Uint64(rec[:8])) + codec := rec[8] + size := int64(binary.BigEndian.Uint32(rec[9:])) + if index < 0 || index >= blocks { + return "", fmt.Errorf("the base stream sends block %d of a disk of %d", index, blocks) + } + want := blockLen(hdr.Size, index) + var data []byte + switch codec { + case baseCodecRaw: + if size != want { + return "", fmt.Errorf("block %d is %d bytes raw, not %d", index, size, want) + } + data = block[:want] + if _, err := io.ReadFull(br, data); err != nil { + return "", fmt.Errorf("the base stream ended in block %d: %w", index, err) + } + case baseCodecLZ4: + if size <= 0 || size > int64(len(comp)) { + return "", fmt.Errorf("block %d is %d bytes compressed", index, size) + } + if _, err := io.ReadFull(br, comp[:size]); err != nil { + return "", fmt.Errorf("the base stream ended in block %d: %w", index, err) + } + got, err := lz4.UncompressBlock(comp[:size], block) + if err != nil || int64(got) != want { + return "", fmt.Errorf("block %d does not decompress to its %d bytes (%d, %v)", index, want, got, err) + } + data = block[:want] + default: + return "", fmt.Errorf("block %d has codec %d", index, codec) + } + // The digest refuses a block out of order before it is written. + if err := d.add(index, data); err != nil { + return "", err + } + if _, err := out.WriteAt(data, index*diskBlockSize); err != nil { + return "", err + } + } + if _, err := br.ReadByte(); !errors.Is(err, io.EOF) { + return "", errors.New("the base stream goes on after its end") + } + if err := out.Sync(); err != nil { + return "", err + } + return d.sum(), nil +} + +// destTakeBase is migrate-take-base: it keeps the base the stream on in +// carries, if its digest is sum. A base already held is answered at once. +func destTakeBase(s *store.Store, sum string, in io.Reader) *migrateReply { + if err := checkDiskDigest(sum); err != nil { + return refuse(codeFailed, err) + } + reply := func() *migrateReply { + p := &migrateProbe{} + destHolds(s, p) + return &migrateReply{OK: true, Probe: p} + } + if baseHeld(s, sum) { + return reply() + } + if err := os.MkdirAll(s.BasesDir(), 0o700); err != nil { + return refuse(codeFailed, err) + } + var rnd [8]byte + _, _ = rand.Read(rnd[:]) + tmp := filepath.Join(s.BasesDir(), ".in-"+hex.EncodeToString(rnd[:])) + out, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return refuse(codeFailed, err) + } + got, err := readBaseStream(in, out, sum) + if cerr := out.Close(); err == nil { + err = cerr + } + if err == nil && got != sum { + err = fmt.Errorf("the base arrived with digest %s, not %s", got, sum) + } + if err == nil { + err = sealBase(s, tmp, sum) + } + if err != nil { + _ = os.Remove(tmp) + return refuse(codeVerifyFailed, fmt.Errorf("take base %s: %w", sum, err)) + } + return reply() +} + +// receivedKernelDir is where a destination keeps the profile kernels a +// prepare sent it, by sha256, so their install record names a file that +// stays. +func receivedKernelDir(s *store.Store) string { + return filepath.Join(s.BasesDir(), "kernels") +} + +// destTakeKernel is migrate-take-kernel: it installs the kernel on in as the +// kernel of profile, if it has sha256 sum and size bytes. +func destTakeKernel(s *store.Store, profile, sum string, size int64, in io.Reader) *migrateReply { + if _, err := bootassets.NormalizeProfile(profile); err != nil || profile == "" { + return refuse(codeFailed, fmt.Errorf("profile %q: %v", profile, err)) + } + if !diskDigestPattern.MatchString(sum) { + return refuse(codeFailed, fmt.Errorf("%q is not a sha256", sum)) + } + if size <= 0 || size > 256<<20 { + return refuse(codeFailed, fmt.Errorf("a kernel of %d bytes", size)) + } + reply := func() *migrateReply { + p := &migrateProbe{} + destHolds(s, p) + return &migrateReply{OK: true, Probe: p} + } + if installedProfileKernel(s.RootDir(), profile) == sum { + return reply() + } + dir := receivedKernelDir(s) + if err := os.MkdirAll(dir, 0o700); err != nil { + return refuse(codeFailed, err) + } + path := filepath.Join(dir, sum+".Image") + tmp := path + ".part" + _ = os.Remove(tmp) + out, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return refuse(codeFailed, err) + } + h := sha256.New() + n, err := io.Copy(io.MultiWriter(out, h), io.LimitReader(in, size+1)) + if err == nil { + err = out.Sync() + } + if cerr := out.Close(); err == nil { + err = cerr + } + if err == nil && (n != size || hex.EncodeToString(h.Sum(nil)) != sum) { + err = fmt.Errorf("the kernel arrived with %d bytes and sha256 %x, not %d and %s", n, h.Sum(nil), size, sum) + } + if err == nil { + err = os.Rename(tmp, path) + } + if err == nil { + err = bootassets.ImportProfileKernel(s.RootDir(), profile, path) + } + if err != nil { + _ = os.Remove(tmp) + return refuse(codeFailed, fmt.Errorf("install the %s kernel: %w", profile, err)) + } + return reply() +} + +// destTakeImage is migrate-take-image: the image of digest is pulled by ref +// unless the store has it. The move itself does not need it (the disk holds +// the root filesystem); a later `hull run` of the same image there does. +func destTakeImage(s *store.Store, ref, digest string) *migrateReply { + if digest == "" || strings.ContainsAny(digest, "/\\") || strings.Contains(digest, "..") { + return refuse(codeFailed, fmt.Errorf("%q is not an image digest", digest)) + } + if s.ImageComplete(digest) { + return &migrateReply{OK: true} + } + if ref == "" || strings.HasPrefix(ref, "-") { + return refuse(codeFailed, fmt.Errorf("image %s is not here, and the source knows no reference to pull it by", digest)) + } + self, err := os.Executable() + if err != nil { + return refuse(codeFailed, err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + var out bytes.Buffer + c := exec.CommandContext(ctx, self, "--store-dir", s.RootDir(), "pull", "--", ref) + c.Stdout, c.Stderr = &out, &out + if err := c.Run(); err != nil { + text := strings.TrimSpace(out.String()) + if len(text) > 1000 { + text = text[len(text)-1000:] + } + return refuse(codeFailed, fmt.Errorf("hull pull %s: %v: %s", ref, err, text)) + } + if !s.ImageComplete(digest) { + return refuse(codeFailed, fmt.Errorf("pulled %s, which is not image %s any more", ref, digest)) + } + return &migrateReply{OK: true, Code: codePulled} +} + +// codePulled is migrate-take-image's answer when it pulled the image, not +// found it there. +const codePulled = "pulled" + +// prepareOptions are the flags of `hull migrate-prepare`. +type prepareOptions struct { + remote migrateRemote + out io.Writer + json bool +} + +// prepareStep is one line of what a prepare did. +type prepareStep struct { + What string `json:"what"` + Result string `json:"result"` // "held", "sent", "skipped", "failed" + Detail string `json:"detail,omitempty"` + Bytes int64 `json:"bytes,omitempty"` + Seconds float64 `json:"seconds,omitempty"` +} + +// prepareResult is `hull migrate-prepare --json`'s output. +type prepareResult struct { + Name string `json:"name"` + To string `json:"to"` + Steps []prepareStep `json:"steps"` + Holds *migrateProbe `json:"holds,omitempty"` + Stream bool `json:"stream"` + Why string `json:"why,omitempty"` +} + +// sourcePrepare is `hull migrate-prepare NAME --to TARGET`. +func sourcePrepare(ctx context.Context, s *store.Store, name string, opt prepareOptions, peer movePeer) error { + if err := store.ValidateInstanceID(name); err != nil { + return err + } + if err := checkSSHTarget(opt.remote.Target); err != nil { + return err + } + state, err := s.GetInstance(name) + if err != nil { + return fmt.Errorf("instance not found: %s", name) + } + if !isHviInstance(state) { + return fmt.Errorf("instance %s does not run on hvi; only an hvi instance moves", name) + } + rec := &migrationRecord{Name: name, Peer: opt.remote.Target, Remote: &opt.remote} + p := peerFor(rec, peer) + res, err := prepareDestination(ctx, s, state, p, opt.remote, opt.out) + if res != nil && opt.json { + _ = json.NewEncoder(os.Stdout).Encode(res) + } + return err +} + +// prepareDestination makes the destination hold what a streamed move of +// state needs and reports each step. It fails only when a step the move +// needs failed: the base. The image and the kernel are reported. +func prepareDestination(ctx context.Context, s *store.Store, state *store.InstanceState, p movePeer, + remote migrateRemote, w io.Writer) (*prepareResult, error) { + res := &prepareResult{Name: state.ID, To: remote.Target} + logf := func(format string, args ...any) { _, _ = fmt.Fprintf(w, format+"\n", args...) } + reply, err := p.call(ctx, "migrate-probe", probeArgs(remote, ""), nil) + if err != nil { + return res, fmt.Errorf("probe %s: %w", remote.Target, err) + } + probe := reply.Probe + if !probe.hasCapability(moveCapStream) { + res.Why = fmt.Sprintf("the hull on %s (%s) takes no streamed move", remote.Target, probe.HullVersion) + logf("%s: nothing to prepare: %s; a move sends the whole checkpoint and disk", remote.Target, res.Why) + res.Holds = probe + return res, nil + } + var failed []string + step := func(st prepareStep) { + res.Steps = append(res.Steps, st) + line := fmt.Sprintf(" %s: %s", st.What, st.Result) + if st.Detail != "" { + line += " (" + st.Detail + ")" + } + logf("%s", line) + } + logf("Preparing %s for %s", remote.Target, state.ID) + + // The image: a later run there needs it; the move does not. + if state.ImageDigest != "" { + started := time.Now() + ref := "" + if img, err := s.GetImage(state.ImageDigest); err == nil { + ref = img.Ref + } + reply, err := p.call(ctx, "migrate-take-image", []string{"--digest", state.ImageDigest, "--ref", ref}, nil) + switch { + case err != nil: + step(prepareStep{What: "image " + digestHead(state.ImageDigest), Result: "failed", Detail: refusalDetail(err)}) + case reply.Code == codePulled: + step(prepareStep{What: "image " + digestHead(state.ImageDigest), Result: "sent", + Detail: "pulled there as " + ref, Seconds: time.Since(started).Seconds()}) + default: + step(prepareStep{What: "image " + digestHead(state.ImageDigest), Result: "held"}) + } + } + + // The profile's kernel: a later boot there needs it, and brig's + // sessions run portable. + if state.Profile != "" { + want := installedProfileKernel(s.RootDir(), state.Profile) + what := state.Profile + " kernel" + switch { + case want == "": + step(prepareStep{What: what, Result: "skipped", Detail: "none is installed here to send"}) + case probe.ProfileKernels[state.Profile] == want: + step(prepareStep{What: what, Result: "held", Detail: digestHead(want)}) + default: + st, err := sendProfileKernel(ctx, s, p, state.Profile, want) + if err != nil { + step(prepareStep{What: what, Result: "failed", Detail: refusalDetail(err)}) + } else { + step(st) + } + } + } + + // The base: what lets the move send only the guest's own writes. + var baseWhat string + switch b := state.Base; { + case b == nil: + baseWhat = "base" + res.Why = "the instance keeps no base (it was started before bases, or not on a block rootfs)" + step(prepareStep{What: baseWhat, Result: "skipped", Detail: res.Why}) + case !baseHeld(s, b.SHA256): + baseWhat = "base " + digestHead(b.SHA256) + res.Why = "its base is gone from this store" + step(prepareStep{What: baseWhat, Result: "skipped", Detail: res.Why}) + case probe.holdsBase(b.SHA256): + step(prepareStep{What: "base " + digestHead(b.SHA256), Result: "held"}) + default: + baseWhat = "base " + digestHead(b.SHA256) + st, err := sendBase(ctx, s, p, b.SHA256) + if err != nil { + failed = append(failed, baseWhat+": "+refusalDetail(err)) + step(prepareStep{What: baseWhat, Result: "failed", Detail: refusalDetail(err)}) + } else { + step(st) + } + } + + if remote.GatewaySock != "" { + if probe.GatewayOK { + step(prepareStep{What: "gateway " + remote.GatewaySock, Result: "held", Detail: "it answers"}) + } else { + step(prepareStep{What: "gateway " + remote.GatewaySock, Result: "failed", Detail: probe.GatewayDetail}) + } + } + + if reply, err := p.call(ctx, "migrate-probe", probeArgs(remote, ""), nil); err == nil { + res.Holds = reply.Probe + } + res.Stream = state.Base != nil && res.Holds.holdsBase(state.Base.SHA256) && res.Holds.HviStream + switch { + case len(failed) > 0: + return res, fmt.Errorf("prepare %s for %s: %s", remote.Target, state.ID, strings.Join(failed, "; ")) + case res.Stream: + logf("%s is prepared: a move of %s sends only what the guest changed", remote.Target, state.ID) + case res.Holds != nil && !res.Holds.HviStream: + res.Why = fmt.Sprintf("its hvi (%s) cannot restore from a stream", res.Holds.Hvi) + logf("%s holds what it can, but %s; a move sends the whole checkpoint and disk", remote.Target, res.Why) + default: + logf("%s holds what it can; a move of %s sends the whole checkpoint and disk (%s)", remote.Target, + state.ID, res.Why) + } + return res, nil +} + +// probeArgs are migrate-probe's arguments for remote and, when not "", the +// instance name the move would use. +func probeArgs(remote migrateRemote, name string) []string { + var args []string + if name != "" { + args = append(args, "--name", name) + } + if remote.Hvi != "" { + args = append(args, "--hvi", remote.Hvi) + } + if remote.GatewaySock != "" { + args = append(args, "--gateway-sock", remote.GatewaySock) + } + return args +} + +// digestHead is the head of a digest, for a person. +func digestHead(d string) string { + d = strings.TrimPrefix(d, "sha256:") + return d[:min(len(d), 12)] +} + +// sendBase streams the base of digest sum to the destination. +func sendBase(ctx context.Context, s *store.Store, p movePeer, sum string) (prepareStep, error) { + started := time.Now() + pr, pw := io.Pipe() + type sent struct { + st baseSendStats + err error + } + done := make(chan sent, 1) + go func() { + st, err := writeBaseStream(pw, s.BasePath(sum), sum) + _ = pw.CloseWithError(err) + done <- sent{st, err} + }() + _, err := p.call(ctx, "migrate-take-base", []string{"--sha256", sum}, pr) + _ = pr.CloseWithError(errors.New("the destination stopped reading")) + got := <-done + if err != nil { + return prepareStep{}, err + } + took := time.Since(started).Seconds() + if got.err != nil && !errors.Is(got.err, io.ErrClosedPipe) { + return prepareStep{}, fmt.Errorf("send: %w", got.err) + } + st := prepareStep{What: "base " + digestHead(sum), Result: "sent", Bytes: got.st.WireBytes, Seconds: took} + if got.err == nil { + st.Detail = fmt.Sprintf("%s of data as %s, %.1f s, %s/s", humanBytes(got.st.DataBytes), + humanBytes(got.st.WireBytes), took, humanBytes(int64(float64(got.st.WireBytes)/max(took, 1e-3)))) + } else { + // It answered before reading it all: it held it already. + st.Result, st.Detail = "held", "the destination already had it" + } + return st, nil +} + +// sendProfileKernel sends this store's kernel of profile, whose record says +// sha256 sum, to be installed there. +func sendProfileKernel(ctx context.Context, s *store.Store, p movePeer, profile, sum string) (prepareStep, error) { + started := time.Now() + kernel, _, err := bootassets.ProfilePaths(s.RootDir(), profile) + if err != nil { + return prepareStep{}, err + } + f, err := os.Open(kernel) + if err != nil { + return prepareStep{}, err + } + defer func() { _ = f.Close() }() + info, err := f.Stat() + if err != nil { + return prepareStep{}, err + } + _, err = p.call(ctx, "migrate-take-kernel", []string{"--profile", profile, "--sha256", sum, + "--size", strconv.FormatInt(info.Size(), 10)}, f) + if err != nil { + return prepareStep{}, err + } + return prepareStep{What: profile + " kernel", Result: "sent", Detail: digestHead(sum), Bytes: info.Size(), + Seconds: time.Since(started).Seconds()}, nil +} + +// migratePrepareCommand is `hull migrate-prepare`. Given NAME and --to it is +// the operator's command above. Given --txid it is the destination's hidden +// step of a move, which brings a received checkpoint up paused; a source +// hull runs that one over ssh, and it keeps its name for every hull that +// already does. +func migratePrepareCommand() *cli.Command { + remote := migrateRemoteVerb("migrate-prepare", "", []cli.Flag{}, + func(cmd *cli.Command, s *store.Store) *migrateReply { return destPrepare(s, cmd.String("txid")) }) + return &cli.Command{ + Name: "migrate-prepare", + Usage: "put on another host, ahead of a move and while the guest runs, what a fast move needs: " + + "the instance's base image, its machine profile's kernel and its image; safe to run again", + ArgsUsage: "NAME --to USER@HOST", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "to", Usage: "USER@HOST to prepare, over ssh"}, + &cli.StringFlag{Name: "remote-store", Usage: "the destination hull's --store-dir (default: its own default)"}, + &cli.StringFlag{Name: "remote-hull", Usage: "path of hull on the destination (default: hull on its PATH)"}, + &cli.StringFlag{Name: "remote-hvi", Usage: "path of hvi on the destination (default: the one its hull would run)"}, + &cli.StringFlag{Name: "remote-gateway-sock", Usage: "a network gateway on the destination to check"}, + &cli.BoolFlag{Name: "json", Usage: "print what was done and what the destination holds as JSON on stdout"}, + &cli.StringFlag{Name: "txid", Hidden: true, Usage: "a move's txid (the destination's step of a move)"}, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if cmd.IsSet("txid") { + if cmd.Args().Len() != 0 || cmd.IsSet("to") { + return errors.New("--txid takes no NAME or --to") + } + return remote.Action(ctx, cmd) + } + if cmd.Args().Len() != 1 || cmd.String("to") == "" { + return errors.New("usage: hull migrate-prepare NAME --to USER@HOST") + } + s, err := globalStore(cmd) + if err != nil { + return err + } + return sourcePrepare(ctx, s, cmd.Args().First(), prepareOptions{ + remote: migrateRemote{Target: cmd.String("to"), Store: cmd.String("remote-store"), + Hull: cmd.String("remote-hull"), Hvi: cmd.String("remote-hvi"), + GatewaySock: cmd.String("remote-gateway-sock")}, + out: os.Stderr, json: cmd.Bool("json"), + }, nil) + }, + } +} + +// migratePrepareRemoteCommands are the destination's prepare verbs, hidden. +func migratePrepareRemoteCommands() []*cli.Command { + return []*cli.Command{ + migrateRemoteVerb("migrate-take-base", "keep the base image the stream on stdin carries", + []cli.Flag{&cli.StringFlag{Name: "sha256", Required: true}}, + func(cmd *cli.Command, s *store.Store) *migrateReply { + return destTakeBase(s, cmd.String("sha256"), os.Stdin) + }), + migrateRemoteVerb("migrate-take-kernel", "install the profile kernel on stdin", + []cli.Flag{&cli.StringFlag{Name: "profile", Required: true}, &cli.StringFlag{Name: "sha256", Required: true}, + &cli.Int64Flag{Name: "size", Required: true}}, + func(cmd *cli.Command, s *store.Store) *migrateReply { + return destTakeKernel(s, cmd.String("profile"), cmd.String("sha256"), cmd.Int64("size"), os.Stdin) + }), + migrateRemoteVerb("migrate-take-image", "pull an image unless the store has it", + []cli.Flag{&cli.StringFlag{Name: "digest", Required: true}, &cli.StringFlag{Name: "ref"}}, + func(cmd *cli.Command, s *store.Store) *migrateReply { + return destTakeImage(s, cmd.String("ref"), cmd.String("digest")) + }), + } +} diff --git a/cmd/hull/migrate_prepare_test.go b/cmd/hull/migrate_prepare_test.go new file mode 100644 index 00000000..07859edc --- /dev/null +++ b/cmd/hull/migrate_prepare_test.go @@ -0,0 +1,239 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "encoding/binary" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/urfave/cli/v3" + + "github.com/brig-sh/hull/internal/bootassets" + "github.com/brig-sh/hull/pkg/store" +) + +// baseStream writes the base stream of a fresh sparse image and returns it +// with the image's digest. +func baseStream(t *testing.T) ([]byte, string, string) { + t.Helper() + img := filepath.Join(t.TempDir(), "img") + writeSparseDisk(t, img, 8<<20) + // Something that compresses and something that does not. + f, err := os.OpenFile(img, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt(bytes.Repeat([]byte("compressible "), 20000), 1<<20); err != nil { + t.Fatal(err) + } + noise := make([]byte, diskBlockSize) + for i := range noise { + noise[i] = byte(i*7919 + i>>3) + } + if _, err := f.WriteAt(noise, 5<<20); err != nil { + t.Fatal(err) + } + _ = f.Close() + sum, _, err := diskDigestFile(img) + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + st, err := writeBaseStream(&buf, img, sum) + if err != nil { + t.Fatal(err) + } + if st.WireBytes != int64(buf.Len()) || st.Blocks == 0 || st.WireBytes >= st.DataBytes { + t.Fatalf("stats %+v for %d bytes on the wire", st, buf.Len()) + } + return buf.Bytes(), sum, img +} + +// A base stream reads back into the same content, holes kept; a stream +// that is cut, damaged, or names another base is refused. +func TestBaseStreamRoundTrip(t *testing.T) { + stream, sum, img := baseStream(t) + dir := t.TempDir() + read := func(name string, data []byte, want string) error { + out, err := os.Create(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + defer func() { _ = out.Close() }() + got, err := readBaseStream(bytes.NewReader(data), out, want) + if err == nil && got != want { + return io.ErrUnexpectedEOF + } + return err + } + if err := read("ok", stream, sum); err != nil { + t.Fatal(err) + } + assertSameContent(t, img, filepath.Join(dir, "ok")) + + cut := stream[:len(stream)-1] + if err := read("cut", cut, sum); err == nil || !strings.Contains(err.Error(), "ended") { + t.Errorf("a stream without its end: %v", err) + } + damaged := bytes.Clone(stream) + damaged[len(damaged)-200] ^= 0x40 + if err := read("damaged", damaged, sum); err == nil { + t.Error("a damaged block was taken") + } + if err := read("other", stream, strings.Repeat("0", 64)); err == nil { + t.Error("a stream of another base was taken") + } + trailing := append(bytes.Clone(stream), 'E') + if err := read("trailing", trailing, sum); err == nil { + t.Error("bytes after the end were taken") + } + // A block index past the disk is refused before anything is written. + hdrEnd := len(baseStreamMagic) + 4 + int(binary.BigEndian.Uint32(stream[len(baseStreamMagic):])) + bad := bytes.Clone(stream) + binary.BigEndian.PutUint64(bad[hdrEnd+1:], 1<<40) + if err := read("past", bad, sum); err == nil || !strings.Contains(err.Error(), "block") { + t.Errorf("a block past the end: %v", err) + } +} + +// prepareFixture is a source store with a running hvi instance that kept a +// base and runs the portable profile, with that profile's kernel +// installed, and an empty destination store. +func prepareFixture(t *testing.T) (*moveFixture, *store.InstanceState) { + t.Helper() + for _, k := range []string{"HULL_BOOT_ASSETS", "BRIG_BOOT_ASSETS", bootassets.VerifyModeEnv} { + t.Setenv(k, "") + } + f := newMoveFixture(t, true) + sum, err := keepBase(f.src, f.disk) + if err != nil { + t.Fatal(err) + } + seedAssetDir(t, filepath.Join(f.src.RootDir(), "assets", bootassets.PortableArm64V1), + map[string]string{bootassets.ProfileKernelName: "the portable kernel"}) + state := mustInstance(t, f.src, f.name) + state.Base = &store.BaseState{SHA256: sum, FromBoot: true} + state.Profile = bootassets.PortableArm64V1 + if err := f.src.SaveInstance(state); err != nil { + t.Fatal(err) + } + return f, state +} + +// migrate-prepare puts the base and the profile kernel on the destination, +// and run again finds both there and sends nothing; the probe lists them. +// An image it cannot pull is reported and does not fail the prepare. +func TestPrepareIsIdempotent(t *testing.T) { + f, state := prepareFixture(t) + peer := newLocalPeer(f.dst) + remote := migrateRemote{Target: "pm@dest", Hvi: f.hvi} + var out bytes.Buffer + res, err := prepareDestination(context.Background(), f.src, state, peer, remote, &out) + if err != nil { + t.Fatalf("%v\n%s", err, out.String()) + } + results := map[string]string{} + for _, st := range res.Steps { + results[strings.Fields(st.What)[0]] = st.Result + } + if results["base"] != "sent" || results[bootassets.PortableArm64V1] != "sent" || results["image"] != "failed" { + t.Fatalf("first prepare: %+v\n%s", res.Steps, out.String()) + } + if !baseHeld(f.dst, state.Base.SHA256) { + t.Fatal("the destination does not hold the base") + } + if got, _, err := diskDigestFile(f.dst.BasePath(state.Base.SHA256)); err != nil || got != state.Base.SHA256 { + t.Fatalf("the destination's base hashes %s (%v)", got, err) + } + if !res.Holds.holdsBase(state.Base.SHA256) || res.Holds.ProfileKernels[bootassets.PortableArm64V1] == "" { + t.Fatalf("the probe does not list what was sent: %+v", res.Holds) + } + if !bootassets.ProfileKernelPresent(f.dst.RootDir(), bootassets.PortableArm64V1) { + t.Fatal("the kernel is not installed on the destination") + } + + sent := peer.called("migrate-take-base") + peer.called("migrate-take-kernel") + out.Reset() + res, err = prepareDestination(context.Background(), f.src, state, peer, remote, &out) + if err != nil { + t.Fatal(err) + } + for _, st := range res.Steps { + if st.Result == "sent" { + t.Errorf("the second prepare sent %s again", st.What) + } + } + if again := peer.called("migrate-take-base") + peer.called("migrate-take-kernel"); again != sent { + t.Errorf("the second prepare called the take verbs %d more times", again-sent) + } +} + +// A destination hull from before streamed moves is not prepared: the +// prepare says a move sends the whole checkpoint, and fails nothing. +func TestPrepareOfAnOldDestination(t *testing.T) { + f, state := prepareFixture(t) + peer := newLocalPeer(f.dst) + peer.answer["migrate-probe"] = &migrateReply{OK: true, Probe: &migrateProbe{HullVersion: "old"}} + var out bytes.Buffer + res, err := prepareDestination(context.Background(), f.src, state, peer, + migrateRemote{Target: "pm@dest"}, &out) + if err != nil || res.Stream || len(res.Steps) != 0 || !strings.Contains(out.String(), "whole checkpoint") { + t.Fatalf("err %v res %+v out %s", err, res, out.String()) + } +} + +// `hull migrate-prepare --txid T` is still the destination's step of a +// move, answering one JSON reply, and the operator's form needs NAME and +// --to. +func TestMigratePrepareCommandDispatch(t *testing.T) { + s := shortStore(t) + // The macOS store marker, so globalStore takes the directory as is. + if err := os.WriteFile(filepath.Join(s.RootDir(), ".case-sensitive-apfs"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + app := func() *cli.Command { + return &cli.Command{Name: "hull", Flags: []cli.Flag{&cli.StringFlag{Name: "store-dir", Value: s.RootDir()}}, + Commands: migrateRemoteCommands()} + } + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + stdout := os.Stdout + os.Stdout = w + runErr := app().Run(context.Background(), []string{"hull", "migrate-prepare", "--txid", strings.Repeat("ab", 16)}) + os.Stdout = stdout + _ = w.Close() + data, _ := io.ReadAll(r) + if runErr != nil { + t.Fatal(runErr) + } + reply, err := parseMigrateReply(data) + if err != nil || reply.OK || reply.Code != codeUnknownTxID { + t.Fatalf("reply %s: %+v %v", data, reply, err) + } + for _, args := range [][]string{{}, {"vm1"}, {"--to", "pm@h"}} { + err := app().Run(context.Background(), append([]string{"hull", "migrate-prepare"}, args...)) + if err == nil || !strings.Contains(err.Error(), "NAME --to") { + t.Errorf("%v: %v", args, err) + } + } +} diff --git a/cmd/hull/migrate_ssh.go b/cmd/hull/migrate_ssh.go index df199598..429c0b2b 100644 --- a/cmd/hull/migrate_ssh.go +++ b/cmd/hull/migrate_ssh.go @@ -80,6 +80,17 @@ type migrateProbe struct { // HviProfile is whether the destination's hvi takes --profile: one // that does not cannot restore a portable instance. HviProfile bool `json:"hviProfile,omitempty"` + // HviStream is whether the destination's hvi restores from a stream + // (`restore --from-stream`), which a streamed move needs. + HviStream bool `json:"hviStream,omitempty"` + // Capabilities are the move features this hull takes beyond protocol + // 1, such as moveCapStream. A hull from before them lists none. + Capabilities []string `json:"capabilities,omitempty"` + // Bases are the disk digests of the bases the store holds. + Bases []string `json:"bases,omitempty"` + // ProfileKernels maps a machine profile to the sha256 of the kernel + // installed for it. + ProfileKernels map[string]string `json:"profileKernels,omitempty"` } // Reply codes a source branches on. diff --git a/go.mod b/go.mod index ccd3554a..4129581a 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/inetaf/tcpproxy v0.0.0-20250222171855-c4b9df066048 github.com/miekg/dns v1.1.72 github.com/opencontainers/runtime-spec v1.2.1 + github.com/pierrec/lz4/v4 v4.1.14 github.com/sirupsen/logrus v1.9.4 github.com/urfave/cli/v3 v3.10.1 github.com/urunc-dev/urunc v0.8.1-0.20261006162953-6c062f39c5bb @@ -71,7 +72,6 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/runc v1.3.6 // indirect github.com/opencontainers/selinux v1.13.1 // indirect - github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rs/zerolog v1.35.1 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect From 61a285f567a81cc81d865adad43778b4b3a0b6cd Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Thu, 15 Oct 2026 01:55:40 +0300 Subject: [PATCH 15/33] feat(hvi): restore an hvi instance from a stream hvi's streamed checkpoint (format 2) is read by `hvi restore --from-stream -` on its standard input, and hull had no way to give a VMM one: a detached VMM always read /dev/null, and a restore always named a bundle directory and checked its manifest first. launchVMM now takes the file a detached VMM reads as stdin. A restore reads a stream in two cases. A streamed move's own restore on the destination (an incoming instance whose command line restores from a stream) reads its own stdin, which `migrate-receive --stream` feeds, and starts while the move is still receiving: hvi checks every component as it reads, and the commit still waits for the move to be verified. A source that takes back a streamed move reads the stream it spooled, but only once that spool was sealed; one that never was is refused. The recorded --from-stream is dropped when the command line is mapped into the next restore, as --from is. Signed-off-by: Panagiotis Moustafellos --- cmd/hull/checkpoint.go | 2 +- cmd/hull/hvirestore.go | 79 ++++++++++++++++++++++++++++++-- cmd/hull/hvirestore_test.go | 52 +++++++++++++++++++++ cmd/hull/launch_cancel_test.go | 4 +- cmd/hull/launch_creating_test.go | 4 +- cmd/hull/orphan_instance_test.go | 4 +- cmd/hull/run.go | 14 +++++- 7 files changed, 147 insertions(+), 12 deletions(-) diff --git a/cmd/hull/checkpoint.go b/cmd/hull/checkpoint.go index 3d2fdc6d..89186588 100644 --- a/cmd/hull/checkpoint.go +++ b/cmd/hull/checkpoint.go @@ -360,7 +360,7 @@ func vzRestoreInstance(ctx context.Context, cmd *cli.Command) error { // The guest resumes with its previous network state in memory, so the // recorded IP stays valid (same MAC, same lease/static address) — skip // rediscovery by reporting no NAT networking to the launcher. - _, err = launchVMM(ctx, cmd, s, state, cmdArgs, gatewayFiles, hypervisors.VzVmm, detach, "none", "") + _, err = launchVMM(ctx, cmd, s, state, cmdArgs, gatewayFiles, hypervisors.VzVmm, detach, "none", "", nil) return err } diff --git a/cmd/hull/hvirestore.go b/cmd/hull/hvirestore.go index d116e9d6..4c6d3b25 100644 --- a/cmd/hull/hvirestore.go +++ b/cmd/hull/hvirestore.go @@ -25,6 +25,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strings" "sync" "syscall" @@ -161,7 +162,7 @@ func hviRestoreArgv(cmdLine []string, from, controlSock, profilePath string) ([] } drop := map[string]bool{ "--kernel": true, "--initramfs": true, "--cmdline": true, "--mem-mib": true, - "--cpus": true, "--fs-uid": true, "--fs-gid": true, "--from": true, + "--cpus": true, "--fs-uid": true, "--fs-gid": true, "--from": true, hviFromStreamFlag: true, "--control-sock": true, "--profile-out": true, // The base digest describes the boot; a restored disk has moved on // from it, and the bundle's disk.json carries what hvi recorded. @@ -235,6 +236,61 @@ func hviHasRestore(binary string) (bool, error) { return !strings.Contains(out.String(), `unknown subcommand "restore"`), nil } +// Files a streamed move keeps in its spool directory on the source. +const ( + // spoolStream is hvi's migrate stream as hvi wrote it: what `hvi + // restore --from-stream` reads, should an abort bring the guest back + // here. + spoolStream = "stream" + // spoolSealed holds hvi's reply once the stream is complete and synced. + spoolSealed = "sealed" +) + +// isStreamRestore reports whether cmdLine is an hvi restore from a stream: +// what a streamed move records for the instance it brings in. +func isStreamRestore(cmdLine []string) bool { + return len(cmdLine) > 1 && cmdLine[1] == "restore" && slices.Contains(cmdLine, hviFromStreamFlag) +} + +// restoreStreamSource is where a restore reads a streamed checkpoint from, +// or nil for a bundle directory: standard input for a streamed move's own +// restore on its destination, which `migrate-receive --stream` feeds, and +// the spooled stream for a source that takes back a streamed move it +// aborted. +func restoreStreamSource(state *store.InstanceState, opt hviRestoreOptions) (*os.File, error) { + m := state.Migration + if opt.move != "" && isStreamRestore(state.CmdLine) && m.State == store.MigrationIncoming { + if !opt.detach { + return nil, errors.New("a streamed move's restore is detached") + } + return os.Stdin, nil + } + spool := filepath.Join(m.BundleDir, spoolStream) + if _, err := os.Stat(spool); err != nil { + return nil, nil + } + if _, err := os.Stat(filepath.Join(m.BundleDir, spoolSealed)); err != nil { + return nil, fmt.Errorf("the streamed checkpoint %s was never sealed", m.BundleDir) + } + if !opt.detach { + return nil, errors.New("a checkpoint kept as a stream restores detached (--detach)") + } + return os.Open(spool) +} + +// streamRestoreArgv turns `restore --from DIR ...` into `restore +// --from-stream - ...`. +func streamRestoreArgv(argv []string) []string { + out := slices.Clone(argv) + for i := 0; i+1 < len(out); i++ { + if out[i] == "--from" { + out[i], out[i+1] = hviFromStreamFlag, hviFromStreamStdin + break + } + } + return out +} + // hviRestoreTakesStream reports whether the hvi at binary restores from a // stream (`restore --from-stream`), which a streamed move hands it on stdin. // Asked as the other probes are: hvi's restore parser answers an argument it @@ -440,9 +496,20 @@ func hviRestoreInstance(ctx context.Context, cmd *cli.Command, s *store.Store, s if _, err := os.Stat(bundle); err != nil { return fmt.Errorf("cannot restore %s: its bundle %s is gone", id, bundle) } - if err := checkBundleManifest(bundle, state.Migration.ManifestSHA256); err != nil { + stream, err := restoreStreamSource(state, opt) + if err != nil { return fmt.Errorf("cannot restore %s: %w", id, err) } + if stream != nil && stream != os.Stdin { + defer func() { _ = stream.Close() }() + } + // A move's streamed restore starts before its manifest has arrived; + // hvi checks every component of the stream against it as it reads. + if stream != os.Stdin { + if err := checkBundleManifest(bundle, state.Migration.ManifestSHA256); err != nil { + return fmt.Errorf("cannot restore %s: %w", id, err) + } + } if err := hviCheckpointable(state.CmdLine); err != nil { return err } @@ -464,6 +531,9 @@ func hviRestoreInstance(ctx context.Context, cmd *cli.Command, s *store.Store, s if err != nil { return fmt.Errorf("cannot restore %s: %w", id, err) } + if stream != nil { + argv = streamRestoreArgv(argv) + } // hvi relays the guest's NIC to the gateway socket by path, so a restore // on the same host re-joins the gateway the instance left, with the same // MAC and so the same lease. A gateway that is not there would leave the @@ -504,7 +574,7 @@ func hviRestoreInstance(ctx context.Context, cmd *cli.Command, s *store.Store, s }() // The IP the record holds stays the guest's: it resumes with its network // state in memory and the same MAC. So no rediscovery, as for Vz. - started, launchErr := launchVMM(ctx, cmd, s, state, argv, nil, hypervisors.HviVmm, opt.detach, "none", "") + started, launchErr := launchVMM(ctx, cmd, s, state, argv, nil, hypervisors.HviVmm, opt.detach, "none", "", stream) // In the foreground launchVMM returns when the VMM has exited, and on a // failed start there never was one. Either way nothing will answer on // the socket, and the wait must end now rather than at its deadline. @@ -731,6 +801,9 @@ func checkMoveRestore(s *store.Store, state *store.InstanceState, opt hviRestore switch { case rec.TxID != opt.move || rec.Role != roleDestination: return fmt.Errorf("the instance is not the destination of move %s", opt.move) + case rec.State == dstReceiving && isStreamRestore(state.CmdLine): + // A streamed move restores while the stream arrives; its commit + // still waits for the stream to be verified. case rec.State != dstVerified && rec.State != dstRestoredPaused: return fmt.Errorf("move %s is %s here, and only a verified one is restored", opt.move, rec.State) case m == nil || m.MoveTxID != opt.move || m.BundleDir == "": diff --git a/cmd/hull/hvirestore_test.go b/cmd/hull/hvirestore_test.go index 2e167769..78518a4a 100644 --- a/cmd/hull/hvirestore_test.go +++ b/cmd/hull/hvirestore_test.go @@ -435,3 +435,55 @@ func TestAwaitRestoredIgnoresTheSourcesStalePid(t *testing.T) { t.Errorf("result = %q, %v, want restored: the source's dead pid ended the wait", got, err) } } + +// A checkpoint kept as hvi's stream restores with `--from-stream -`, its +// stream on stdin: a move's restore reads the stream it is being sent, and +// a source that takes back a streamed move reads its sealed spool; a spool +// that was never sealed is refused, and a bundle directory restores as it +// always did. +func TestRestoreFromAStream(t *testing.T) { + streamed := []string{"/bin/hvi", "restore", "--from-stream", "-", "--disk", "/d", "--checkpoint-root", "/r"} + if !isStreamRestore(streamed) || isStreamRestore([]string{"/bin/hvi", "boot", "--disk", "/d"}) || + isStreamRestore([]string{"/bin/hvi", "restore", "--from", "/b", "--disk", "/d"}) { + t.Fatal("isStreamRestore") + } + argv, err := hviRestoreArgv(streamed, "/b", "/c", "/p") + if err != nil { + t.Fatal(err) + } + if got := strings.Join(streamRestoreArgv(argv), " "); got != + "/bin/hvi restore --from-stream - --disk /d --checkpoint-root /r --control-sock /c --profile-out /p" { + t.Fatalf("argv %s", got) + } + + dir := t.TempDir() + incoming := &store.InstanceState{CmdLine: streamed, Migration: &store.MigrationState{ + State: store.MigrationIncoming, BundleDir: dir, MoveTxID: "t"}} + if f, err := restoreStreamSource(incoming, hviRestoreOptions{move: "t", paused: true, detach: true}); err != nil || f != os.Stdin { + t.Fatalf("a move's restore reads %v, %v", f, err) + } + if _, err := restoreStreamSource(incoming, hviRestoreOptions{move: "t", paused: true}); err == nil { + t.Fatal("a streamed move's restore in the foreground was allowed") + } + + fenced := &store.InstanceState{CmdLine: []string{"/bin/hvi", "boot", "--disk", "/d"}, + Migration: &store.MigrationState{State: store.MigrationFenced, BundleDir: dir}} + if f, err := restoreStreamSource(fenced, hviRestoreOptions{detach: true}); f != nil || err != nil { + t.Fatalf("a bundle directory reads %v, %v", f, err) + } + if err := os.WriteFile(filepath.Join(dir, spoolStream), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := restoreStreamSource(fenced, hviRestoreOptions{detach: true}); err == nil || + !strings.Contains(err.Error(), "never sealed") { + t.Fatalf("an unsealed spool: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, spoolSealed), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + f, err := restoreStreamSource(fenced, hviRestoreOptions{detach: true}) + if err != nil || f == nil || f.Name() != filepath.Join(dir, spoolStream) { + t.Fatalf("a sealed spool reads %v, %v", f, err) + } + _ = f.Close() +} diff --git a/cmd/hull/launch_cancel_test.go b/cmd/hull/launch_cancel_test.go index 2b5de428..af5a0597 100644 --- a/cmd/hull/launch_cancel_test.go +++ b/cmd/hull/launch_cancel_test.go @@ -70,7 +70,7 @@ func TestCanceledLaunchDoesNotSpawn(t *testing.T) { cancel() started, launchErr := launchVMM(ctx, &cli.Command{}, s, state, []string{"/bin/sh", "-c", `printf 'spawned\n' > "$1"`, "hull-canceled-launch", marker}, - nil, hypervisors.VzVmm, true, "none", "") + nil, hypervisors.VzVmm, true, "none", "", nil) if started && state.PID > 1 && state.PID != os.Getpid() { // A regression can spawn this short-lived command. Reap it before // checking the marker, without signalling any recorded process. @@ -169,7 +169,7 @@ func TestCanceledDetachedLaunchStopsAndReapsChild(t *testing.T) { Action: func(ctx context.Context, cmd *cli.Command) error { var err error started, err = launchVMM(ctx, cmd, s, state, args, - []*os.File{readyWrite, controlRead}, hypervisors.VzVmm, true, "nat", "") + []*os.File{readyWrite, controlRead}, hypervisors.VzVmm, true, "nat", "", nil) return err }, } diff --git a/cmd/hull/launch_creating_test.go b/cmd/hull/launch_creating_test.go index 88dfb20a..6bce6db5 100644 --- a/cmd/hull/launch_creating_test.go +++ b/cmd/hull/launch_creating_test.go @@ -132,7 +132,7 @@ func TestLaunchCreatingPublishesRunningAndStops(t *testing.T) { wantMetadata := *state before := time.Now() started, launchErr := launchVMM(context.Background(), &cli.Command{}, s, state, args, - []*os.File{readyWrite, controlRead}, hypervisors.VzVmm, true, "none", "") + []*os.File{readyWrite, controlRead}, hypervisors.VzVmm, true, "none", "", nil) after := time.Now() // Establish safe cleanup before deadline/scanner operations can fail. // The readiness pipe below remains independent evidence of the actual PID. @@ -238,7 +238,7 @@ func TestFailedLaunchRestoresCreatingRecord(t *testing.T) { } state := &store.InstanceState{ID: id, ImageDigest: "sha256:failed-launch-test", LogFile: s.InstanceLogFile(id), BundleDir: s.InstanceBundleDir(id)} started, err := launchVMM(context.Background(), &cli.Command{}, s, state, - []string{filepath.Join(t.TempDir(), "no-such-vmm")}, nil, hypervisors.VzVmm, true, "none", "") + []string{filepath.Join(t.TempDir(), "no-such-vmm")}, nil, hypervisors.VzVmm, true, "none", "", nil) if started || err == nil || !strings.Contains(err.Error(), "failed to start VMM:") || !errors.Is(err, os.ErrNotExist) { t.Fatalf("missing executable: started=%v, err=%v", started, err) } diff --git a/cmd/hull/orphan_instance_test.go b/cmd/hull/orphan_instance_test.go index 3069753a..47f32a80 100644 --- a/cmd/hull/orphan_instance_test.go +++ b/cmd/hull/orphan_instance_test.go @@ -96,7 +96,7 @@ func TestNoVMMIsSpawnedWithoutARecordOfIt(t *testing.T) { marker := filepath.Join(t.TempDir(), "vmm-started") state := &store.InstanceState{ID: "vm1", LogFile: s.InstanceLogFile("vm1")} started, err := launchVMM(context.Background(), &cli.Command{}, s, state, standInVMM(t, marker), - nil, vzVmm, true, "none", "") + nil, vzVmm, true, "none", "", nil) if err == nil { t.Errorf("launchVMM reported success with a store it could not write to; "+ @@ -227,7 +227,7 @@ func TestFailedSpawnLeavesNoStartingRecord(t *testing.T) { started, err := launchVMM(context.Background(), &cli.Command{}, s, state, []string{filepath.Join(t.TempDir(), "no-such-vmm")}, nil, - vzVmm, true, "none", "") + vzVmm, true, "none", "", nil) if err == nil { t.Fatal("launchVMM succeeded with a VMM binary that does not exist") } diff --git a/cmd/hull/run.go b/cmd/hull/run.go index f60d758b..00b4c1f5 100644 --- a/cmd/hull/run.go +++ b/cmd/hull/run.go @@ -1597,7 +1597,8 @@ exec %s "$@" Profile: profile, Base: instanceBase, } - started, err := launchVMM(ctx, cmd, s, state, cmdArgs, append(gatewayFiles, shareFiles...), vmmType, detach, netMode, gatewayIP) + started, err := launchVMM(ctx, cmd, s, state, cmdArgs, append(gatewayFiles, shareFiles...), vmmType, detach, netMode, + gatewayIP, nil) instanceStarted = started return err } @@ -2520,7 +2521,13 @@ func containerHosts(rootfsDir string, entries []string) (string, error) { // until the VMM exits. Shared by run and restore. The returned bool reports // whether the VMM process was started — once it was, the caller must keep // the instance directory even on error. -func launchVMM(ctx context.Context, cmd *cli.Command, s *store.Store, state *store.InstanceState, cmdArgs []string, gatewayFiles []*os.File, vmmType hypervisors.VmmType, detach bool, netMode, gatewayIP string) (bool, error) { +// +// stdin is what a detached VMM reads on its standard input: nil, /dev/null, +// for every VMM but hvi restoring from a stream (`restore --from-stream -`), +// which reads the checkpoint there. +func launchVMM(ctx context.Context, cmd *cli.Command, s *store.Store, state *store.InstanceState, cmdArgs []string, + gatewayFiles []*os.File, vmmType hypervisors.VmmType, detach bool, netMode, gatewayIP string, + stdin *os.File) (bool, error) { // Register before spawning, including when called by restore. A canceled // preparation cannot start a VMM; a signal racing the spawn must stop the // child through the same graceful shutdown path as a foreground signal. @@ -2599,6 +2606,9 @@ func launchVMM(ctx context.Context, cmd *cli.Command, s *store.Store, state *sto } vmmCmd.Stdout = logFd vmmCmd.Stderr = logFd + if stdin != nil { + vmmCmd.Stdin = stdin + } } // Publish the record BEFORE the spawn, not after it. From b6b8b0f3c2ed1dc4e2d3308173480db2ad2d5c53 Mon Sep 17 00:00:00 2001 From: Panagiotis Moustafellos Date: Thu, 15 Oct 2026 01:55:19 +0300 Subject: [PATCH 16/33] feat(hvi): move an instance as a stream with pre-copy rounds A whole-bundle move stops the guest for the whole transfer: 5.3 to 9.4 s for the 1 GiB ubuntu guest between the Pis, 46 to 54 s for the claude-code guest from the Mac. hvi now streams its checkpoint with RAM pre-copy rounds, so the guest only has to stop for the last one. When both hosts can, `hull migrate` runs one ssh session to `migrate-receive --stream` in the HULLMIG2 format: disk batches that differ from the base and hvi's components relayed with hvi's own headers and checked against hvi's sha256, then a trailer with a running hash and an account of every disk round that the source checks against hvi's digests. The source spools the stream and seals it, so a fenced guest whose move fails can be restored from the spool, and holds off SIGINT, SIGTERM and SIGHUP while hvi's stream is not on record. The destination accepts the header before hvi's migrate begins, so a refusal there costs no fence. The downtime fixes move every synced write they can out of the pause: the spool's and the disk's syncs, the fence record and hvi's documents, leaving one synced write per host between the seal and the commit, with writeback started as the spool and the disk grow so the host does not throttle. The last disk pre-copy round runs right before hvi's migrate. A fake hvi models hvi's restore and pre-boot writes for the tests. migration.md, cli.md and storage.md describe streamed moves, bases, migrate-prepare and what a streamed move records. Signed-off-by: Panagiotis Moustafellos --- cmd/hull/hvirestore.go | 7 +- cmd/hull/hvistream.go | 294 +++++++ cmd/hull/migrate_dest.go | 5 +- cmd/hull/migrate_diskcheck_test.go | 164 ++++ cmd/hull/migrate_fixture_test.go | 15 +- cmd/hull/migrate_prepare.go | 7 +- cmd/hull/migrate_record.go | 144 +++- cmd/hull/migrate_record_test.go | 21 + cmd/hull/migrate_signals.go | 108 +++ cmd/hull/migrate_signals_test.go | 94 ++ cmd/hull/migrate_source.go | 253 +++++- cmd/hull/migrate_spool.go | 293 +++++++ cmd/hull/migrate_spool_test.go | 388 +++++++++ cmd/hull/migrate_ssh.go | 8 + cmd/hull/migrate_stream2.go | 542 ++++++++++++ cmd/hull/migrate_stream2_test.go | 315 +++++++ cmd/hull/migrate_streamdst.go | 782 +++++++++++++++++ cmd/hull/migrate_streamed_test.go | 985 +++++++++++++++++++++ cmd/hull/migrate_streamsrc.go | 1286 ++++++++++++++++++++++++++++ cmd/hull/migrate_transport.go | 108 +++ cmd/hull/writeback_darwin.go | 24 + cmd/hull/writeback_linux.go | 32 + docs/cli.md | 26 +- docs/migration.md | 339 +++++++- docs/storage.md | 15 +- 25 files changed, 6217 insertions(+), 38 deletions(-) create mode 100644 cmd/hull/hvistream.go create mode 100644 cmd/hull/migrate_diskcheck_test.go create mode 100644 cmd/hull/migrate_signals.go create mode 100644 cmd/hull/migrate_signals_test.go create mode 100644 cmd/hull/migrate_spool.go create mode 100644 cmd/hull/migrate_spool_test.go create mode 100644 cmd/hull/migrate_stream2.go create mode 100644 cmd/hull/migrate_stream2_test.go create mode 100644 cmd/hull/migrate_streamdst.go create mode 100644 cmd/hull/migrate_streamed_test.go create mode 100644 cmd/hull/migrate_streamsrc.go create mode 100644 cmd/hull/migrate_transport.go create mode 100644 cmd/hull/writeback_darwin.go create mode 100644 cmd/hull/writeback_linux.go diff --git a/cmd/hull/hvirestore.go b/cmd/hull/hvirestore.go index 4c6d3b25..d1a258ed 100644 --- a/cmd/hull/hvirestore.go +++ b/cmd/hull/hvirestore.go @@ -243,6 +243,7 @@ const ( // here. spoolStream = "stream" // spoolSealed holds hvi's reply once the stream is complete and synced. + // A spool without it may still be whole: see verifySpool. spoolSealed = "sealed" ) @@ -269,8 +270,10 @@ func restoreStreamSource(state *store.InstanceState, opt hviRestoreOptions) (*os if _, err := os.Stat(spool); err != nil { return nil, nil } - if _, err := os.Stat(filepath.Join(m.BundleDir, spoolSealed)); err != nil { - return nil, fmt.Errorf("the streamed checkpoint %s was never sealed", m.BundleDir) + // A spool hull did not live to seal is sealed here if it checks out + // whole, frame by frame against its manifest. + if err := openSpool(m.BundleDir); err != nil { + return nil, err } if !opt.detach { return nil, errors.New("a checkpoint kept as a stream restores detached (--detach)") diff --git a/cmd/hull/hvistream.go b/cmd/hull/hvistream.go new file mode 100644 index 00000000..2c8d966b --- /dev/null +++ b/cmd/hull/hvistream.go @@ -0,0 +1,294 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "time" +) + +// hvi's streaming ops on the control socket, which a streamed move drives: +// `migrate` (RAM rounds, the pause, the machine's state and the manifest) +// and `disk-blocks` (the bytes of disk blocks). Each answers with a run of +// components, every one a JSON header line naming its size and sha256 and +// then its bytes, and ends with the op's reply line, the one object with a +// "v" key (hvi docs/checkpoint.md). `disk-dirty` answers with one line, as +// the other ops do, but its bitmap can outgrow hviControlMaxReply. + +// hviStreamIdle is how long a streaming op may go without a byte from hvi +// before hull gives up on it. +const hviStreamIdle = 2 * time.Minute + +// hviDiskDirtyMaxReply bounds a disk-dirty reply: the bitmap of a 4 TiB disk +// in 64 KiB blocks is 8 MiB, 11 MiB as base64. +const hviDiskDirtyMaxReply = 12 << 20 + +// idleConn extends the connection's deadline before every read and write, +// so a stream that keeps moving is never cut, and one that stalls is. +type idleConn struct { + net.Conn + idle time.Duration +} + +func (c *idleConn) Read(p []byte) (int, error) { + _ = c.SetDeadline(time.Now().Add(c.idle)) + return c.Conn.Read(p) +} + +func (c *idleConn) Write(p []byte) (int, error) { + _ = c.SetDeadline(time.Now().Add(c.idle)) + return c.Conn.Write(p) +} + +// hviStreamOp sends one request on sock and returns the connection and a +// reader for what hvi streams back. The caller closes the connection. +func hviStreamOp(sock string, req map[string]any, idle time.Duration) (net.Conn, *bufio.Reader, error) { + msg := map[string]any{"v": hviControlVersion} + for k, v := range req { + msg[k] = v + } + line, err := json.Marshal(msg) + if err != nil { + return nil, nil, err + } + raw, err := net.DialTimeout("unix", sock, hviControlTimeout) + if err != nil { + return nil, nil, fmt.Errorf("connect to hvi's control socket %s: %w", sock, err) + } + conn := &idleConn{Conn: raw, idle: idle} + if _, err := conn.Write(append(line, '\n')); err != nil { + _ = raw.Close() + return nil, nil, fmt.Errorf("send %v to hvi: %w", req["op"], err) + } + return conn, bufio.NewReaderSize(conn, 1<<20), nil +} + +// readHviLine reads one line of at most max bytes, without its newline. +func readHviLine(br *bufio.Reader, max int) ([]byte, error) { + var line []byte + for { + chunk, err := br.ReadSlice('\n') + line = append(line, chunk...) + if len(line) > max { + return nil, fmt.Errorf("hvi sent a line longer than %d bytes", max) + } + if err == nil { + return bytes.TrimRight(line, "\r\n"), nil + } + if !errors.Is(err, bufio.ErrBufferFull) { + if errors.Is(err, io.EOF) { + return nil, errors.New("hvi closed the connection before its reply") + } + return nil, err + } + } +} + +// readHviComponents reads hvi's components from br, handing each to fn, +// which must read body in full or fail, and returns hvi's reply line once +// it comes. A reply that refuses is returned with an *hviRefusal. +func readHviComponents(op string, br *bufio.Reader, + fn func(line []byte, hdr hviComponentHeader, body io.Reader) error) (*hviReply, json.RawMessage, error) { + for { + line, err := readHviLine(br, migrateMaxHeader) + if err != nil { + return nil, nil, fmt.Errorf("hvi's %s stream: %w", op, err) + } + var probe map[string]json.RawMessage + if err := json.Unmarshal(line, &probe); err != nil { + return nil, nil, fmt.Errorf("hvi's %s stream has a line that is not JSON: %w", op, err) + } + if _, isReply := probe["v"]; isReply { + var reply hviReply + if err := json.Unmarshal(line, &reply); err != nil { + return nil, nil, fmt.Errorf("hvi's reply to %s: %w", op, err) + } + if reply.V != hviControlVersion { + return nil, nil, fmt.Errorf("hvi replied to %s with protocol version %d", op, reply.V) + } + if !reply.OK { + return &reply, line, &hviRefusal{Op: op, Code: reply.Code, Message: reply.Error} + } + return &reply, line, nil + } + hdr, err := parseHviComponentHeader(line) + if err != nil { + return nil, nil, err + } + body := io.LimitReader(br, hdr.Size) + if err := fn(line, hdr, body); err != nil { + return nil, nil, err + } + if n, _ := io.Copy(io.Discard, body); n != 0 { + return nil, nil, fmt.Errorf("hull left %d bytes of hvi's %s unread", n, hdr.Component) + } + } +} + +// hviDiskDirtyReply is disk-dirty's answer. +type hviDiskDirtyReply struct { + V int `json:"v"` + OK bool `json:"ok"` + Code string `json:"code,omitempty"` + Error string `json:"error,omitempty"` + Since string `json:"since"` + Round int `json:"round"` + BlockSize int64 `json:"block_size"` + Blocks int64 `json:"blocks"` + Dirty int64 `json:"dirty"` + Generation uint64 `json:"generation"` + BitmapB64 string `json:"bitmap_b64"` + // BitmapSHA256 is the sha256 of the bitmap's bytes, from an hvi that + // records each round's digests. + BitmapSHA256 string `json:"bitmap_sha256,omitempty"` +} + +// dirtyBlocks decodes the bitmap into the block numbers it sets, ascending, +// held to the disk's size. +func (r *hviDiskDirtyReply) dirtyBlocks(diskSize int64) ([]int64, error) { + if r.BlockSize != diskBlockSize { + return nil, fmt.Errorf("hvi tracks %d-byte disk blocks, hull %d", r.BlockSize, diskBlockSize) + } + if r.Blocks != diskBlocks(diskSize) { + return nil, fmt.Errorf("hvi's disk has %d blocks, the disk file %d", r.Blocks, diskBlocks(diskSize)) + } + bits, err := base64.StdEncoding.DecodeString(r.BitmapB64) + if err != nil { + return nil, fmt.Errorf("hvi's dirty bitmap: %w", err) + } + if int64(len(bits)) != (r.Blocks+7)/8 { + return nil, fmt.Errorf("hvi's dirty bitmap is %d bytes for %d blocks", len(bits), r.Blocks) + } + if r.BitmapSHA256 != "" && sha256Hex(bits) != r.BitmapSHA256 { + return nil, fmt.Errorf("hvi's dirty bitmap of round %d has sha256 %s, its reply says %s", r.Round, + sha256Hex(bits), r.BitmapSHA256) + } + var out []int64 + for i, b := range bits { + for j := 0; b != 0; j++ { + if b&1 != 0 { + block := int64(i)*8 + int64(j) + if block >= r.Blocks { + return nil, fmt.Errorf("hvi's dirty bitmap sets block %d of %d", block, r.Blocks) + } + out = append(out, block) + } + b >>= 1 + } + } + if int64(len(out)) != r.Dirty { + return nil, fmt.Errorf("hvi's dirty bitmap sets %d blocks and says %d", len(out), r.Dirty) + } + return out, nil +} + +// hviDiskDirty asks hvi for the disk blocks written since boot or since the +// last read ("boot" or "round"), which starts a round. +func hviDiskDirty(sock, since string) (*hviDiskDirtyReply, error) { + conn, br, err := hviStreamOp(sock, map[string]any{"op": "disk-dirty", "since": since}, 30*time.Second) + if err != nil { + return nil, err + } + defer func() { _ = conn.Close() }() + line, err := readHviLine(br, hviDiskDirtyMaxReply) + if err != nil { + return nil, fmt.Errorf("hvi's reply to disk-dirty: %w", err) + } + var reply hviDiskDirtyReply + if err := json.Unmarshal(line, &reply); err != nil { + return nil, fmt.Errorf("hvi's reply to disk-dirty: %w", err) + } + if reply.V != hviControlVersion { + return nil, fmt.Errorf("hvi replied to disk-dirty with protocol version %d", reply.V) + } + if !reply.OK { + return &reply, &hviRefusal{Op: "disk-dirty", Code: reply.Code, Message: reply.Error} + } + return &reply, nil +} + +// hviLacksOp reports whether err is hvi saying it has no such op: one from +// before it. +func hviLacksOp(err error) bool { + code := hviRefusalCode(err) + return code == "unknown_op" || code == "not_implemented" +} + +// hviDiskBlocksReply is what hull reads of disk-blocks' reply. +type hviDiskBlocksReply struct { + Round *int `json:"round"` + Blocks int64 `json:"blocks"` + Bytes int64 `json:"bytes"` + Batches int64 `json:"batches"` + SentSHA256 string `json:"sent_sha256"` +} + +// hviDiskBlocks streams the blocks of round from hvi, handing each batch to +// fn with its first block and its bytes, checked against its sha256, and +// holds hvi's reply to what came: as many blocks, bytes and batches, and, +// from an hvi that gives it, the same sent_sha256, the sha256 of the +// batches' digests in order. It returns that digest. +func hviDiskBlocks(sock string, round int, fn func(first, count int64, data []byte) error) (string, error) { + conn, br, err := hviStreamOp(sock, map[string]any{"op": "disk-blocks", "round": round}, hviStreamIdle) + if err != nil { + return "", err + } + defer func() { _ = conn.Close() }() + buf := make([]byte, moveMaxBatchBlocks*diskBlockSize) + var got diskRoundDigest + var bytes int64 + _, line, err := readHviComponents("disk-blocks", br, func(_ []byte, hdr hviComponentHeader, body io.Reader) error { + if hdr.First == nil || hdr.Count == nil || *hdr.Count < 1 || *hdr.Count > moveMaxBatchBlocks || + hdr.Size > *hdr.Count*diskBlockSize { + return fmt.Errorf("hvi sent a disk batch of %d bytes that names no blocks", hdr.Size) + } + data := buf[:hdr.Size] + if _, err := io.ReadFull(body, data); err != nil { + return fmt.Errorf("hvi's disk batch at block %d: %w", *hdr.First, err) + } + if got := sha256Hex(data); got != hdr.SHA256 { + return fmt.Errorf("hvi's disk batch at block %d has sha256 %s, its header says %s", *hdr.First, got, hdr.SHA256) + } + got.add(*hdr.Count, hdr.SHA256) + bytes += hdr.Size + return fn(*hdr.First, *hdr.Count, data) + }) + if err != nil { + return "", err + } + var reply hviDiskBlocksReply + if err := json.Unmarshal(line, &reply); err != nil { + return "", fmt.Errorf("hvi's reply to disk-blocks: %w", err) + } + sent := got.done() + switch { + case reply.Round != nil && *reply.Round != round: + return "", fmt.Errorf("hvi sent the blocks of round %d, not %d", *reply.Round, round) + case reply.Blocks != sent.Blocks || reply.Batches != sent.Batches || reply.Bytes != bytes: + return "", fmt.Errorf("hvi says it sent %d blocks (%d bytes) in %d batches of round %d; %d (%d bytes) in "+ + "%d came", reply.Blocks, reply.Bytes, reply.Batches, round, sent.Blocks, bytes, sent.Batches) + case reply.SentSHA256 != "" && reply.SentSHA256 != sent.SentSHA256: + return "", fmt.Errorf("hvi's sent_sha256 of round %d is %s; the batches that came make %s", round, + reply.SentSHA256, sent.SentSHA256) + } + return sent.SentSHA256, nil +} diff --git a/cmd/hull/migrate_dest.go b/cmd/hull/migrate_dest.go index c0a3ab4c..03eace58 100644 --- a/cmd/hull/migrate_dest.go +++ b/cmd/hull/migrate_dest.go @@ -889,8 +889,11 @@ func migrateRemoteCommands() []*cli.Command { return destProbe(s, cmd.String("name"), cmd.String("hvi"), cmd.String("gateway-sock")) }), migrateRemoteVerb("migrate-receive", "reserve a name for a move, or receive its stream on stdin", - []cli.Flag{txidFlag, nameFlag, &cli.BoolFlag{Name: "reserve"}}, + []cli.Flag{txidFlag, nameFlag, &cli.BoolFlag{Name: "reserve"}, &cli.BoolFlag{Name: "stream"}}, func(cmd *cli.Command, s *store.Store) *migrateReply { + if cmd.Bool("stream") { + return destReceiveStream(s, cmd.String("txid"), cmd.String("name"), os.Stdin) + } if cmd.Bool("reserve") { var req migrateReserveRequest if err := json.NewDecoder(io.LimitReader(os.Stdin, 1<<20)).Decode(&req); err != nil { diff --git a/cmd/hull/migrate_diskcheck_test.go b/cmd/hull/migrate_diskcheck_test.go new file mode 100644 index 00000000..32d10ee2 --- /dev/null +++ b/cmd/hull/migrate_diskcheck_test.go @@ -0,0 +1,164 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// diskRoundsCase builds what a destination holds after a live move with +// hvi's rounds 0 and 1 before the fence and round 2 after it: the +// trailer's account, the batches applied, and hvi's disk.json. +func diskRoundsCase() (*streamDiskRounds, map[int]*diskRoundDigest, map[string]any) { + applied := map[int]*diskRoundDigest{} + t := &streamDiskRounds{Engine: "hvi"} + var listed []map[string]any + for round, batches := range [][]string{{"a", "b"}, {"c"}, {"d", "e", "f"}} { + d := &diskRoundDigest{round: streamDiskRound{Round: round, Final: round == 2}} + for _, b := range batches { + d.add(2, sha256Hex([]byte(b))) + } + applied[round] = d + want := d.done() + want.From, want.Generation, want.BitmapSHA256 = "hvi", uint64(10*(round+1)), strings.Repeat("b", 64) + t.Rounds = append(t.Rounds, want) + if round < 2 { + sent := want.SentSHA256 + listed = append(listed, map[string]any{"round": round, "generation": want.Generation, + "blocks": want.Blocks, "bitmap_sha256": want.BitmapSHA256, "sent_sha256": sent}) + } + } + return t, applied, map[string]any{"version": 1, "generation": 30, "rounds": listed} +} + +// The destination holds the disk rounds it applied to the trailer, and +// hvi's rounds to what hvi recorded in disk.json: a batch lost on the way, +// a round hvi read and the source never sent, a round sent twice, a final +// round skipped or read before the guest's last write all refuse. +func TestCheckDiskRounds(t *testing.T) { + check := func(edit func(*streamDiskRounds, map[int]*diskRoundDigest, map[string]any)) error { + tr, applied, doc := diskRoundsCase() + if edit != nil { + edit(tr, applied, doc) + } + js, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + return checkDiskRounds(false, tr, applied, js) + } + if err := check(nil); err != nil { + t.Fatalf("a whole move: %v", err) + } + listed := func(doc map[string]any) []map[string]any { return doc["rounds"].([]map[string]any) } + for name, edit := range map[string]func(*streamDiskRounds, map[int]*diskRoundDigest, map[string]any){ + "a batch lost": func(_ *streamDiskRounds, a map[int]*diskRoundDigest, _ map[string]any) { + d := &diskRoundDigest{round: streamDiskRound{Round: 0}} + d.add(2, sha256Hex([]byte("a"))) + a[0] = d + }, + "batches reordered": func(_ *streamDiskRounds, a map[int]*diskRoundDigest, _ map[string]any) { + d := &diskRoundDigest{round: streamDiskRound{Round: 0}} + d.add(2, sha256Hex([]byte("b"))) + d.add(2, sha256Hex([]byte("a"))) + a[0] = d + }, + "a round not in the trailer": func(_ *streamDiskRounds, a map[int]*diskRoundDigest, _ map[string]any) { + a[7] = &diskRoundDigest{round: streamDiskRound{Round: 7}} + }, + "the final round skipped": func(tr *streamDiskRounds, a map[int]*diskRoundDigest, _ map[string]any) { + tr.Rounds = tr.Rounds[:2] + delete(a, 2) + }, + "the final round read early": func(tr *streamDiskRounds, _ map[int]*diskRoundDigest, doc map[string]any) { + doc["generation"] = 31 + }, + "a round hvi read, never sent": func(_ *streamDiskRounds, _ map[int]*diskRoundDigest, doc map[string]any) { + doc["rounds"] = append(listed(doc), map[string]any{"round": 2, "generation": 30, "blocks": 0, + "bitmap_sha256": strings.Repeat("b", 64), "sent_sha256": nil}) + }, + "a round hvi never streamed": func(_ *streamDiskRounds, _ map[int]*diskRoundDigest, doc map[string]any) { + listed(doc)[1]["sent_sha256"] = nil + }, + "hvi's digest differs": func(_ *streamDiskRounds, _ map[int]*diskRoundDigest, doc map[string]any) { + listed(doc)[1]["sent_sha256"] = strings.Repeat("0", 64) + }, + "a pre-copy round hvi never recorded": func(_ *streamDiskRounds, _ map[int]*diskRoundDigest, doc map[string]any) { + doc["rounds"] = listed(doc)[:1] + }, + "no disk.json rounds at all": func(_ *streamDiskRounds, _ map[int]*diskRoundDigest, doc map[string]any) { + delete(doc, "rounds") + }, + "two final rounds": func(tr *streamDiskRounds, a map[int]*diskRoundDigest, _ map[string]any) { + tr.Rounds[1].Final = true + a[1].round.Final = true + }, + } { + if err := check(edit); err == nil { + t.Errorf("%s: the destination took the disk", name) + } + } + // A replay sends the disk by comparison, which disk.json does not + // describe: only the trailer's account holds. + tr, applied, _ := diskRoundsCase() + if err := checkDiskRounds(true, tr, applied, []byte(`{"generation":99}`)); err != nil { + t.Errorf("a replay: %v", err) + } + if err := checkDiskRounds(false, nil, applied, nil); err == nil { + t.Error("disk rounds with no account of them were taken") + } +} + +// A destination base whose blocks do not hash to its name is set aside +// once the move has read it, and the move sends the base again and goes +// on from the spool. +func TestStreamedMoveResendsACorruptBase(t *testing.T) { + f := newStreamFixture(t, true, true) + st := mustInstance(t, f.src, f.name) + if _, err := sendBase(t.Context(), f.src, f.peer, st.Base.SHA256); err != nil { + t.Fatal(err) + } + path := f.dst.BasePath(st.Base.SHA256) + if err := os.Chmod(path, 0o600); err != nil { + t.Fatal(err) + } + fh, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := fh.WriteAt([]byte("rot"), 4096); err != nil { + t.Fatal(err) + } + _ = fh.Close() + if err := f.migrate(t, ""); err != nil { + t.Fatal(err) + } + rec := f.srcRecord(t) + if rec.State != srcCommitted || rec.Stream.Mode != "replay" { + t.Fatalf("source %s, stream %+v", rec.State, rec.Stream) + } + aside, _ := filepath.Glob(path + ".bad-*") + if len(aside) != 1 || !baseHeld(f.dst, st.Base.SHA256) { + t.Errorf("set aside %v; base held again %v", aside, baseHeld(f.dst, st.Base.SHA256)) + } + if got, _, err := diskDigestFile(path); err != nil || got != st.Base.SHA256 { + t.Errorf("the base sent again hashes to %s, %v", got, err) + } + f.checkArrived(t, rec) +} diff --git a/cmd/hull/migrate_fixture_test.go b/cmd/hull/migrate_fixture_test.go index b97339de..c602570f 100644 --- a/cmd/hull/migrate_fixture_test.go +++ b/cmd/hull/migrate_fixture_test.go @@ -276,6 +276,9 @@ type localPeer struct { down bool answer map[string]*migrateReply calls []string + // mangle, when set, stands between a verb and its stdin, to damage or + // cut what crosses the link. + mangle func(verb string, args []string, in io.Reader) io.Reader } func newLocalPeer(s *store.Store) *localPeer { @@ -304,9 +307,12 @@ func (p *localPeer) call(_ context.Context, verb string, args []string, stdin io } return canned, nil } + if p.mangle != nil && stdin != nil { + stdin = p.mangle(verb, args, stdin) + } flags := map[string]string{} for i := 0; i < len(args); i++ { - if i+1 < len(args) && args[i] != "--reserve" { + if i+1 < len(args) && args[i] != "--reserve" && args[i] != "--stream" { flags[args[i]] = args[i+1] i++ } else { @@ -318,7 +324,12 @@ func (p *localPeer) call(_ context.Context, verb string, args []string, stdin io case "migrate-probe": reply = destProbe(p.s, flags["--name"], flags["--hvi"], flags["--gateway-sock"]) case "migrate-receive": - if flags["--reserve"] == "true" { + if flags["--stream"] == "true" { + reply = destReceiveStream(p.s, flags["--txid"], flags["--name"], stdin) + // As ssh: the call ends with the remote command, and what the + // source still writes goes nowhere. + go func() { _, _ = io.Copy(io.Discard, stdin) }() + } else if flags["--reserve"] == "true" { var req migrateReserveRequest if err := json.NewDecoder(stdin).Decode(&req); err != nil { return nil, err diff --git a/cmd/hull/migrate_prepare.go b/cmd/hull/migrate_prepare.go index 7270bc24..fe5530e2 100644 --- a/cmd/hull/migrate_prepare.go +++ b/cmd/hull/migrate_prepare.go @@ -50,8 +50,11 @@ import ( // moveCapStream is the capability a destination's probe lists when it takes // a streamed move (`migrate-receive --stream`) and the prepare verbs. A hull -// from before them lists nothing and gets the whole-bundle move. -const moveCapStream = "stream-v1" +// from before them lists nothing and gets the whole-bundle move. v2 is the +// stream whose header names an attempt the destination accepts in its +// record before hvi's migrate starts, and whose trailer accounts for the +// disk rounds; a v1 hull refuses that trailer, so it gets the whole bundle. +const moveCapStream = "stream-v2" // probeMaxBases bounds the bases a probe lists. const probeMaxBases = 512 diff --git a/cmd/hull/migrate_record.go b/cmd/hull/migrate_record.go index 3f0f8d46..89500d86 100644 --- a/cmd/hull/migrate_record.go +++ b/cmd/hull/migrate_record.go @@ -91,6 +91,29 @@ var migrateTransitions = map[string]map[string][]string{ }, } +// streamSourceTransitions are a streamed move's source transitions. It goes +// from reserved straight to transferring: hvi fences the guest inside the +// stream, after the pre-copy rounds, and hull records that fence on the +// instance when hvi seals the stream. Past transferring they are the +// whole-bundle move's. +var streamSourceTransitions = func() map[string][]string { + t := map[string][]string{} + for from, to := range migrateTransitions[roleSource] { + t[from] = to + } + t[srcReserved] = []string{srcTransferring, migAborted} + delete(t, srcFenced) + return t +}() + +// transitions is the table rec moves by. +func (rec *migrationRecord) transitions() map[string][]string { + if rec.Role == roleSource && rec.Transport == transportStream { + return streamSourceTransitions + } + return migrateTransitions[rec.Role] +} + // migrateStateOrder ranks the states of role, for "has the move got this far // yet". aborted ranks with none of them. func migrateStateOrder(role, state string) int { @@ -143,6 +166,22 @@ type migrationRecord struct { Verify string `json:"verify,omitempty"` // Timings are the milliseconds each step took, by step name. Timings map[string]int64 `json:"timingsMs,omitempty"` + // Transport is how the move carries the guest: "stream" for the + // streamed move (pre-copy rounds, only the disk blocks that differ + // from the base), "" for the whole fenced bundle and disk. Chosen at + // preflight and kept, so a retry takes the same path. + Transport string `json:"transport,omitempty"` + // BaseSHA256 is the base a streamed move's disk blocks apply to. + BaseSHA256 string `json:"baseSha256,omitempty"` + // TransportWhy says why a move took the whole-bundle path. + TransportWhy string `json:"transportWhy,omitempty"` + // Stream is what a streamed move carried, round by round, and how + // long the guest was stopped. + Stream *migrateStreamStats `json:"stream,omitempty"` + // StreamAccepted is, on the destination, the attempt of a streamed + // move whose header it took and whose base it copied: the source + // waits for it before hvi's migrate fences the guest. + StreamAccepted string `json:"streamAccepted,omitempty"` // Error is the last step's failure, kept for `hull migrate --status`. Error string `json:"error,omitempty"` @@ -214,6 +253,79 @@ type migrateTransfer struct { StartedAt time.Time `json:"startedAt"` } +// Transports a move takes. +const ( + transportBundle = "" + transportStream = "stream" +) + +// migrateStreamStats are a streamed move's numbers. +type migrateStreamStats struct { + // Mode is "live" (hvi's migrate op) or "replay" (the source's copy of + // a stream hvi sealed in an earlier attempt). + Mode string `json:"mode"` + // DiskEngine is how the changed disk blocks were found: "hvi" (hvi's + // dirty-block rounds) or "compare" (hull compared the disk with the + // base). + DiskEngine string `json:"diskEngine"` + Rounds []streamRound `json:"rounds"` + // Chain is the stream's trailer hash, which the destination's record + // of what it received must match. + Chain string `json:"chain,omitempty"` + // Bytes by kind: disk blocks as data and on the wire, RAM rounds, and + // the machine's state. + DiskBytes int64 `json:"diskBytes"` + DiskWireBytes int64 `json:"diskWireBytes"` + RAMBytes int64 `json:"ramBytes"` + StateBytes int64 `json:"stateBytes"` + WireBytes int64 `json:"wireBytes"` + Seconds float64 `json:"seconds"` + // PausedAt is when the last RAM round began to arrive, the nearest + // this host sees to hvi pausing the guest; SealedMs is from then to + // hvi's reply, FinalDiskMs the disk round taken after it. + PausedAt time.Time `json:"pausedAt,omitempty"` + SealedMs int64 `json:"sealedMs,omitempty"` + FinalDiskMs int64 `json:"finalDiskMs,omitempty"` + // FinalDiskBytes is the last disk round's data, and DiskWriteRate the + // fastest the guest wrote to its disk during the disk rounds before + // it, in bytes a second. + FinalDiskBytes int64 `json:"finalDiskBytes,omitempty"` + DiskWriteRate float64 `json:"diskWriteRate,omitempty"` + // DowntimeMs is from PausedAt to the destination's resume, on two + // hosts' clocks. + DowntimeMs int64 `json:"downtimeMs,omitempty"` + // HviReply is hvi's reply to the migrate op, as hvi wrote it. + HviReply json.RawMessage `json:"hviReply,omitempty"` + // Steps are when this host reached each step from the pause on, by + // its own clock, so the downtime can be read step by step. + Steps []streamStep `json:"steps,omitempty"` +} + +// streamStep is when a host reached one step of a streamed move. +type streamStep struct { + Step string `json:"step"` + At time.Time `json:"at"` +} + +// step records that this host reached step now. +func (st *migrateStreamStats) step(name string) { + st.Steps = append(st.Steps, streamStep{Step: name, At: time.Now().UTC()}) +} + +// streamRound is one round of a streamed move. +type streamRound struct { + Kind string `json:"kind"` // "disk", "ram" or "state" + Name string `json:"name"` + Final bool `json:"final,omitempty"` + Blocks int64 `json:"blocks,omitempty"` + Bytes int64 `json:"bytes"` + WireBytes int64 `json:"wireBytes"` + Seconds float64 `json:"seconds"` + // DirtyPerSec is, for a disk round after the first, the blocks the + // guest wrote per second during the round before. + DirtyPerSec float64 `json:"dirtyPerSec,omitempty"` +} + // migrateHistory is one state change. type migrateHistory struct { State string `json:"state"` @@ -251,7 +363,7 @@ func (rec *migrationRecord) advance(path, to string) error { return nil } allowed := false - for _, next := range migrateTransitions[rec.Role][rec.State] { + for _, next := range rec.transitions()[rec.State] { if next == to { allowed = true break @@ -266,6 +378,36 @@ func (rec *migrationRecord) advance(path, to string) error { return writeMigrationRecord(path, rec) } +// advanceThrough moves rec through each of states in turn, as advance +// would, and records them all in one write: a step whose effects several +// states promise at once pays for one synced write, not one per state. +func (rec *migrationRecord) advanceThrough(path string, states ...string) error { + moved := false + for _, to := range states { + if rec.State == to { + continue + } + allowed := false + for _, next := range rec.transitions()[rec.State] { + if next == to { + allowed = true + break + } + } + if !allowed { + return fmt.Errorf("move %s: %s cannot go from %s to %s", rec.TxID, rec.Role, rec.State, to) + } + rec.State = to + rec.Error = "" + rec.History = append(rec.History, migrateHistory{State: to, At: time.Now().UTC()}) + moved = true + } + if !moved { + return nil + } + return writeMigrationRecord(path, rec) +} + // fail records why the current step stopped, without moving the state. func (rec *migrationRecord) fail(path string, err error) error { rec.Error = err.Error() diff --git a/cmd/hull/migrate_record_test.go b/cmd/hull/migrate_record_test.go index 3657c01c..b5134b64 100644 --- a/cmd/hull/migrate_record_test.go +++ b/cmd/hull/migrate_record_test.go @@ -67,6 +67,27 @@ func TestMoveTransitions(t *testing.T) { t.Errorf("%s %s -> %s was allowed", c.role, c.from, c.to) } } + // A streamed move fences inside its transfer: from reserved it goes + // to transferring and nowhere else, and no further back than the + // whole-bundle move past it. + for _, c := range []struct { + from, to string + ok bool + }{ + {srcReserved, srcTransferring, true}, + {srcReserved, srcFenced, false}, + {srcReserved, migAborted, true}, + {srcTransferring, srcReserved, false}, + {srcTransferring, srcTransferred, true}, + {srcCommitting, srcTransferred, false}, + {srcCommitted, migAborted, false}, + } { + rec := &migrationRecord{TxID: strings.Repeat("a", 32), Role: roleSource, State: c.from, + Transport: transportStream} + if err := rec.advance(path, c.to); (err == nil) != c.ok { + t.Errorf("stream %s -> %s: err %v, want allowed=%v", c.from, c.to, err, c.ok) + } + } } func TestMoveRecordJSON(t *testing.T) { diff --git a/cmd/hull/migrate_signals.go b/cmd/hull/migrate_signals.go new file mode 100644 index 00000000..8b8792d9 --- /dev/null +++ b/cmd/hull/migrate_signals.go @@ -0,0 +1,108 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "io" + "os" + "os/signal" + "sync" + "syscall" +) + +// moveSignals is how `hull migrate` takes the signals that would end it. +// SIGINT, SIGTERM and SIGHUP cancel the move's context, which stops it at +// its next step with its record saying where, except while the move holds +// them: from hvi's migrate op until hull has recorded the fence that op +// ends in. hvi fences the guest as soon as its manifest is written, and +// until the fence is recorded the stream hull is reading and spooling is +// the guest, so a signal there waits, and acts once the fence is on +// record. SIGPIPE is taken too, so a write to a reader that went away +// (brig, a pipe to tee) fails as an error rather than killing hull. +type moveSignals struct { + mu sync.Mutex + held int + pending os.Signal + cancel context.CancelFunc + out io.Writer +} + +// watchMoveSignals returns ctx with the cancel the signals call, the +// signals to hold, and the function that stops watching. +func watchMoveSignals(ctx context.Context, out io.Writer) (context.Context, *moveSignals, func()) { + ctx, cancel := context.WithCancel(ctx) + ms := &moveSignals{cancel: cancel, out: out} + ch := make(chan os.Signal, 8) + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGPIPE) + done := make(chan struct{}) + go func() { + for { + select { + case sig := <-ch: + ms.got(sig) + case <-done: + return + } + } + }() + return ctx, ms, func() { + signal.Stop(ch) + close(done) + cancel() + } +} + +func (ms *moveSignals) got(sig os.Signal) { + if sig == syscall.SIGPIPE { + return + } + ms.mu.Lock() + defer ms.mu.Unlock() + if ms.held == 0 { + ms.cancel() + return + } + if ms.pending == nil { + _, _ = fmt.Fprintf(ms.out, "move: %v: hvi may have fenced the guest, and its stream is not on record "+ + "yet; the move stops once it is\n", sig) + } + ms.pending = sig +} + +// hold defers the signals until the matching release. A nil moveSignals +// (a move run by a test) holds nothing. +func (ms *moveSignals) hold() { + if ms == nil { + return + } + ms.mu.Lock() + ms.held++ + ms.mu.Unlock() +} + +// release ends a hold, and acts on a signal that came during it. +func (ms *moveSignals) release() { + if ms == nil { + return + } + ms.mu.Lock() + defer ms.mu.Unlock() + ms.held-- + if ms.held == 0 && ms.pending != nil { + ms.cancel() + } +} diff --git a/cmd/hull/migrate_signals_test.go b/cmd/hull/migrate_signals_test.go new file mode 100644 index 00000000..86ac9861 --- /dev/null +++ b/cmd/hull/migrate_signals_test.go @@ -0,0 +1,94 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +// syncBuffer is a bytes.Buffer the signal goroutine may write to. +type syncBuffer struct { + mu sync.Mutex + b bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.b.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.b.String() +} + +// A signal that would end hull cancels the move at once, except while it +// is held: from hvi's migrate op until the fence is recorded. One that +// comes then waits for the release. SIGPIPE never ends the move. +func TestMoveSignalsWaitForTheFenceRecord(t *testing.T) { + var out syncBuffer + ctx, ms, stop := watchMoveSignals(context.Background(), &out) + defer stop() + ms.hold() + for _, sig := range []syscall.Signal{syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE} { + if err := syscall.Kill(syscall.Getpid(), sig); err != nil { + t.Fatal(err) + } + } + select { + case <-ctx.Done(): + t.Fatal("a held signal cancelled the move") + case <-time.After(200 * time.Millisecond): + } + if !strings.Contains(out.String(), "stops once it is") { + t.Errorf("no word of the held signal: %q", out.String()) + } + ms.release() + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("the held signal was lost at the release") + } + + ctx, _, stop2 := watchMoveSignals(context.Background(), &out) + defer stop2() + if err := syscall.Kill(syscall.Getpid(), syscall.SIGPIPE); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("SIGPIPE cancelled the move") + case <-time.After(200 * time.Millisecond): + } + if err := syscall.Kill(syscall.Getpid(), syscall.SIGHUP); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("SIGHUP outside a hold did not cancel the move") + } + var none *moveSignals + none.hold() + none.release() +} diff --git a/cmd/hull/migrate_source.go b/cmd/hull/migrate_source.go index 35bc5b2c..fb1a995c 100644 --- a/cmd/hull/migrate_source.go +++ b/cmd/hull/migrate_source.go @@ -26,8 +26,10 @@ import ( "path/filepath" "runtime" "slices" + "sort" "strconv" "strings" + "sync" "time" "github.com/urfave/cli/v3" @@ -48,8 +50,12 @@ type migrateOptions struct { txid string dryRun bool fenceTimout time.Duration + // wholeBundle forces the whole-bundle move. + wholeBundle bool // out takes the progress lines. out io.Writer + // signals are held while hvi's stream is not on record; nil in tests. + signals *moveSignals } func migrateCommand() *cli.Command { @@ -68,6 +74,8 @@ func migrateCommand() *cli.Command { "hvi's no-egress stub NIC instead)"}, &cli.StringFlag{Name: "txid", Usage: "resume the move with this txid from where it stopped, or start one with it"}, &cli.BoolFlag{Name: "dry-run", Usage: "check both hosts and print the plan; change nothing on either"}, + &cli.BoolFlag{Name: "whole-bundle", Usage: "send the whole fenced checkpoint and disk, as before " + + "streamed moves, even when both hosts can stream"}, &cli.BoolFlag{Name: "abort", Usage: "abort the move NAME or TXID names, if it has not committed, and run the instance here again"}, &cli.BoolFlag{Name: "status", Usage: "print the move's record on both hosts and what it means"}, &cli.IntFlag{Name: "timeout", Value: 120, Usage: "seconds to wait for hvi to write the fenced checkpoint"}, @@ -105,9 +113,13 @@ func migrateCommand() *cli.Command { }, txid: cmd.String("txid"), dryRun: cmd.Bool("dry-run"), + wholeBundle: cmd.Bool("whole-bundle"), fenceTimout: time.Duration(cmd.Int("timeout")) * time.Second, out: os.Stderr, } + ctx, signals, stop := watchMoveSignals(ctx, os.Stderr) + defer stop() + opt.signals = signals return sourceMigrate(ctx, s, arg, opt, nil) }, } @@ -189,18 +201,19 @@ func sourceMigrate(ctx context.Context, s *store.Store, name string, opt migrate return fmt.Errorf("move %s goes to %s, not %s", rec.TxID, rec.Remote.Target, opt.to) } - probed := false + var probe *migrateProbe if rec == nil { - if rec, err = startMove(ctx, s, state, opt, peer); err != nil || rec == nil { + if rec, probe, err = startMove(ctx, s, state, opt, peer); err != nil || rec == nil { return err } - probed = true } else { _, _ = fmt.Fprintf(opt.out, "Resuming move %s of %s to %s at %s\n", rec.TxID, name, rec.Peer, rec.State) } m := &moveSource{s: s, name: name, path: path, rec: rec, peer: peerFor(rec, peer), out: opt.out, - fenceTimeout: opt.fenceTimout, probed: probed} - return m.run(ctx) + fenceTimeout: opt.fenceTimout, probed: probe != nil, probe: probe, signals: opt.signals} + err = m.run(ctx) + m.waitSpool() + return err } // describeRecordError is ", because ..." for a record whose last step @@ -216,25 +229,25 @@ func describeRecordError(rec *migrationRecord) string { // dry run, writes the new move's record in preflight. It returns nil and no // error for a dry run. func startMove(ctx context.Context, s *store.Store, state *store.InstanceState, opt migrateOptions, - peer movePeer) (*migrationRecord, error) { + peer movePeer) (*migrationRecord, *migrateProbe, error) { if opt.to == "" { - return nil, errors.New("--to USER@HOST is required") + return nil, nil, errors.New("--to USER@HOST is required") } if err := checkSSHTarget(opt.to); err != nil { - return nil, err + return nil, nil, err } txid := opt.txid if txid == "" { var err error if txid, err = newMoveTxID(); err != nil { - return nil, err + return nil, nil, err } } else if err := checkTxID(txid); err != nil { - return nil, err + return nil, nil, err } spec, err := sourcePreflight(s, state, opt) if err != nil { - return nil, fmt.Errorf("cannot move %s: %w", state.ID, err) + return nil, nil, fmt.Errorf("cannot move %s: %w", state.ID, err) } rec := &migrationRecord{ TxID: txid, Role: roleSource, Name: state.ID, Epoch: state.OwnerEpoch + 1, State: srcPreflight, @@ -244,18 +257,28 @@ func startMove(ctx context.Context, s *store.Store, state *store.InstanceState, // the instance leaves nothing behind on either host to resume or abort. probe, err := probeDestination(ctx, peerFor(rec, peer), rec, s, state) if err != nil { - return nil, fmt.Errorf("cannot move %s: %w", state.ID, err) + return nil, nil, fmt.Errorf("cannot move %s: %w", state.ID, err) + } + rec.Transport, rec.TransportWhy = decideTransport(moveTransportFacts(s, state, probe, opt.wholeBundle)) + if rec.Transport == transportStream { + rec.BaseSHA256 = state.Base.SHA256 } if opt.dryRun { printMovePlan(opt.out, rec, state, probe) - return nil, nil + return nil, nil, nil } rec.History = []migrateHistory{{State: srcPreflight, At: time.Now().UTC()}} if err := writeMigrationRecord(s.InstanceMigrationRecord(state.ID), rec); err != nil { - return nil, err + return nil, nil, err } _, _ = fmt.Fprintf(opt.out, "Move %s of %s to %s, epoch %d\n", txid, state.ID, opt.to, rec.Epoch) - return rec, nil + if rec.Transport == transportStream { + _, _ = fmt.Fprintf(opt.out, "Transport: streamed (pre-copy rounds; the disk blocks that differ from base %s)\n", + digestHead(rec.BaseSHA256)) + } else { + _, _ = fmt.Fprintf(opt.out, "Transport: the whole checkpoint and disk, because %s\n", rec.TransportWhy) + } + return rec, probe, nil } // sourcePreflight checks that the instance can be moved at all, and returns @@ -444,6 +467,16 @@ func printMovePlan(w io.Writer, rec *migrationRecord, state *store.InstanceState } _, _ = fmt.Fprintf(w, ", MAC %s\n", rec.Spec.MAC) _, _ = fmt.Fprintf(w, " epoch %d -> %d\n", state.OwnerEpoch, rec.Epoch) + if rec.Transport == transportStream { + held := "it holds the base" + if !p.holdsBase(rec.BaseSHA256) { + held = "the base is sent first, while the guest runs" + } + _, _ = fmt.Fprintf(w, " transport: streamed, the disk blocks that differ from base %s (%s)\n", + digestHead(rec.BaseSHA256), held) + } else { + _, _ = fmt.Fprintf(w, " transport: the whole checkpoint and disk, because %s\n", rec.TransportWhy) + } _, _ = fmt.Fprintln(w, " steps: reserve the name there, fence this VM (it never runs here again unless the "+ "move is aborted), stream the checkpoint and the disk, verify, restore paused there, commit") if !p.HviRestore { @@ -461,8 +494,73 @@ type moveSource struct { peer movePeer out io.Writer fenceTimeout time.Duration - // probed is set when this run has already probed the destination. + // probed is set when this run has already probed the destination, + // and probe is what it answered. probed bool + probe *migrateProbe + // destPaused is set when the destination's reply to a streamed + // receive says the guest is restored paused there already, so the + // prepare step has nothing to ask. + destPaused bool + // signals are held while hvi's stream is not on record. + signals *moveSignals + // spoolSealing reports the seal of a streamed move's spool, which + // runs on behind the move; fenceRecord the instance's record of the + // fence, written beside the end of the move, and fenceRecordedAt when. + spoolSealing chan error + fenceRecord chan error + fenceRecordedAt *time.Time + // sealGate holds the spool's sync until releaseSeal. + sealGate chan struct{} + sealGateOnce sync.Once +} + +// releaseSeal lets the spool's seal sync it: once the commit is answered, +// or the move stops, or hull exits. +func (m *moveSource) releaseSeal() { + if m.sealGate != nil { + m.sealGateOnce.Do(func() { close(m.sealGate) }) + } +} + +// waitFenceRecord waits for the instance's record of a streamed move's +// fence, when one is being written, and puts when it was made among the +// move's steps. +func (m *moveSource) waitFenceRecord() error { + if m.fenceRecord == nil { + return nil + } + err := <-m.fenceRecord + m.fenceRecord = nil + if err != nil { + return fmt.Errorf("hvi fenced %s, but recording it on the instance failed: %w", m.name, err) + } + if st := m.rec.Stream; st != nil && m.fenceRecordedAt != nil { + st.Steps = append(st.Steps, streamStep{Step: "instance's fence recorded", At: *m.fenceRecordedAt}) + sort.Slice(st.Steps, func(i, j int) bool { return st.Steps[i].At.Before(st.Steps[j].At) }) + } + return nil +} + +// waitSpool waits for the seal of the spool a streamed move left running +// behind it, so the process does not end before its writes are synced. +func (m *moveSource) waitSpool() { + m.releaseSeal() + if err := m.waitFenceRecord(); err != nil { + m.logf("warning: %v", err) + } + if m.spoolSealing == nil { + return + } + started := time.Now() + err := <-m.spoolSealing + m.spoolSealing = nil + if err != nil { + m.logf("warning: the spooled stream of move %s was not sealed (%v); a re-run or an abort checks it "+ + "frame by frame before using it", m.rec.TxID, err) + } else if took := time.Since(started); took > time.Second { + m.logf("move: the spool was synced %.1f s after the move ended", took.Seconds()) + } } func (m *moveSource) logf(format string, args ...any) { @@ -487,11 +585,20 @@ func (m *moveSource) run(ctx context.Context) error { case srcPreflight: err = m.reserve(ctx) case srcReserved: - err = m.fence() + if m.rec.Transport == transportStream { + // hvi fences the guest inside the stream. + err = m.rec.advance(m.path, srcTransferring) + } else { + err = m.fence() + } case srcFenced: err = m.rec.advance(m.path, srcTransferring) case srcTransferring: - err = m.transfer(ctx) + if m.rec.Transport == transportStream { + err = m.streamTransferWithBase(ctx) + } else { + err = m.transfer(ctx) + } case srcTransferred: err = m.prepare(ctx) case srcCommitting: @@ -509,14 +616,27 @@ func (m *moveSource) run(ctx context.Context) error { // stopped records why the move stopped and says what the operator can do. func (m *moveSource) stopped(err error) error { + m.releaseSeal() err = m.rec.fail(m.path, err) fenced := migrateStateOrder(roleSource, m.rec.State) >= migrateStateOrder(roleSource, srcFenced) + if m.rec.Transport == transportStream && m.rec.State == srcTransferring { + // hvi fences the guest only once it has sealed the stream; a + // stream that failed before that left it running. + state, serr := m.s.GetInstance(m.name) + fenced = serr != nil || fencedFor(state, m.rec.TxID) || hviVMState(state) == store.MigrationFenced + } next := fmt.Sprintf("`hull migrate %s --txid %s` resumes it from here", m.name, m.rec.TxID) if m.rec.State == srcCommitting { return fmt.Errorf("move %s stopped at %s: %w\nThe source stays fenced. The destination may "+ "have committed; %s and records the commit, or `hull migrate --abort %s` ends the move "+ "if the destination confirms it is not running", m.rec.TxID, m.rec.State, err, next, m.rec.TxID) } + var lost *lostStreamError + if errors.As(err, &lost) { + // Neither a re-run's replay nor an abort can use a spool that is + // not whole, so neither is offered as if it could. + return fmt.Errorf("move %s stopped at %s: %w", m.rec.TxID, m.rec.State, err) + } if fenced { return fmt.Errorf("move %s stopped at %s: %w\nThe source stays fenced and will not run "+ "until the move commits or is aborted. %s; `hull migrate --abort %s` brings the instance "+ @@ -546,6 +666,11 @@ func (m *moveSource) reserve(ctx context.Context) error { return fmt.Errorf("reserve %s on %s: %w", m.name, m.rec.Peer, err) } m.logf("Reserved %s on %s", m.name, m.rec.Peer) + if m.rec.Transport == transportStream && m.probe != nil && !m.probe.holdsBase(m.rec.BaseSHA256) { + if err := m.sendBaseFirst(ctx); err != nil { + return err + } + } return m.rec.advance(m.path, srcReserved) } @@ -847,6 +972,10 @@ func (m *moveSource) sourceDiskSum() error { // worked does the source record committing: a restore the destination // refuses is learned while an abort can still bring the instance back here. func (m *moveSource) prepare(ctx context.Context) error { + if m.destPaused { + // The streamed receive ended with the guest restored paused. + return m.rec.advance(m.path, srcCommitting) + } started := time.Now() m.logf("Restoring %s paused on %s", m.name, m.rec.Peer) if _, err := m.peer.call(ctx, "migrate-prepare", []string{"--txid", m.rec.TxID}, nil); err != nil { @@ -868,8 +997,12 @@ func (m *moveSource) prepare(ctx context.Context) error { // again sends the commit again, which the destination answers the same way. func (m *moveSource) commit(ctx context.Context) error { started := time.Now() + if st := m.rec.Stream; st != nil && m.rec.Transport == transportStream { + st.step("commit sent") + } reply, err := m.peer.call(ctx, "migrate-commit", []string{"--txid", m.rec.TxID, "--epoch", strconv.FormatUint(m.rec.Epoch, 10)}, nil) + m.releaseSeal() if reply == nil || reply.Record == nil || reply.Record.State != dstRunning || reply.Record.TxID != m.rec.TxID || reply.Record.Epoch != m.rec.Epoch { if err == nil { @@ -881,6 +1014,28 @@ func (m *moveSource) commit(ctx context.Context) error { m.logf("warning: %s took the move over, but: %v", m.rec.Peer, err) } m.timing("commit", started) + if st := m.rec.Stream; st != nil && !st.PausedAt.IsZero() { + for _, h := range reply.Record.History { + if h.State == dstRunning { + // Recorded just before the resume was sent; the resume's + // own time is added. + st.DowntimeMs = h.At.Sub(st.PausedAt).Milliseconds() + reply.Record.Timings["resume"] + } + } + } + // The instance's record of the fence is most often done by now; its + // time goes on the record with the commit. + if m.fenceRecord != nil { + select { + case err := <-m.fenceRecord: + m.fenceRecord = make(chan error, 1) + m.fenceRecord <- err + if err == nil { + _ = m.waitFenceRecord() + } + default: + } + } return m.rec.advance(m.path, srcCommitted) } @@ -888,10 +1043,21 @@ func (m *moveSource) commit(ctx context.Context) error { // again: the instance record marks the checkpoint stale, and the fenced hvi, // which nothing will run again, is stopped. func (m *moveSource) finish() error { + // A streamed move's instance record of the fence may still be on its + // way to disk; it is the record finish turns into "migrated". + if err := m.waitFenceRecord(); err != nil { + m.logf("warning: %v", err) + } state, err := m.s.GetInstance(m.name) if err != nil { return err } + if state.Migration == nil && m.rec.Transport == transportStream { + // Its record of the fence was lost (a hull that died after the + // commit): the move's record has what it needs. + state.Migration = &store.MigrationState{BundleDir: m.rec.BundleDir, TxID: m.rec.TxID, + ManifestSHA256: m.rec.ManifestSHA256} + } if state.Migration != nil && state.Migration.State != store.MigrationMigrated { state.Migration.State = store.MigrationMigrated state.Migration.MoveTxID = m.rec.TxID @@ -911,6 +1077,16 @@ func (m *moveSource) finish() error { m.logf("Timings (ms): fence %d, transfer %d, restore paused %d, commit %d", t["fence"], t["transfer"], t["prepare"], t["commit"]) } + if st := m.rec.Stream; st != nil { + m.logf("move: %s in %.1f s; disk %s, RAM %s, state %s; sealed %d ms after the last RAM round "+ + "began, last disk round %s in %d ms; downtime about %d ms", humanBytes(st.WireBytes), st.Seconds, + humanBytes(st.DiskWireBytes), humanBytes(st.RAMBytes), humanBytes(st.StateBytes), st.SealedMs, + humanBytes(st.FinalDiskBytes), st.FinalDiskMs, st.DowntimeMs) + if st.DiskWriteRate > 0 { + m.logf("move: the guest wrote up to %s/s to its disk during the disk rounds", + humanBytes(int64(st.DiskWriteRate))) + } + } return nil } @@ -954,10 +1130,41 @@ func sourceAbort(ctx context.Context, s *store.Store, name string, w io.Writer, // fence whose reply was lost before hull recorded it. fenced := migrateStateOrder(roleSource, rec.State) >= migrateStateOrder(roleSource, srcFenced) || fencedFor(state, rec.TxID) + streamed := rec.Transport == transportStream + if streamed { + // A streamed move fences inside its transfer, once hvi seals the + // stream: before that the guest still runs here. + fenced = fencedFor(state, rec.TxID) || + migrateStateOrder(roleSource, rec.State) >= migrateStateOrder(roleSource, srcTransferred) + } + if vm := hviVMState(state); !fenced && streamed && (vm == "" || vm == "?") { + // hvi does not answer (gone with a host that went down, say) and + // nothing recorded a fence; a whole spool of this move says hvi + // did fence it. + if root, ok := flagValue(state.CmdLine, "--checkpoint-root"); ok { + if _, err := verifySpool(filepath.Join(root, rec.TxID)); err == nil { + m := &moveSource{s: s, name: name, path: path, rec: rec, out: w} + if err := m.adoptStreamFence(state); err != nil { + return fmt.Errorf("cannot abort move %s: %w. Nothing was changed", rec.TxID, err) + } + fenced = true + } + } + } if !fenced && hviVMState(state) == store.MigrationFenced { // Adopted before anything changes: the abort must have the bundle // to restore the instance from, or it has nothing to bring back. - if err := adoptFenceForAbort(s, name, state, rec, w); err != nil { + var err error + if streamed { + m := &moveSource{s: s, name: name, path: path, rec: rec, out: w} + if err = m.adoptStreamFence(state); err != nil { + err = fmt.Errorf("cannot abort move %s: %w. Nothing was changed", rec.TxID, err) + _ = rec.fail(path, err) + } + } else { + err = adoptFenceForAbort(s, name, state, rec, w) + } + if err != nil { return err } fenced = true @@ -1076,6 +1283,14 @@ func abortLocally(s *store.Store, name string, rec *migrationRecord, w io.Writer "`hull migrate --abort %s` run again finishes", rec.TxID, name, err, rec.TxID) } } + if rec.Transport == transportStream && state.Migration != nil && state.Migration.TxID == rec.TxID { + // The fenced hvi is stopped only once the spool is known whole: a + // spool that is not, while hvi lives, is taken from hvi again. + m := &moveSource{s: s, name: name, rec: rec, out: w} + if _, err := m.ensureStreamSpool(state, state.Migration.BundleDir); err != nil { + return fmt.Errorf("aborted move %s, but %s cannot be restored here yet: %w", rec.TxID, name, err) + } + } if state.Status == "running" && state.PID > 0 && processIsAVMM(state.PID, state) { if err := stopInstanceIn(s, name, 10); err != nil { return fmt.Errorf("aborted move %s, but stopping the fenced VMM failed: %w; `hull stop %s`, "+ diff --git a/cmd/hull/migrate_spool.go b/cmd/hull/migrate_spool.go new file mode 100644 index 00000000..ef73cce8 --- /dev/null +++ b/cmd/hull/migrate_spool.go @@ -0,0 +1,293 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "io" + "os" + "path/filepath" + + "github.com/brig-sh/hull/pkg/store" +) + +// A streamed move keeps hvi's stream in a spool on the source, which an +// abort restores the guest from and a re-run sends again. hvi fences the +// guest once its manifest is in the socket's buffer, before hull has read +// it, so a hull that dies between that and the `sealed` marker leaves a +// fenced guest whose spool may be whole, cut short, or missing. Nothing +// here trusts the marker alone: a spool without one is checked frame by +// frame against the manifest it ends with, and one that does not check out +// is taken again from hvi, which streams a fenced guest again unchanged +// (the guest has not run since), for as long as that hvi is alive. + +// spoolContents is what a spooled stream holds, once every frame checked. +type spoolContents struct { + // files are the components that are not RAM rounds, by name. + files map[string][]byte + manifest []byte +} + +// hviStreamManifest is what hull reads of hvi's manifest.json: the +// components before it, each with its size, sha256 and frame count. +type hviStreamManifest struct { + Components []struct { + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Frames int64 `json:"frames"` + } `json:"components"` +} + +// spoolComponent is one component as the spool's frames add up to it. +type spoolComponent struct { + name string + size int64 + frames int64 + h hash.Hash +} + +// verifySpool reads the spooled stream in dir and checks every frame's +// sha256, that the stream ends with manifest.json and nothing after it, +// and that the manifest lists exactly the components the frames make up, +// in order, with their sizes, digests and frame counts. +func verifySpool(dir string) (*spoolContents, error) { + f, err := os.Open(filepath.Join(dir, spoolStream)) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + br := bufio.NewReaderSize(f, 1<<20) + c := &spoolContents{files: map[string][]byte{}} + var comps []*spoolComponent + var kept int64 + for { + if _, err := br.Peek(1); errors.Is(err, io.EOF) { + break + } + if c.manifest != nil { + return nil, errors.New("the spooled stream goes on after its manifest") + } + line, err := readHviLine(br, migrateMaxHeader) + if err != nil { + return nil, fmt.Errorf("the spooled stream ends in a frame header: %w", err) + } + hdr, err := parseHviComponentHeader(line) + if err != nil { + return nil, fmt.Errorf("the spooled stream: %w", err) + } + var cur *spoolComponent + if n := len(comps); n > 0 && comps[n-1].name == hdr.Component { + cur = comps[n-1] + } else if hdr.Component != "manifest.json" { + cur = &spoolComponent{name: hdr.Component, h: sha256.New()} + comps = append(comps, cur) + } + frame := sha256.New() + dst := []io.Writer{frame} + var keep *bytesBuffer + if !isRAMRound(hdr.Component) { + if kept+hdr.Size > spooledStateMax { + return nil, fmt.Errorf("the spooled stream's %s of %d bytes is too large to be a document", + hdr.Component, hdr.Size) + } + kept += hdr.Size + keep = &bytesBuffer{} + dst = append(dst, keep) + } + if cur != nil { + dst = append(dst, cur.h) + } + n, err := io.Copy(io.MultiWriter(dst...), io.LimitReader(br, hdr.Size)) + if err != nil { + return nil, fmt.Errorf("the spooled stream's %s: %w", hdr.Component, err) + } + if n != hdr.Size { + return nil, fmt.Errorf("the spooled stream ends in %s, at %d of its %d bytes", hdr.Component, n, hdr.Size) + } + if got := hex.EncodeToString(frame.Sum(nil)); got != hdr.SHA256 { + return nil, fmt.Errorf("a frame of %s in the spooled stream has sha256 %s, its header says %s", + hdr.Component, got, hdr.SHA256) + } + if cur != nil { + cur.size += hdr.Size + cur.frames++ + } + if keep != nil { + c.files[hdr.Component] = append(c.files[hdr.Component], keep.b...) + } + if hdr.Component == "manifest.json" { + c.manifest = keep.b + } + } + if c.manifest == nil { + return nil, errors.New("the spooled stream has no manifest.json: it was cut short") + } + var man hviStreamManifest + if err := json.Unmarshal(c.manifest, &man); err != nil { + return nil, fmt.Errorf("the spooled manifest.json: %w", err) + } + if len(man.Components) != len(comps) { + return nil, fmt.Errorf("the spooled manifest lists %d components, the stream holds %d", + len(man.Components), len(comps)) + } + for i, want := range man.Components { + got := comps[i] + sum := hex.EncodeToString(got.h.Sum(nil)) + if want.Name != got.name || want.Size != got.size || want.SHA256 != sum || + (want.Frames != 0 && want.Frames != got.frames) { + return nil, fmt.Errorf("the spooled stream's %s (%d bytes in %d frames, sha256 %s) is not the "+ + "manifest's %s (%d bytes in %d frames, sha256 %s)", got.name, got.size, got.frames, sum, + want.Name, want.Size, want.Frames, want.SHA256) + } + } + return c, nil +} + +// spoolIsSealed reports whether the spool in dir was sealed: its stream +// synced, its documents written beside it and the marker last. +func spoolIsSealed(dir string) bool { + _, err := os.Stat(filepath.Join(dir, spoolSealed)) + return err == nil +} + +// sealCheckedSpool seals a spool that verifySpool checked, whose seal hull +// did not live to write: the stream is synced, the documents written beside +// it, and the marker records that hvi's reply was lost. +func sealCheckedSpool(dir string, c *spoolContents) error { + f, err := os.OpenFile(filepath.Join(dir, spoolStream), os.O_RDWR, 0) + if err != nil { + return err + } + err = f.Sync() + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return err + } + for name, data := range c.files { + if err := writeFileSynced(filepath.Join(dir, name), data); err != nil { + return err + } + } + reply, err := json.Marshal(map[string]any{"v": hviControlVersion, "ok": true, "state": store.MigrationFenced, + "manifest_sha256": sha256Hex(c.manifest), + "note": "hvi's reply was lost with the hull that read it; hull checked the spooled stream frame by frame"}) + if err != nil { + return err + } + if err := writeFileSynced(filepath.Join(dir, spoolSealed), append(reply, '\n')); err != nil { + return err + } + return syncDir(dir) +} + +// openSpool returns the sealed spool in dir, sealing it first when it was +// never sealed but checks out whole. It refuses a spool that does not. +func openSpool(dir string) error { + if spoolIsSealed(dir) { + return nil + } + c, err := verifySpool(dir) + if err != nil { + return fmt.Errorf("the streamed checkpoint %s was never sealed, and it is not whole: %w", dir, err) + } + return sealCheckedSpool(dir, c) +} + +// lostStreamError is a fenced guest with no whole copy of its stream on +// this host. While its hvi is alive the guest is still there, and only +// there; once that hvi is gone, so is the guest, unless the destination +// verified the stream. +type lostStreamError struct { + dir string + why error + hviAlive bool + pid int + again error +} + +func (e *lostStreamError) Error() string { + if e.hviAlive { + return fmt.Sprintf("the guest is fenced and no whole copy of its stream is in %s (%v); hvi (pid %d) "+ + "still holds the guest, but streaming it again failed: %v. Do not stop that hvi: it is the only "+ + "copy of the guest. Running the move again retries", e.dir, e.why, e.pid, e.again) + } + return fmt.Sprintf("the guest is fenced, no whole copy of its stream is in %s (%v), and the hvi that "+ + "held it is gone: this host cannot bring the guest back. If the destination verified the stream "+ + "(`hull migrate-status`), running the move again commits it there", e.dir, e.why) +} + +// ensureStreamSpool makes sure the spool dir holds a sealed, whole copy of +// the stream hvi fenced the instance's guest with for this move, and +// returns its manifest. A spool that was never sealed is checked and +// sealed; one that is not whole is taken again from hvi when hvi still +// holds the fenced guest. +func (m *moveSource) ensureStreamSpool(state *store.InstanceState, dir string) ([]byte, error) { + if spoolIsSealed(dir) { + return os.ReadFile(filepath.Join(dir, "manifest.json")) + } + c, verr := verifySpool(dir) + if verr == nil { + m.logf("The spooled stream %s is whole but was never sealed (hull stopped before it could); sealing it", dir) + if err := sealCheckedSpool(dir, c); err != nil { + return nil, fmt.Errorf("seal the spooled stream %s: %w", dir, err) + } + return c.manifest, nil + } + lost := &lostStreamError{dir: dir, why: verr, pid: state.PID} + if hviVMState(state) != store.MigrationFenced { + return nil, lost + } + lost.hviAlive = true + m.logf("The spooled stream %s is not whole (%v); hvi still holds the fenced guest and streams it "+ + "again, unchanged, into the spool", dir, verr) + manifest, err := m.restreamFenced(state, dir) + if err != nil { + lost.again = err + return nil, lost + } + return manifest, nil +} + +// restreamFenced has hvi stream the fenced guest again into a fresh spool +// in dir, and seals it. The guest has not run since hvi fenced it, so this +// is the same machine as the stream that fenced it. +func (m *moveSource) restreamFenced(state *store.InstanceState, dir string) ([]byte, error) { + sp, err := newSpool(dir) + if err != nil { + return nil, err + } + ss := &streamSender{m: m, state: state, sw: newMoveStreamWriter(io.Discard), link: &softLink{w: io.Discard}, + stats: &migrateStreamStats{Mode: "restream"}, spoolDir: dir} + reply, files, err := ss.relayHvi(sp) + if err != nil { + sp.close() + return nil, err + } + if err := sp.seal(files, reply); err != nil { + sp.close() + return nil, err + } + sp.close() + return ss.manifest, nil +} diff --git a/cmd/hull/migrate_spool_test.go b/cmd/hull/migrate_spool_test.go new file mode 100644 index 00000000..53087939 --- /dev/null +++ b/cmd/hull/migrate_spool_test.go @@ -0,0 +1,388 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/brig-sh/hull/pkg/store" +) + +// spoolOf writes data as the spooled stream of a fresh directory. +func spoolOf(t *testing.T, data []byte) string { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, spoolStream), data, 0o600); err != nil { + t.Fatal(err) + } + return dir +} + +// fakeHviStream is one whole stream as the fake hvi writes it, and the +// offset at which its manifest's frame starts. +func fakeHviStream(t *testing.T) ([]byte, int) { + t.Helper() + f := newStreamFixture(t, true, true) + conn, br, err := hviStreamOp(f.hvi.path, map[string]any{"op": "migrate", "then": "fence"}, hviStreamIdle) + if err != nil { + t.Fatal(err) + } + _, _ = io.ReadAll(br) + _ = conn.Close() + f.hvi.mu.Lock() + defer f.hvi.mu.Unlock() + data := append([]byte{}, f.hvi.emitted...) + return data, bytes.LastIndex(data, []byte(`{"component":"manifest.json"`)) +} + +// A spool is whole only when every frame checks and the manifest it ends +// with lists exactly what the frames add up to. +func TestVerifySpool(t *testing.T) { + data, man := fakeHviStream(t) + c, err := verifySpool(spoolOf(t, data)) + if err != nil { + t.Fatal(err) + } + if c.manifest == nil || c.files["cpu.json"] == nil || c.files["ram-round-0"] != nil { + t.Fatalf("contents: manifest %d bytes, files %d", len(c.manifest), len(c.files)) + } + // The first frame of ram-round-0 ends where its second header begins. + second := bytes.Index(data[1:], []byte(`{"component":"ram-round-0"`)) + 1 + second = bytes.Index(data[second+1:], []byte(`{"component":"ram-round-0"`)) + second + 1 + flipped := append([]byte{}, data...) + flipped[man-10] ^= 0xff + for name, bad := range map[string][]byte{ + "cut in the manifest": data[:len(data)-3], + "cut before it": data[:man], + "a corrupted frame": flipped, + "bytes after the end": append(append([]byte{}, data...), 'x'), + "a frame dropped": append(append([]byte{}, data[:second]...), data[bytes.Index(data[second+1:], []byte(`{"component"`))+second+1:]...), + "empty": nil, + "a header with no body": []byte("{\"component\":\"profile.json\",\"size\":9,\"sha256\":\"" + strings.Repeat("0", 64) + "\"}\n"), + } { + if _, err := verifySpool(spoolOf(t, bad)); err == nil { + t.Errorf("%s: a spool that is not whole checked out", name) + } + } + // openSpool seals a whole spool, and refuses one that is not. + dir := spoolOf(t, data) + if err := openSpool(dir); err != nil || !spoolIsSealed(dir) { + t.Fatalf("a whole spool was not sealed: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "manifest.json")); err != nil { + t.Error(err) + } + dir = spoolOf(t, data[:man]) + if err := openSpool(dir); err == nil || spoolIsSealed(dir) { + t.Fatalf("a cut spool was sealed: %v", err) + } +} + +// The environment a crashing move's child reads its fixture from. +const ( + crashEnvSrc = "HULL_TEST_CRASH_SRC" + crashEnvDst = "HULL_TEST_CRASH_DST" + crashEnvName = "HULL_TEST_CRASH_NAME" + crashEnvHvi = "HULL_TEST_CRASH_HVI" + crashEnvAt = "HULL_TEST_CRASH_AT" +) + +// TestStreamedMoveCrashChild is not a test of its own: run by +// crashMoveAfterSeal in a child process, it moves the parent's instance and +// dies, with no deferred call run, as soon as hvi has sealed the stream and +// fenced the guest, before hull records anything: a hull killed in that +// window. +func TestStreamedMoveCrashChild(t *testing.T) { + src := os.Getenv(crashEnvSrc) + if src == "" { + t.Skip("run by the crash tests") + } + s, err := store.New(src) + if err != nil { + t.Fatal(err) + } + d, err := store.New(os.Getenv(crashEnvDst)) + if err != nil { + t.Fatal(err) + } + (&fakeDestRestore{}).install(t) + streamTestHook = func(stage string) { + if stage == os.Getenv(crashEnvAt) { + os.Exit(3) + } + } + opt := migrateOptions{to: "pm@dest", remote: migrateRemote{Target: "pm@dest", Hvi: os.Getenv(crashEnvHvi)}, + out: io.Discard} + err = sourceMigrate(context.Background(), s, os.Getenv(crashEnvName), opt, newLocalPeer(d)) + t.Fatalf("the move ended without dying in the window: %v", err) +} + +// crashMoveAfterSeal runs the move of f in a child that dies once hvi has +// fenced the guest, and returns the spool the move was writing. +func crashMoveAfterSeal(t *testing.T, f *streamFixture) string { + t.Helper() + return crashMoveAt(t, f, "relayed") +} + +// crashMoveAt runs the move of f in a child that dies at stage (see +// streamTestHook), and returns the spool the move was writing. +func crashMoveAt(t *testing.T, f *streamFixture, stage string) string { + t.Helper() + c := exec.Command(os.Args[0], "-test.run=^TestStreamedMoveCrashChild$", "-test.count=1") + c.Env = append(os.Environ(), crashEnvSrc+"="+f.src.RootDir(), crashEnvDst+"="+f.dst.RootDir(), + crashEnvName+"="+f.name, crashEnvHvi+"="+f.moveFixture.hvi, crashEnvAt+"="+stage) + out, err := c.CombinedOutput() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 3 { + t.Fatalf("the child did not die in the window: %v\n%s", err, out) + } + if f.hvi.state != "fenced" { + t.Fatalf("hvi is %s after the child died", f.hvi.state) + } + rec := f.srcRecord(t) + st := mustInstance(t, f.src, f.name) + if recorded := fencedFor(st, rec.TxID); rec.State != srcTransferring || recorded != (stage == "recorded") { + t.Fatalf("the child died at %s with the move %s, the instance's migration %+v", stage, rec.State, st.Migration) + } + root, _ := flagValue(mustInstance(t, f.src, f.name).CmdLine, "--checkpoint-root") + return filepath.Join(root, rec.TxID) +} + +// A hull killed between hvi's seal and its own leaves the guest fenced with +// an unsealed spool. The abort checks the spool frame by frame, seals it, +// and brings the guest back from it, without asking hvi again; when the +// spool was cut short, hvi, which still holds the fenced guest, streams it +// again first. +func TestStreamedMoveAbortAfterAKillInTheSealWindow(t *testing.T) { + for _, tc := range []struct { + name, stage string + cut bool + }{{"whole", "relayed", false}, {"cut", "relayed", true}, {"recorded", "recorded", false}, + {"recorded-cut", "recorded", true}} { + cut := tc.cut + t.Run(tc.name, func(t *testing.T) { + f := newStreamFixture(t, true, true) + dir := crashMoveAt(t, f, tc.stage) + if spoolIsSealed(dir) { + t.Fatal("the dead hull sealed its spool") + } + if cut { + info, err := os.Stat(filepath.Join(dir, spoolStream)) + if err != nil { + t.Fatal(err) + } + if err := os.Truncate(filepath.Join(dir, spoolStream), info.Size()/2); err != nil { + t.Fatal(err) + } + } + var restoredFrom string + localRestore = func(s *store.Store, name string) error { + st, err := s.GetInstance(name) + if err != nil { + return err + } + r, err := restoreStreamSource(st, hviRestoreOptions{detach: true}) + if r != nil { + restoredFrom = r.Name() + _ = r.Close() + } + return err + } + var out bytes.Buffer + if err := sourceAbort(context.Background(), f.src, f.name, &out, f.peer); err != nil { + t.Fatalf("%v\n%s", err, out.String()) + } + if restoredFrom != filepath.Join(dir, spoolStream) { + t.Fatalf("restored from %q\n%s", restoredFrom, out.String()) + } + if want := map[bool]int{false: 1, true: 2}[cut]; f.hvi.migrations != want { + t.Errorf("hvi streamed %d times, want %d", f.hvi.migrations, want) + } + if _, err := verifySpool(dir); err != nil || !spoolIsSealed(dir) { + t.Errorf("the spool the guest came back from: sealed %v, %v", spoolIsSealed(dir), err) + } + }) + } +} + +// The same kill, then the move run again: it records the fence it finds +// and finishes the move from the spool. +func TestStreamedMoveRerunAfterAKillInTheSealWindow(t *testing.T) { + f := newStreamFixture(t, true, true) + dir := crashMoveAfterSeal(t, f) + rec := f.srcRecord(t) + if err := f.migrate(t, rec.TxID); err != nil { + t.Fatal(err) + } + rec = f.srcRecord(t) + if rec.State != srcCommitted || rec.BundleDir != dir || rec.Stream.Mode != "replay" { + t.Fatalf("source record %s %s %+v", rec.State, rec.BundleDir, rec.Stream) + } + if f.hvi.migrations != 1 { + t.Errorf("hvi streamed %d times; the whole spool needed none", f.hvi.migrations) + } + f.checkArrived(t, rec) +} + +// With the spool cut short and an hvi that cannot stream the fenced guest +// again, nothing can bring the guest back but that hvi: the abort changes +// nothing, does not stop it, and does not tell the operator to abort. +func TestStreamedMoveAbortRefusesWithNoWholeStream(t *testing.T) { + f := newStreamFixture(t, true, true) + dir := crashMoveAfterSeal(t, f) + if err := os.Truncate(filepath.Join(dir, spoolStream), 100); err != nil { + t.Fatal(err) + } + f.hvi.mu.Lock() + f.hvi.refuseFenced = true + f.hvi.mu.Unlock() + restored := false + localRestore = func(*store.Store, string) error { restored = true; return nil } + err := sourceAbort(context.Background(), f.src, f.name, io.Discard, f.peer) + if err == nil || !strings.Contains(err.Error(), "Do not stop that hvi") || strings.Contains(err.Error(), "--abort") { + t.Fatalf("err = %v", err) + } + if restored || syscall.Kill(f.vm.pid, 0) != nil || f.hvi.state != "fenced" { + t.Errorf("restored %v; the fenced hvi was stopped or changed: %s", restored, f.hvi.state) + } + if rec := f.srcRecord(t); rec.State != srcTransferring { + t.Errorf("the abort moved the record to %s", rec.State) + } + // A re-run says the same, and offers no abort either. + err = f.migrate(t, f.srcRecord(t).TxID) + if err == nil || !strings.Contains(err.Error(), "Do not stop that hvi") || strings.Contains(err.Error(), "--abort") { + t.Fatalf("re-run: %v", err) + } +} + +// The spool's sync is not part of the downtime: with the sync of the +// spooled stream held, the move still commits, and hull waits for the +// sync only before it exits. The destination's last disk sync runs while +// hvi restores, and the move waits for it before it is verified. +func TestStreamedMoveCommitsWhileTheSpoolSyncs(t *testing.T) { + f := newStreamFixture(t, true, true) + f.hvi.writesAt["ram-round-0"] = []int64{3} + release := make(chan struct{}) + held := make(chan struct{}, 1) + prev := syncFile + t.Cleanup(func() { syncFile = prev }) + syncFile = func(file *os.File) error { + if filepath.Base(file.Name()) == spoolStream { + select { + case held <- struct{}{}: + default: + } + <-release + } + return file.Sync() + } + done := make(chan error, 1) + go func() { done <- f.migrate(t, "") }() + <-held + deadline := time.Now().Add(10 * time.Second) + for { + rec, err := readMigrationRecord(f.src.InstanceMigrationRecord(f.name)) + if err == nil && rec.State == srcCommitted { + break + } + if time.Now().After(deadline) { + close(release) + t.Fatalf("the move did not commit while the spool synced: %+v", rec) + } + time.Sleep(10 * time.Millisecond) + } + select { + case err := <-done: + t.Fatalf("hull ended before the spool was synced: %v", err) + case <-time.After(100 * time.Millisecond): + } + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } + rec := f.srcRecord(t) + if !spoolIsSealed(rec.BundleDir) { + t.Error("the spool was never sealed") + } + f.checkArrived(t, rec) + steps := map[string]bool{} + for _, s := range rec.Stream.Steps { + steps[s.Step] = true + } + for _, want := range []string{"hvi sealed the stream", "instance's fence recorded", "trailer sent", + "destination answered", "commit sent"} { + if !steps[want] { + t.Errorf("no step %q in %+v", want, rec.Stream.Steps) + } + } + dst := f.dstRecord(t, rec.TxID) + order := []string{} + for _, s := range dst.Stream.Steps { + order = append(order, s.Step) + } + if strings.Join(order, ",") != "base checked,trailer read,hvi restored the guest paused,disk synced" { + t.Errorf("destination steps %v", order) + } +} + +// A host that goes down right after hvi's seal takes hvi with it and may +// leave no record of the fence: the abort finds the whole spool of the +// move and brings the guest back from it. +func TestStreamedMoveAbortAfterHviIsGone(t *testing.T) { + f := newStreamFixture(t, true, true) + dir := crashMoveAt(t, f, "relayed") + if err := syscall.Kill(f.vm.pid, syscall.SIGKILL); err != nil { + t.Fatal(err) + } + <-f.vm.done + f.hvi.mu.Lock() + f.hvi.state = "stopped" + f.hvi.mu.Unlock() + st := mustInstance(t, f.src, f.name) + st.ControlSock = filepath.Join(t.TempDir(), "gone.sock") + if err := f.src.SaveInstance(st); err != nil { + t.Fatal(err) + } + var restoredFrom string + localRestore = func(s *store.Store, name string) error { + st, err := s.GetInstance(name) + if err != nil { + return err + } + r, err := restoreStreamSource(st, hviRestoreOptions{detach: true}) + if r != nil { + restoredFrom = r.Name() + _ = r.Close() + } + return err + } + var out bytes.Buffer + if err := sourceAbort(context.Background(), f.src, f.name, &out, f.peer); err != nil { + t.Fatalf("%v\n%s", err, out.String()) + } + if restoredFrom != filepath.Join(dir, spoolStream) { + t.Fatalf("restored from %q\n%s", restoredFrom, out.String()) + } +} diff --git a/cmd/hull/migrate_ssh.go b/cmd/hull/migrate_ssh.go index 429c0b2b..e9791f1b 100644 --- a/cmd/hull/migrate_ssh.go +++ b/cmd/hull/migrate_ssh.go @@ -25,6 +25,7 @@ import ( "os/exec" "regexp" "strings" + "syscall" "time" "unicode" ) @@ -105,6 +106,9 @@ const ( codeNotReady = "not_ready" codeVerifyFailed = "verify_failed" codeFailed = "failed" + // codeNoBase is a streamed receive refused because the destination + // does not hold the base the disk blocks apply to. + codeNoBase = "no_base" ) // errMoveNoAnswer is a remote verb that produced no reply: ssh could not @@ -221,6 +225,10 @@ func (p *sshPeer) call(ctx context.Context, verb string, args []string, stdin io c.Stdout = &stdout c.Stderr = &stderr c.Stdin = stdin + // In a process group of its own, so a Ctrl-C at the terminal reaches + // hull alone, which holds it while hvi's stream is not on record + // (moveSignals) and otherwise ends ssh through ctx. + c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} // A destination that dies mid-stream must not leave the copy blocked // on a pipe nobody reads. c.WaitDelay = 5 * time.Second diff --git a/cmd/hull/migrate_stream2.go b/cmd/hull/migrate_stream2.go new file mode 100644 index 00000000..fbd9268c --- /dev/null +++ b/cmd/hull/migrate_stream2.go @@ -0,0 +1,542 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "io" + "regexp" + "strings" + + "github.com/pierrec/lz4/v4" +) + +// The streamed move's wire format: what `hull migrate` writes into one ssh +// session and `hull migrate-receive --stream` reads (docs/migration.md, +// "The streamed move"). It multiplexes two sources into one stream: the +// disk blocks the guest changed since its base, which hull reads, and the +// components of hvi's migrate stream (RAM rounds, then the machine's state +// and the manifest), which hull relays as hvi wrote them. +// +// "HULLMIG2" +// 'H' u32 len, JSON moveStreamHeader +// 'D' u32 len, JSON diskBatchHeader, then size bytes (disk blocks) +// 'C' u32 len, hvi's component header line, then size bytes +// 'T' u32 len, JSON moveStreamTrailer +// +// Integers are big-endian. Every record's payload carries its own sha256 +// in its header, checked before the payload is used (a disk batch) or as +// it is relayed (a component, which hvi checks again). The trailer carries +// the number of records and a running sha256 over every record's kind and +// header, so a record lost, added, reordered or altered fails the move, and +// a stream that ends without its trailer is refused. +const moveStreamMagic = "HULLMIG2" + +// Limits a reader holds the stream to before it allocates or writes. +const ( + moveMaxRecords = 1 << 22 + // moveMaxComponents bounds hvi's frames: a RAM round comes in frames + // of about 1 MiB, so a 64 GiB guest's first round alone is 65536. + moveMaxComponents = 1 << 21 + // moveMaxBatchBlocks is the most blocks one disk batch carries; hvi's + // own batches are at most 16. + moveMaxBatchBlocks = 64 +) + +// hviComponentName is a name an hvi component may carry. +var hviComponentName = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,63}$`) + +// moveStreamHeader is the 'H' record. +type moveStreamHeader struct { + TxID string `json:"txid"` + Name string `json:"name"` + Epoch uint64 `json:"epoch"` + BaseSHA256 string `json:"baseSha256"` + DiskSize int64 `json:"diskSize"` + BlockSize int64 `json:"blockSize"` + // Replay says the hvi components come from the source's own copy of a + // stream hvi already sealed, sent again after a failed attempt. + Replay bool `json:"replay,omitempty"` + // Attempt names this attempt of the move; the destination records it + // as accepted once it has taken the header and copied the base. + Attempt string `json:"attempt,omitempty"` +} + +// diskBatchHeader is a 'D' record: count adjacent blocks from first, as +// rawSize bytes whose sha256 is SHA256, sent as size bytes in codec. +type diskBatchHeader struct { + Round int `json:"round"` + Final bool `json:"final,omitempty"` + First int64 `json:"first"` + Count int64 `json:"count"` + Codec string `json:"codec"` + RawSize int64 `json:"rawSize"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` +} + +// Disk batch codecs. +const ( + codecRaw = "raw" + codecLZ4 = "lz4" +) + +// hviComponentHeader is what hull reads of an hvi component's header line; +// the line itself goes on as hvi wrote it. +type hviComponentHeader struct { + Component string `json:"component"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + // First and Count are set on hvi's disk-blocks batches. + First *int64 `json:"first,omitempty"` + Count *int64 `json:"count,omitempty"` +} + +// moveStreamTrailer is the 'T' record. +type moveStreamTrailer struct { + Records int64 `json:"records"` + Chain string `json:"chain"` + // HviReply is hvi's reply to the migrate op, after its manifest. + HviReply json.RawMessage `json:"hviReply,omitempty"` + // Disk is what the disk rounds were, for the destination to hold the + // batches it applied to, and to hvi's own digests in disk.json. + Disk *streamDiskRounds `json:"disk,omitempty"` +} + +// streamDiskRounds is the trailer's account of the disk rounds. +type streamDiskRounds struct { + // Engine is the source's disk engine: "hvi" or "compare". + Engine string `json:"engine"` + // Rounds are every disk round sent, in order, the final one last. + Rounds []streamDiskRound `json:"rounds"` +} + +// streamDiskRound is one disk round as sent. +type streamDiskRound struct { + Round int `json:"round"` + Final bool `json:"final,omitempty"` + // From is "hvi" for a round of hvi's dirty blocks, "compare" for one + // hull found by comparing the disk with the base. + From string `json:"from"` + Blocks int64 `json:"blocks"` + Batches int64 `json:"batches"` + // SentSHA256 is the sha256 of the round's batch digests (each the hex + // sha256 of its raw bytes) in order, nothing between them: for a round + // from hvi, hvi's own sent_sha256 for it, which the source checked. + SentSHA256 string `json:"sentSha256"` + // Generation and BitmapSHA256 are from hvi's disk-dirty reply that + // started a round from hvi. + Generation uint64 `json:"generation,omitempty"` + BitmapSHA256 string `json:"bitmapSha256,omitempty"` +} + +// diskRoundDigest gathers a round's batches as they are sent or applied. +type diskRoundDigest struct { + round streamDiskRound + digests strings.Builder +} + +// add counts one batch of count blocks whose raw bytes have sha256 sum. +func (d *diskRoundDigest) add(count int64, sum string) { + d.round.Blocks += count + d.round.Batches++ + d.digests.WriteString(sum) +} + +// done is the round with its sent_sha256. +func (d *diskRoundDigest) done() streamDiskRound { + r := d.round + r.SentSHA256 = sha256Hex([]byte(d.digests.String())) + return r +} + +// moveChain is the running hash of the records' kinds and headers. +type moveChain struct { + h hash.Hash + n int64 +} + +func newMoveChain() *moveChain { return &moveChain{h: sha256.New()} } + +func (c *moveChain) add(kind byte, hdr []byte) { + var b [5]byte + b[0] = kind + binary.BigEndian.PutUint32(b[1:], uint32(len(hdr))) + c.h.Write(b[:]) + c.h.Write(hdr) + c.n++ +} + +func (c *moveChain) sum() string { return hex.EncodeToString(c.h.Sum(nil)) } + +// moveStreamWriter writes one streamed move. +type moveStreamWriter struct { + cw *countingWriter + bw *bufio.Writer + chain *moveChain + lz lz4.Compressor + comp []byte +} + +func newMoveStreamWriter(w io.Writer) *moveStreamWriter { + cw := &countingWriter{w: w} + return &moveStreamWriter{cw: cw, bw: bufio.NewWriterSize(cw, 1<<20), chain: newMoveChain()} +} + +// written is the bytes of the stream so far. +func (sw *moveStreamWriter) written() int64 { return sw.cw.n + int64(sw.bw.Buffered()) } + +// flush hands what is buffered to the underlying writer. +func (sw *moveStreamWriter) flush() error { return sw.bw.Flush() } + +func (sw *moveStreamWriter) record(kind byte, hdr []byte) error { + if len(hdr) > migrateMaxHeader { + return fmt.Errorf("a %q record header of %d bytes is over the limit", kind, len(hdr)) + } + var b [5]byte + b[0] = kind + binary.BigEndian.PutUint32(b[1:], uint32(len(hdr))) + if _, err := sw.bw.Write(b[:]); err != nil { + return err + } + if _, err := sw.bw.Write(hdr); err != nil { + return err + } + if kind != 'T' { + sw.chain.add(kind, hdr) + } + return nil +} + +func (sw *moveStreamWriter) header(h moveStreamHeader) error { + if _, err := sw.bw.WriteString(moveStreamMagic); err != nil { + return err + } + js, err := json.Marshal(h) + if err != nil { + return err + } + return sw.record('H', js) +} + +// disk writes count adjacent blocks from first, raw being their bytes, and +// returns the bytes it put on the wire and the hex sha256 of raw. +func (sw *moveStreamWriter) disk(round int, final bool, first, count int64, raw []byte) (int64, string, error) { + sum := sha256.Sum256(raw) + hdr := diskBatchHeader{Round: round, Final: final, First: first, Count: count, Codec: codecRaw, + RawSize: int64(len(raw)), Size: int64(len(raw)), SHA256: hex.EncodeToString(sum[:])} + payload := raw + if bound := lz4.CompressBlockBound(len(raw)); len(sw.comp) < bound { + sw.comp = make([]byte, bound) + } + if n, err := sw.lz.CompressBlock(raw, sw.comp); err == nil && n > 0 && n < len(raw) { + hdr.Codec, hdr.Size, payload = codecLZ4, int64(n), sw.comp[:n] + } + js, err := json.Marshal(hdr) + if err != nil { + return 0, "", err + } + before := sw.written() + if err := sw.record('D', js); err != nil { + return 0, "", err + } + if _, err := sw.bw.Write(payload); err != nil { + return 0, "", err + } + return sw.written() - before, hdr.SHA256, nil +} + +// component relays one hvi component: line is its header line as hvi wrote +// it (without the newline), body its bytes. The bytes are checked against +// the line's sha256 as they pass, and a component that does not match fails +// the stream: a source never sends what hvi did not seal. +func (sw *moveStreamWriter) component(line []byte, hdr hviComponentHeader, body io.Reader, also io.Writer) error { + if err := sw.record('C', line); err != nil { + return err + } + h := sha256.New() + dst := io.MultiWriter(sw.bw, h) + if also != nil { + dst = io.MultiWriter(sw.bw, h, also) + } + n, err := io.Copy(dst, io.LimitReader(body, hdr.Size)) + if err != nil { + return fmt.Errorf("relay %s: %w", hdr.Component, err) + } + if n != hdr.Size { + return fmt.Errorf("hvi's %s ended at %d of its %d bytes", hdr.Component, n, hdr.Size) + } + if got := hex.EncodeToString(h.Sum(nil)); got != hdr.SHA256 { + return fmt.Errorf("hvi's %s has sha256 %s, and its header says %s", hdr.Component, got, hdr.SHA256) + } + return nil +} + +// trailer seals the stream and flushes it. +func (sw *moveStreamWriter) trailer(hviReply json.RawMessage, disk *streamDiskRounds) error { + js, err := json.Marshal(moveStreamTrailer{Records: sw.chain.n, Chain: sw.chain.sum(), HviReply: hviReply, + Disk: disk}) + if err != nil { + return err + } + if err := sw.record('T', js); err != nil { + return err + } + return sw.bw.Flush() +} + +// moveStreamHandler is what a reader hands each record to, in order. +type moveStreamHandler interface { + // header is the 'H' record. + header(h moveStreamHeader) error + // disk is a disk batch, decompressed and checked against its sha256. + disk(hdr diskBatchHeader, raw []byte) error + // component is an hvi component: line is hvi's header line, body its + // bytes, which the handler reads in full. They are checked against the + // line's sha256 once the handler returns, and a mismatch fails the + // stream: the handler must not act on a component before then except + // to relay it to an hvi that checks it too. + component(line []byte, hdr hviComponentHeader, body io.Reader) error + // componentVerified follows component once its bytes matched. + componentVerified(hdr hviComponentHeader) error +} + +// moveStreamResult is what a reader received. +type moveStreamResult struct { + Header moveStreamHeader + Trailer moveStreamTrailer + Bytes int64 + Disk struct{ Batches, Blocks, Bytes int64 } + // Components are the hvi components by name, in order. + Components []hviComponentHeader +} + +// readMoveStream reads a streamed move from r, handing each record to h in +// order. Nothing in the stream is trusted: every length, name, index and +// count is held to the limits before anything is allocated or handed on, +// and a stream with no trailer, a trailer that disagrees, or bytes after it +// is refused. +func readMoveStream(r io.Reader, h moveStreamHandler) (*moveStreamResult, error) { + cr := &countingReader{r: r} + br := bufio.NewReaderSize(cr, 1<<20) + magic := make([]byte, len(moveStreamMagic)) + if _, err := io.ReadFull(br, magic); err != nil { + return nil, fmt.Errorf("the stream ended before it began: %w", err) + } + if string(magic) != moveStreamMagic { + return nil, fmt.Errorf("the stream does not start with %s", moveStreamMagic) + } + res := &moveStreamResult{} + chain := newMoveChain() + kind, js, err := readStreamRecord(br) + if err != nil { + return nil, err + } + if kind != 'H' { + return nil, fmt.Errorf("the stream starts with a %q record, not its header", kind) + } + if err := strictUnmarshal(js, &res.Header); err != nil { + return nil, fmt.Errorf("stream header: %w", err) + } + if err := checkMoveHeader(res.Header); err != nil { + return nil, err + } + chain.add(kind, js) + if err := h.header(res.Header); err != nil { + return nil, err + } + blocks := diskBlocks(res.Header.DiskSize) + raw := make([]byte, moveMaxBatchBlocks*diskBlockSize) + var comp []byte + for { + kind, js, err := readStreamRecord(br) + if err != nil { + return nil, err + } + if chain.n >= moveMaxRecords { + return nil, fmt.Errorf("the stream has more than %d records", moveMaxRecords) + } + switch kind { + case 'D': + var d diskBatchHeader + if err := strictUnmarshal(js, &d); err != nil { + return nil, fmt.Errorf("disk batch: %w", err) + } + want, err := checkDiskBatch(d, res.Header.DiskSize, blocks) + if err != nil { + return nil, err + } + chain.add(kind, js) + data := raw[:want] + switch d.Codec { + case codecRaw: + if _, err := io.ReadFull(br, data); err != nil { + return nil, fmt.Errorf("the stream ended in a disk batch: %w", err) + } + case codecLZ4: + if int64(len(comp)) < d.Size { + comp = make([]byte, lz4.CompressBlockBound(len(raw))) + } + if _, err := io.ReadFull(br, comp[:d.Size]); err != nil { + return nil, fmt.Errorf("the stream ended in a disk batch: %w", err) + } + n, err := lz4.UncompressBlock(comp[:d.Size], data) + if err != nil || int64(n) != want { + return nil, fmt.Errorf("disk batch at block %d does not decompress to its %d bytes", d.First, want) + } + } + if sum := sha256.Sum256(data); hex.EncodeToString(sum[:]) != d.SHA256 { + return nil, fmt.Errorf("disk batch at block %d (round %d) arrived corrupted", d.First, d.Round) + } + if err := h.disk(d, data); err != nil { + return nil, err + } + res.Disk.Batches++ + res.Disk.Blocks += d.Count + res.Disk.Bytes += want + case 'C': + c, err := parseHviComponentHeader(js) + if err != nil { + return nil, err + } + if len(res.Components) >= moveMaxComponents { + return nil, fmt.Errorf("the stream carries more than %d hvi components", moveMaxComponents) + } + chain.add(kind, js) + sum := sha256.New() + counted := &countingReader{r: io.LimitReader(br, c.Size)} + body := io.TeeReader(counted, sum) + herr := h.component(js, c, body) + // Whatever the handler left unread is read here, so the + // stream stays in step and the hash covers every byte. + _, cerr := io.Copy(io.Discard, body) + if cerr == nil && counted.n != c.Size { + cerr = io.ErrUnexpectedEOF + } + if cerr != nil { + return nil, fmt.Errorf("the stream ended in hvi's %s: %w", c.Component, cerr) + } + if herr != nil { + return nil, herr + } + if got := hex.EncodeToString(sum.Sum(nil)); got != c.SHA256 { + return nil, fmt.Errorf("hvi's %s arrived corrupted: sha256 %s, its header says %s", c.Component, got, c.SHA256) + } + if err := h.componentVerified(c); err != nil { + return nil, err + } + res.Components = append(res.Components, c) + case 'T': + var t moveStreamTrailer + if err := strictUnmarshal(js, &t); err != nil { + return nil, fmt.Errorf("stream trailer: %w", err) + } + if t.Records != chain.n || t.Chain != chain.sum() { + return nil, fmt.Errorf("the trailer counts %d records with chain %s; %d arrived with chain %s", + t.Records, t.Chain, chain.n, chain.sum()) + } + if _, err := br.ReadByte(); !errors.Is(err, io.EOF) { + return nil, errors.New("the stream goes on after its trailer") + } + res.Trailer = t + res.Bytes = cr.n + return res, nil + default: + return nil, fmt.Errorf("the stream has a record of unknown kind %q", kind) + } + } +} + +// checkMoveHeader holds the 'H' record to what a move can be. +func checkMoveHeader(h moveStreamHeader) error { + switch { + case checkTxID(h.TxID) != nil: + return fmt.Errorf("the stream names move %q", h.TxID) + case h.BlockSize != diskBlockSize: + return fmt.Errorf("the stream's disk blocks are %d bytes, not %d", h.BlockSize, diskBlockSize) + case h.DiskSize <= 0 || h.DiskSize > migrateMaxDiskSize: + return fmt.Errorf("the stream says the disk is %d bytes", h.DiskSize) + case checkDiskDigest(h.BaseSHA256) != nil: + return fmt.Errorf("the stream names base %q", h.BaseSHA256) + } + return nil +} + +// checkDiskBatch holds a disk batch to the disk and returns its raw length. +func checkDiskBatch(d diskBatchHeader, size, blocks int64) (int64, error) { + if d.Count < 1 || d.Count > moveMaxBatchBlocks || d.First < 0 || d.First >= blocks || d.Count > blocks-d.First { + return 0, fmt.Errorf("a disk batch of %d blocks at %d is outside the %d blocks of the disk", d.Count, d.First, blocks) + } + if d.Round < 0 { + return 0, fmt.Errorf("a disk batch of round %d", d.Round) + } + want := min(d.Count*diskBlockSize, size-d.First*diskBlockSize) + if d.RawSize != want { + return 0, fmt.Errorf("a disk batch of %d blocks at %d says %d bytes, not %d", d.Count, d.First, d.RawSize, want) + } + switch d.Codec { + case codecRaw: + if d.Size != want { + return 0, fmt.Errorf("a raw disk batch at %d is %d bytes, not %d", d.First, d.Size, want) + } + case codecLZ4: + if d.Size <= 0 || d.Size > int64(lz4.CompressBlockBound(int(want))) { + return 0, fmt.Errorf("a compressed disk batch at %d says %d bytes", d.First, d.Size) + } + default: + return 0, fmt.Errorf("a disk batch at %d has codec %q", d.First, d.Codec) + } + if len(d.SHA256) != 64 { + return 0, fmt.Errorf("a disk batch at %d has no sha256", d.First) + } + return want, nil +} + +// parseHviComponentHeader reads an hvi component's header line and holds it +// to the limits. Fields hull does not know are left to hvi, which reads the +// line again on the destination. +func parseHviComponentHeader(line []byte) (hviComponentHeader, error) { + var c hviComponentHeader + dec := json.NewDecoder(bytes.NewReader(line)) + if err := dec.Decode(&c); err != nil { + return c, fmt.Errorf("an hvi component header: %w", err) + } + if dec.More() { + return c, errors.New("an hvi component header has more than one object") + } + switch { + case !hviComponentName.MatchString(c.Component) || strings.Contains(c.Component, ".."): + return c, fmt.Errorf("an hvi component is named %q", c.Component) + case c.Size < 0 || c.Size > migrateMaxFileSize: + return c, fmt.Errorf("hvi's %s says %d bytes", c.Component, c.Size) + case len(c.SHA256) != 64: + return c, fmt.Errorf("hvi's %s has no sha256", c.Component) + } + return c, nil +} + +// sha256Hex is the sha256 of b, as lowercase hex. +func sha256Hex(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} diff --git a/cmd/hull/migrate_stream2_test.go b/cmd/hull/migrate_stream2_test.go new file mode 100644 index 00000000..985c9390 --- /dev/null +++ b/cmd/hull/migrate_stream2_test.go @@ -0,0 +1,315 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "fmt" + "io" + "strings" + "testing" +) + +// hviFrame is one component as hvi frames it, written here by hand rather +// than with hull's own types, so the tests check hull against the format +// hvi documents and not against itself. +func hviFrame(name string, body []byte) []byte { + sum := sha256.Sum256(body) + line := fmt.Sprintf(`{"component":%q,"size":%d,"sha256":"%s"}`+"\n", name, len(body), hex.EncodeToString(sum[:])) + return append([]byte(line), body...) +} + +// recordingHandler keeps what a reader hands it. +type recordingHandler struct { + gotHeader moveStreamHeader + blocks map[int64][]byte + components []string + bodies map[string][]byte + verified []string + failAt string +} + +func newRecordingHandler() *recordingHandler { + return &recordingHandler{blocks: map[int64][]byte{}, bodies: map[string][]byte{}} +} + +func (h *recordingHandler) header(m moveStreamHeader) error { h.gotHeader = m; return nil } + +func (h *recordingHandler) disk(d diskBatchHeader, raw []byte) error { + for i := int64(0); i < d.Count; i++ { + lo := i * diskBlockSize + h.blocks[d.First+i] = bytes.Clone(raw[lo:min(lo+diskBlockSize, int64(len(raw)))]) + } + return nil +} + +func (h *recordingHandler) component(_ []byte, c hviComponentHeader, body io.Reader) error { + if c.Component == h.failAt { + return fmt.Errorf("the handler refuses %s", c.Component) + } + b, err := io.ReadAll(body) + h.components = append(h.components, c.Component) + h.bodies[c.Component] = b + return err +} + +func (h *recordingHandler) componentVerified(c hviComponentHeader) error { + h.verified = append(h.verified, c.Component) + return nil +} + +// testMoveStream builds a stream: two disk batches (one that compresses, +// one that does not, the second ending at a short last block), hvi's +// components, and the trailer. +func testMoveStream(t *testing.T) ([]byte, map[int64][]byte, map[string][]byte) { + t.Helper() + const size = 40*diskBlockSize + 1000 + var buf bytes.Buffer + sw := newMoveStreamWriter(&buf) + if err := sw.header(moveStreamHeader{TxID: strings.Repeat("ab", 16), Name: "vm1", Epoch: 2, + BaseSHA256: strings.Repeat("cd", 32), DiskSize: size, BlockSize: diskBlockSize}); err != nil { + t.Fatal(err) + } + disk := map[int64][]byte{} + a := bytes.Repeat([]byte("abcd"), 2*diskBlockSize/4) + disk[3], disk[4] = a[:diskBlockSize], a[diskBlockSize:] + if _, _, err := sw.disk(0, false, 3, 2, a); err != nil { + t.Fatal(err) + } + noise := make([]byte, diskBlockSize+1000) + for i := range noise { + noise[i] = byte(i*7919 + i>>5) + } + disk[39], disk[40] = noise[:diskBlockSize], noise[diskBlockSize:] + if _, _, err := sw.disk(1, true, 39, 2, noise); err != nil { + t.Fatal(err) + } + comps := map[string][]byte{ + "profile.json": []byte(`{"profile_version":1,"checkpoint_format":2}`), + "ram-round-0": bytes.Repeat([]byte{7}, 100000), + "cpu.json": []byte(`{"version":1}`), + "manifest.json": []byte(`{"checkpoint_format":2}`), + } + for _, name := range []string{"profile.json", "ram-round-0", "cpu.json", "manifest.json"} { + frame := hviFrame(name, comps[name]) + br := bufio.NewReader(bytes.NewReader(frame)) + line, _ := br.ReadBytes('\n') + line = bytes.TrimSuffix(line, []byte("\n")) + hdr, err := parseHviComponentHeader(line) + if err != nil { + t.Fatal(err) + } + if err := sw.component(line, hdr, br, nil); err != nil { + t.Fatal(err) + } + } + if err := sw.trailer([]byte(`{"v":1,"ok":true}`), nil); err != nil { + t.Fatal(err) + } + return buf.Bytes(), disk, comps +} + +// The demultiplexer hands back exactly what the multiplexer was given: the +// disk blocks by index and hvi's components in order, each checked. +func TestMoveStreamRoundTrip(t *testing.T) { + stream, disk, comps := testMoveStream(t) + h := newRecordingHandler() + res, err := readMoveStream(bytes.NewReader(stream), h) + if err != nil { + t.Fatal(err) + } + if res.Bytes != int64(len(stream)) || res.Disk.Blocks != 4 || res.Disk.Batches != 2 || len(res.Components) != 4 { + t.Errorf("result %+v", res) + } + if h.gotHeader.Name != "vm1" || h.gotHeader.Epoch != 2 { + t.Errorf("header %+v", h.gotHeader) + } + for index, want := range disk { + if !bytes.Equal(h.blocks[index], want) { + t.Errorf("disk block %d differs", index) + } + } + if strings.Join(h.components, ",") != "profile.json,ram-round-0,cpu.json,manifest.json" || + strings.Join(h.verified, ",") != strings.Join(h.components, ",") { + t.Errorf("components %v verified %v", h.components, h.verified) + } + for name, want := range comps { + if !bytes.Equal(h.bodies[name], want) { + t.Errorf("%s differs", name) + } + } + if string(res.Trailer.HviReply) != `{"v":1,"ok":true}` { + t.Errorf("hvi reply %s", res.Trailer.HviReply) + } +} + +// moveStreamRecords returns the offset of every record in stream, after the +// magic: kind, header start, header end, payload end. +func moveStreamRecords(t *testing.T, stream []byte) [][4]int { + t.Helper() + var out [][4]int + off := len(moveStreamMagic) + for off < len(stream) { + kind := int(stream[off]) + n := int(binary.BigEndian.Uint32(stream[off+1:])) + hdrStart, hdrEnd := off+5, off+5+n + end := hdrEnd + switch kind { + case 'D': + var d diskBatchHeader + if err := strictUnmarshal(stream[hdrStart:hdrEnd], &d); err != nil { + t.Fatal(err) + } + end += int(d.Size) + case 'C': + c, err := parseHviComponentHeader(stream[hdrStart:hdrEnd]) + if err != nil { + t.Fatal(err) + } + end += int(c.Size) + } + out = append(out, [4]int{kind, hdrStart, hdrEnd, end}) + off = end + } + return out +} + +// A stream that was damaged, cut short or rearranged on the way is refused, +// and the refusal names what is wrong; nothing after the damage is handed +// on. +func TestMoveStreamRefusesDamage(t *testing.T) { + stream, _, _ := testMoveStream(t) + recs := moveStreamRecords(t, stream) + find := func(kind byte, nth int) [4]int { + for _, r := range recs { + if r[0] == int(kind) { + if nth == 0 { + return r + } + nth-- + } + } + t.Fatalf("no record %q", kind) + return [4]int{} + } + ram := find('C', 1) + disk := find('D', 1) + cases := []struct { + name string + mut func([]byte) []byte + want string + }{ + {"a corrupted RAM round", func(b []byte) []byte { b[ram[2]+500] ^= 1; return b }, "ram-round-0 arrived corrupted"}, + {"a corrupted disk batch", func(b []byte) []byte { b[disk[3]-10] ^= 1; return b }, "disk batch"}, + {"a corrupted header", func(b []byte) []byte { + h := find('C', 2) + i := bytes.Index(b[h[1]:h[2]], []byte(`"cpu.json"`)) + b[h[1]+i+1] = 'x' // "xpu.json": a component the trailer does not know + return b + }, "chain"}, + {"no trailer", func(b []byte) []byte { return b[:find('T', 0)[1]-5] }, "ended without its trailer"}, + {"cut inside a RAM round", func(b []byte) []byte { return b[:ram[2]+100] }, "ram-round-0"}, + {"cut inside a disk batch", func(b []byte) []byte { return b[:disk[2]+3] }, "ended in a disk batch"}, + {"cut inside a header", func(b []byte) []byte { return b[:ram[1]+5] }, "ended inside a record header"}, + {"a record dropped", func(b []byte) []byte { + c := find('C', 2) + return append(b[:c[1]-5:c[1]-5], b[c[3]:]...) + }, "trailer counts"}, + {"bytes after the trailer", func(b []byte) []byte { return append(b, 0) }, "goes on after its trailer"}, + {"another magic", func(b []byte) []byte { copy(b, "HULLMIG1"); return b }, "does not start with"}, + } + for _, tc := range cases { + h := newRecordingHandler() + _, err := readMoveStream(bytes.NewReader(tc.mut(bytes.Clone(stream))), h) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: err = %v, want one naming %q", tc.name, err, tc.want) + } + } + // A component the handler refuses ends the stream there. + h := newRecordingHandler() + h.failAt = "cpu.json" + if _, err := readMoveStream(bytes.NewReader(stream), h); err == nil || !strings.Contains(err.Error(), "refuses cpu.json") { + t.Errorf("handler refusal: %v", err) + } + if len(h.components) != 2 { + t.Errorf("components after the refusal: %v", h.components) + } +} + +// The writer refuses to relay a component whose bytes do not match the +// sha256 hvi gave it: a source never sends what hvi did not seal. +func TestMoveStreamWriterChecksHvi(t *testing.T) { + sw := newMoveStreamWriter(io.Discard) + frame := hviFrame("ram-round-0", []byte("the guest's memory")) + br := bufio.NewReader(bytes.NewReader(frame)) + line, _ := br.ReadBytes('\n') + line = bytes.TrimSuffix(line, []byte("\n")) + hdr, err := parseHviComponentHeader(line) + if err != nil { + t.Fatal(err) + } + damaged := bytes.NewReader([]byte("the guest's memorx")) + if err := sw.component(line, hdr, damaged, nil); err == nil || !strings.Contains(err.Error(), "ram-round-0 has sha256") { + t.Errorf("a damaged component was relayed: %v", err) + } + short := bytes.NewReader([]byte("the guest")) + if err := sw.component(line, hdr, short, nil); err == nil || !strings.Contains(err.Error(), "ended at") { + t.Errorf("a short component was relayed: %v", err) + } +} + +// Limits hold before anything is allocated: a disk batch outside the disk, +// a component of a name hvi would not use, a header that is not a move's. +func TestMoveStreamLimits(t *testing.T) { + const size = 4 * diskBlockSize + for _, d := range []diskBatchHeader{ + {First: 4, Count: 1, Codec: codecRaw, RawSize: diskBlockSize, Size: diskBlockSize, SHA256: strings.Repeat("0", 64)}, + {First: 3, Count: 2, Codec: codecRaw, RawSize: 2 * diskBlockSize, Size: 2 * diskBlockSize, SHA256: strings.Repeat("0", 64)}, + {First: 0, Count: 1, Codec: codecRaw, RawSize: 10, Size: 10, SHA256: strings.Repeat("0", 64)}, + {First: 0, Count: 1, Codec: "zstd", RawSize: diskBlockSize, Size: 10, SHA256: strings.Repeat("0", 64)}, + {First: 0, Count: 1, Codec: codecLZ4, RawSize: diskBlockSize, Size: 1 << 30, SHA256: strings.Repeat("0", 64)}, + {First: 0, Count: moveMaxBatchBlocks + 1, Codec: codecRaw, RawSize: 0, Size: 0}, + {Round: -1, First: 0, Count: 1, Codec: codecRaw, RawSize: diskBlockSize, Size: diskBlockSize, SHA256: strings.Repeat("0", 64)}, + } { + if _, err := checkDiskBatch(d, size, diskBlocks(size)); err == nil { + t.Errorf("batch %+v passed", d) + } + } + for _, line := range []string{ + `{"component":"../ram","size":1,"sha256":"` + strings.Repeat("0", 64) + `"}`, + `{"component":"ram","size":-1,"sha256":"` + strings.Repeat("0", 64) + `"}`, + `{"component":"ram","size":1}`, + `{"component":"Ram","size":1,"sha256":"` + strings.Repeat("0", 64) + `"}`, + `{"component":"ram","size":1,"sha256":"` + strings.Repeat("0", 64) + `"}{}`, + } { + if _, err := parseHviComponentHeader([]byte(line)); err == nil { + t.Errorf("component header %s passed", line) + } + } + for _, h := range []moveStreamHeader{ + {TxID: "x", BaseSHA256: strings.Repeat("a", 64), DiskSize: 1, BlockSize: diskBlockSize}, + {TxID: strings.Repeat("a", 32), BaseSHA256: strings.Repeat("a", 64), DiskSize: 1, BlockSize: 4096}, + {TxID: strings.Repeat("a", 32), BaseSHA256: strings.Repeat("a", 64), DiskSize: 0, BlockSize: diskBlockSize}, + {TxID: strings.Repeat("a", 32), BaseSHA256: "nope", DiskSize: 1, BlockSize: diskBlockSize}, + } { + if err := checkMoveHeader(h); err == nil { + t.Errorf("header %+v passed", h) + } + } +} diff --git a/cmd/hull/migrate_streamdst.go b/cmd/hull/migrate_streamdst.go new file mode 100644 index 00000000..d560aa30 --- /dev/null +++ b/cmd/hull/migrate_streamdst.go @@ -0,0 +1,782 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" + + "github.com/brig-sh/hull/pkg/store" +) + +// The destination's half of a streamed move: `migrate-receive --stream` +// reads the stream on stdin, applies the disk blocks to a copy of the base +// that becomes the instance's disk, and relays hvi's components to an `hvi +// restore --from-stream` it starts for the instance. RAM rounds go to hvi +// as they arrive, so no RAM lands on this host's disk; the machine's state +// and the manifest are held until the stream's trailer has been checked +// and the last disk round is on disk, so hvi rebuilds the machine only from +// a stream hull has verified whole, over a disk that is complete. The move +// is verified, and the guest restored paused, when both hull and hvi have +// checked everything; only the source's commit resumes it. + +// streamRestoreTimeout bounds a streamed restore: it lasts as long as the +// stream, pre-copy rounds included. +const streamRestoreTimeout = time.Hour + +// streamRestore is the hvi restore a streamed receive feeds. +type streamRestore struct { + in io.WriteCloser + wait func() error + kill func() +} + +// destStreamRestore starts the restore of the incoming instance name for +// move txid, reading hvi's stream from what is written to its in. A +// variable so the tests can stand in for hvi. +var destStreamRestore = startDestStreamRestore + +// startDestStreamRestore runs `hull restore --detach --paused --move TXID +// NAME` as a child, as a move's restore always runs, with a pipe as its +// stdin, which the hvi it starts inherits. wait returns once hvi reports the +// guest restored paused, or why it did not. +func startDestStreamRestore(s *store.Store, name, txid string) (*streamRestore, error) { + self, err := os.Executable() + if err != nil { + return nil, err + } + pr, pw, err := os.Pipe() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), streamRestoreTimeout) + var out bytes.Buffer + c := exec.CommandContext(ctx, self, "--store-dir", s.RootDir(), "restore", "--detach", "--paused", + "--timeout", fmt.Sprint(int(streamRestoreTimeout.Seconds())), "--move", txid, name) + c.Stdin, c.Stdout, c.Stderr = pr, &out, &out + if err := c.Start(); err != nil { + cancel() + _ = pr.Close() + _ = pw.Close() + return nil, err + } + _ = pr.Close() + return &streamRestore{ + in: pw, + wait: func() error { + defer cancel() + if err := c.Wait(); err != nil { + text := strings.TrimSpace(out.String()) + if i := strings.LastIndex(text, "error: "); i >= 0 { + text = text[i+len("error: "):] + } + return fmt.Errorf("hull restore %s: %s", name, text) + } + return nil + }, + kill: func() { + _ = pw.Close() + if c.Process != nil { + _ = c.Process.Kill() + } + _ = c.Wait() + cancel() + stopIncomingVMM(s, name) + }, + }, nil +} + +// stopIncomingVMM kills the hvi a failed streamed receive started, if its +// record names one. +func stopIncomingVMM(s *store.Store, name string) { + state, err := s.GetInstance(name) + if err != nil || state.PID <= 0 || !processIsAVMM(state.PID, state) { + return + } + _ = syscall.Kill(state.PID, syscall.SIGKILL) + for i := 0; i < 50 && syscall.Kill(state.PID, 0) == nil; i++ { + time.Sleep(20 * time.Millisecond) + } + state.Status, state.PID = "stopped", 0 + _ = s.SaveInstance(state) +} + +// streamReceiver is the handler a streamed receive reads the stream with. +type streamReceiver struct { + s *store.Store + rec *migrationRecord + path string + instDir string + root string + hvi string + diskPath string + diskFile *os.File + size int64 + base string + + restore *streamRestore + // stage: 0 before profile.json, 1 after it, 2 in the RAM rounds, 3 in + // the machine's state. + stage int + // pending is the component being read, when it is kept whole. + pending *bytesBuffer + // held are the state components, framed for hvi, held until the + // trailer; files the small components kept beside the record. + held [][]byte + heldBytes int64 + files map[string][]byte + manifest []byte + zero []byte + // restoreErr is why hvi's restore failed, once it did. + restoreErr error + // profileLine is hvi's header line of profile.json, held with it. + profileLine []byte + // unsynced is what was written to the disk since the last background + // sync was asked for. syncing waits for the syncs in the background; + // syncMu guards whether one runs, whether another is asked for, and + // the first error one met. + unsynced int64 + syncing sync.WaitGroup + syncMu sync.Mutex + syncRunning bool + syncAgain bool + syncErr error + // filesDone reports the documents written beside the record. + filesDone chan error + // applied is each disk round's batches as applied, by round. + applied map[int]*diskRoundDigest + // baseChecked reports the check of the base's digest, which runs while + // the stream arrives. + baseChecked chan error +} + +// maybeSyncDisk starts the disk's writeback, without waiting, once +// spoolSyncEvery bytes were written to it since the last (where the host +// cannot, it syncs the disk in the background instead): the sync the guest +// waits for, after the stream, then has only the last round left, and no +// sync runs to hold up the records written while the guest is stopped. +func (r *streamReceiver) maybeSyncDisk(n int64) { + r.unsynced += n + if r.unsynced < spoolSyncEvery { + return + } + if canStartWriteback { + r.unsynced = 0 + startWriteback(r.diskFile, 0, 0) + return + } + r.syncDisk() +} + +// syncDisk syncs the disk in the background. One asked for while another +// runs runs after it, so every write made before the call is covered once +// syncing is done. +func (r *streamReceiver) syncDisk() { + r.unsynced = 0 + r.syncMu.Lock() + defer r.syncMu.Unlock() + if r.syncRunning { + r.syncAgain = true + return + } + r.syncRunning = true + r.syncing.Add(1) + f := r.diskFile + go func() { + defer r.syncing.Done() + for { + err := syncFile(f) + r.syncMu.Lock() + if err != nil && r.syncErr == nil { + r.syncErr = err + } + if !r.syncAgain { + r.syncRunning = false + r.syncMu.Unlock() + return + } + r.syncAgain = false + r.syncMu.Unlock() + } + }() +} + +// toHvi writes data to hvi's stdin. A write that fails means hvi stopped +// reading, which it does when it refuses the restore: its reason is the +// one to report. +func (r *streamReceiver) toHvi(what string, data []byte) error { + if _, err := r.restore.in.Write(data); err != nil { + _ = r.restore.in.Close() + if werr := r.restore.wait(); werr != nil { + r.restoreErr = werr + return werr + } + return fmt.Errorf("hand %s to hvi: %w", what, err) + } + return nil +} + +func (r *streamReceiver) header(h moveStreamHeader) error { + switch { + case h.TxID != r.rec.TxID: + return fmt.Errorf("the stream is for move %s, not %s", h.TxID, r.rec.TxID) + case h.Name != r.rec.Name: + return fmt.Errorf("the stream is for %s, not %s", h.Name, r.rec.Name) + case h.Epoch != r.rec.Epoch: + return fmt.Errorf("the stream hands over epoch %d, not %d", h.Epoch, r.rec.Epoch) + case !baseHeld(r.s, h.BaseSHA256): + return &noBaseError{sum: h.BaseSHA256} + } + r.base = h.BaseSHA256 + r.size = h.DiskSize + // The base is trusted by its name only until this check, which reads + // it while the stream's rounds arrive and is waited for before hvi + // gets the machine. + r.baseChecked = make(chan error, 1) + go func(path, want string) { + got, _, err := diskDigestFile(path) + if err == nil && got != want { + err = fmt.Errorf("its blocks hash to %s", got) + } + r.baseChecked <- err + }(r.s.BasePath(h.BaseSHA256), h.BaseSHA256) + // A fresh copy of the base on every attempt: whatever an earlier one + // applied is gone with it. + if err := os.MkdirAll(filepath.Dir(r.diskPath), 0o700); err != nil { + return err + } + _ = os.Remove(r.diskPath) + if _, err := copyDisk(r.s.BasePath(h.BaseSHA256), r.diskPath); err != nil { + return fmt.Errorf("copy the base: %w", err) + } + if err := os.Chmod(r.diskPath, 0o600); err != nil { + return err + } + f, err := os.OpenFile(r.diskPath, os.O_RDWR, 0) + if err != nil { + return err + } + r.diskFile = f + info, err := f.Stat() + if err != nil { + return err + } + if info.Size() != h.DiskSize { + return fmt.Errorf("the base is %d bytes and the guest's disk %d", info.Size(), h.DiskSize) + } + r.zero = make([]byte, diskBlockSize) + // The source waits for this before hvi's migrate fences the guest: a + // refusal from here on cannot be one the header would have told. + if h.Attempt != "" { + r.rec.StreamAccepted = h.Attempt + if err := writeMigrationRecord(r.path, r.rec); err != nil { + return err + } + } + return nil +} + +// checkBase waits for the check of the base's digest. A base whose blocks +// do not hash to its name is set aside, so the source sends it again, as +// for a base this host never held. +func (r *streamReceiver) checkBase() error { + if r.baseChecked == nil { + return errors.New("the stream named no base") + } + err := <-r.baseChecked + r.baseChecked = nil + if err == nil { + r.rec.Stream.step("base checked") + return nil + } + path := r.s.BasePath(r.base) + aside := fmt.Sprintf("%s.bad-%d", path, time.Now().UnixNano()) + if rerr := os.Rename(path, aside); rerr != nil { + return fmt.Errorf("base %s is corrupt (%v), and setting it aside failed: %w", r.base, err, rerr) + } + log.Warnf("Base %s is corrupt (%v); set aside as %s", r.base, err, aside) + return &noBaseError{sum: r.base} +} + +// checkDiskRounds holds the disk rounds the destination applied to the +// trailer's account of them: every round, with as many blocks and batches, +// and the same digest of its batches' digests. For the rounds from hvi's +// dirty blocks it then holds that account to what hvi itself recorded in +// disk.json, which the manifest covers: each round hvi read went out +// whole, as hvi streamed it, and the final round, read after the fence, +// is the one after the last listed, at the generation hvi captured the +// disk at, so no write of the guest was left out. A replay sends the disk +// by comparison, which disk.json does not describe. +func checkDiskRounds(replay bool, t *streamDiskRounds, applied map[int]*diskRoundDigest, diskJSON []byte) error { + if t == nil { + if len(applied) > 0 { + return errors.New("the stream's trailer gives no account of its disk rounds") + } + return nil + } + sent := map[int]streamDiskRound{} + for _, want := range t.Rounds { + if _, dup := sent[want.Round]; dup { + return fmt.Errorf("the trailer lists disk round %d twice", want.Round) + } + sent[want.Round] = want + got := (&diskRoundDigest{}).done() + if a := applied[want.Round]; a != nil { + got = a.done() + if got.Final != want.Final { + return fmt.Errorf("disk round %d arrived final %v, the trailer says %v", want.Round, got.Final, want.Final) + } + } + if got.Blocks != want.Blocks || got.Batches != want.Batches || got.SentSHA256 != want.SentSHA256 { + return fmt.Errorf("disk round %d: %d blocks in %d batches arrived (digest %s); the source sent %d "+ + "in %d (digest %s)", want.Round, got.Blocks, got.Batches, got.SentSHA256, want.Blocks, + want.Batches, want.SentSHA256) + } + } + for round := range applied { + if _, ok := sent[round]; !ok { + return fmt.Errorf("disk round %d arrived, and the trailer does not list it", round) + } + } + var final *streamDiskRound + fromHvi := false + for i, r := range t.Rounds { + fromHvi = fromHvi || r.From == "hvi" + if r.Final { + if final != nil { + return errors.New("the trailer lists two final disk rounds") + } + final = &t.Rounds[i] + } + } + if replay || !fromHvi { + return nil + } + var doc struct { + Generation uint64 `json:"generation"` + Rounds []struct { + Round int `json:"round"` + Generation uint64 `json:"generation"` + Blocks int64 `json:"blocks"` + BitmapSHA256 string `json:"bitmap_sha256"` + SentSHA256 *string `json:"sent_sha256"` + } `json:"rounds"` + } + if diskJSON == nil { + return errors.New("the disk rounds came from hvi, and its stream has no disk.json to hold them to") + } + if err := json.Unmarshal(diskJSON, &doc); err != nil { + return fmt.Errorf("hvi's disk.json: %w", err) + } + last := -1 + listed := map[int]bool{} + for _, l := range doc.Rounds { + listed[l.Round] = true + last = max(last, l.Round) + s, ok := sent[l.Round] + switch { + case !ok: + return fmt.Errorf("hvi read disk round %d, and the stream does not carry it", l.Round) + case s.From != "hvi": + continue + case l.SentSHA256 == nil || *l.SentSHA256 != s.SentSHA256: + return fmt.Errorf("hvi recorded disk round %d as sent with digest %v; the stream carried %s", + l.Round, l.SentSHA256, s.SentSHA256) + case l.Generation != s.Generation || l.Blocks != s.Blocks || + (l.BitmapSHA256 != "" && l.BitmapSHA256 != s.BitmapSHA256): + return fmt.Errorf("hvi recorded disk round %d at generation %d with %d blocks; the source sent it "+ + "as read at %d with %d", l.Round, l.Generation, l.Blocks, s.Generation, s.Blocks) + } + } + for _, s := range t.Rounds { + if s.From == "hvi" && !s.Final && !listed[s.Round] { + return fmt.Errorf("disk round %d is not one hvi recorded before the fence", s.Round) + } + } + switch { + case final == nil || final.From != "hvi": + return errors.New("the stream has no final disk round from hvi: the guest's last writes would be missing") + case final.Round != last+1: + return fmt.Errorf("the final disk round is %d; hvi's last recorded round was %d", final.Round, last) + case final.Generation != doc.Generation: + return fmt.Errorf("the final disk round was read at generation %d, and hvi captured the disk at %d: "+ + "a write of the guest is missing", final.Generation, doc.Generation) + } + return nil +} + +// noBaseError is a stream whose base this store does not hold. +type noBaseError struct{ sum string } + +func (e *noBaseError) Error() string { + return fmt.Sprintf("this host does not hold base %s: `hull migrate-prepare` sends it", e.sum) +} + +func (r *streamReceiver) disk(d diskBatchHeader, raw []byte) error { + if r.applied == nil { + r.applied = map[int]*diskRoundDigest{} + } + round := r.applied[d.Round] + if round == nil { + round = &diskRoundDigest{round: streamDiskRound{Round: d.Round, Final: d.Final}} + r.applied[d.Round] = round + } else if round.round.Final != d.Final { + return fmt.Errorf("disk round %d is sent as both final and not", d.Round) + } + round.add(d.Count, d.SHA256) + cur := make([]byte, diskBlockSize) + for i := int64(0); i < d.Count; i++ { + index := d.First + i + lo := i * diskBlockSize + block := raw[lo:min(lo+diskBlockSize, int64(len(raw)))] + off := index * diskBlockSize + if isZero(block) { + // A zero block over one that is zero already (a hole of the + // base) is left alone, so the disk stays sparse. + n, err := r.diskFile.ReadAt(cur[:len(block)], off) + if err != nil && !errors.Is(err, io.EOF) { + return err + } + if isZero(cur[:n]) { + continue + } + } + if _, err := r.diskFile.WriteAt(block, off); err != nil { + return fmt.Errorf("write disk block %d: %w", index, err) + } + r.maybeSyncDisk(int64(len(block))) + } + return nil +} + +func (r *streamReceiver) component(line []byte, hdr hviComponentHeader, body io.Reader) error { + name := hdr.Component + switch { + case r.stage == 0: + if name != "profile.json" { + return fmt.Errorf("hvi's stream starts with %s, not profile.json", name) + } + if hdr.Size > 1<<20 { + return fmt.Errorf("profile.json is %d bytes", hdr.Size) + } + r.profileLine = append([]byte{}, line...) + r.pending = &bytesBuffer{} + _, err := io.Copy(r.pending, body) + return err + case isRAMRound(name): + if r.stage == 3 { + return fmt.Errorf("hvi's %s comes after the machine's state", name) + } + r.stage = 2 + if err := r.toHvi(name, append(append([]byte{}, line...), '\n')); err != nil { + return err + } + buf := make([]byte, 1<<20) + for { + n, err := body.Read(buf) + if n > 0 { + if werr := r.toHvi(name, buf[:n]); werr != nil { + return werr + } + } + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + } + } + r.stage = 3 + if r.heldBytes+hdr.Size > spooledStateMax { + return fmt.Errorf("hvi's %s of %d bytes is not a RAM round and too large to hold", name, hdr.Size) + } + r.heldBytes += hdr.Size + r.pending = &bytesBuffer{} + _, err := io.Copy(r.pending, body) + if err == nil { + r.held = append(r.held, append(append(append([]byte{}, line...), '\n'), r.pending.b...)) + } + return err +} + +func (r *streamReceiver) componentVerified(hdr hviComponentHeader) error { + name := hdr.Component + if isRAMRound(name) { + return nil + } + if r.manifest != nil { + return fmt.Errorf("hvi's %s comes after its manifest", name) + } + r.files[name] = append(r.files[name], r.pending.b...) + switch name { + case "manifest.json": + r.manifest = r.pending.b + case "profile.json": + if r.stage != 0 { + return errors.New("hvi's stream carries profile.json twice") + } + if err := checkStreamProfile(r.pending.b, r.rec.Spec.Profile); err != nil { + return err + } + if err := writeStreamIncomingInstance(r.s, r.rec, r.hvi, r.root, r.diskPath, r.base); err != nil { + return err + } + restore, err := destStreamRestore(r.s, r.rec.Name, r.rec.TxID) + if err != nil { + return fmt.Errorf("start hvi restore: %w", err) + } + r.restore = restore + r.stage = 1 + // The disk rounds that run with the guest running are all in: sync + // them while the RAM rounds arrive, so the sync after the stream + // has only the last round to write. + if r.diskFile != nil { + r.syncDisk() + } + if err := r.toHvi(name, append(append(r.profileLine, '\n'), r.pending.b...)); err != nil { + return err + } + } + r.pending = nil + return nil +} + +// checkStreamProfile refuses a stream whose machine profile is not the one +// the move's spec names, as checkBundleProfile does for a bundle. +func checkStreamProfile(data []byte, want string) error { + dir, err := os.MkdirTemp("", "hull-profile-") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(dir) }() + if err := os.WriteFile(filepath.Join(dir, "profile.json"), data, 0o600); err != nil { + return err + } + return checkBundleProfile(dir, want) +} + +// finish hands hvi the state it held, once the stream is verified whole +// and the last disk round is written, and waits for hvi to report the +// guest restored paused. hvi reads the disk through the page cache, so +// the sync that makes the last round durable runs while hvi rebuilds the +// machine rather than before it; finish returns once both are done. The +// documents kept beside the record are written meanwhile too, unsynced. +func (r *streamReceiver) finish() error { + if r.restore == nil || r.manifest == nil { + return errors.New("the stream carried no hvi checkpoint") + } + r.rec.Stream.step("trailer read") + r.syncDisk() + r.filesDone = make(chan error, 1) + go func() { + // Kept for the record only: hvi has the machine from the stream, + // so they are not synced, which on a busy disk would cost the + // stopped guest a sync each. + for name, data := range r.files { + if err := os.WriteFile(filepath.Join(r.rec.BundleDir, name), data, 0o600); err != nil { + r.filesDone <- err + return + } + } + r.filesDone <- nil + }() + for _, c := range r.held { + if err := r.toHvi("the machine's state", c); err != nil { + return err + } + } + if err := r.restore.in.Close(); err != nil { + return err + } + if err := r.restore.wait(); err != nil { + r.restoreErr = err + return err + } + r.rec.Stream.step("hvi restored the guest paused") + r.syncing.Wait() + r.rec.Stream.step("disk synced") + if r.syncErr != nil { + return fmt.Errorf("sync the disk: %w", r.syncErr) + } + err := r.diskFile.Close() + r.diskFile = nil + if err != nil { + return err + } + if err := <-r.filesDone; err != nil { + return fmt.Errorf("keep hvi's documents: %w", err) + } + r.filesDone = nil + return nil +} + +// abandon ends a receive that failed: hvi's input is cut, and the restore +// and any hvi it started are stopped. +func (r *streamReceiver) abandon() { + if r.baseChecked != nil { + <-r.baseChecked + } + r.syncing.Wait() + if r.filesDone != nil { + <-r.filesDone + } + if r.diskFile != nil { + _ = r.diskFile.Close() + } + if r.restore != nil { + r.restore.kill() + } +} + +// destStreamCmdLine is the command line a streamed move records for the +// instance it brings in: a restore from a stream, with the destination's +// own disk, sockets and checkpoint root. +func destStreamCmdLine(s *store.Store, rec *migrationRecord, hvi, root, disk string) []string { + argv := destCmdLine(s, rec, hvi, root) + for i := 0; i+1 < len(argv); i++ { + switch argv[i] { + case "--from": + argv[i], argv[i+1] = hviFromStreamFlag, hviFromStreamStdin + case "--disk": + argv[i+1] = disk + } + } + return argv +} + +// writeStreamIncomingInstance writes the incoming instance's state.json for +// a streamed move: stopped, its checkpoint the move's, which only the +// move's restore may bring up. +func writeStreamIncomingInstance(s *store.Store, rec *migrationRecord, hvi, root, disk, base string) error { + rec.BundleDir = filepath.Join(root, rec.TxID) + if err := os.MkdirAll(rec.BundleDir, 0o700); err != nil { + return err + } + rec.Disk = &migrateDisk{Path: disk} + spec := rec.Spec + state := &store.InstanceState{ + ID: rec.Name, + ImageDigest: spec.ImageDigest, + Status: "stopped", + LogFile: s.InstanceLogFile(rec.Name), + CmdLine: destStreamCmdLine(s, rec, hvi, root, disk), + BundleDir: s.InstanceBundleDir(rec.Name), + MAC: spec.MAC, + IP: spec.IP, + Backend: string(hypervisors.HviVmm), + Profile: spec.Profile, + StartTime: time.Now(), + Migration: &store.MigrationState{State: store.MigrationIncoming, BundleDir: rec.BundleDir, + TxID: rec.TxID, MoveTxID: rec.TxID, Peer: rec.Peer}, + Base: &store.BaseState{SHA256: base}, + } + return s.SaveInstance(state) +} + +// destReceiveStream is `migrate-receive --stream`. +func destReceiveStream(s *store.Store, txid, name string, in io.Reader) *migrateReply { + rec, path, unlock, no := destRecord(s, txid) + if no != nil { + return no + } + if unlock != nil { + defer unlock() + } + switch { + case rec.State == migAborted: + return refuseWith(codeAborted, fmt.Errorf("move %s was aborted here", txid), rec) + case rec.Name != name: + return refuseWith(codeFailed, fmt.Errorf("move %s is for %s here, not %s", txid, rec.Name, name), rec) + case migrateStateOrder(roleDestination, rec.State) >= migrateStateOrder(roleDestination, dstVerified): + return &migrateReply{OK: true, Record: rec} + } + // An earlier attempt may have left its hvi running. + stopIncomingVMM(s, name) + if err := rec.advance(path, dstReceiving); err != nil { + return refuseWith(codeFailed, err, rec) + } + hvi, err := destHviPath(rec.Spec.Hvi) + if err != nil { + return refuseWith(codeFailed, rec.fail(path, err), rec) + } + instDir := s.InstanceDir(name) + root, err := hviCheckpointRoot(instDir) + if err != nil { + return refuseWith(codeFailed, rec.fail(path, err), rec) + } + rec.StreamAccepted = "" + r := &streamReceiver{s: s, rec: rec, path: path, instDir: instDir, root: root, hvi: hvi, + diskPath: filepath.Join(instDir, "bundle", "rootfs.ext4"), files: map[string][]byte{}} + rec.Stream = &migrateStreamStats{} + started := time.Now() + res, err := readMoveStream(in, r) + if err == nil { + err = r.checkBase() + } + if err == nil { + err = checkDiskRounds(res.Header.Replay, res.Trailer.Disk, r.applied, r.files["disk.json"]) + } + if err == nil { + err = r.finish() + } + if err != nil { + r.abandon() + if r.restoreErr != nil { + return refuseWith(codeRestoreRefused, rec.fail(path, fmt.Errorf("restore: %w", r.restoreErr)), rec) + } + var nb *noBaseError + if errors.As(err, &nb) { + return refuseWith(codeNoBase, rec.fail(path, err), rec) + } + return refuseWith(codeVerifyFailed, rec.fail(path, fmt.Errorf("receive: %w", err)), rec) + } + took := time.Since(started) + rec.ManifestSHA256 = sha256Hex(r.manifest) + rec.BaseSHA256 = r.base + rec.Transport = transportStream + rec.Disk = &migrateDisk{Path: r.diskPath, Size: r.size, DataBytes: res.Disk.Bytes} + rec.Transfer = &migrateTransfer{Bytes: res.Bytes, Seconds: took.Seconds(), StartedAt: started.UTC(), + BytesPerSec: perSec(res.Bytes, took.Seconds())} + rec.Stream = &migrateStreamStats{Chain: res.Trailer.Chain, DiskBytes: res.Disk.Bytes, + WireBytes: res.Bytes, Seconds: took.Seconds(), HviReply: res.Trailer.HviReply, Steps: rec.Stream.Steps} + rec.Verify = fmt.Sprintf("stream: %d hvi components and %d disk blocks in %d batches checked; hvi restored "+ + "the guest paused", len(res.Components), res.Disk.Blocks, res.Disk.Batches) + if state, err := s.GetInstance(name); err == nil && state.Migration != nil { + state.Migration.ManifestSHA256 = rec.ManifestSHA256 + if err := s.SaveInstance(state); err != nil { + return refuseWith(codeFailed, rec.fail(path, err), rec) + } + } + // One write for the three: each is true once hvi has restored the + // guest from a stream hull checked whole. + if err := rec.advanceThrough(path, dstReceived, dstVerified, dstRestoredPaused); err != nil { + return refuseWith(codeFailed, err, rec) + } + return &migrateReply{OK: true, Record: rec, VM: store.MigrationRestoredPaused} +} diff --git a/cmd/hull/migrate_streamed_test.go b/cmd/hull/migrate_streamed_test.go new file mode 100644 index 00000000..8adf08b0 --- /dev/null +++ b/cmd/hull/migrate_streamed_test.go @@ -0,0 +1,985 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "io" + "net" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/brig-sh/hull/pkg/store" +) + +// fakeStreamHvi is a source hvi that speaks the streaming ops a streamed +// move drives, as hvi documents them, written here independently of hull's +// parser: disk-dirty and disk-blocks over a real disk file, and migrate, +// which streams RAM rounds, pauses, writes the state and the manifest and +// fences. Its "guest" writes disk blocks between the RAM rounds. It serves +// one connection at a time, as hvi does. +type fakeStreamHvi struct { + t *testing.T + path string + disk string + size int64 + + mu sync.Mutex + // state is running, fenced or stopped. + state string + diskOps bool + sinceBoot map[int64]bool + sinceRead map[int64]bool + round int + last []int64 + sealFail bool + // refuseFenced makes migrate refuse a fenced guest, as no hvi does + // today; hvi streams one again. + refuseFenced bool + // generation counts the guest's writes; rounds are the disk-dirty + // reads since the last since-boot one, as disk.json lists them. + generation uint64 + rounds []fakeDiskRound + // diskWritesAt are blocks the guest writes while hvi sends the disk + // blocks of a round, by round: they land in the next round. + diskWritesAt map[int][]int64 + // migrateReq is the last migrate request. + migrateReq map[string]any + writesAt map[string][]int64 + emitted []byte + ops []string + migrations int + // pid is the process standing in for this hvi, which stop ends. + pid int +} + +func newFakeStreamHvi(t *testing.T, disk string, diskOps bool) *fakeStreamHvi { + t.Helper() + info, err := os.Stat(disk) + if err != nil { + t.Fatal(err) + } + h := &fakeStreamHvi{t: t, path: filepath.Join(shortSocketDir(t), "c.sock"), disk: disk, size: info.Size(), + state: "running", diskOps: diskOps, sinceBoot: map[int64]bool{}, sinceRead: map[int64]bool{}, round: -1, + writesAt: map[string][]int64{}, diskWritesAt: map[int][]int64{}} + ln, err := net.Listen("unix", h.path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + h.serve(conn) + _ = conn.Close() + } + }() + return h +} + +// fakeDiskRound is one round as hvi records it in disk.json. +type fakeDiskRound struct { + Round int `json:"round"` + Generation uint64 `json:"generation"` + Blocks int `json:"blocks"` + BitmapSHA256 string `json:"bitmap_sha256"` + SentSHA256 *string `json:"sent_sha256"` +} + +// guestWrite has the guest write block index, as virtio-blk would: marked, +// then written. +func (h *fakeStreamHvi) guestWrite(index int64, b byte) { + h.generation++ + h.sinceBoot[index] = true + h.sinceRead[index] = true + f, err := os.OpenFile(h.disk, os.O_WRONLY, 0) + if err != nil { + h.t.Error(err) + return + } + defer func() { _ = f.Close() }() + n := min(diskBlockSize, h.size-index*diskBlockSize) + if _, err := f.WriteAt(bytes.Repeat([]byte{b}, int(n)), index*diskBlockSize); err != nil { + h.t.Error(err) + } +} + +func (h *fakeStreamHvi) serve(conn net.Conn) { + line, err := bufio.NewReader(conn).ReadBytes('\n') + if err != nil { + return + } + var req map[string]any + if json.Unmarshal(line, &req) != nil { + return + } + h.mu.Lock() + defer h.mu.Unlock() + op, _ := req["op"].(string) + h.ops = append(h.ops, op) + w := bufio.NewWriter(conn) + defer func() { _ = w.Flush() }() + reply := func(format string, args ...any) { _, _ = fmt.Fprintf(w, format+"\n", args...) } + switch op { + case "status": + reply(`{"v":1,"ok":true,"state":%q,"vcpus":1,"vcpus_parked":0}`, h.state) + case "profile": + reply(`{"v":1,"ok":true,"profile":{"profile_version":1,"checkpoint_format":2}}`) + case "disk-dirty": + if !h.diskOps { + reply(`{"v":1,"ok":false,"code":"unknown_op","error":"no op disk-dirty"}`) + return + } + var set map[int64]bool + switch req["since"] { + case "boot": + h.round, set = 0, h.sinceBoot + case "round": + if h.round < 0 { + reply(`{"v":1,"ok":false,"code":"refused","error":"no round 0 yet"}`) + return + } + h.round++ + set = h.sinceRead + } + h.sinceRead = map[int64]bool{} + h.last = h.last[:0] + blocks := (h.size + diskBlockSize - 1) / diskBlockSize + bits := make([]byte, (blocks+7)/8) + for b := range set { + bits[b/8] |= 1 << (b % 8) + h.last = append(h.last, b) + } + sort.Slice(h.last, func(i, j int) bool { return h.last[i] < h.last[j] }) + if h.round == 0 { + h.rounds = nil + } + bitmapSum := sha256.Sum256(bits) + h.rounds = append(h.rounds, fakeDiskRound{Round: h.round, Generation: h.generation, Blocks: len(set), + BitmapSHA256: hex.EncodeToString(bitmapSum[:])}) + reply(`{"v":1,"ok":true,"since":%q,"round":%d,"block_size":65536,"blocks":%d,"dirty":%d,"generation":%d,"bitmap_sha256":"%s","bitmap_b64":%q}`, + req["since"], h.round, blocks, len(set), h.generation, hex.EncodeToString(bitmapSum[:]), + base64.StdEncoding.EncodeToString(bits)) + case "disk-blocks": + if int(req["round"].(float64)) != h.round { + reply(`{"v":1,"ok":false,"code":"refused","error":"not the last round"}`) + return + } + f, err := os.Open(h.disk) + if err != nil { + h.t.Error(err) + return + } + defer func() { _ = f.Close() }() + batch, sent := 0, 0 + var digests strings.Builder + for i := 0; i < len(h.last); { + first, count := h.last[i], int64(1) + for i+int(count) < len(h.last) && h.last[i+int(count)] == first+count && count < 16 { + count++ + } + end := min((first+count)*diskBlockSize, h.size) + data := make([]byte, end-first*diskBlockSize) + _, _ = f.ReadAt(data, first*diskBlockSize) + sum := sha256.Sum256(data) + reply(`{"component":"disk-blocks","batch":%d,"first":%d,"count":%d,"offset":%d,"size":%d,"sha256":"%s"}`, + batch, first, count, first*diskBlockSize, len(data), hex.EncodeToString(sum[:])) + _, _ = w.Write(data) + digests.WriteString(hex.EncodeToString(sum[:])) + batch++ + sent += len(data) + i += int(count) + } + for _, b := range h.diskWritesAt[h.round] { + h.guestWrite(b, byte(0xd0+h.round)) + } + chain := sha256.Sum256([]byte(digests.String())) + sentSum := hex.EncodeToString(chain[:]) + if n := len(h.rounds); n > 0 && h.rounds[n-1].Round == h.round { + h.rounds[n-1].SentSHA256 = &sentSum + } + reply(`{"v":1,"ok":true,"round":%d,"blocks":%d,"bytes":%d,"batches":%d,"generation":%d,"max_lock_us":3,"sent_sha256":"%s"}`, + h.round, len(h.last), sent, batch, h.generation, sentSum) + case "migrate": + // hvi streams a fenced guest again, unchanged: it has not run since + // the stream that fenced it. refuseFenced stands for an hvi that + // would not. + if h.state == "fenced" && h.refuseFenced { + reply(`{"v":1,"ok":false,"code":"fenced","error":"the VM was fenced after a checkpoint"}`) + return + } + h.migrations++ + h.migrateReq = req + h.emitted = nil + // The manifest lists each component once, as hvi's does: its + // frames' bytes, their sha256 in order, and how many. + type listing struct { + name string + size int + frames int + h hash.Hash + } + var comps []*listing + // As hvi: a stream the client stops reading before the seal is a + // failed capture, and the guest runs on. + lost := false + emit := func(name string, body []byte) { + if lost { + return + } + sum := sha256.Sum256(body) + frame := fmt.Sprintf(`{"component":%q,"size":%d,"sha256":"%s"}`+"\n", name, len(body), hex.EncodeToString(sum[:])) + _, _ = w.WriteString(frame) + _, _ = w.Write(body) + h.emitted = append(h.emitted, frame...) + h.emitted = append(h.emitted, body...) + if name != "manifest.json" { + if n := len(comps); n == 0 || comps[n-1].name != name { + comps = append(comps, &listing{name: name, h: sha256.New()}) + } + c := comps[len(comps)-1] + c.size += len(body) + c.frames++ + c.h.Write(body) + } + if w.Flush() != nil { + lost = true + return + } + for _, b := range h.writesAt[name] { + h.guestWrite(b, byte(len(name)+h.migrations)) + } + } + emit("profile.json", []byte(`{"profile_version":1,"checkpoint_format":2}`)) + // hvi cuts a round into frames of about 1 MiB, all of one name. + for i := 0; i < 3; i++ { + emit("ram-round-0", bytes.Repeat([]byte{'p', byte(i)}, 48<<10)) + } + emit("ram-round-1", bytes.Repeat([]byte("dirt"), 4<<10)) + if h.sealFail { + reply(`{"v":1,"ok":false,"code":"capture_failed","error":"the capture failed; the VM runs on"}`) + return + } + emit("ram-round-2", bytes.Repeat([]byte("last"), 1<<10)) + for _, name := range []string{"cpu.json", "platform.json", "devices.json", "disk.json", "session.json"} { + doc := []byte(`{"version":1,"of":"` + name + `"}`) + if name == "disk.json" { + // As hvi captures it with the guest fenced: the + // generation and every round read so far. + doc, _ = json.Marshal(map[string]any{"version": 1, "generation": h.generation, + "rounds": append([]fakeDiskRound{}, h.rounds...)}) + } + emit(name, doc) + } + var listed []string + for _, c := range comps { + listed = append(listed, fmt.Sprintf(`{"name":%q,"size":%d,"sha256":"%s","frames":%d}`, c.name, c.size, + hex.EncodeToString(c.h.Sum(nil)), c.frames)) + } + man := []byte(`{"checkpoint_format":2,"components":[` + strings.Join(listed, ",") + `]}`) + emit("manifest.json", man) + if lost { + return + } + h.state = "fenced" + sum := sha256.Sum256(man) + now := time.Now().UnixMilli() + reply(`{"v":1,"ok":true,"state":"fenced","manifest_sha256":"%s","stats":{"pause_ms":4,`+ + `"precopy_end_unix_ms":%d,"sealed_at_unix_ms":%d}}`, hex.EncodeToString(sum[:]), now-7, now) + case "stop": + h.state = "stopped" + if h.pid > 0 { + _ = syscall.Kill(h.pid, syscall.SIGKILL) + } + reply(`{"v":1,"ok":true,"state":"stopping"}`) + default: + reply(`{"v":1,"ok":false,"code":"unknown_op","error":"?"}`) + } +} + +// fakeDestRestore stands in for `hvi restore --from-stream` on the +// destination: it reads hvi's framing from its stdin, checking every +// component's sha256 as hvi would, and once its input ends with a manifest +// it reports the guest restored paused, as the bundle fixture's restore +// does. failNext makes the next restore refuse after reading its input. +type fakeDestRestore struct { + mu sync.Mutex + got [][]byte + failNext int + restores int + // refuseProfile makes the next restore stop reading after + // profile.json, as hvi does when the machine is not one it can build. + refuseProfile bool +} + +func (r *fakeDestRestore) install(t *testing.T) { + prev := destStreamRestore + t.Cleanup(func() { destStreamRestore = prev }) + destStreamRestore = func(s *store.Store, name, txid string) (*streamRestore, error) { + pr, pw := io.Pipe() + done := make(chan error, 1) + r.mu.Lock() + refuse := r.refuseProfile + r.refuseProfile = false + r.mu.Unlock() + go func() { + if refuse { + line, _ := bufio.NewReader(pr).ReadBytes('\n') + _ = pr.CloseWithError(errors.New("closed")) + done <- fmt.Errorf("hvi refused the profile %s: the host is not compatible", bytes.TrimSpace(line)) + return + } + data, err := io.ReadAll(pr) + if err == nil { + err = checkHviFraming(data) + } + r.mu.Lock() + r.got = append(r.got, data) + r.mu.Unlock() + done <- err + }() + return &streamRestore{ + in: pw, + wait: func() error { + if err := <-done; err != nil { + return err + } + r.mu.Lock() + fail := r.failNext > 0 + if fail { + r.failNext-- + } + r.restores++ + r.mu.Unlock() + if fail { + return errors.New("hvi refused the restore") + } + return fakeRestoredPaused(t, s, name, txid) + }, + kill: func() { _ = pw.CloseWithError(errors.New("killed")) }, + }, nil + } +} + +// checkHviFraming reads hvi's framing as hvi would and checks each +// component, and that the manifest comes last. +func checkHviFraming(data []byte) error { + br := bufio.NewReader(bytes.NewReader(data)) + last := "" + for { + line, err := br.ReadBytes('\n') + if errors.Is(err, io.EOF) && len(line) == 0 { + break + } + if err != nil { + return fmt.Errorf("hvi's stream ended in a header: %w", err) + } + var c struct { + Component string `json:"component"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + } + if err := json.Unmarshal(line, &c); err != nil { + return err + } + body := make([]byte, c.Size) + if _, err := io.ReadFull(br, body); err != nil { + return fmt.Errorf("%s: %w", c.Component, err) + } + if sum := sha256.Sum256(body); hex.EncodeToString(sum[:]) != c.SHA256 { + return fmt.Errorf("%s arrived corrupted", c.Component) + } + last = c.Component + } + if last != "manifest.json" { + return fmt.Errorf("hvi's stream ends with %q, not the manifest", last) + } + return nil +} + +// fakeRestoredPaused records the instance as an hvi restore that reached +// restored-paused would: a process named hvi with a control socket. +func fakeRestoredPaused(t *testing.T, s *store.Store, name, txid string) error { + var mu sync.Mutex + resumed := false + stub := newStubHvi(t, func(req map[string]any) stubAnswer { + mu.Lock() + defer mu.Unlock() + switch req["op"] { + case "resume": + resumed = true + return stubAnswer{reply: `{"v":1,"ok":true,"state":"running"}`} + case "status": + if resumed { + return stubAnswer{reply: `{"v":1,"ok":true,"state":"running"}`} + } + return stubAnswer{reply: `{"v":1,"ok":true,"state":"restored-paused"}`} + case "stop": + return stubAnswer{reply: `{"v":1,"ok":true,"state":"stopping"}`} + } + return stubAnswer{reply: `{"v":1,"ok":false,"code":"unknown_op","error":"?"}`} + }) + pid, _ := startProcessAs(t, "hvi", "/bin/sleep", "60") + st, err := s.GetInstance(name) + if err != nil { + return err + } + if err := checkMoveRestore(s, st, hviRestoreOptions{move: txid, paused: true}); err != nil { + return err + } + st.Status, st.PID, st.ControlSock = "running", pid, stub.path + st.Migration.State = store.MigrationRestoredPaused + return s.SaveInstance(st) +} + +// streamFixture is the move fixture with a source that streams: an hvi of +// checkpoint format 2 behind the fake above, a base kept before the guest +// wrote, and a guest that wrote two blocks since its boot. +type streamFixture struct { + *moveFixture + hvi *fakeStreamHvi + dest *fakeDestRestore + peer *localPeer +} + +func newStreamFixture(t *testing.T, diskOps, fromBoot bool) *streamFixture { + t.Helper() + f := newMoveFixture(t, true) + sum, err := keepBase(f.src, f.disk) + if err != nil { + t.Fatal(err) + } + h := newFakeStreamHvi(t, f.disk, diskOps) + h.pid = f.vm.pid + // Written since the boot: one where the base has data, one in a hole, + // and one zeroed where the base had data. + h.guestWrite(0, 0xa1) + h.guestWrite(700, 0xa2) + h.guestWrite(48, 0) + state := mustInstance(t, f.src, f.name) + state.ControlSock = h.path + state.ProfilePath = filepath.Join(f.src.InstanceDir(f.name), "profile.json") + if err := os.WriteFile(state.ProfilePath, []byte(`{"profile_version":1,"checkpoint_format":2}`), 0o600); err != nil { + t.Fatal(err) + } + state.Base = &store.BaseState{SHA256: sum, FromBoot: fromBoot} + if err := f.src.SaveInstance(state); err != nil { + t.Fatal(err) + } + dest := &fakeDestRestore{} + dest.install(t) + return &streamFixture{moveFixture: f, hvi: h, dest: dest, peer: newLocalPeer(f.dst)} +} + +func (f *streamFixture) migrate(t *testing.T, txid string) error { + t.Helper() + opt := f.options(false) + opt.txid = txid + return sourceMigrate(context.Background(), f.src, f.name, opt, f.peer) +} + +// checkArrived checks what the destination ended with: the source's disk, +// byte for byte, and the components hvi emitted, as it emitted them. +func (f *streamFixture) checkArrived(t *testing.T, rec *migrationRecord) { + t.Helper() + assertSameContent(t, f.disk, filepath.Join(f.dst.InstanceDir(f.name), "bundle", "rootfs.ext4")) + f.dest.mu.Lock() + got := f.dest.got[len(f.dest.got)-1] + f.dest.mu.Unlock() + f.hvi.mu.Lock() + emitted := f.hvi.emitted + f.hvi.mu.Unlock() + if !bytes.Equal(got, emitted) { + t.Errorf("the destination's hvi read %d bytes, the source's hvi wrote %d", len(got), len(emitted)) + } + dst := f.dstRecord(t, rec.TxID) + if dst.Transport != transportStream || dst.Stream == nil || dst.Stream.Chain != rec.Stream.Chain { + t.Errorf("destination record %+v", dst) + } +} + +// A streamed move with a fake hvi on both ends: the base goes first, the +// disk rounds carry what the guest wrote since boot and during the RAM +// rounds, hvi's stream arrives whole at the destination's hvi, and the move +// commits with its numbers on record. +func TestStreamedMove(t *testing.T) { + f := newStreamFixture(t, true, true) + // The guest writes during the RAM rounds: these land in the last disk + // round, read after hvi fenced it. + f.hvi.writesAt["ram-round-0"] = []int64{1, 2, 900} + if err := f.migrate(t, ""); err != nil { + t.Fatal(err) + } + rec := f.srcRecord(t) + if rec.State != srcCommitted || rec.Transport != transportStream || rec.Stream == nil { + t.Fatalf("source record %+v", rec) + } + st := rec.Stream + if st.DiskEngine != "hvi" || st.Mode != "live" { + t.Errorf("engine %s mode %s", st.DiskEngine, st.Mode) + } + var final *streamRound + ram := 0 + for i, r := range st.Rounds { + switch { + case r.Kind == "disk" && r.Final: + final = &st.Rounds[i] + case r.Kind == "ram": + ram++ + } + } + if ram != 3 || final == nil || final.Blocks != 3 || st.Rounds[0].Blocks != 3 { + t.Errorf("rounds %+v", st.Rounds) + } + if st.PausedAt.IsZero() || st.SealedMs != 7 || st.DowntimeMs <= 0 || st.RAMBytes != 3*96<<10+16<<10+4<<10 || + st.WireBytes == 0 { + t.Errorf("numbers %+v", st) + } + if !baseHeld(f.dst, rec.BaseSHA256) || f.peer.called("migrate-take-base") != 1 { + t.Errorf("the base was not sent first: held %v", baseHeld(f.dst, rec.BaseSHA256)) + } + f.checkArrived(t, rec) + if f.peer.called("migrate-prepare") != 0 { + t.Error("the source asked for a restore the streamed receive had done") + } + // The source keeps the stream it sent, sealed, for an abort that + // never came, and the destination keeps hvi's small documents. + for _, name := range []string{spoolStream, spoolSealed, "manifest.json"} { + if _, err := os.Stat(filepath.Join(rec.BundleDir, name)); err != nil { + t.Errorf("source spool: %v", err) + } + } + dst := f.dstRecord(t, rec.TxID) + if _, err := os.Stat(filepath.Join(dst.BundleDir, "cpu.json")); err != nil { + t.Errorf("destination documents: %v", err) + } + if dst.State != dstRunning { + t.Errorf("destination %s", dst.State) + } + state := mustInstance(t, f.dst, f.name) + if state.Base == nil || state.Base.SHA256 != rec.BaseSHA256 || state.Base.FromBoot || + !isStreamRestore(state.CmdLine) { + t.Errorf("destination instance %+v %v", state.Base, state.CmdLine) + } + if _, err := hviRestoreArgv(state.CmdLine, "/b", "/c", "/p"); err != nil { + t.Errorf("the moved instance cannot be checkpointed and moved on: %v", err) + } +} + +// Without hvi's dirty-block rounds, or when the running hvi did not boot +// from the base and does not say it carries the base, hull compares the +// disk with the base instead; a restored hvi whose profile names the base +// carries its blocks since boot, and its rounds are used. Either way the +// destination ends with the source's disk. +func TestStreamedMoveComparesWithoutHviRounds(t *testing.T) { + for _, tc := range []struct { + name string + diskOps, fromBoot bool + carries bool + engine string + }{ + {"no disk ops", false, true, false, "compare"}, + {"restored hvi", true, false, false, "compare"}, + {"restored hvi carrying the base", true, false, true, "hvi"}, + {"files written before boot", true, false, false, "compare"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newStreamFixture(t, tc.diskOps, tc.fromBoot) + switch tc.name { + case "restored hvi": + // hvi restored the guest and, its profile naming no + // base, started its since-boot blocks afresh: the + // writes before the restore are only on the disk. + f.hvi.mu.Lock() + f.hvi.sinceBoot = map[int64]bool{} + f.hvi.mu.Unlock() + case "files written before boot": + // As `hull run` writes an instance's own files over its + // image's base before hvi boots: on the disk, in no + // bitmap of hvi's. + f.hvi.mu.Lock() + f.hvi.sinceBoot = map[int64]bool{} + f.hvi.mu.Unlock() + fh, err := os.OpenFile(f.disk, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := fh.WriteAt(bytes.Repeat([]byte{0x5e}, 4096), 9*diskBlockSize); err != nil { + t.Fatal(err) + } + _ = fh.Close() + } + if tc.carries { + st := mustInstance(t, f.src, f.name) + prof := fmt.Sprintf(`{"profile_version":1,"checkpoint_format":2,"disk_base_sha256":%q}`, st.Base.SHA256) + if err := os.WriteFile(st.ProfilePath, []byte(prof), 0o600); err != nil { + t.Fatal(err) + } + } + f.hvi.writesAt["ram-round-1"] = []int64{3, 701} + if err := f.migrate(t, ""); err != nil { + t.Fatal(err) + } + rec := f.srcRecord(t) + if rec.State != srcCommitted || rec.Stream.DiskEngine != tc.engine { + t.Fatalf("record %s engine %s", rec.State, rec.Stream.DiskEngine) + } + f.checkArrived(t, rec) + }) + } +} + +// A capture that fails before hvi seals its stream leaves the guest running +// here: the move stops in transferring, not fenced, and run again it +// streams from the start and commits. +func TestStreamedMoveCaptureFailsBeforeTheSeal(t *testing.T) { + f := newStreamFixture(t, true, true) + f.hvi.sealFail = true + err := f.migrate(t, "") + if err == nil || !strings.Contains(err.Error(), "capture_failed") || !strings.Contains(err.Error(), "still runs here") { + t.Fatalf("err = %v", err) + } + rec := f.srcRecord(t) + if rec.State != srcTransferring || fencedFor(mustInstance(t, f.src, f.name), rec.TxID) { + t.Fatalf("after the failed capture: %s", rec.State) + } + if dst := f.dstRecord(t, rec.TxID); dst.State != dstReceiving || dst.Error == "" { + t.Errorf("destination %s %q", dst.State, dst.Error) + } + f.hvi.sealFail = false + if err := f.migrate(t, rec.TxID); err != nil { + t.Fatal(err) + } + rec = f.srcRecord(t) + if rec.State != srcCommitted || rec.Stream.Mode != "live" { + t.Fatalf("the retry: %s %s", rec.State, rec.Stream.Mode) + } + f.checkArrived(t, rec) +} + +// damageAt returns a mangle that flips one byte at offset off of a verb's +// stdin, or, with cut, ends it there. +func damageAt(verb string, off int64, cut bool, times *int) func(string, []string, io.Reader) io.Reader { + return func(v string, args []string, in io.Reader) io.Reader { + if v != verb || *times == 0 || (verb == "migrate-receive" && !slices.Contains(args, "--stream")) { + return in + } + *times-- + if cut { + return io.LimitReader(in, off) + } + return &flipReader{r: in, at: off} + } +} + +type flipReader struct { + r io.Reader + at int64 + pos int64 +} + +func (f *flipReader) Read(p []byte) (int, error) { + n, err := f.r.Read(p) + if f.at >= f.pos && f.at < f.pos+int64(n) { + p[f.at-f.pos] ^= 0x20 + } + f.pos += int64(n) + return n, err +} + +// A stream damaged or cut on the way, after hvi sealed it, is refused by +// the destination, which keeps nothing running. The source is fenced with +// its spooled copy: run again, the move replays the spool, compares the +// disk with the base, and commits. +func TestStreamedMoveRefusedInTransitThenReplayed(t *testing.T) { + for _, tc := range []struct { + name string + cut bool + want string + }{{"corrupted", false, "corrupted"}, {"truncated", true, "ended"}} { + t.Run(tc.name, func(t *testing.T) { + f := newStreamFixture(t, true, true) + // Into the RAM rounds: past the header and the first disk + // round (three blocks, compressible), well before the end. + times := 1 + f.peer.mangle = damageAt("migrate-receive", 120<<10, tc.cut, ×) + err := f.migrate(t, "") + if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "stays fenced") { + t.Fatalf("err = %v", err) + } + rec := f.srcRecord(t) + state := mustInstance(t, f.src, f.name) + if rec.State != srcTransferring || !fencedFor(state, rec.TxID) { + t.Fatalf("source %s, instance %+v", rec.State, state.Migration) + } + dst := f.dstRecord(t, rec.TxID) + if dst.State != dstReceiving || f.dest.restores != 0 { + t.Fatalf("destination %s, restores %d", dst.State, f.dest.restores) + } + if err := f.migrate(t, rec.TxID); err != nil { + t.Fatal(err) + } + rec = f.srcRecord(t) + if rec.State != srcCommitted || rec.Stream.Mode != "replay" || rec.Stream.DiskEngine != "compare" { + t.Fatalf("the replay: %s %+v", rec.State, rec.Stream) + } + f.checkArrived(t, rec) + if f.hvi.migrations != 1 { + t.Errorf("hvi ran migrate %d times; the replay must not ask a fenced hvi again", f.hvi.migrations) + } + }) + } +} + +// A destination whose hvi refuses the restore after the seal leaves the +// source fenced; the abort brings the guest back from the spooled stream. +func TestStreamedMoveAbortRestoresFromTheSpool(t *testing.T) { + f := newStreamFixture(t, true, true) + f.dest.failNext = 1 + var restoredFrom *os.File + localRestore = func(s *store.Store, name string) error { + st, err := s.GetInstance(name) + if err != nil { + return err + } + restoredFrom, err = restoreStreamSource(st, hviRestoreOptions{detach: true}) + return err + } + err := f.migrate(t, "") + if err == nil || !strings.Contains(err.Error(), "refused the restore") { + t.Fatalf("err = %v", err) + } + rec := f.srcRecord(t) + if err := sourceAbort(context.Background(), f.src, f.name, io.Discard, f.peer); err != nil { + t.Fatal(err) + } + if restoredFrom == nil || restoredFrom.Name() != filepath.Join(rec.BundleDir, spoolStream) { + t.Fatalf("the abort restored from %v", restoredFrom) + } + _ = restoredFrom.Close() + state := mustInstance(t, f.src, f.name) + if state.Migration.State != store.MigrationFenced || state.Migration.BundleDir != rec.BundleDir { + t.Errorf("instance %+v", state.Migration) + } + if _, err := os.Stat(f.dst.InstanceDir(f.name)); !os.IsNotExist(err) { + t.Errorf("the destination kept the instance: %v", err) + } + argv := streamRestoreArgv([]string{"hvi", "restore", "--from", rec.BundleDir, "--disk", "/d"}) + if strings.Join(argv, " ") != "hvi restore --from-stream - --disk /d" { + t.Errorf("restore argv %v", argv) + } +} + +// A move that never got as far as hvi's seal aborts without touching the +// guest, which still runs here. +func TestStreamedMoveAbortBeforeTheSeal(t *testing.T) { + f := newStreamFixture(t, true, true) + f.hvi.sealFail = true + if err := f.migrate(t, ""); err == nil { + t.Fatal("the move went through") + } + restored := false + localRestore = func(*store.Store, string) error { restored = true; return nil } + var out bytes.Buffer + if err := sourceAbort(context.Background(), f.src, f.name, &out, f.peer); err != nil { + t.Fatal(err) + } + if restored || !strings.Contains(out.String(), "never stopped running here") { + t.Errorf("restored %v; %s", restored, out.String()) + } + if f.hvi.state != "running" || syscall.Kill(f.vm.pid, 0) != nil { + t.Errorf("the guest is %s", f.hvi.state) + } +} + +// The transport is decided from what both hosts can do, and the reason is +// said when it is the whole bundle. +func TestDecideTransport(t *testing.T) { + all := transportFacts{SourceFormat: 2, HasBase: true, DestStream: true, DestHviStream: true} + if got, why := decideTransport(all); got != transportStream || why != "" { + t.Fatalf("all there: %q %q", got, why) + } + for _, tc := range []struct { + mut func(*transportFacts) + want string + }{ + {func(f *transportFacts) { f.WholeBundle = true }, "--whole-bundle"}, + {func(f *transportFacts) { f.DestStream = false; f.DestHull = "v0.9" }, "hull there (v0.9) takes no streamed"}, + {func(f *transportFacts) { f.DestHviStream = false }, "cannot restore from a stream"}, + {func(f *transportFacts) { f.SourceFormat = 1 }, "checkpoint format 1"}, + {func(f *transportFacts) { f.SourceFormat = 0 }, "checkpoint format 0"}, + {func(f *transportFacts) { f.HasBase = false; f.BaseReason = "no base kept" }, "no base kept"}, + } { + f := all + tc.mut(&f) + got, why := decideTransport(f) + if got != transportBundle || !strings.Contains(why, tc.want) { + t.Errorf("%+v: %q %q, want the bundle because %q", f, got, why, tc.want) + } + } +} + +// The dry run of a streamed move says how it would go; one to a hull from +// before streamed moves says it would send the whole bundle, and why. +func TestStreamedMoveDryRun(t *testing.T) { + f := newStreamFixture(t, true, true) + var out bytes.Buffer + opt := f.options(false) + opt.dryRun, opt.out = true, &out + if err := sourceMigrate(context.Background(), f.src, f.name, opt, f.peer); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "transport: streamed") || !strings.Contains(out.String(), "base is sent first") { + t.Errorf("dry run: %s", out.String()) + } + probe := destProbe(f.dst, f.name, f.moveFixture.hvi, "") + probe.Probe.Capabilities = nil + f.peer.answer["migrate-probe"] = probe + out.Reset() + if err := sourceMigrate(context.Background(), f.src, f.name, opt, f.peer); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "takes no streamed move") { + t.Errorf("dry run to an old hull: %s", out.String()) + } +} + +// A destination hvi that refuses the machine stops reading, and its +// refusal is the move's reason. The source still reads hvi's stream to its +// end into the spool: hvi may seal it, and fence the guest, from what sits +// in the socket's buffer, and a fenced source must have the whole stream +// to restore from. Here hvi seals; the abort brings the guest back from +// the spool. +func TestStreamedMoveDestinationHviRefuses(t *testing.T) { + f := newStreamFixture(t, true, true) + f.dest.refuseProfile = true + err := f.migrate(t, "") + if err == nil || !strings.Contains(err.Error(), "the host is not compatible") || + !strings.Contains(err.Error(), codeRestoreRefused) || !strings.Contains(err.Error(), "stays fenced") { + t.Fatalf("err = %v", err) + } + rec := f.srcRecord(t) + if !fencedFor(mustInstance(t, f.src, f.name), rec.TxID) { + t.Fatal("the fence hvi sealed was not recorded") + } + if err := checkHviFramingFile(filepath.Join(rec.BundleDir, spoolStream)); err != nil { + t.Fatalf("the spool: %v", err) + } + restored := false + localRestore = func(*store.Store, string) error { restored = true; return nil } + if err := sourceAbort(context.Background(), f.src, f.name, io.Discard, f.peer); err != nil || !restored { + t.Fatalf("abort: %v, restored %v", err, restored) + } +} + +// checkHviFramingFile is checkHviFraming of a file. +func checkHviFramingFile(path string) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + return checkHviFraming(data) +} + +// A guest that writes to its disk while the disk rounds run: each round +// carries what was written during the one before, the rounds go on until +// one is small, and the last, read after the fence, holds only what the +// guest wrote during hvi's RAM rounds. +func TestStreamedMoveDiskRoundsFollowTheGuestsWrites(t *testing.T) { + f := newStreamFixture(t, true, true) + // 40 blocks during round 0 (more than a round may settle at), then + // 3 during round 1, then none. + for b := int64(100); b < 140; b++ { + f.hvi.diskWritesAt[0] = append(f.hvi.diskWritesAt[0], b) + } + f.hvi.diskWritesAt[1] = []int64{200, 201, 202} + f.hvi.writesAt["ram-round-1"] = []int64{300} + if err := f.migrate(t, ""); err != nil { + t.Fatal(err) + } + rec := f.srcRecord(t) + st := rec.Stream + var blocks []int64 + for _, r := range st.Rounds { + if r.Kind == "disk" { + blocks = append(blocks, r.Blocks) + } + } + if fmt.Sprint(blocks) != "[3 40 3 1]" { + t.Errorf("disk rounds of %v blocks", blocks) + } + if st.DiskWriteRate <= 0 { + t.Errorf("no disk write rate on record: %+v", st) + } + if _, ok := f.hvi.migrateReq["max_secs"]; ok { + t.Errorf("hvi's pre-copy was limited: %v", f.hvi.migrateReq) + } + if st.FinalDiskBytes != diskBlockSize { + t.Errorf("final disk round %d bytes", st.FinalDiskBytes) + } + f.checkArrived(t, rec) +} + +// A destination that refuses the stream's header (here: it lost the base +// after the source checked) refuses it while the guest still runs: hvi's +// migrate never starts, the source sends the base, and the move goes on +// live, by itself. +func TestStreamedMoveSendsTheBaseOnNoBase(t *testing.T) { + f := newStreamFixture(t, true, true) + st := mustInstance(t, f.src, f.name) + removed := 0 + f.peer.mangle = func(verb string, args []string, in io.Reader) io.Reader { + if verb == "migrate-receive" && slices.Contains(args, "--stream") && removed == 0 { + removed++ + if err := os.Remove(f.dst.BasePath(st.Base.SHA256)); err != nil { + t.Error(err) + } + } + return in + } + if err := f.migrate(t, ""); err != nil { + t.Fatal(err) + } + rec := f.srcRecord(t) + if rec.State != srcCommitted || rec.Stream.Mode != "live" { + t.Fatalf("source %s, stream %+v", rec.State, rec.Stream) + } + if f.hvi.migrations != 1 || removed != 1 { + t.Errorf("hvi streamed %d times, the base was removed %d times", f.hvi.migrations, removed) + } + if n := f.peer.called("migrate-take-base"); n != 2 { + t.Errorf("the base was sent %d times, want 2: %v", n, f.peer.calls) + } + f.checkArrived(t, rec) +} diff --git a/cmd/hull/migrate_streamsrc.go b/cmd/hull/migrate_streamsrc.go new file mode 100644 index 00000000..218e0800 --- /dev/null +++ b/cmd/hull/migrate_streamsrc.go @@ -0,0 +1,1286 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/brig-sh/hull/pkg/store" +) + +// The source's half of a streamed move (docs/migration.md, "The streamed +// move"): one ssh session to `migrate-receive --stream`, whose stdin +// carries, in order, the disk blocks that differ from the base (round 0, +// then a few rounds of what the guest wrote meanwhile), hvi's migrate +// stream (RAM rounds while the guest runs, then the pause, the last RAM +// round, the machine's state and the manifest, after which hvi has fenced +// the guest), the disk blocks the guest wrote during the RAM rounds, and +// the trailer. +// +// hvi serves one control connection at a time, so the disk rounds hull +// asks for cannot run while the migrate op holds the socket: the last disk +// round is read once hvi has sealed the stream. The guest is fenced by +// then, and its disk synced, so that round and the RAM describe the same +// machine. + +// Disk pre-copy policy: stop when a round finds at most this many blocks +// (2 MiB), or after streamRoundCap rounds. +const ( + streamDiskSettle = 32 + streamRoundCap = 5 +) + +// blockHash is what one disk block holds, as far as a comparison needs. +type blockHash struct { + zero bool + sum [32]byte +} + +// diskCompare finds the blocks of a disk that differ from what the +// destination holds: its copy of the base, plus every block sent since. +// It is the disk engine when hvi has no dirty-block rounds, or when the +// running hvi did not boot from the base, and it reads the whole disk on +// every pass. +type diskCompare struct { + size int64 + base map[int64][32]byte + sent map[int64]blockHash +} + +// newDiskCompare hashes the base's blocks. +func newDiskCompare(basePath string, size int64) (*diskCompare, error) { + f, err := os.Open(basePath) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + info, err := f.Stat() + if err != nil { + return nil, err + } + if info.Size() != size { + return nil, fmt.Errorf("the base is %d bytes and the disk %d", info.Size(), size) + } + c := &diskCompare{size: size, base: map[int64][32]byte{}, sent: map[int64]blockHash{}} + err = forEachDataBlock(f, size, func(index int64, data []byte) error { + c.base[index] = sha256.Sum256(data) + return nil + }) + return c, err +} + +// held is what the destination holds at block index. +func (c *diskCompare) held(index int64) blockHash { + if h, ok := c.sent[index]; ok { + return h + } + if sum, ok := c.base[index]; ok { + return blockHash{sum: sum} + } + return blockHash{zero: true} +} + +// diff calls emit for every block of disk that differs from what the +// destination holds, in index order, and records it as sent. +func (c *diskCompare) diff(disk *os.File, emit func(index int64, data []byte) error) error { + seen := map[int64]bool{} + err := forEachDataBlock(disk, c.size, func(index int64, data []byte) error { + seen[index] = true + h := blockHash{sum: sha256.Sum256(data)} + if c.held(index) == h { + return nil + } + c.sent[index] = h + return emit(index, data) + }) + if err != nil { + return err + } + // Blocks that are zero on the disk now but hold data there. + var zeroed []int64 + for index := range c.base { + if !seen[index] && !c.held(index).zero { + zeroed = append(zeroed, index) + } + } + for index, h := range c.sent { + if !seen[index] && !h.zero { + if _, inBase := c.base[index]; !inBase { + zeroed = append(zeroed, index) + } + } + } + sortInt64(zeroed) + zero := make([]byte, diskBlockSize) + for _, index := range zeroed { + c.sent[index] = blockHash{zero: true} + if err := emit(index, zero[:blockLen(c.size, index)]); err != nil { + return err + } + } + return nil +} + +// sortInt64 sorts s ascending. +func sortInt64(s []int64) { + for i := 1; i < len(s); i++ { + for j := i; j > 0 && s[j] < s[j-1]; j-- { + s[j], s[j-1] = s[j-1], s[j] + } + } +} + +// batcher gathers adjacent blocks into disk batches of up to 16. +type batcher struct { + size int64 + first int64 + count int64 + buf []byte + flush func(first, count int64, data []byte) error +} + +func newBatcher(size int64, flush func(first, count int64, data []byte) error) *batcher { + return &batcher{size: size, buf: make([]byte, 0, 16*diskBlockSize), flush: flush} +} + +func (b *batcher) add(index int64, data []byte) error { + if b.count > 0 && (index != b.first+b.count || b.count == 16) { + if err := b.done(); err != nil { + return err + } + } + if b.count == 0 { + b.first = index + } + b.buf = append(b.buf, data...) + b.count++ + return nil +} + +func (b *batcher) done() error { + if b.count == 0 { + return nil + } + err := b.flush(b.first, b.count, b.buf) + b.buf, b.count = b.buf[:0], 0 + return err +} + +// streamSender writes one streamed move into sw. +type streamSender struct { + m *moveSource + state *store.InstanceState + sw *moveStreamWriter + disk *os.File + size int64 + stats *migrateStreamStats + // hviDisk is set while hvi answers disk-dirty; cmp is the compare + // engine, when it is the one in use. + hviDisk bool + cmp *diskCompare + // lastRound is the round hvi numbered the last disk-dirty read. + lastRound int + // spoolDir is where hvi's stream is kept on this host. + spoolDir string + // manifest is hvi's manifest.json, once relayed. + manifest []byte + // roundTotal is the blocks of the disk round being sent, when known. + roundTotal int64 + // link is what sw writes to; its error is the destination's failure. + link *softLink + // diskRounds gather each disk round's batches for the trailer. + diskRounds []*diskRoundDigest + // sealDone reports the spool's seal, which runs on behind the move + // once the stream is out; recorded the record of the fence, which the + // commit waits for, and recordedAt when it was made. + sealDone chan error + recorded chan error + recordedAt time.Time + // sealGate opens when the spool's seal may sync it. + sealGate chan struct{} + // ctx is the move's; attempt names this attempt, and callDone is + // closed once the receive's ssh session has ended. + ctx context.Context + attempt string + callDone chan struct{} +} + +func (ss *streamSender) progress(format string, args ...any) { + ss.m.logf("move: "+format, args...) +} + +// beginDiskRound starts the trailer's account of a disk round: from hvi's +// dirty blocks when dirty is hvi's reply that started it, else found by +// comparison. +func (ss *streamSender) beginDiskRound(round int, final bool, dirty *hviDiskDirtyReply) *diskRoundDigest { + d := &diskRoundDigest{round: streamDiskRound{Round: round, Final: final, From: "compare"}} + if dirty != nil { + d.round.From, d.round.Generation, d.round.BitmapSHA256 = "hvi", dirty.Generation, dirty.BitmapSHA256 + } + ss.diskRounds = append(ss.diskRounds, d) + return d +} + +// diskTrailer is the trailer's account of the disk rounds sent. +func (ss *streamSender) diskTrailer() *streamDiskRounds { + t := &streamDiskRounds{Engine: ss.stats.DiskEngine, Rounds: []streamDiskRound{}} + for _, d := range ss.diskRounds { + t.Rounds = append(t.Rounds, d.done()) + } + return t +} + +// sendBatch writes one disk batch and counts it in round r, of total +// blocks when that is known (0 when not), and in the trailer's account d. +func (ss *streamSender) sendBatch(r *streamRound, d *diskRoundDigest, round int, + final bool) func(first, count int64, data []byte) error { + started := time.Now() + last := started + return func(first, count int64, data []byte) error { + if err := ss.link.err; err != nil { + return err + } + wire, sum, err := ss.sw.disk(round, final, first, count, data) + if err != nil { + return err + } + d.add(count, sum) + r.Blocks += count + r.Bytes += int64(len(data)) + r.WireBytes += wire + if now := time.Now(); now.Sub(last) >= progressEvery { + last = now + line := fmt.Sprintf("%s: %d blocks so far, %s/s", r.Name, r.Blocks, + humanBytes(perSec(r.WireBytes, now.Sub(started).Seconds()))) + if ss.roundTotal > 0 { + line += fmt.Sprintf(" (%d%% of %d)", r.Blocks*100/ss.roundTotal, ss.roundTotal) + } + ss.progress("%s", line) + } + return nil + } +} + +// endRound records a round and prints it. +func (ss *streamSender) endRound(r streamRound, started time.Time) { + r.Seconds = time.Since(started).Seconds() + ss.stats.Rounds = append(ss.stats.Rounds, r) + switch r.Kind { + case "disk": + ss.stats.DiskBytes += r.Bytes + ss.stats.DiskWireBytes += r.WireBytes + line := fmt.Sprintf("%s: %d blocks, %s (%s on the wire) in %.2f s, %s/s", r.Name, r.Blocks, + humanBytes(r.Bytes), humanBytes(r.WireBytes), r.Seconds, humanBytes(perSec(r.WireBytes, r.Seconds))) + if r.DirtyPerSec > 0 { + line += fmt.Sprintf(", guest wrote %.0f blocks/s", r.DirtyPerSec) + } + ss.progress("%s", line) + case "ram": + ss.stats.RAMBytes += r.Bytes + ss.progress("%s: %s in %.2f s, %s/s", r.Name, humanBytes(r.Bytes), r.Seconds, + humanBytes(perSec(r.Bytes, r.Seconds))) + default: + ss.stats.StateBytes += r.Bytes + } +} + +// perSec is n per second over secs. +func perSec(n int64, secs float64) int64 { return int64(float64(n) / max(secs, 1e-3)) } + +// diskPreCopy sends the disk rounds that run while the guest does: round 0, +// every block that differs from the base, then the blocks written since, +// until a round is small or the cap is reached. +func (ss *streamSender) diskPreCopy(basePath string) error { + started := time.Now() + r := streamRound{Kind: "disk", Name: "disk round 0"} + read, err := ss.diskDirty("boot") + switch { + case err == nil: + ss.hviDisk = true + ss.lastRound = read.Round + case hviLacksOp(err): + default: + return fmt.Errorf("hvi's disk-dirty: %w", err) + } + fromBoot := ss.state.Base != nil && (ss.state.Base.FromBoot || hviCarriesBase(ss.state, ss.state.Base.SHA256)) + if ss.hviDisk && fromBoot { + // hvi's blocks since boot are every difference from the base. + ss.stats.DiskEngine = "hvi" + if err := ss.hviRoundBlocks(&r, read, false); err != nil { + return err + } + } else { + // The running hvi did not boot from the base (it was restored), + // or keeps no rounds: compare. With hvi's rounds, the read above + // started round 0, so whatever the guest writes from here is in + // the next round, the comparison's torn reads included. + ss.stats.DiskEngine = "compare" + cmp, err := newDiskCompare(basePath, ss.size) + if err != nil { + return fmt.Errorf("read the base: %w", err) + } + ss.cmp = cmp + b := newBatcher(ss.size, ss.sendBatch(&r, ss.beginDiskRound(0, false, nil), 0, false)) + if err := cmp.diff(ss.disk, b.add); err != nil { + return err + } + if err := b.done(); err != nil { + return err + } + } + ss.endRound(r, started) + if !ss.hviDisk { + return ss.sw.flush() + } + prev := time.Since(started).Seconds() + for round := 1; round < streamRoundCap; round++ { + started := time.Now() + read, err := ss.diskDirty("round") + if err != nil { + return fmt.Errorf("hvi's disk-dirty: %w", err) + } + ss.lastRound = read.Round + r := streamRound{Kind: "disk", Name: fmt.Sprintf("disk round %d", read.Round), + DirtyPerSec: float64(read.Dirty) / max(prev, 1e-3)} + if err := ss.hviRoundBlocks(&r, read, false); err != nil { + return err + } + ss.endRound(r, started) + prev = time.Since(started).Seconds() + ss.stats.DiskWriteRate = max(ss.stats.DiskWriteRate, r.DirtyPerSec*diskBlockSize) + if read.Dirty <= streamDiskSettle { + break + } + } + return ss.sw.flush() +} + +// hviCarriesBase reports whether the running hvi's machine profile names +// base: an hvi that does carries its blocks written since boot across a +// restore (from disk.json), so they still cover every difference from the +// base on a restored guest. +func hviCarriesBase(state *store.InstanceState, base string) bool { + if state.ProfilePath == "" { + return false + } + data, err := os.ReadFile(state.ProfilePath) + if err != nil { + return false + } + var p struct { + DiskBase string `json:"disk_base_sha256"` + } + return json.Unmarshal(data, &p) == nil && p.DiskBase != "" && p.DiskBase == base +} + +// diskDirty reads hvi's dirty blocks since boot or since the last round, +// and holds its bitmap to the disk this host sees: the same block size, the +// same number of blocks, as many set as it says. +func (ss *streamSender) diskDirty(since string) (*hviDiskDirtyReply, error) { + read, err := hviDiskDirty(ss.state.ControlSock, since) + if err != nil { + return nil, err + } + if _, err := read.dirtyBlocks(ss.size); err != nil { + return nil, err + } + return read, nil +} + +// hviRoundBlocks sends hvi's blocks of the round dirty started, which the +// compare engine then holds as sent. A round with no blocks is asked for +// too, so hvi records its sent_sha256 like any other's (disk.json), and a +// destination can tell it went out. +func (ss *streamSender) hviRoundBlocks(r *streamRound, dirty *hviDiskDirtyReply, final bool) error { + round := dirty.Round + ss.roundTotal = dirty.Dirty + defer func() { ss.roundTotal = 0 }() + d := ss.beginDiskRound(round, final, dirty) + send := ss.sendBatch(r, d, round, final) + sent, err := hviDiskBlocks(ss.state.ControlSock, round, func(first, count int64, data []byte) error { + if ss.cmp != nil { + for i := int64(0); i < count; i++ { + lo := i * diskBlockSize + block := data[lo:min(lo+diskBlockSize, int64(len(data)))] + h := blockHash{sum: sha256.Sum256(block)} + if isZero(block) { + h = blockHash{zero: true} + } + ss.cmp.sent[first+i] = h + } + } + return send(first, count, data) + }) + if err != nil { + return fmt.Errorf("hvi's disk-blocks of round %d: %w", round, err) + } + if got := d.done(); got.SentSHA256 != sent { + return fmt.Errorf("round %d went out as batches whose digests make %s, hvi sent %s", round, + got.SentSHA256, sent) + } + return nil +} + +// diskFinal sends the blocks the guest wrote since the last disk round, +// now that hvi has fenced it and synced its disk. +func (ss *streamSender) diskFinal() error { + started := time.Now() + r := streamRound{Kind: "disk", Name: "disk final round", Final: true} + if ss.hviDisk { + read, err := ss.diskDirty("round") + if err != nil { + return fmt.Errorf("hvi's disk-dirty after the fence: %w", err) + } + r.Name = fmt.Sprintf("disk round %d (final)", read.Round) + if err := ss.hviRoundBlocks(&r, read, true); err != nil { + return err + } + } else { + b := newBatcher(ss.size, ss.sendBatch(&r, ss.beginDiskRound(streamRoundCap, true, nil), streamRoundCap, true)) + if err := ss.cmp.diff(ss.disk, b.add); err != nil { + return err + } + if err := b.done(); err != nil { + return err + } + } + ss.endRound(r, started) + ss.stats.FinalDiskMs = time.Since(started).Milliseconds() + ss.stats.FinalDiskBytes = r.Bytes + return nil +} + +// spoolWriter keeps hvi's stream on this host as hvi wrote it, and the +// small components beside it as files. Nothing syncs it while the stream +// runs: on a busy disk a sync of the spool held up the synced records the +// stopped guest waits for. The seal syncs it, behind the move. Every +// spoolSyncEvery bytes its writeback is started (startWriteback), with no +// wait: without that a 2 GiB stream left so many dirty pages that the +// kernel throttled every write on the host, the move's included. +type spoolWriter struct { + dir string + f *os.File + cw *countingWriter + bw *bufio.Writer + // started is how far the writeback has been started. + started int64 +} + +// writeback starts the writeback of what was written since the last call, +// once that is spoolSyncEvery bytes or more. +func (sp *spoolWriter) writeback() { + if sp.cw.n+int64(sp.bw.Buffered())-sp.started < spoolSyncEvery { + return + } + if sp.bw.Flush() != nil { + return // the seal reports it + } + startWriteback(sp.f, sp.started, sp.cw.n-sp.started) + sp.started = sp.cw.n +} + +// spoolSyncEvery is how much the source writes to its spool between the +// starts of its writeback, and a destination to the incoming disk between +// the syncs it runs in the background while the stream goes on. +const spoolSyncEvery = 32 << 20 + +func newSpool(dir string) (*spoolWriter, error) { + emptyDir(dir) + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + f, err := os.OpenFile(filepath.Join(dir, spoolStream), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return nil, err + } + cw := &countingWriter{w: f} + return &spoolWriter{dir: dir, f: f, cw: cw, bw: bufio.NewWriterSize(cw, 1<<20)}, nil +} + +// seal flushes and syncs the stream, writes the small components and the +// reply, and syncs the directory: from here an abort can restore from it. +func (sp *spoolWriter) seal(files map[string][]byte, reply []byte) error { + if err := sp.bw.Flush(); err != nil { + return err + } + if err := syncFile(sp.f); err != nil { + return err + } + if err := sp.f.Close(); err != nil { + return err + } + for name, data := range files { + if err := writeFileSynced(filepath.Join(sp.dir, name), data); err != nil { + return err + } + } + if err := writeFileSynced(filepath.Join(sp.dir, spoolSealed), append(reply, '\n')); err != nil { + return err + } + return syncDir(sp.dir) +} + +func (sp *spoolWriter) close() { + if sp.f != nil { + _ = sp.f.Close() + } +} + +// syncFile syncs f; a variable so a test can hold a sync and see what +// waits for it. +var syncFile = func(f *os.File) error { return f.Sync() } + +// writeFileSynced writes data to path and syncs it. +func writeFileSynced(path string, data []byte) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + _, err = f.Write(data) + if err == nil { + err = f.Sync() + } + if cerr := f.Close(); err == nil { + err = cerr + } + return err +} + +// progressEvery is how often a long round prints how far it is. +const progressEvery = 2 * time.Second + +// isRAMRound reports whether an hvi component is one of the RAM rounds. +func isRAMRound(name string) bool { return strings.HasPrefix(name, "ram-round-") } + +// spooledStateMax bounds the non-RAM components kept in memory to be +// written beside the spool. +const spooledStateMax = 64 << 20 + +// roundFolder folds hvi's frames into rounds for the record and the +// progress lines: hvi cuts a RAM round into frames of about 1 MiB, all +// named after the round. +type roundFolder struct { + ss *streamSender + cur *streamRound + started time.Time + printed time.Time + lastRAM time.Time + ramSeen bool +} + +// begin starts the frame of component name, which ends the round before +// it when it is another one. +func (f *roundFolder) begin(name string) { + if f.cur != nil && f.cur.Name == name { + return + } + f.flush() + now := time.Now() + kind := "state" + if isRAMRound(name) { + kind = "ram" + f.lastRAM, f.ramSeen = now, true + } else if f.ramSeen && f.ss.stats.PausedAt.IsZero() { + // The first document after the RAM rounds: the round before it + // was the last, which hvi takes with the guest paused. + f.ss.stats.PausedAt = f.lastRAM.UTC() + } + f.cur = &streamRound{Kind: kind, Name: name} + f.started, f.printed = now, now +} + +// add counts a frame of bytes, wire bytes on the wire. +func (f *roundFolder) add(bytes, wire int64) { + f.cur.Bytes += bytes + f.cur.WireBytes += wire + if now := time.Now(); f.cur.Kind == "ram" && now.Sub(f.printed) >= progressEvery { + f.printed = now + f.ss.progress("%s: %s so far, %s/s", f.cur.Name, humanBytes(f.cur.Bytes), + humanBytes(perSec(f.cur.WireBytes, now.Sub(f.started).Seconds()))) + } +} + +// flush records the round in progress. +func (f *roundFolder) flush() { + if f.cur != nil { + f.ss.endRound(*f.cur, f.started) + f.cur = nil + } +} + +// relayFrame relays one of hvi's frames into the stream, and into also +// when that is not nil, and folds it into its round. +func (ss *streamSender) relayFrame(f *roundFolder, line []byte, hdr hviComponentHeader, body io.Reader, + also io.Writer) error { + f.begin(hdr.Component) + before := ss.sw.written() + if err := ss.sw.component(line, hdr, body, also); err != nil { + return err + } + f.add(hdr.Size, ss.sw.written()-before) + return nil +} + +// relayHvi runs hvi's migrate op and relays its frames into the stream and +// the spool. It returns hvi's reply line once hvi has sealed the stream and +// fenced the guest. +func (ss *streamSender) relayHvi(sp *spoolWriter) (json.RawMessage, map[string][]byte, error) { + m := ss.m + conn, br, err := hviStreamOp(ss.state.ControlSock, map[string]any{"op": "migrate", "then": hviThenFence, + "txid": m.rec.TxID, "epoch": m.rec.Epoch}, hviStreamIdle) + if err != nil { + return nil, nil, err + } + defer func() { _ = conn.Close() }() + files := map[string][]byte{} + var kept int64 + folder := &roundFolder{ss: ss} + reply, line, err := readHviComponents("migrate", br, func(line []byte, hdr hviComponentHeader, body io.Reader) error { + if _, err := sp.bw.Write(append(append([]byte{}, line...), '\n')); err != nil { + return err + } + var also io.Writer = sp.bw + var keep *bytesBuffer + if !isRAMRound(hdr.Component) { + if kept+hdr.Size > spooledStateMax { + return fmt.Errorf("hvi's %s of %d bytes is not a RAM round and too large to keep", hdr.Component, hdr.Size) + } + kept += hdr.Size + keep = &bytesBuffer{} + also = io.MultiWriter(sp.bw, keep) + } + if err := ss.relayFrame(folder, line, hdr, body, also); err != nil { + return err + } + sp.writeback() + if keep != nil { + files[hdr.Component] = append(files[hdr.Component], keep.b...) + if hdr.Component == "manifest.json" { + ss.manifest = keep.b + // hvi fences the guest once the manifest is in the + // socket's buffer. From here the spool is the guest: every + // byte goes to the file now, so a hull that dies before the + // seal leaves a stream a re-run or an abort can check whole + // (verifySpool). + if err := sp.bw.Flush(); err != nil { + return fmt.Errorf("spool the stream: %w", err) + } + } + } + return nil + }) + folder.flush() + if err != nil { + return nil, nil, err + } + if ss.manifest == nil { + return nil, nil, errors.New("hvi sealed its migrate stream without a manifest.json") + } + ss.takeHviTimes(reply) + return line, files, nil +} + +// takeHviTimes takes, from hvi's reply, when the pre-copy rounds ended and +// when hvi sealed the stream, by this host's clock: closer to the pause than +// what hull sees of the frames. +func (ss *streamSender) takeHviTimes(reply *hviReply) { + var st struct { + PrecopyEnd int64 `json:"precopy_end_unix_ms"` + SealedAt int64 `json:"sealed_at_unix_ms"` + } + if reply != nil && json.Unmarshal(reply.Stats, &st) == nil && st.PrecopyEnd > 0 { + ss.stats.PausedAt = time.UnixMilli(st.PrecopyEnd).UTC() + if st.SealedAt >= st.PrecopyEnd { + ss.stats.SealedMs = st.SealedAt - st.PrecopyEnd + return + } + } + if !ss.stats.PausedAt.IsZero() { + ss.stats.SealedMs = time.Since(ss.stats.PausedAt).Milliseconds() + } +} + +// bytesBuffer collects what is written to it. +type bytesBuffer struct{ b []byte } + +func (b *bytesBuffer) Write(p []byte) (int, error) { + b.b = append(b.b, p...) + return len(p), nil +} + +// replayHvi relays the spooled stream of an earlier attempt that hvi +// sealed: the guest is fenced already, and its RAM exists only there. +func (ss *streamSender) replayHvi(dir string) (json.RawMessage, error) { + f, err := os.Open(filepath.Join(dir, spoolStream)) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + reply, err := os.ReadFile(filepath.Join(dir, spoolSealed)) + if err != nil { + return nil, fmt.Errorf("the spooled stream %s was never sealed: %w", dir, err) + } + br := bufio.NewReaderSize(f, 1<<20) + folder := &roundFolder{ss: ss} + for { + if _, err := br.Peek(1); errors.Is(err, io.EOF) { + break + } + line, err := readHviLine(br, migrateMaxHeader) + if err != nil { + return nil, fmt.Errorf("the spooled stream: %w", err) + } + hdr, err := parseHviComponentHeader(line) + if err != nil { + return nil, fmt.Errorf("the spooled stream: %w", err) + } + var keep *bytesBuffer + var also io.Writer + if hdr.Component == "manifest.json" { + keep = &bytesBuffer{} + also = keep + } + if err := ss.relayFrame(folder, line, hdr, br, also); err != nil { + return nil, fmt.Errorf("the spooled stream: %w", err) + } + if keep != nil { + ss.manifest = keep.b + } + } + folder.flush() + if ss.manifest == nil { + return nil, errors.New("the spooled stream has no manifest.json") + } + return json.RawMessage(strings.TrimSpace(string(reply))), nil +} + +// streamTransfer is transfer for a streamed move. A destination that has +// verified the move already is not sent it again. A guest that hvi has not +// fenced yet is sent live; one whose stream hvi sealed in an earlier +// attempt is sent from the spool, with the disk compared against the base. +func (m *moveSource) streamTransfer(ctx context.Context) (retErr error) { + if reply, err := m.peer.call(ctx, "migrate-status", []string{"--txid", m.rec.TxID}, nil); err == nil && + reply.Record != nil && + migrateStateOrder(roleDestination, reply.Record.State) >= migrateStateOrder(roleDestination, dstVerified) { + if m.rec.Disk == nil || m.rec.Stream == nil { + return errors.New("the destination has verified this move, but this host has no record of what it sent") + } + if err := m.checkDiskUnchanged(); err != nil { + return err + } + if err := m.compareStreamReceived(reply.Record); err != nil { + return err + } + m.destPaused = reply.Record.State == dstRestoredPaused + return m.rec.advance(m.path, srcTransferred) + } + state, err := m.s.GetInstance(m.name) + if err != nil { + return err + } + replay := fencedFor(state, m.rec.TxID) + if !replay && hviVMState(state) == store.MigrationFenced { + if err := m.adoptStreamFence(state); err != nil { + return err + } + if state, err = m.s.GetInstance(m.name); err != nil { + return err + } + replay = true + } + if !replay { + if err := requireRunningHvi(state, "migrate"); err != nil { + return err + } + switch vm := hviVMState(state); vm { + case "running", "paused": + default: + return fmt.Errorf("its VM is %s; a move takes a running or paused one", vm) + } + } else if err := m.checkDiskUnchanged(); err != nil { + return err + } + if replay && m.rec.BundleDir != "" { + // A hull that died after recording the fence may not have lived + // to seal the spool. + if _, err := m.ensureStreamSpool(state, m.rec.BundleDir); err != nil { + return err + } + } + disk, ok := flagValue(state.CmdLine, "--disk") + if !ok { + return errors.New("the instance has no --disk") + } + if !baseHeld(m.s, m.rec.BaseSHA256) { + return fmt.Errorf("the base %s is gone from this store", m.rec.BaseSHA256) + } + f, err := os.Open(disk) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + info, err := f.Stat() + if err != nil { + return err + } + root, ok := flagValue(state.CmdLine, "--checkpoint-root") + if !ok { + return errors.New("the instance has no checkpoint root") + } + spoolDir := filepath.Join(root, m.rec.TxID) + mode := "live" + if replay { + mode = "replay" + } + stats := &migrateStreamStats{Mode: mode} + m.logf("Streaming %s to %s (%s)", m.name, m.rec.Peer, mode) + started := time.Now() + pr, pw := io.Pipe() + link := &softLink{w: pw} + sw := newMoveStreamWriter(link) + attempt, err := newMoveTxID() + if err != nil { + return err + } + m.sealGate, m.sealGateOnce = make(chan struct{}), sync.Once{} + ss := &streamSender{m: m, state: state, sw: sw, link: link, disk: f, size: info.Size(), stats: stats, + spoolDir: spoolDir, ctx: ctx, attempt: attempt, callDone: make(chan struct{}), sealGate: m.sealGate} + type sendResult struct { + hviReply json.RawMessage + err error + } + sent := make(chan sendResult, 1) + go func() { + reply, err := ss.send(replay) + _ = pw.CloseWithError(err) + sent <- sendResult{reply, err} + }() + reply, err := m.peer.call(ctx, "migrate-receive", []string{"--stream", "--txid", m.rec.TxID, "--name", m.name}, pr) + close(ss.callDone) + _ = pr.CloseWithError(errors.New("the destination stopped reading")) + res := <-sent + m.spoolSealing = ss.sealDone + if ss.recorded != nil { + m.fenceRecord, m.fenceRecordedAt = ss.recorded, &ss.recordedAt + defer func() { + // A move that stops here leaves the instance's record of the + // fence, and its sealed spool, for the abort and the re-run. + if retErr != nil { + m.releaseSeal() + if err := m.waitFenceRecord(); err != nil { + retErr = errors.Join(retErr, err) + } + } + }() + } + if res.err == nil { + stats.Chain = sw.chain.sum() + stats.WireBytes = sw.written() + stats.Seconds = time.Since(started).Seconds() + stats.HviReply = res.hviReply + } + stats.step("destination answered") + // What was sent is on record before the move goes on: a reply that is + // lost leaves a re-run to compare the destination's record with it. + // A step that fails from here records it (stopped), and one that + // succeeds records it with the next state, in one write. + m.rec.Stream = stats + if res.err != nil && !errors.Is(res.err, io.ErrClosedPipe) && err == nil { + err = fmt.Errorf("send: %w", res.err) + } + if err != nil { + if res.err != nil && !errors.Is(res.err, io.ErrClosedPipe) && !strings.Contains(err.Error(), res.err.Error()) { + err = fmt.Errorf("%w (sending: %v)", err, res.err) + } + return fmt.Errorf("transfer to %s: %w", m.rec.Peer, err) + } + if reply.Record == nil { + return errors.New("the destination's receive reply has no record") + } + if err := m.checkDiskUnchanged(); err != nil { + return err + } + if err := m.compareStreamReceived(reply.Record); err != nil { + return err + } + m.rec.Transfer = &migrateTransfer{Bytes: stats.WireBytes, Seconds: stats.Seconds, StartedAt: started.UTC(), + BytesPerSec: perSec(stats.WireBytes, stats.Seconds)} + m.timing("transfer", started) + m.destPaused = reply.Record.State == dstRestoredPaused + m.logf("Sent %s in %.1f s (%s/s): disk %s as %s, RAM %s, state %s; destination: %s", + humanBytes(stats.WireBytes), stats.Seconds, humanBytes(perSec(stats.WireBytes, stats.Seconds)), + humanBytes(stats.DiskBytes), humanBytes(stats.DiskWireBytes), humanBytes(stats.RAMBytes), + humanBytes(stats.StateBytes), reply.Record.Verify) + if m.destPaused { + // The destination holds the guest restored paused: the source's + // commit point, recorded with the transfer in one write, as the + // prepare step would record it. + return m.rec.advanceThrough(m.path, srcTransferred, srcCommitting) + } + return m.rec.advance(m.path, srcTransferred) +} + +// send writes the whole stream. Live, it ends with hvi having fenced the +// guest, the fence recorded on the instance and in the move's record. +func (ss *streamSender) send(replay bool) (json.RawMessage, error) { + m := ss.m + if err := ss.sw.header(moveStreamHeader{TxID: m.rec.TxID, Name: m.name, Epoch: m.rec.Epoch, + BaseSHA256: m.rec.BaseSHA256, DiskSize: ss.size, BlockSize: diskBlockSize, Replay: replay, + Attempt: ss.attempt}); err != nil { + return nil, err + } + if err := ss.sw.flush(); err != nil { + return nil, err + } + if !replay { + // A refusal of the header (no base, another profile, no room + // for the base's copy) must come while the guest still runs: + // once hvi's migrate has begun, hull reads it to the end. + if err := ss.waitAccepted(); err != nil { + return nil, err + } + } + basePath := m.s.BasePath(m.rec.BaseSHA256) + if replay { + // The guest is fenced: one comparison is the whole disk round. + ss.stats.DiskEngine = "compare" + cmp, err := newDiskCompare(basePath, ss.size) + if err != nil { + return nil, fmt.Errorf("read the base: %w", err) + } + ss.cmp = cmp + started := time.Now() + r := streamRound{Kind: "disk", Name: "disk round 0 (final)", Final: true} + b := newBatcher(ss.size, ss.sendBatch(&r, ss.beginDiskRound(0, true, nil), 0, true)) + if err := cmp.diff(ss.disk, b.add); err != nil { + return nil, err + } + if err := b.done(); err != nil { + return nil, err + } + ss.endRound(r, started) + reply, err := ss.replayHvi(ss.spoolDir) + if err != nil { + return nil, err + } + if err := ss.sw.trailer(reply, ss.diskTrailer()); err != nil { + return nil, err + } + return reply, ss.link.err + } + + // Everything the migrate op needs is set up first, so the last disk + // round runs right before it: the guest's writes from then on are all + // the final round holds. + unlock, err := lockInstanceMigration(m.s, m.name) + if err != nil { + return nil, err + } + sp, err := newSpool(ss.spoolDir) + if err != nil { + unlock() + return nil, err + } + if err := ss.diskPreCopy(basePath); err != nil { + sp.close() + unlock() + return nil, err + } + if err := ss.link.err; err != nil { + sp.close() + unlock() + return nil, err + } + select { + case <-ss.callDone: + // The destination's receive ended before hvi's migrate began, + // which it never does unless it failed: its reply says why. + sp.close() + unlock() + return nil, errors.New("the destination stopped receiving before hvi's migrate began") + default: + } + // A restored guest that was never resumed may be resumed by the + // migrate op: hvi resumes the guest when the stream fails before its + // seal. + err = markRestoredGuestRunning(m.s, m.name) + var reply json.RawMessage + var files map[string][]byte + if err == nil { + // From the migrate op until the fence is recorded, a signal that + // would end hull waits (moveSignals). + m.signals.hold() + reply, files, err = ss.relayHvi(sp) + if err != nil { + m.signals.release() + } + } + if err != nil { + sp.close() + unlock() + return nil, err + } + if streamTestHook != nil { + streamTestHook("relayed") + } + ss.stats.HviReply = reply + ss.stats.step("hvi sealed the stream") + // hvi has fenced the guest, and every byte of its stream is in the + // spool's file. The move's record takes the fence in memory, and + // writes it with the commit point, the one synced write the commit + // waits for. The instance's records of the fence are written beside + // the last disk round, the trailer and the destination's checks, and + // the spool is synced behind the move: on a busy disk every synced + // write is a wait the stopped guest would otherwise sit through. A hull + // that dies before they are done leaves a spool that a re-run or an + // abort checks whole before using it (verifySpool). + if err := ss.noteFence(); err != nil { + m.signals.release() + sp.close() + unlock() + return nil, err + } + ss.progress("hvi sealed the stream and fenced %s (%d ms after the last RAM round began)", m.name, + ss.stats.SealedMs) + ss.recorded = make(chan error, 1) + fenceDone := make(chan struct{}) + go func() { + err := ss.recordInstanceFence() + ss.recordedAt = time.Now().UTC() + m.signals.release() + close(fenceDone) + if err == nil && streamTestHook != nil { + streamTestHook("recorded") + } + ss.recorded <- err + }() + sendErr := ss.link.err + if sendErr == nil { + sendErr = ss.diskFinal() + } + if sendErr == nil { + sendErr = ss.sw.trailer(reply, ss.diskTrailer()) + ss.stats.step("trailer sent") + } + ss.sealDone = make(chan error, 1) + go func() { + // The seal's sync of the whole spool waits until the commit is + // answered or the move stops (moveSource.releaseSeal): run beside + // the commit point's synced record, it held that record up for + // seconds, the guest stopped all the while. + <-ss.sealGate + err := sp.seal(files, reply) + sp.close() + <-fenceDone + unlock() + ss.sealDone <- err + }() + if sendErr != nil { + return nil, sendErr + } + return reply, ss.link.err +} + +// acceptPoll is how often the source asks whether the destination took the +// stream's header. +const acceptPoll = 50 * time.Millisecond + +// waitAccepted waits until the destination's record says it took this +// attempt's header and copied the base, and fails if its receive ends +// first. +func (ss *streamSender) waitAccepted() error { + m := ss.m + started := time.Now() + for { + reply, err := m.peer.call(ss.ctx, "migrate-status", []string{"--txid", m.rec.TxID}, nil) + if err == nil && reply.Record != nil && reply.Record.StreamAccepted == ss.attempt { + ss.stats.step("destination accepted") + if took := time.Since(started); took > time.Second { + ss.progress("the destination took the stream in %.1f s (it copied the base)", took.Seconds()) + } + return nil + } + select { + case <-ss.callDone: + return errors.New("the destination refused the stream before hvi's migrate began") + case <-ss.ctx.Done(): + return ss.ctx.Err() + case <-time.After(acceptPoll): + } + } +} + +// streamTestHook, when set, is called at points of a streamed move a test +// stops it at: "relayed" is once hvi has sealed its stream and fenced the +// guest, and hull has read it all, before anything is recorded; "recorded" +// is once the fence is recorded, maybe before the spool is sealed. +var streamTestHook func(stage string) + +// softLink is the writer between the stream and the destination. Once a +// write to the destination fails it keeps the error and takes every later +// write without sending it, so a source that is reading hvi's migrate +// stream reads it to the end into its spool whatever became of the +// destination. Stopping half way would not stop hvi: the rest of a small +// last round, the state and the manifest can all sit in the socket's +// buffer, and hvi seals, and fences the guest, once they are written there. +// A source that left them unread would be fenced with no whole copy of the +// guest anywhere. +type softLink struct { + w io.Writer + err error +} + +func (l *softLink) Write(p []byte) (int, error) { + if l.err != nil { + return len(p), nil + } + if _, err := l.w.Write(p); err != nil { + l.err = err + } + return len(p), nil +} + +// noteFence puts the fence hvi made on the move's record, in memory: the +// spool, the manifest it ends with, the disk as fenced. The record carries +// them to disk with its next write, which before the commit is the one +// that records the commit point; until then the move's record, synced +// before the stream began, already keeps the instance from running here +// (moveRefusal), and a spool a re-run or an abort finds without its record +// is checked whole before it is used. +func (ss *streamSender) noteFence() error { + m := ss.m + disk, _ := flagValue(ss.state.CmdLine, "--disk") + size, mod, err := fileModNs(disk) + if err != nil { + return err + } + m.rec.BundleDir = ss.spoolDir + m.rec.ManifestSHA256 = sha256Hex(ss.manifest) + m.rec.Disk = &migrateDisk{Path: disk, Size: size, ModNs: mod} + return nil +} + +// recordInstanceFence records the fence on the instance: the spool as its +// checkpoint, taken over by this move. latest.json goes first, as a +// checkpoint writes it: a restore that finds it naming a checkpoint the +// record never took in refuses rather than rewinding past it. It runs +// beside the end of the move; the commit does not wait for it. +func (ss *streamSender) recordInstanceFence() error { + m := ss.m + sum := sha256Hex(ss.manifest) + if err := writeHviCheckpointRecord(instanceCheckpointDir(m.s, m.name), hviCheckpointRecord{ + Dir: ss.spoolDir, ManifestSHA256: sum, Then: hviThenFence, Stats: ss.stats.HviReply, + SavedAt: time.Now().UTC(), TxID: m.rec.TxID, + }); err != nil { + return fmt.Errorf("the stream is sealed in %s but recording it failed: %w", ss.spoolDir, err) + } + state, err := m.s.GetInstance(m.name) + if err != nil { + return err + } + state.Migration = &store.MigrationState{State: store.MigrationMigrating, BundleDir: ss.spoolDir, + TxID: m.rec.TxID, ManifestSHA256: sum, MoveTxID: m.rec.TxID, Peer: m.rec.Peer} + return m.s.SaveInstance(state) +} + +// adoptStreamFence records a stream hvi sealed for this move when hull did +// not live to record it. The spool must be whole: sealed, or checked frame +// by frame, or taken again from the hvi that still holds the fenced guest. +func (m *moveSource) adoptStreamFence(state *store.InstanceState) error { + root, ok := flagValue(state.CmdLine, "--checkpoint-root") + if !ok { + return errors.New("the instance has no checkpoint root") + } + dir := filepath.Join(root, m.rec.TxID) + man, err := m.ensureStreamSpool(state, dir) + if err != nil { + return err + } + state.Migration = &store.MigrationState{State: store.MigrationMigrating, BundleDir: dir, TxID: m.rec.TxID, + ManifestSHA256: sha256Hex(man), MoveTxID: m.rec.TxID, Peer: m.rec.Peer} + if err := m.s.SaveInstance(state); err != nil { + return err + } + disk, _ := flagValue(state.CmdLine, "--disk") + size, mod, err := fileModNs(disk) + if err != nil { + return err + } + m.rec.BundleDir, m.rec.ManifestSHA256 = dir, state.Migration.ManifestSHA256 + m.rec.Disk = &migrateDisk{Path: disk, Size: size, ModNs: mod} + m.logf("Adopted the stream hvi fenced the guest with, spooled in %s", dir) + return writeMigrationRecord(m.path, m.rec) +} + +// compareStreamReceived checks the destination's record of a streamed move +// against what the source sent. +func (m *moveSource) compareStreamReceived(dst *migrationRecord) error { + if dst.TxID != m.rec.TxID || dst.Epoch != m.rec.Epoch { + return fmt.Errorf("the destination recorded move %s epoch %d, not %s epoch %d", dst.TxID, dst.Epoch, + m.rec.TxID, m.rec.Epoch) + } + if m.rec.ManifestSHA256 == "" || dst.ManifestSHA256 != m.rec.ManifestSHA256 { + return fmt.Errorf("the destination has manifest sha256 %s, the source %s", dst.ManifestSHA256, m.rec.ManifestSHA256) + } + if m.rec.Stream == nil || dst.Stream == nil || dst.Stream.Chain == "" || dst.Stream.Chain != m.rec.Stream.Chain { + return errors.New("the destination's record of the stream is not the stream the source sent") + } + if dst.BaseSHA256 != m.rec.BaseSHA256 { + return fmt.Errorf("the destination applied the disk to base %s, not %s", dst.BaseSHA256, m.rec.BaseSHA256) + } + return nil +} + +// sendBaseFirst puts the move's base on the destination before the guest +// stops, as `hull migrate-prepare` would have. A base that cannot be sent +// turns the move into a whole-bundle one: nothing is fenced yet. +func (m *moveSource) sendBaseFirst(ctx context.Context) error { + m.logf("Sending base %s to %s (once per base; `hull migrate-prepare` does it ahead of a move)", + digestHead(m.rec.BaseSHA256), m.rec.Peer) + st, err := sendBase(ctx, m.s, m.peer, m.rec.BaseSHA256) + if err != nil { + m.rec.Transport, m.rec.BaseSHA256 = transportBundle, "" + m.rec.TransportWhy = "sending the base failed: " + refusalDetail(err) + m.logf("Transport: the whole checkpoint and disk, because %s", m.rec.TransportWhy) + return writeMigrationRecord(m.path, m.rec) + } + m.logf(" base: %s (%s)", st.Result, st.Detail) + return nil +} + +// streamTransferWithBase is streamTransfer, which sends the base and tries +// once more when the destination answers that it does not hold it. +func (m *moveSource) streamTransferWithBase(ctx context.Context) error { + err := m.streamTransfer(ctx) + if refusalCode(err) != codeNoBase { + return err + } + m.logf("%s does not hold base %s", m.rec.Peer, digestHead(m.rec.BaseSHA256)) + if _, err := sendBase(ctx, m.s, m.peer, m.rec.BaseSHA256); err != nil { + return fmt.Errorf("send base %s: %w", digestHead(m.rec.BaseSHA256), err) + } + return m.streamTransfer(ctx) +} diff --git a/cmd/hull/migrate_transport.go b/cmd/hull/migrate_transport.go new file mode 100644 index 00000000..1ff9c13f --- /dev/null +++ b/cmd/hull/migrate_transport.go @@ -0,0 +1,108 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/brig-sh/hull/pkg/store" +) + +// streamCheckpointFormat is the first hvi checkpoint format with the +// streamed migrate op. +const streamCheckpointFormat = 2 + +// transportFacts are what decides how a move carries the guest. +type transportFacts struct { + // WholeBundle is the operator asking for the whole-bundle move. + WholeBundle bool + // SourceFormat is the checkpoint_format of the source's hvi, 0 when + // unknown. + SourceFormat int + // HasBase says the instance records a base and this store still + // holds it. + HasBase bool + BaseReason string + // DestStream is the destination hull taking a streamed move, and + // DestHviStream its hvi restoring from a stream. + DestStream bool + DestHviStream bool + DestHull string + DestHvi string +} + +// decideTransport is how a move carries the guest, and why when it is the +// whole bundle. The streamed move needs every piece: an hvi here that +// streams, a hull and an hvi there that take a stream, and a base the disk +// blocks apply to (the destination need not hold it yet: the move sends it +// before the guest stops). +func decideTransport(f transportFacts) (string, string) { + switch { + case f.WholeBundle: + return transportBundle, "--whole-bundle asked for it" + case !f.DestStream: + return transportBundle, fmt.Sprintf("the hull there (%s) takes no streamed move", f.DestHull) + case !f.DestHviStream: + return transportBundle, fmt.Sprintf("the hvi there (%s) cannot restore from a stream", f.DestHvi) + case f.SourceFormat < streamCheckpointFormat: + return transportBundle, fmt.Sprintf("the hvi here writes checkpoint format %d, and a streamed "+ + "move needs %d", f.SourceFormat, streamCheckpointFormat) + case !f.HasBase: + return transportBundle, f.BaseReason + } + return transportStream, "" +} + +// hviCheckpointFormat reads the checkpoint format of the running hvi from +// the machine profile it wrote, or asks it; 0 when neither answers. +func hviCheckpointFormat(state *store.InstanceState) int { + var p struct { + CheckpointFormat int `json:"checkpoint_format"` + } + if state.ProfilePath != "" { + if data, err := os.ReadFile(state.ProfilePath); err == nil && json.Unmarshal(data, &p) == nil && + p.CheckpointFormat > 0 { + return p.CheckpointFormat + } + } + if state.ControlSock != "" { + if reply, err := hviOp(state.ControlSock, "profile"); err == nil && json.Unmarshal(reply.Profile, &p) == nil { + return p.CheckpointFormat + } + } + return 0 +} + +// moveTransportFacts gathers the facts for a move of state to the probed +// destination. +func moveTransportFacts(s *store.Store, state *store.InstanceState, probe *migrateProbe, wholeBundle bool) transportFacts { + f := transportFacts{WholeBundle: wholeBundle, SourceFormat: hviCheckpointFormat(state)} + if probe != nil { + f.DestStream = probe.hasCapability(moveCapStream) + f.DestHviStream = probe.HviStream + f.DestHull, f.DestHvi = probe.HullVersion, probe.Hvi + } + switch b := state.Base; { + case b == nil: + f.BaseReason = "the instance keeps no base (it was started before bases, or not on a block rootfs)" + case !baseHeld(s, b.SHA256): + f.BaseReason = fmt.Sprintf("its base %s is gone from this store", digestHead(b.SHA256)) + default: + f.HasBase = true + } + return f +} diff --git a/cmd/hull/writeback_darwin.go b/cmd/hull/writeback_darwin.go new file mode 100644 index 00000000..805d3256 --- /dev/null +++ b/cmd/hull/writeback_darwin.go @@ -0,0 +1,24 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "os" + +// canStartWriteback says startWriteback does something here. +const canStartWriteback = false + +// startWriteback starts nothing on macOS, which has no call that starts a +// file's writeback without waiting for it. +func startWriteback(*os.File, int64, int64) {} diff --git a/cmd/hull/writeback_linux.go b/cmd/hull/writeback_linux.go new file mode 100644 index 00000000..a1497b12 --- /dev/null +++ b/cmd/hull/writeback_linux.go @@ -0,0 +1,32 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// canStartWriteback says startWriteback does something here. +const canStartWriteback = true + +// startWriteback asks the kernel to start writing n bytes of f from off to +// disk, and returns at once: no wait, no sync of the metadata or of the +// journal; n 0 means to the end of the file. A long stream written through the page cache then never piles +// up enough dirty pages for the kernel to throttle the writer. +func startWriteback(f *os.File, off, n int64) { + _ = unix.SyncFileRange(int(f.Fd()), off, n, unix.SYNC_FILE_RANGE_WRITE) +} diff --git a/docs/cli.md b/docs/cli.md index 15d99fc2..da1cd4a3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -29,9 +29,10 @@ for the one build that reports `dev`. There are twenty-two top-level commands you call, plus hidden ones. `network-gateway` is hidden: it does not appear in `hull --help`, but it runs -and prints its own help when named. The six `migrate-*` verbs are hidden too: -`hull migrate` runs them on the destination over ssh (see below). The one -hidden flag is `restore --move`, which only a move's own restore passes. +and prints its own help when named. The `migrate-*` verbs other than +`migrate-prepare` are hidden too: `hull migrate` runs them on the +destination over ssh (see below). The one hidden flag is `restore --move`, +which only a move's own restore passes. | Command | What it does | |---|---| @@ -45,6 +46,7 @@ hidden flag is `restore --move`, which only a move's own restore passes. | `checkpoint ` | `vz`: pause, save VM and disk state, resume. `hvi`: write a checkpoint bundle, then continue or fence | | `restore ` | restore a stopped instance: `vz` from its saved state; `hvi` from a fenced bundle, with an `hvi` that has a `restore` subcommand (on macOS, from 4ea836f on `hvi`'s `feat/teleport`; a KVM bundle cannot be restored yet) | | `migrate --to USER@HOST` | move a running `hvi` instance to another host over ssh; see [migration.md](migration.md) | +| `migrate-prepare --to USER@HOST` | put the instance's base, its profile's kernel and its image on another host ahead of a move; see [migration.md](migration.md#preparing-a-destination) | | `rm ` | remove a stopped instance | | `logs ` | show an instance's log | | `inspect ` | print instance details as JSON | @@ -57,7 +59,7 @@ hidden flag is `restore --move`, which only a move's own restore passes. | `telemetry` | control usage and crash telemetry | | `version` | print the version and the build behind this binary | | `network-gateway` | run the user-mode network gateway daemon (hidden) | -| `migrate-probe`, `migrate-receive`, `migrate-prepare`, `migrate-commit`, `migrate-status`, `migrate-abort` | the destination's half of a move (hidden) | +| `migrate-probe`, `migrate-receive`, `migrate-commit`, `migrate-status`, `migrate-abort`, `migrate-take-base`, `migrate-take-kernel`, `migrate-take-image` | the destination's half of a move and of a prepare (hidden); `migrate-prepare --txid` is the destination's step too | ## `hull run` @@ -164,7 +166,8 @@ as a default that an explicit `--env TERM` beats. | `pause `, `resume ` | none | | `checkpoint ` | `--timeout `, default 60; `hvi` only: `--then continue\|fence`, default `continue`; `--dir `, an empty directory for the bundle, below `/checkpoint` | | `restore ` | `--detach`, `-d`; `--stop-grace `, default 10; `--wait-ip`; `--gateway-sock ` (`vz` only); `hvi` only: `--paused`, `--timeout `, default 120, `--adopt-latest` | -| `migrate ` | `--to USER@HOST`; `--remote-store `; `--remote-hull `; `--remote-hvi `; `--remote-gateway-sock `; `--txid ` resume a move; `--dry-run`; `--abort` and `--status` (with a name or a txid); `--timeout `, default 120, for the fence; `--allow-no-restore`, testing only | +| `migrate ` | `--to USER@HOST`; `--remote-store `; `--remote-hull `; `--remote-hvi `; `--remote-gateway-sock `; `--txid ` resume a move; `--dry-run`; `--abort` and `--status` (with a name or a txid); `--timeout `, default 120, for the fence; `--allow-no-restore`, testing only; `--whole-bundle` send the whole checkpoint and disk even when both hosts can stream | +| `migrate-prepare ` | `--to USER@HOST`; `--remote-store `; `--remote-hull `; `--remote-hvi `; `--remote-gateway-sock `, checked, not started; `--json` prints what was done and what the destination holds | | `assets show` | `--profile NAME` shows that profile's kernel and the initrd it boots with | | `assets dir` | `--profile NAME` prints the profile's kernel directory instead. Prints the directory and nothing else, for scripts | | `assets pull [REF]` | `--force` download even when the assets are present. `--profile NAME` installs the profile's kernel from REF, `HULL_PORTABLE_KERNEL` or `HULL_PORTABLE_KERNEL_REF` (see [images.md](images.md#a-profiles-kernel)); without `--force` it checks the installed one against its record. Plain `--force` leaves the profile's kernel alone: replace it with `--profile NAME --force` | @@ -194,7 +197,10 @@ signals as for any VMM. `hull migrate` moves a running `hvi` instance to `--to USER@HOST`: it fences the guest here, streams the checkpoint and the disk over ssh, has the destination restore it paused, and only then commits, after which the -destination resumes it and the copy here is stale. While a move holds an +destination resumes it and the copy here is stale. When both hosts can, it +sends the guest's RAM in rounds while the guest runs and only the disk +blocks that differ from the instance's base, and the guest stops only for +the last round (see [migration.md](migration.md#the-streamed-move)). While a move holds an instance, `checkpoint`, `resume`, `restore` and `rm` of it are refused on both hosts, and what an aborted move left on the destination can only be removed. The protocol, its states and its recovery are in @@ -205,12 +211,15 @@ The destination's verbs each print one JSON object on stdout: | Verb | Flags | What it does | |---|---|---| -| `migrate-probe` | `--name`, `--hvi`, `--gateway-sock` | reports hull and hvi versions, whether hvi takes the control flags and `--profile` (a stand-in for "can restore a portable bundle"; see [migration.md](migration.md#usage)) and has `restore`, the hypervisor, free space, whether the name is taken, and whether the gateway answers | -| `migrate-receive` | `--txid`, `--name`, `--reserve` | with `--reserve`, claims the name (a JSON request on stdin); without, reads the move stream on stdin and ends `verified` | +| `migrate-probe` | `--name`, `--hvi`, `--gateway-sock` | reports hull and hvi versions, whether hvi takes the control flags and `--profile` (a stand-in for "can restore a portable bundle"; see [migration.md](migration.md#usage)), has `restore` and restores from a stream, the hypervisor, free space, whether the name is taken, whether the gateway answers, the bases and profile kernels the store holds, and the capability `stream-v2` | +| `migrate-receive` | `--txid`, `--name`, `--reserve`, `--stream` | with `--reserve`, claims the name (a JSON request on stdin); with `--stream`, reads a streamed move on stdin, restores it paused as it arrives and ends `restored-paused`; with neither, reads the whole-bundle stream and ends `verified` | | `migrate-prepare` | `--txid` | restores the received checkpoint paused | | `migrate-commit` | `--txid`, `--epoch` | records `running`, then resumes the guest | | `migrate-status` | `--txid` | prints the move's record and the VM's state | | `migrate-abort` | `--txid` | refused once running; otherwise records `aborted`, stops what the move restored and frees the name | +| `migrate-take-base` | `--sha256` | keeps the base the stream on stdin carries, if it has that digest | +| `migrate-take-kernel` | `--profile`, `--sha256`, `--size` | installs the profile kernel on stdin, if it has that sha256 and size | +| `migrate-take-image` | `--digest`, `--ref` | pulls the image by its reference unless the store has its digest | `hull images` prints `No images found` and `hull ps` prints `No instances found` on an empty store. Both exit 0. `hull prune` prints @@ -350,6 +359,7 @@ See [telemetry.md](telemetry.md). | `TERM` | forwarded into the guest for a pty `exec` | | `COMPOSE_FILE`, `COMPOSE_PROJECT_NAME`, `COMPOSE_PROFILES`, `COMPOSE_DISABLE_ENV_FILE` | compose equivalents of the flags above | | `HULL_MIGRATE_SSH` | a program `hull migrate` runs instead of `ssh`, with the same arguments; for testing a move on one machine (see [migration.md](migration.md#testing-a-move)) | +| `HULL_MIGRATE_SSH_MUX` | `off` gives every step of a move its own ssh connection. Otherwise the steps share one (`ControlMaster`, socket `/tmp/hull-ssh-%C`), kept a minute after the last, so the commit pays for no handshake | A session counts as interactive only when `CI` is unset **and** both stdin and stderr are terminals. diff --git a/docs/migration.md b/docs/migration.md index a7a6fad3..8a218cca 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -30,6 +30,15 @@ what can go wrong, and how to recover. > pair](#the-portable-profile-on-the-linux-pair)). A portable move between > this Mac and a Pi has not run. +> **Streamed moves, 2026-10-15.** With an `hvi` that streams its +> checkpoint (`feat/teleport-fast` merged with `feat/teleport-fast-disk`, +> built as ef8833a in a scratch clone), `hull migrate` sends the guest's +> RAM in pre-copy rounds while it runs and only the disk blocks that differ +> from its base. Between rp1 and rp2 the 1 GiB ubuntu guest moved in 5.4 s +> and 5.7 s, and was stopped for 279 ms and 340 ms as the guest's own clock +> counts it; the whole-bundle move of the same guest on the same hosts and +> binaries stopped it for 7.43 s. See [The streamed move](#the-streamed-move). + ## Usage ```bash @@ -40,6 +49,8 @@ hull migrate vm1 --to pm@rp2 \ hull migrate vm1 --txid 3f...e1 # resume a move that stopped hull migrate --status vm1 # this host's record, and the other's hull migrate --abort 3f...e1 # end a move that has not committed +hull migrate-prepare vm1 --to pm@rp2 # send the base, kernel and image ahead of a move +hull migrate vm1 --to pm@rp2 --whole-bundle # the whole checkpoint and disk, as before streaming ``` | Flag | Meaning | @@ -55,6 +66,7 @@ hull migrate --abort 3f...e1 # end a move that has not com | `--status` | with NAME or TXID: print the move's record here and on the other host | | `--timeout SECS` | how long hvi may take to write the fenced checkpoint; default 120 | | `--allow-no-restore` | testing only: go on although an `hvi` on either host cannot restore | +| `--whole-bundle` | send the whole fenced checkpoint and disk even when both hosts can stream | The instance must be an `hvi` instance on a block rootfs, with no shared host directory, running or paused, started with a control socket. Both hosts @@ -95,11 +107,17 @@ reply on stdout and nothing else. Either side can also end in `aborted`. +A streamed move (see [The streamed move](#the-streamed-move)) has no +separate fence step. The source goes from `reserved` to `transferring`, and +hvi fences the guest inside the transfer, once it has sent the RAM rounds. +The destination restores the guest paused while it receives, so the +streamed receive ends in `restored-paused`, and the source skips step 5. + The plan listed the transfer before the fence. In this version the fence comes first, because the bundle a move carries is the fenced checkpoint: hvi has no pre-copy yet, so there is nothing to send until the guest has stopped. -Pre-copy (hvi H7) would send most of the RAM and the disk while the guest -runs, then fence and send what changed. +The streamed move below does that: it sends most of the RAM and of the +disk while the guest runs, then fences it and sends what changed. ### Epochs and txids @@ -254,6 +272,315 @@ each bundle file against the manifest as it reads it. `migration.json` records the stream's bytes, its duration and its rate. +## The streamed move + +A move streams when both hosts can, and otherwise takes the whole-bundle +path above and says why on its second line (`Transport: ...`). It needs: + +- an `hvi` here of checkpoint format 2 (the `checkpoint_format` in the + instance's machine profile), which has the `migrate` op; +- a hull there whose probe lists the capability `stream-v2`, and an `hvi` + there that takes `restore --from-stream`; +- a base of the instance's disk in this store (below). The destination + need not hold it yet: the move sends it before the guest stops. + +`--whole-bundle` asks for the old path. A hull from before streamed moves +lists no capabilities, so the two still move instances the old way. A hull +that lists only `stream-v1` (the first streamed moves, whose trailer had no +account of the disk rounds and whose destination did not accept the header +before the fence) gets the whole bundle too. + +### Bases + +A base is the disk image exactly as `hull run` built it, before the guest +first wrote to it. `hull run` keeps one for every `hvi` instance on a block +rootfs that has a control socket: it clones the new image (APFS +`clonefile` on macOS, `FICLONE` on Linux, else a copy that keeps the +holes) into `/bases/`, hashes the copy, makes it read-only and names +it `.ext4`. The digest is recorded in `state.json` (`base`) and +passed to an `hvi` that takes `--disk-base-sha256`, which records it in +its profile and in `disk.json`. On the Pis keeping the 111 MiB ubuntu base +took 0.45 s of `hull run` (a copy: ext4 has no reflink). + +The disk digest is the sha256 of `HULLDISK1\0`, the image's size and the +block size (64 KiB, big-endian u64 each), then, for every 64 KiB block on +an absolute boundary that holds a byte that is not zero, the block's index +(u64) and its bytes; the last block may be short. A hole and a block of +written zeros hash alike, so a clone, a sparse copy and a received copy of +one image have one digest. It is not the image's plain sha256, which would +have to read every hole. hvi's own docs call the value of +`--disk-base-sha256` the disk's SHA-256; hvi does not read the disk to +check it, and from hull it is this digest. + +`mke2fs` writes a fresh UUID and timestamps into every image it builds, so +two runs of one OCI image get two bases. A base therefore serves one +instance and the moves of it, not every instance of an image. Nothing +removes a base yet: `hull rm` and `hull prune` leave `/bases` alone. + +### Preparing a destination + +```bash +hull migrate-prepare vm1 --to pm@rp2 [--remote-store DIR] [--remote-hull PATH] [--json] +``` + +puts on the destination, while the guest keeps running, what a streamed +move then does not carry: + +| Step | What it does | +|---|---| +| image | `hull pull REF` there by the reference this store pulled it by, unless that store has the digest. The move does not need it; a later `hull run` there does. A failure is reported and fails nothing | +| profile kernel | for a `--profile` instance, this store's kernel of that profile, unless the destination's record has the same sha256. It is checked against the sha256 and installed through the same import as `HULL_PORTABLE_KERNEL`, from a copy kept under `bases/kernels/` | +| base | the base, as lz4-compressed non-zero blocks (`HULLBAS1`); the destination writes it to a temporary file, keeps it only if what arrived has the base's digest, and makes it read-only | +| gateway | with `--remote-gateway-sock`, whether it answers. hull does not start one: brig starts the gateway with the session's policy | + +Each step finds its effect already there and sends nothing, so the command +can run any number of times; the destination's probe lists the bases and +the profile kernels it holds. On rp2 the first run took 5.9 s (the image +pull 3.7 s, the 53 MB kernel 0.8 s, the base 110.8 MiB of data as 49.2 MiB +in 1.1 s) and the second 0.24 s, every step `held`. + +The destination's own step of a move keeps the name: `migrate-prepare +--txid T` still brings a received checkpoint up paused. The prepare verbs +it runs there are hidden: `migrate-take-image`, `migrate-take-kernel` and +`migrate-take-base`. + +### The order on the wire + +One ssh session runs `hull migrate-receive --stream --txid T --name N`, and +its stdin carries, in this order: + +1. The disk rounds while the guest runs. Round 0 is every block that + differs from the base: hvi's blocks written since boot when the running + `hvi` booted from the base or names it in its profile (an `hvi` that + does carries those blocks across a restore), otherwise every block of + the disk compared with the base. Rounds 1 to 4 are hvi's blocks written + since the round before, read with `disk-dirty` and `disk-blocks`, until + a round finds at most 32 blocks (2 MiB). Without hvi's rounds there is + only round 0. +2. hvi's `migrate` stream, relayed frame by frame as hvi writes it: RAM + rounds while the guest runs, then the pause, the last RAM round, the + machine's documents and the manifest. hvi fences the guest once the + manifest is written to its socket. +3. The last disk round: hvi's blocks written since the last round, read + once hvi has fenced the guest and synced its disk; or, without hvi's + rounds, the whole disk compared with what the destination holds. +4. The trailer. + +hvi serves one control connection at a time, so hull cannot read disk +rounds while `migrate` holds the socket: the last disk round is read after +the seal, not inside the pause, and it holds every block the guest wrote +while hvi's RAM rounds ran. That is the guest's disk write rate times the +time those rounds take, capped by how many distinct blocks the guest +rewrites, and it is part of the downtime. hull sets the spool and the lock +up before the disk rounds, so the last one runs right before `migrate`, and +records the fastest rate the guest wrote at during the disk rounds +(`diskWriteRate`) and the last round's size (`finalDiskBytes`). + +Nothing else bounds it. Limiting hvi's RAM rounds by that rate (`max_secs`) +was tried and made the downtime worse: hvi checks the limit between rounds, +so it stopped after RAM round 0 and took round 1 (30 MiB) with the guest +paused, while the last disk round stayed at the same 484 blocks (the guest +rewrote one 30 MiB region, however long the rounds ran). The fix is hvi's: +disk rounds beside the RAM rounds, either as `disk-blocks` components inside +the `migrate` stream with the last one taken inside the pause, or the disk +ops served on a second connection while `migrate` runs. The guest never runs again after the +pause, so that round and the RAM still describe one machine. It also means +the `disk.json` hvi writes inside the pause does not list the last round, +and the destination's restore cannot be given `--disk-generation`, whose +value comes from that round. hull makes the same check itself (below). + +### The format + +``` +"HULLMIG2" +'H' u32 len {"txid", "name", "epoch", "baseSha256", "diskSize", "blockSize": 65536, "replay"?} +'D' u32 len {"round", "final"?, "first", "count", "codec": "lz4"|"raw", "rawSize", "size", "sha256"} + then size bytes: count adjacent blocks from first, lz4 when smaller +'C' u32 len hvi's frame header line, as hvi wrote it, then its bytes +'T' u32 len {"records", "chain", "hviReply", "disk"} +``` + +A disk batch's sha256 is of its raw bytes and is checked before a byte is +written. An hvi frame is relayed with hvi's own header line, checked by the +source as it reads it from hvi and by the destination as it passes it on; +hvi checks it a third time. `chain` is a running sha256 over every record's +kind and header, and `records` their number, so a record lost, added, +moved or altered fails the move even when each payload is intact. The +reader holds every length, index and name to limits before it allocates, +and refuses a stream without its trailer or with bytes after it. + +The trailer's `disk` is the source's account of the disk rounds: +`{"engine", "rounds": [{"round", "final"?, "from": "hvi"|"compare", +"blocks", "batches", "sentSha256", "generation"?, "bitmapSha256"?}]}`. +`sentSha256` is the sha256 of the round's batch digests (the hex `sha256` +of each `D` record) in order, with nothing between them. For a round from +hvi it is hvi's own `sent_sha256`, which the source checks against the +batches it read before it sends them; the source also checks each +`disk-dirty` bitmap against its `bitmap_sha256`, and every `disk-blocks` +reply's counts. The destination counts the batches of each round as it +applies them and refuses, before hvi gets the machine: + +- a round whose blocks, batches or digest differ from the trailer's, or + one the trailer does not list; +- for a live move with hvi's rounds, a round `disk.json` lists that the + stream does not carry, or whose `sent_sha256`, generation, block count + or bitmap digest there differ from the trailer's; `disk.json` is covered + by hvi's manifest; +- a live move with hvi's rounds whose final round is missing, is not the + round after the last one `disk.json` lists, or was read at a generation + other than the one hvi captured the disk at: a write of the guest would + be missing. This is the check `hvi restore --disk-generation` makes. + +A replay sends the disk by comparison, which `disk.json` does not +describe, so only the trailer's own account is checked there. + +### The destination + +`migrate-receive --stream` copies the base to the instance's disk, applies +each disk batch to it, and starts `hull restore --detach --paused --move T` +with a pipe as its stdin once `profile.json` has arrived and matched the +move's machine profile; that restore runs `hvi restore --from-stream -`, +which reads the pipe. While the stream arrives it reads the base and +checks that its blocks hash to the digest it is named by; one that does +not is set aside as `.ext4.bad-