From 2fbc60cd1c47f195d1c9855e4de71f87f13769b8 Mon Sep 17 00:00:00 2001 From: chris lee Date: Sun, 11 Oct 2026 17:49:10 -0400 Subject: [PATCH] macOS guests 4/6: stopped snapshots, same-instance standby and restore, and stopped clones Stopped snapshots after a graceful stop, same-instance restore through a staged disk swap, experimental same-instance standby, and clones from a stopped source with a fresh machine identifier, MAC and SSH host keys. --- cmd/vz-shim/server.go | 16 + docs/macos-snapshots.md | 122 +++++++ docs/macos-support.md | 16 +- lib/hypervisor/snapshot_support.go | 8 + lib/hypervisor/snapshot_support_test.go | 34 ++ lib/hypervisor/tracing.go | 12 + lib/hypervisor/vz/client.go | 35 +- lib/hypervisor/vz/client_test.go | 93 +++++ lib/hypervisor/vz/shimconfig/config.go | 2 - lib/instances/create.go | 2 +- lib/instances/fork.go | 22 +- lib/instances/macos.go | 51 +-- lib/instances/macos_admission_test.go | 42 +++ lib/instances/macos_boot_admission_test.go | 84 +++++ lib/instances/macos_identity.go | 258 ++++++++++++++ lib/instances/macos_identity_buffer_test.go | 20 ++ lib/instances/macos_identity_darwin.go | 16 + lib/instances/macos_identity_other.go | 9 + lib/instances/macos_identity_test.go | 83 +++++ lib/instances/macos_memory_snapshot_test.go | 377 ++++++++++++++++++++ lib/instances/macos_shim_gone_test.go | 24 ++ lib/instances/macos_snapshot_plan_test.go | 55 +++ lib/instances/macos_test.go | 67 +++- lib/instances/manager.go | 10 + lib/instances/restore.go | 12 +- lib/instances/snapshot.go | 50 ++- lib/instances/snapshot_compression.go | 3 + lib/instances/snapshot_macos.go | 354 ++++++++++++++++++ lib/instances/snapshot_macos_darwin_test.go | 72 ++++ lib/instances/snapshot_macos_test.go | 164 +++++++++ lib/instances/snapshot_schedule.go | 4 +- lib/instances/snapshot_schedule_test.go | 6 +- lib/instances/standby.go | 26 +- lib/instances/start.go | 12 + lib/instances/stop.go | 5 + lib/instances/types.go | 6 + lib/instances/vm_config_validation.go | 3 + lib/network/allocate.go | 11 + 38 files changed, 2107 insertions(+), 79 deletions(-) create mode 100644 docs/macos-snapshots.md create mode 100644 lib/hypervisor/snapshot_support.go create mode 100644 lib/hypervisor/snapshot_support_test.go create mode 100644 lib/instances/macos_admission_test.go create mode 100644 lib/instances/macos_boot_admission_test.go create mode 100644 lib/instances/macos_identity.go create mode 100644 lib/instances/macos_identity_buffer_test.go create mode 100644 lib/instances/macos_identity_darwin.go create mode 100644 lib/instances/macos_identity_other.go create mode 100644 lib/instances/macos_identity_test.go create mode 100644 lib/instances/macos_memory_snapshot_test.go create mode 100644 lib/instances/macos_snapshot_plan_test.go create mode 100644 lib/instances/snapshot_macos.go create mode 100644 lib/instances/snapshot_macos_darwin_test.go create mode 100644 lib/instances/snapshot_macos_test.go diff --git a/cmd/vz-shim/server.go b/cmd/vz-shim/server.go index 74640b8eb..621c0f3b2 100644 --- a/cmd/vz-shim/server.go +++ b/cmd/vz-shim/server.go @@ -59,6 +59,7 @@ func (s *ShimServer) Handler() http.Handler { mux.HandleFunc("PUT /api/v1/vm.resume", s.handleResume) mux.HandleFunc("PUT /api/v1/vm.shutdown", s.handleShutdown) mux.HandleFunc("PUT /api/v1/vm.snapshot", s.handleSnapshot) + mux.HandleFunc("GET /api/v1/vm.snapshot-support", s.handleSnapshotSupport) mux.HandleFunc("GET /api/v1/vm.balloon", s.handleGetBalloon) mux.HandleFunc("PUT /api/v1/vm.balloon", s.handleSetBalloon) mux.HandleFunc("PUT /api/v1/vm.power-button", s.handlePowerButton) @@ -138,6 +139,21 @@ func (s *ShimServer) handleShutdown(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } +func (s *ShimServer) handleSnapshotSupport(w http.ResponseWriter, r *http.Request) { + s.mu.RLock() + defer s.mu.RUnlock() + response := struct { + Supported bool `json:"supported"` + Reason string `json:"reason,omitempty"` + }{Supported: true} + if err := validateSaveRestoreSupport(s.vmConfig); err != nil { + response.Supported = false + response.Reason = err.Error() + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + func (s *ShimServer) handleSnapshot(w http.ResponseWriter, r *http.Request) { s.mu.Lock() defer s.mu.Unlock() diff --git a/docs/macos-snapshots.md b/docs/macos-snapshots.md new file mode 100644 index 000000000..164f99bbf --- /dev/null +++ b/docs/macos-snapshots.md @@ -0,0 +1,122 @@ +# macOS snapshots, schedules and restore (PR 4, draft) + +Supported experimentally: stopped disk snapshots, same-instance memory checkpoints, +snapshot schedules, and cold forks from gracefully stopped instances or snapshots. +Independent warm memory forks remain unsupported. Native validation of the new +memory-snapshot and schedule paths is still required. + +## Stopped snapshots and cold forks + +- `kind: Stopped` requires a stopped source and a graceful shutdown receipt: + acknowledged guest shutdown with confirmed owned VMM exit. A forced stop is + refused with 409. +- Restore targets are Stopped or Running. The source instance, hardware model, + machine identifier and MAC must match the snapshot. +- A stopped-snapshot fork requires a graceful receipt and an enabled guest agent. + Disk and auxiliary storage are copied, the clone gets a fresh machine identifier + and MAC, and SSH host keys are rekeyed on first boot. The default target is + Stopped; Running cold-boots and completes rekeying before returning. +- Forking a stopped instance uses the same identity/rekey contract. Hostnames, + account credentials and application-specific identities are not rekeyed. +- Failed rekeying stops the clone and leaves rekey pending for a later retry. + +## Same-instance memory snapshots + +Use `POST /instances/{id}/snapshots` with `{"kind":"Standby"}`. + +- A Running source goes through pause, VZ machine-state save and confirmed VMM + shutdown. The disk, auxiliary storage and machine state are copied together + while the VMM is down; then the source resumes from its own checkpoint. +- A Standby source is copied without being resumed. Stopped and other source + states are rejected with 409. +- Before pausing, the shim validates the exact VM configuration using Apple's + `validateSaveRestoreSupport`. Host/architecture capability checks alone are + not proof that the guest's configured devices support save/restore. +- Pause waits for raw Paused state, not transitional Pausing state, before save. + Error-path resume and source recovery use bounded cleanup contexts that survive + cancellation of the snapshot request. If recovery fails, the API reports the + failure; the source may need explicit recovery. +- VZ machine-state files stay uncompressed. Explicit enabled compression is + rejected before capture; Linux server compression defaults do not apply. + +Restore using `POST /instances/{id}/snapshots/{snapshot_id}/restore`. + +- This is a same-instance operation, not a clone. Restore requires an inactive + Stopped or Standby source; Running, Paused and other live states are refused. +- The default target for a memory snapshot is Running. Explicit Standby installs + the checkpoint without resuming it. Stopped discards memory and does not create + a graceful-shutdown receipt; a subsequent clean boot/stop is required before + making a stopped snapshot or cold fork. +- Preflight checks saved CPU/RAM, hardware model, machine identifier, NIC MAC, + disk/auxiliary paths and the complete memory/disk/auxiliary bundle before + replacing source storage. Invalid payloads leave current storage untouched. +- Restored metadata is staged with the payload, including the current instance + name and expiration. Directory-swap recovery therefore commits or rolls back + the matching metadata and storage together. Host phase accounting is preserved, + rather than replayed from the checkpoint; memory restore retains its boot time. + An explicit Stopped target suppresses automatic restart. +- Restoring never mutates the persistent snapshot. Framework restore failure + after installation leaves the restored checkpoint available in Standby. +- If the API dies during a restore, the shim it started keeps loading the VM. + Until it opens its socket the instance reads Standby, and start or restore + return 409 while that shim runs. Once loaded it reads Paused: `POST /restore` + resumes it and completes the restore (records the shim, removes the consumed + checkpoint), and DELETE kills it. As on Linux, stop does not accept Paused. +- Saved state is not portable: use the originating Mac and OS account, with a + compatible framework version and VM configuration. API availability on macOS + 14+ does not guarantee support for a particular macOS guest configuration. + +## Schedules + +The existing snapshot-schedule API, interval handling and retention policies are +shared with Linux. macOS schedules require host save/restore support; admission +also probes the exact configuration when a live VM is available. + +- A gracefully Stopped source produces a stopped snapshot. +- A Running source produces a memory snapshot and is resumed afterwards. +- A Standby source produces a memory snapshot and remains in Standby. +- Ineligible state, absent graceful receipt, configuration or capture failure is + recorded in `last_error`; no successful snapshot ID is reported for that run. +- Failed runs consume the interval, matching existing scheduler behavior. The + next run follows the configured cadence rather than tight retries. +- Retention and schedule deletion use the existing controller. Scheduling does + not stop a running guest to manufacture a graceful disk snapshot. + +## Warm memory forks: feasibility gate + +`ForkSnapshot` still rejects macOS memory snapshots with 501, and `ForkInstance` +still requires a gracefully stopped macOS source. These guards must not be +removed merely because same-instance restore works. + +VZ restore requires compatible saved device/platform configuration. The current +VZ fork preparation preserves NIC identity, while independent macOS guests need +unique machine identifiers, auxiliary storage and network identities. Hypeman +also rejects concurrently live duplicate machine identifiers or MACs. + +A native experiment must establish a supported identity-transition mechanism +before independent warm forks can be enabled. Reusing the source identity or +rekeying only SSH host keys is not an independent warm fork. + +References: +- [Apple: macOS virtual machines](https://developer.apple.com/documentation/virtualization/running-macos-in-a-virtual-machine-on-apple-silicon) +- [Apple: Mac platform configuration](https://developer.apple.com/documentation/virtualization/vzmacplatformconfiguration) +- [Apple: restoring machine state](https://developer.apple.com/documentation/virtualization/vzvirtualmachine/restoremachinestatefrom(url:completionhandler:)) + +## Evidence and remaining native QA + +- Linux-runnable filesystem/metadata tests: memory-bundle capture from Standby, + same-instance restore, disk/aux/memory isolation, manifest mismatch rejection, + preserved expiration/boot time/phase accounting, stopped-target restart + suppression, schedules, retention, failed-run cadence and crash-swap metadata. +- Mock tests cover configuration rejection before pause/schedule persistence and + recovery contexts after request cancellation. HTTP fixtures cover VZ pause + completion, cancellation and configuration probing. These are not native VMs. +- A Darwin metadata test covers two independent stopped-snapshot forks with fresh + identifiers/MACs, snapshot isolation and duplicate-name rejection. It does not + boot a guest. +- Earlier revisions were tested live for stopped snapshot restore and stopped + instance clones with distinct UUID, MAC, address and SSH keys. Those runs do + not validate the new memory-snapshot/schedule implementation. +- Still required on macOS: exact-configuration probe, Running capture/resume, + same-instance memory rollback, repeated restore, scheduled captures/retention, + cancellation after pause/save, and restart during staged restore. diff --git a/docs/macos-support.md b/docs/macos-support.md index b3f71b461..e00bf3f80 100644 --- a/docs/macos-support.md +++ b/docs/macos-support.md @@ -15,14 +15,18 @@ owns the behavior. | Memory reclaim | Unsupported | Reported per guest as `unsupported`. No balloon change. | PR 1 | | Guest port forwarding | Agent-gated | HTTP and WebSocket to a guest loopback port at `/instances/{id}/ports/{port}/*`. The image starts its own services. See [macos-guest-ports.md](macos-guest-ports.md). | PR 6 | | Builds | Partial | Opt-in `macos_builds`. Restricted recipe. Linux Dockerfile builds unsupported. | PR 5 | -| Stopped snapshots, same-instance restore | Partial | Graceful stop required. Running restore has an open readiness issue. | PR 4 | -| Standby, memory snapshot, memory fork | Unsupported | Rejected with a stable error. | — | -| Stopped clones, fork from snapshot | Unsupported | Needs identity rebinding and guest rekeying. | PR 4 (later) | +| Stopped snapshots, same-instance restore | Experimental | Graceful stop required. Stopped and Running targets; restored metadata and storage are staged together. | PR 4 | +| Standby and restore (same instance) | Experimental | Saves memory, restores the same instance. Restore can fail (about 2% observed, cause unknown); a failed restore leaves the instance in Standby. Recover by deleting, as on Linux. Standby auto-policies rejected: no host-visible connection tracking on NAT. | PR 4 | +| Same-instance memory snapshots | Experimental | Running/Standby capture, with exact VZ configuration validation and an uncompressed disk/aux/memory bundle. Same-instance restore only; new path needs native QA. | PR 4 | +| Independent warm memory forks | Unsupported | Remain rejected: saved-state compatibility and independent machine/NIC identity need native proof. | — | +| Stopped clones (fork from stopped) | Experimental | Rebinds machine identifier and MAC; rekeys SSH host keys on first start. Hostname and credentials not yet checked. | PR 4 | +| Fork from stopped snapshot | Experimental | Graceful receipt and enabled agent required. Fresh machine identifier/MAC; SSH rekey on first cold boot. Stopped and Running targets supported. | PR 4 | | Update (PATCH), VM restart policy | Partial | Expiration/TTL and `restart_policy: always` or `never` are supported. Manual stop suppresses automatic restart. A shim that dies without removing its socket (for example, SIGKILL) reads as Stopped, so `always` restarts it. `on_failure` is rejected: no workload exit status is available. | PR 1 | | Exec health checks | Agent-gated | Supported at create and PATCH; `type: none` disables them. HTTP/TCP checks remain unsupported. | PR 3 | | Volumes, env and commands, credentials and auto-standby policies, passthrough, shaping | Unsupported | Rejected at create, start, or update. | — | | Ingress | Unsupported | Refused in macOS-only mode and for macOS targets. | — | -| Snapshot schedules | Unsupported | Refused at admission. | — | +| Snapshot schedules | Experimental | State-based stopped or memory capture, existing cadence/retention, host save/restore gate and live configuration probe. New path needs native QA. | PR 4 | -Limits that apply everywhere: preserved machine identity (one active instance per -identifier, no rekeying), no clean-base sanitation audit, and no external review verdict. +Limits that apply everywhere: one active VM per machine identifier, no clean-base +sanitation audit, and no external review verdict. Same-instance memory restore +preserves identity; stopped clones get fresh machine identifiers and SSH host keys. diff --git a/lib/hypervisor/snapshot_support.go b/lib/hypervisor/snapshot_support.go new file mode 100644 index 000000000..1afcc89dc --- /dev/null +++ b/lib/hypervisor/snapshot_support.go @@ -0,0 +1,8 @@ +package hypervisor + +import "context" + +// SnapshotSupportValidator checks configuration-specific support before pausing. +type SnapshotSupportValidator interface { + ValidateSnapshotSupport(context.Context) error +} diff --git a/lib/hypervisor/snapshot_support_test.go b/lib/hypervisor/snapshot_support_test.go new file mode 100644 index 000000000..f643d1572 --- /dev/null +++ b/lib/hypervisor/snapshot_support_test.go @@ -0,0 +1,34 @@ +package hypervisor + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +type fakeSnapshotSupportValidator struct { + fakeHypervisor + err error + called bool +} + +func (h *fakeSnapshotSupportValidator) ValidateSnapshotSupport(context.Context) error { + h.called = true + return h.err +} + +func TestTracingPreservesSnapshotSupportValidation(t *testing.T) { + probeErr := errors.New("configuration probe failed") + hv := &fakeSnapshotSupportValidator{err: probeErr} + wrapped := WrapHypervisor(TypeVZ, hv) + validator, ok := wrapped.(SnapshotSupportValidator) + require.True(t, ok) + require.ErrorIs(t, validator.ValidateSnapshotSupport(context.Background()), probeErr) + require.True(t, hv.called) + hv.err = nil + require.NoError(t, validator.ValidateSnapshotSupport(context.Background())) + unsupported := WrapHypervisor(TypeCloudHypervisor, fakeHypervisor{}).(SnapshotSupportValidator) + require.ErrorIs(t, unsupported.ValidateSnapshotSupport(context.Background()), ErrNotSupported) +} diff --git a/lib/hypervisor/tracing.go b/lib/hypervisor/tracing.go index 4694c4e21..4abf828e2 100644 --- a/lib/hypervisor/tracing.go +++ b/lib/hypervisor/tracing.go @@ -219,6 +219,18 @@ func (h *tracingHypervisor) Resume(ctx context.Context) (err error) { return h.next.Resume(ctx) } +func (h *tracingHypervisor) ValidateSnapshotSupport(ctx context.Context) (err error) { + validator, ok := h.next.(SnapshotSupportValidator) + if !ok { + return ErrNotSupported + } + ctx, span := startTraceSpan(ctx, h.tracer, "hypervisor.validate_snapshot_support", + h.spanAttrs(attribute.String("operation", "validate_snapshot_support"))..., + ) + defer func() { finishTraceSpan(span, err) }() + return validator.ValidateSnapshotSupport(ctx) +} + func (h *tracingHypervisor) Snapshot(ctx context.Context, destPath string) (err error) { ctx, span := startTraceSpan(ctx, h.tracer, "hypervisor.snapshot", h.spanAttrs( diff --git a/lib/hypervisor/vz/client.go b/lib/hypervisor/vz/client.go index 1f2bc7ae7..984c7edd6 100644 --- a/lib/hypervisor/vz/client.go +++ b/lib/hypervisor/vz/client.go @@ -262,7 +262,28 @@ func (c *Client) GetVMInfo(ctx context.Context) (*hypervisor.VMInfo, error) { } func (c *Client) Pause(ctx context.Context) error { - return c.doPut(ctx, "/api/v1/vm.pause", nil) + if err := c.doPut(ctx, "/api/v1/vm.pause", nil); err != nil { + return err + } + return c.waitForRawState(ctx, "Paused") +} + +func (c *Client) ValidateSnapshotSupport(ctx context.Context) error { + body, err := c.doGet(ctx, "/api/v1/vm.snapshot-support") + if err != nil { + return err + } + var response struct { + Supported bool `json:"supported"` + Reason string `json:"reason"` + } + if err := json.Unmarshal(body, &response); err != nil { + return fmt.Errorf("decode snapshot support: %w", err) + } + if !response.Supported { + return fmt.Errorf("%w: VZ configuration cannot be saved/restored: %s", hypervisor.ErrNotSupported, response.Reason) + } + return nil } func (c *Client) Resume(ctx context.Context) error { @@ -279,16 +300,16 @@ func (c *Client) Resume(ctx context.Context) error { } const ( - resumeReadyTimeout = 10 * time.Second - resumeReadyPollInterval = 25 * time.Millisecond + stateReadyTimeout = 10 * time.Second + stateReadyPollInterval = 25 * time.Millisecond ) // waitForRawState polls vm.info until the shim's raw state string equals want or -// resumeReadyTimeout elapses. It checks the raw state (not the hypervisor.VMState +// stateReadyTimeout elapses. It checks the raw state (not the hypervisor.VMState // mapping, which collapses "Resuming" into Running) so the post-condition is the // guest actually being in the requested state. func (c *Client) waitForRawState(ctx context.Context, want string) error { - deadline := time.Now().Add(resumeReadyTimeout) + deadline := time.Now().Add(stateReadyTimeout) for { state, err := c.rawVMState(ctx) if err == nil && state == want { @@ -298,12 +319,12 @@ func (c *Client) waitForRawState(ctx context.Context, want string) error { if err != nil { return fmt.Errorf("wait for vm state %q: %w", want, err) } - return fmt.Errorf("vm did not reach state %q within %s (last state %q)", want, resumeReadyTimeout, state) + return fmt.Errorf("vm did not reach state %q within %s (last state %q)", want, stateReadyTimeout, state) } select { case <-ctx.Done(): return ctx.Err() - case <-time.After(resumeReadyPollInterval): + case <-time.After(stateReadyPollInterval): } } } diff --git a/lib/hypervisor/vz/client_test.go b/lib/hypervisor/vz/client_test.go index c3090540f..cba9afcdd 100644 --- a/lib/hypervisor/vz/client_test.go +++ b/lib/hypervisor/vz/client_test.go @@ -13,6 +13,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/kernel/hypeman/lib/hypervisor" ) const connectionCloseTimeout = 2 * time.Second @@ -84,6 +86,97 @@ func (f *unixHTTPFixture) waitForNoConnections(t *testing.T) { } } +func TestPauseWaitsForRawPausedStateBeforeSnapshot(t *testing.T) { + var reads atomic.Int32 + fixture := newUnixHTTPFixture(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/vmm.ping": + _, _ = w.Write([]byte("OK")) + case "/api/v1/vm.pause": + w.WriteHeader(http.StatusNoContent) + case "/api/v1/vm.info": + state := "Pausing" + if reads.Add(1) >= 3 { + state = "Paused" + } + _, _ = w.Write([]byte(`{"state":"` + state + `"}`)) + case "/api/v1/vm.snapshot": + if reads.Load() < 3 { + http.Error(w, "not paused", http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + client, err := NewClient(fixture.socketPath()) + if err != nil { + t.Fatal(err) + } + if err := client.Pause(context.Background()); err != nil { + t.Fatal(err) + } + if err := client.Snapshot(context.Background(), "/snapshot"); err != nil { + t.Fatal(err) + } +} + +func TestPauseWaitHonorsCancellation(t *testing.T) { + fixture := newUnixHTTPFixture(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/vmm.ping": + _, _ = w.Write([]byte("OK")) + case "/api/v1/vm.pause": + w.WriteHeader(http.StatusNoContent) + default: + _, _ = w.Write([]byte(`{"state":"Pausing"}`)) + } + })) + client, err := NewClient(fixture.socketPath()) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if err := client.Pause(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected deadline exceeded, got %v", err) + } +} + +func TestValidateSnapshotSupportUsesConfigurationProbe(t *testing.T) { + for _, test := range []struct { + name string + response string + wantErr bool + }{ + {"supported", `{"supported":true}`, false}, + {"unsupported", `{"supported":false,"reason":"unsupported device"}`, true}, + {"missing-result", `{}`, true}, + } { + t.Run(test.name, func(t *testing.T) { + fixture := newUnixHTTPFixture(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/vmm.ping" { + _, _ = w.Write([]byte("OK")) + return + } + _, _ = w.Write([]byte(test.response)) + })) + client, err := NewClient(fixture.socketPath()) + if err != nil { + t.Fatal(err) + } + err = client.ValidateSnapshotSupport(context.Background()) + if test.wantErr && !errors.Is(err, hypervisor.ErrNotSupported) { + t.Fatalf("expected unsupported configuration, got %v", err) + } + if !test.wantErr && err != nil { + t.Fatal(err) + } + }) + } +} + func TestRepeatedClientsReleaseSuccessfulControlConnections(t *testing.T) { fixture := newUnixHTTPFixture(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { diff --git a/lib/hypervisor/vz/shimconfig/config.go b/lib/hypervisor/vz/shimconfig/config.go index 2cd81671a..039114de8 100644 --- a/lib/hypervisor/vz/shimconfig/config.go +++ b/lib/hypervisor/vz/shimconfig/config.go @@ -1,5 +1,3 @@ -//go:build darwin - // Package shimconfig defines the configuration types shared between // the hypeman API server and the vz-shim subprocess. package shimconfig diff --git a/lib/instances/create.go b/lib/instances/create.go index 25b9b443d..c2ac01562 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -811,7 +811,7 @@ func (m *manager) startAndBootVM( if stored.MacOS != nil { m.macOSBootMu.Lock() defer m.macOSBootMu.Unlock() - if err := m.checkMacOSIdentityAvailable(ctx, stored); err != nil { + if err := m.checkMacOSBootAvailable(ctx, stored); err != nil { return err } } diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 5a5c57847..4ccf52b0f 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -46,6 +46,11 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR return nil, "", false, err } source := m.toInstance(ctx, meta) + // A macOS clone copies the source's disk and aux, so it is taken only from a + // gracefully stopped source, as stopped snapshots are. + if meta.MacOS != nil && (source.State != StateStopped || !meta.StopGraceful) { + return nil, "", false, fmt.Errorf("%w: macOS fork requires a gracefully stopped source (state %s)", ErrInvalidState, source.State) + } targetState, err := resolveForkTargetState(req.TargetState, source.State) if err != nil { return nil, "", false, err @@ -242,7 +247,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin if existsByMetadata { return nil, false, fmt.Errorf("%w: instance name '%s' already exists", ErrAlreadyExists, req.Name) } - if stored.NetworkEnabled { + if stored.NetworkEnabled && stored.MacOS == nil { exists, err := m.networkManager.NameExists(ctx, req.Name, "") if err != nil { return nil, false, fmt.Errorf("check instance name availability: %w", err) @@ -329,6 +334,15 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin forkMeta.IP = "" forkMeta.MAC = "" } + if forkMeta.MacOS != nil { + used, err := m.storedMACs(forkID) + if err != nil { + return nil, false, fmt.Errorf("list MACs for macOS clone: %w", err) + } + if err := prepareMacOSClone(&forkMeta, used); err != nil { + return nil, false, err + } + } restoreNeedsSourceLock := false if source.State == StateStandby { @@ -410,7 +424,7 @@ func (m *manager) saveForkMetadata(ctx context.Context, meta *metadata) error { retErr = fmt.Errorf("%w: instance name '%s' already exists", ErrAlreadyExists, name) return retErr } - if meta.NetworkEnabled { + if meta.NetworkEnabled && meta.MacOS == nil { exists, err := m.networkManager.NameExists(holdCtx, name, "") if err != nil { retErr = fmt.Errorf("check instance name availability: %w", err) @@ -421,6 +435,10 @@ func (m *manager) saveForkMetadata(ctx context.Context, meta *metadata) error { return retErr } } + if err := m.reserveMacOSCloneMAC(&meta.StoredMetadata); err != nil { + retErr = err + return retErr + } if err := m.saveMetadata(meta); err != nil { retErr = fmt.Errorf("save fork metadata: %w", err) return retErr diff --git a/lib/instances/macos.go b/lib/instances/macos.go index 1774bda08..3147ed23b 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -209,31 +209,16 @@ func (m *manager) checkMacOSIdentityAvailable(ctx context.Context, stored *Store if len(pids) > 0 { return fmt.Errorf("%w: a vz-shim (pid %d) is already running for this instance; if an interrupted operation left it, wait for it or delete the instance", ErrInvalidState, pids[0]) } - entries, err := os.ReadDir(m.paths.GuestsDir()) - if err != nil { - return err - } - for _, entry := range entries { - if !entry.IsDir() || entry.Name() == stored.Id { - continue - } - meta, err := m.loadMetadata(entry.Name()) - if errors.Is(err, ErrNotFound) { - // A create in progress or a leftover directory has no instance to admit against. - continue - } - if err != nil { - return fmt.Errorf("check Mac identity admission: %w", err) - } + return m.forEachOtherInstance(stored.Id, "check Mac identity admission", func(meta *metadata) error { if meta.MacOS == nil || !bytes.Equal(meta.MacOS.MachineIdentifier, stored.MacOS.MachineIdentifier) { - continue + return nil } state := m.deriveStateWithoutHydration(ctx, &meta.StoredMetadata).State if state.RequiresVMM() || state == StateUnknown { return fmt.Errorf("%w: macOS template identity is already in use by instance %s; stop it first", ErrInsufficientResources, meta.Id) } - } - return nil + return nil + }) } // GuestAgentEnabled reports whether exec, copy, readiness and shutdown may use the @@ -262,28 +247,16 @@ func guestAgentReadyExecutable(stored *StoredMetadata) string { // macOS guest, so a normal stop can record a graceful receipt and strict export waits. const macOSMinStopTimeout = 30 -// macOSUnsupportedOperations are the instance operations experimental macOS guests -// do not implement. -var macOSUnsupportedOperations = map[string]bool{ - "fork": true, - "restore": true, - "snapshot": true, - "restore snapshot": true, - "snapshot schedules": true, - "standby": true, +// hypervisorProcessLive reports whether the instance's VMM process is still running. +func hypervisorProcessLive(inst *Instance) bool { + pid, err := resolveLiveHypervisorPID(inst.HypervisorProcessIdentity, inst.SocketPath) + return err == nil && pid != 0 } -// admitGuestOperation refuses an operation this server cannot run for the guest: -// any Linux operation on a macOS-only server, and operations macOS guests do not -// implement. Callers check it right after loading the record under the lock. -func (m *manager) admitGuestOperation(stored *StoredMetadata, operation string) error { - if err := m.requireMacOSGuest(stored); err != nil { - return err - } - if stored.MacOS != nil && macOSUnsupportedOperations[operation] { - return fmt.Errorf("%w: %s is not implemented for experimental macOS instances", ErrInvalidRequest, operation) - } - return nil +// admitGuestOperation applies the server's guest-type restriction. Operation-specific +// admission follows at the caller, under the instance lock. +func (m *manager) admitGuestOperation(stored *StoredMetadata, _ string) error { + return m.requireMacOSGuest(stored) } // vmnet stores octets without leading zeroes. A lease is observed addressing, diff --git a/lib/instances/macos_admission_test.go b/lib/instances/macos_admission_test.go new file mode 100644 index 000000000..e0ac462b0 --- /dev/null +++ b/lib/instances/macos_admission_test.go @@ -0,0 +1,42 @@ +package instances + +import ( + "testing" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestClearMacOSStopReceiptPersistsBeforeBoot(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + require.NoError(t, m.ensureDirectories("mac")) + stored := StoredMetadata{Id: "mac", DataDir: p.InstanceDir("mac"), MacOS: &images.MacOSImage{}, StopGraceful: true} + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: stored})) + + require.NoError(t, m.clearMacOSStopReceipt(&stored)) + require.False(t, stored.StopGraceful) + meta, err := m.loadMetadata("mac") + require.NoError(t, err) + require.False(t, meta.StopGraceful, "the cleared receipt must be on disk before the guest boots") +} + +func TestReserveMacOSCloneMACRepicksTakenMAC(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + const taken = "02:00:00:00:00:01" + require.NoError(t, m.ensureDirectories("other")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: "other", DataDir: p.InstanceDir("other"), MAC: taken}})) + + source := &images.MacOSImage{MAC: taken} + clone := StoredMetadata{Id: "clone", NetworkEnabled: true, MAC: taken, MacOS: source} + require.NoError(t, m.reserveMacOSCloneMAC(&clone)) + require.NotEqual(t, taken, clone.MAC) + require.Equal(t, clone.MAC, clone.MacOS.MAC) + require.Equal(t, taken, source.MAC, "the re-pick must not alias another instance's macOS struct") + + free := StoredMetadata{Id: "clone2", NetworkEnabled: true, MAC: "02:00:00:00:00:02", MacOS: &images.MacOSImage{MAC: "02:00:00:00:00:02"}} + require.NoError(t, m.reserveMacOSCloneMAC(&free)) + require.Equal(t, "02:00:00:00:00:02", free.MAC, "a free MAC is kept") +} diff --git a/lib/instances/macos_boot_admission_test.go b/lib/instances/macos_boot_admission_test.go new file mode 100644 index 000000000..9f7f448cf --- /dev/null +++ b/lib/instances/macos_boot_admission_test.go @@ -0,0 +1,84 @@ +package instances + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type macOSBootAdmissionStarter struct { + concurrentForkPrepareTestStarter + manager *manager + started chan string + release <-chan struct{} +} + +func (s *macOSBootAdmissionStarter) StartVM(_ context.Context, _ *paths.Paths, _ string, socket string, _ hypervisor.VMConfig) (int, hypervisor.Hypervisor, error) { + id := filepath.Base(filepath.Dir(socket)) + s.started <- id + <-s.release + if err := os.WriteFile(socket, nil, 0600); err != nil { + return 0, nil, err + } + s.manager.storeCachedHypervisorState(id, hypervisor.StateRunning) + return os.Getpid(), lifecycleNoopHypervisor{state: hypervisor.StateRunning}, nil +} + +func TestMacOSConcurrentBootsRecheckMACUnderBootLock(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + release := make(chan struct{}) + starter := &macOSBootAdmissionStarter{manager: m, started: make(chan string, 2), release: release} + hvType := hypervisor.Type("macos-boot-admission-test") + m.vmStarters = map[hypervisor.Type]hypervisor.VMStarter{hvType: starter} + var guests []*StoredMetadata + for i, id := range []string{"first", "second"} { + require.NoError(t, m.ensureDirectories(id)) + macos := *testMacImage().MacOS + macos.MachineIdentifier = []byte{byte(i + 1)} + stored := &StoredMetadata{Id: id, DataDir: p.InstanceDir(id), SocketPath: p.InstanceSocket(id, starter.SocketName()), HypervisorType: hvType, MacOS: &macos, MAC: macos.MAC, NetworkEnabled: true, Size: int64(macos.Memory), Vcpus: int(macos.CPUs)} + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: *stored})) + guests = append(guests, stored) + } + + // Both advisory checks succeed before either guest enters the boot lock. + for _, stored := range guests { + require.NoError(t, m.checkMacOSMACAvailable(context.Background(), stored)) + require.NoError(t, m.checkMacOSIdentityAvailable(context.Background(), stored)) + } + results := make(chan error, 2) + for _, stored := range guests { + go func() { results <- m.startAndBootVM(context.Background(), stored, nil, nil, false) }() + } + select { + case <-starter.started: + case <-time.After(5 * time.Second): + close(release) + t.Fatal("first boot did not start") + } + close(release) + var succeeded, refused int + for range guests { + select { + case err := <-results: + if err == nil { + succeeded++ + } else { + require.True(t, errors.Is(err, ErrInsufficientResources), "%v", err) + refused++ + } + case <-time.After(5 * time.Second): + t.Fatal("boot admission did not finish") + } + } + require.Equal(t, 1, succeeded) + require.Equal(t, 1, refused) + require.Empty(t, starter.started, "the conflicting guest must never reach StartVM") +} diff --git a/lib/instances/macos_identity.go b/lib/instances/macos_identity.go new file mode 100644 index 000000000..c78cd5134 --- /dev/null +++ b/lib/instances/macos_identity.go @@ -0,0 +1,258 @@ +package instances + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/network" +) + +// macOSIdentityRekeyCommand replaces the guest's SSH host keys. A clone shares its +// source's keys until this runs, so it runs on the clone's first start. +const macOSIdentityRekeyCommand = "rm -f /etc/ssh/ssh_host_* && ssh-keygen -A" + +const ( + macOSRekeyDeadline = 3 * time.Minute + macOSRekeyStderrLimit = 4 << 10 +) + +// boundedBuffer keeps at most limit bytes and reports every write as accepted, so a guest +// cannot grow host memory or stall the exec stream by writing more. +type boundedBuffer struct { + buf []byte + limit int +} + +func (b *boundedBuffer) Write(p []byte) (int, error) { + if room := b.limit - len(b.buf); room > 0 { + if room > len(p) { + room = len(p) + } + b.buf = append(b.buf, p[:room]...) + } + return len(p), nil +} + +func (b *boundedBuffer) String() string { return string(b.buf) } + +// prepareMacOSClone gives a stopped macOS clone its own identity: a fresh Apple machine +// identifier, a MAC unique among stored instances, and a pending SSH host key rekey. +// The macOS struct is copied so the clone never aliases its source. +func prepareMacOSClone(clone *StoredMetadata, usedMACs []string) error { + if clone.MacOS == nil { + return nil + } + if !clone.MacOS.GuestAgent || clone.SkipGuestAgent { + return fmt.Errorf("%w: macOS clone needs the guest agent to rekey its identity", ErrInvalidRequest) + } + macos := *clone.MacOS + macos.HardwareModel = append([]byte(nil), clone.MacOS.HardwareModel...) + id, err := newMacMachineIdentifierData() + if err != nil { + return fmt.Errorf("generate macOS machine identifier: %w", err) + } + macos.MachineIdentifier = id + if clone.NetworkEnabled { + mac, err := network.NewUniqueMAC(usedMACs) + if err != nil { + return fmt.Errorf("allocate macOS clone MAC: %w", err) + } + macos.MAC = mac + clone.MAC = mac + } + clone.IP = "" + clone.MacOS = &macos + clone.IdentityRekeyPending = true + return nil +} + +// forEachOtherInstance calls fn with the metadata of every stored instance except +// exclude, stopping at fn's first error. Directories without metadata are skipped; +// any other load error is wrapped with loadErrorContext when it is set. +func (m *manager) forEachOtherInstance(exclude, loadErrorContext string, fn func(*metadata) error) error { + entries, err := os.ReadDir(m.paths.GuestsDir()) + if err != nil { + return err + } + for _, entry := range entries { + if !entry.IsDir() || entry.Name() == exclude { + continue + } + meta, err := m.loadMetadata(entry.Name()) + if errors.Is(err, ErrNotFound) { + // A create in progress or a leftover directory has no instance to admit against. + continue + } + if err != nil { + if loadErrorContext != "" { + return fmt.Errorf("%s: %w", loadErrorContext, err) + } + return err + } + if err := fn(meta); err != nil { + return err + } + } + return nil +} + +// storedMACs returns every MAC recorded by a stored instance, except exclude. +func (m *manager) storedMACs(exclude string) ([]string, error) { + var macs []string + err := m.forEachOtherInstance(exclude, "", func(meta *metadata) error { + if meta.MAC != "" { + macs = append(macs, meta.MAC) + } + return nil + }) + if err != nil { + return nil, err + } + return macs, nil +} + +// checkMacOSMACAvailable refuses a macOS start whose MAC a live instance already holds. +// Two live guests on one MAC share one vmnet DHCP lease, so the API would report one +// address for both. +func (m *manager) checkMacOSMACAvailable(ctx context.Context, stored *StoredMetadata) error { + if stored.MAC == "" { + return nil + } + return m.forEachOtherInstance(stored.Id, "check macOS MAC availability", func(meta *metadata) error { + if meta.MAC != stored.MAC { + return nil + } + state := m.deriveStateWithoutHydration(ctx, &meta.StoredMetadata).State + if state.RequiresVMM() || state == StateUnknown { + return fmt.Errorf("%w: MAC %s is already in use by instance %s; stop it first", ErrInsufficientResources, stored.MAC, meta.Id) + } + return nil + }) +} + +// completeMacOSIdentityRekey runs on a clone's first start after the guest agent answers. +// It clears the pending flag only after the rekey succeeds and is saved, so a failure +// leaves the flag set and the next start retries. +func (m *manager) completeMacOSIdentityRekey(ctx context.Context, stored *StoredMetadata) error { + log := logger.FromContext(ctx) + if err := ensureGuestAgentReadyForForkPhase(ctx, stored, "before macOS identity rekey"); err != nil { + return err + } + dialer, err := hypervisor.NewVsockDialer(stored.HypervisorType, stored.VsockSocket, stored.VsockCID) + if err != nil { + return fmt.Errorf("create vsock dialer for identity rekey: %w", err) + } + // The rekey runs from the clone's own disk, which the source owner controls. Its output is + // untrusted: stdout is discarded and stderr is kept only up to a small cap. The deadline + // bounds the guest's work even if it stalls or streams without end. + var stderr boundedBuffer + stderr.limit = macOSRekeyStderrLimit + rekeyCtx, cancel := context.WithTimeout(ctx, macOSRekeyDeadline) + defer cancel() + exit, err := guest.ExecIntoInstance(rekeyCtx, dialer, guest.ExecOptions{ + Command: []string{"sh", "-c", macOSIdentityRekeyCommand}, + Stdout: io.Discard, + Stderr: &stderr, + Timeout: int32(macOSRekeyDeadline / time.Second), + WaitForAgent: 120 * time.Second, + }) + if err != nil { + return fmt.Errorf("exec macOS identity rekey: %w", err) + } + if exit.Code != 0 { + return fmt.Errorf("macOS identity rekey failed (exit=%d, stderr=%q)", exit.Code, stderr.String()) + } + stored.IdentityRekeyPending = false + if err := m.saveMetadata(&metadata{StoredMetadata: *stored}); err != nil { + return fmt.Errorf("save macOS identity rekey: %w", err) + } + log.InfoContext(ctx, "macOS clone identity rekeyed", "instance_id", stored.Id) + return nil +} + +// rekeyMacOSCloneOnStart rekeys a macOS clone's guest SSH host keys on its first +// start. A failed rekey stops the VM, and the pending flag stays set so the next +// start retries. +func (m *manager) rekeyMacOSCloneOnStart(ctx context.Context, stored *StoredMetadata) error { + if stored.MacOS == nil || !stored.IdentityRekeyPending { + return nil + } + err := m.completeMacOSIdentityRekey(ctx, stored) + if err == nil { + return nil + } + logger.FromContext(ctx).ErrorContext(ctx, "macOS identity rekey failed; stopping clone", "instance_id", stored.Id, "error", err) + // Teardown runs even if saving the pending flag fails. Otherwise the clone would keep + // running, networked, with its source's SSH host keys. + saveErr := m.saveMetadata(&metadata{StoredMetadata: *stored}) + _, stopErr := m.stopInstance(context.WithoutCancel(ctx), stored.Id) + if saveErr != nil || stopErr != nil { + return fmt.Errorf("macOS clone rekey failed and teardown needs attention: %w", errors.Join(err, saveErr, stopErr)) + } + return err +} + +// checkMacOSBootAvailable checks machine identity and MAC under macOSBootMu. +// A stopped or standby guest does not reserve either against another boot. +func (m *manager) checkMacOSBootAvailable(ctx context.Context, stored *StoredMetadata) error { + if err := m.checkMacOSIdentityAvailable(ctx, stored); err != nil { + return err + } + if !stored.NetworkEnabled { + return nil + } + return m.checkMacOSMACAvailable(ctx, stored) +} + +// clearMacOSStopReceipt persists the end of the last graceful stop before the guest +// boots and can write to its disk. Otherwise a start that fails after boot, or a host +// crash, would leave a receipt that admits a stopped snapshot of a changed disk. +func (m *manager) clearMacOSStopReceipt(stored *StoredMetadata) error { + stored.StopGraceful = false + meta, err := m.loadMetadata(stored.Id) + if err != nil { + return err + } + if !meta.StopGraceful { + return nil + } + meta.StopGraceful = false + if err := m.saveMetadata(meta); err != nil { + return fmt.Errorf("clear macOS stop receipt: %w", err) + } + return nil +} + +// reserveMacOSCloneMAC re-checks a clone's MAC under the fork admission lock. The +// MAC was picked before the lock, so a concurrent fork may have taken it. +func (m *manager) reserveMacOSCloneMAC(clone *StoredMetadata) error { + if clone.MacOS == nil || !clone.NetworkEnabled { + return nil + } + used, err := m.storedMACs(clone.Id) + if err != nil { + return fmt.Errorf("list MACs for macOS clone: %w", err) + } + for _, mac := range used { + if strings.EqualFold(mac, clone.MacOS.MAC) { + fresh, err := network.NewUniqueMAC(used) + if err != nil { + return fmt.Errorf("allocate macOS clone MAC: %w", err) + } + macos := *clone.MacOS + macos.MAC = fresh + clone.MacOS = &macos + clone.MAC = fresh + return nil + } + } + return nil +} diff --git a/lib/instances/macos_identity_buffer_test.go b/lib/instances/macos_identity_buffer_test.go new file mode 100644 index 000000000..b06869930 --- /dev/null +++ b/lib/instances/macos_identity_buffer_test.go @@ -0,0 +1,20 @@ +package instances + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBoundedBufferCapsAndAcceptsEverything(t *testing.T) { + b := boundedBuffer{limit: 8} + n, err := b.Write([]byte("0123456789")) + require.NoError(t, err) + require.Equal(t, 10, n, "writes report full acceptance so the guest stream is never stalled") + for i := 0; i < 1000; i++ { + _, _ = b.Write([]byte(strings.Repeat("x", 1024))) + } + require.Equal(t, "01234567", b.String()) + require.LessOrEqual(t, len(b.buf), 8) +} diff --git a/lib/instances/macos_identity_darwin.go b/lib/instances/macos_identity_darwin.go new file mode 100644 index 000000000..e8cfa7d28 --- /dev/null +++ b/lib/instances/macos_identity_darwin.go @@ -0,0 +1,16 @@ +//go:build darwin && arm64 + +package instances + +import ( + vz "github.com/Code-Hex/vz/v3" +) + +// newMacMachineIdentifierData returns a fresh Apple machine identifier for a cloned guest. +func newMacMachineIdentifierData() ([]byte, error) { + id, err := vz.NewMacMachineIdentifier() + if err != nil { + return nil, err + } + return id.DataRepresentation(), nil +} diff --git a/lib/instances/macos_identity_other.go b/lib/instances/macos_identity_other.go new file mode 100644 index 000000000..2f17f3a50 --- /dev/null +++ b/lib/instances/macos_identity_other.go @@ -0,0 +1,9 @@ +//go:build !(darwin && arm64) + +package instances + +import "errors" + +func newMacMachineIdentifierData() ([]byte, error) { + return nil, errors.New("macOS machine identifiers require darwin/arm64") +} diff --git a/lib/instances/macos_identity_test.go b/lib/instances/macos_identity_test.go new file mode 100644 index 000000000..065740a56 --- /dev/null +++ b/lib/instances/macos_identity_test.go @@ -0,0 +1,83 @@ +//go:build darwin && arm64 + +package instances + +import ( + "net" + "testing" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/network" + "github.com/stretchr/testify/require" +) + +func testMacOSSource() *StoredMetadata { + return &StoredMetadata{ + Id: "source", + NetworkEnabled: true, + MAC: "a6:fd:95:db:a2:62", + MacOS: &images.MacOSImage{ + HardwareModel: []byte("model"), + MachineIdentifier: []byte("source-identifier"), + MAC: "a6:fd:95:db:a2:62", + GuestAgent: true, + }, + } +} + +func TestPrepareMacOSCloneRebindsIdentityWithoutAliasingSource(t *testing.T) { + source := testMacOSSource() + clone := cloneStoredMetadata(*source) + + require.NoError(t, prepareMacOSClone(&clone, []string{source.MacOS.MAC})) + + require.True(t, clone.IdentityRekeyPending) + require.NotEqual(t, []byte("source-identifier"), clone.MacOS.MachineIdentifier) + require.NotEmpty(t, clone.MacOS.MachineIdentifier) + require.NotEqual(t, source.MacOS.MAC, clone.MacOS.MAC) + require.Equal(t, clone.MacOS.MAC, clone.MAC) + require.Empty(t, clone.IP) + + // The source keeps its own identity; the clone's struct is independent. + require.Equal(t, []byte("source-identifier"), source.MacOS.MachineIdentifier) + require.Equal(t, "a6:fd:95:db:a2:62", source.MacOS.MAC) + require.False(t, source.IdentityRekeyPending) +} + +func TestPrepareMacOSCloneRefusesGuestWithoutAgent(t *testing.T) { + clone := testMacOSSource() + clone.MacOS.GuestAgent = false + err := prepareMacOSClone(clone, nil) + require.ErrorIs(t, err, ErrInvalidRequest) + require.False(t, clone.IdentityRekeyPending) +} + +func TestPrepareMacOSCloneRefusesSkippedAgent(t *testing.T) { + clone := testMacOSSource() + clone.SkipGuestAgent = true + require.ErrorIs(t, prepareMacOSClone(clone, nil), ErrInvalidRequest) + require.False(t, clone.IdentityRekeyPending) +} + +func TestPrepareMacOSCloneWithoutNetworkKeepsNoMAC(t *testing.T) { + clone := testMacOSSource() + clone.NetworkEnabled = false + clone.MAC = "" + clone.MacOS.MAC = "" + require.NoError(t, prepareMacOSClone(clone, nil)) + require.Empty(t, clone.MacOS.MAC) + require.True(t, clone.IdentityRekeyPending) +} + +func TestNewUniqueMACAvoidsUsedAddresses(t *testing.T) { + used := []string{"02:00:00:00:00:01", "02:00:00:00:00:02"} + for i := 0; i < 64; i++ { + mac, err := network.NewUniqueMAC(used) + require.NoError(t, err) + require.NotContains(t, used, mac) + hw, err := net.ParseMAC(mac) + require.NoError(t, err) + require.NotZero(t, hw[0]&0x02, "locally administered bit set") + require.Zero(t, hw[0]&0x01, "unicast") + } +} diff --git a/lib/instances/macos_memory_snapshot_test.go b/lib/instances/macos_memory_snapshot_test.go new file mode 100644 index 000000000..ce45f8075 --- /dev/null +++ b/lib/instances/macos_memory_snapshot_test.go @@ -0,0 +1,377 @@ +package instances + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/kernel/hypeman/lib/instances/phasetracking" + "github.com/kernel/hypeman/lib/paths" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/stretchr/testify/require" +) + +const macOSMemoryTestHypervisor hypervisor.Type = "macos-memory-snapshot-test" + +func macOSMemorySnapshotFixture(t *testing.T) (*manager, *metadata) { + t.Helper() + hypervisor.RegisterCapabilities(macOSMemoryTestHypervisor, hypervisor.Capabilities{SupportsMacOSBoot: true, SupportsSnapshot: true}) + m := &manager{paths: paths.New(t.TempDir()), macOSOnly: true, now: time.Now, lifecycleEvents: newLifecycleSubscribers(), vmStarters: map[hypervisor.Type]hypervisor.VMStarter{macOSMemoryTestHypervisor: concurrentForkPrepareTestStarter{}}} + m.deleteSnapshotFn = m.deleteSnapshot + const id = "mac-memory-source" + createStoppedSnapshotSourceFixture(t, m, id, id, macOSMemoryTestHypervisor) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.MacOS = macOSGuestMetadata(2).MacOS + meta.Size, meta.Vcpus = int64(meta.MacOS.Memory), int(meta.MacOS.CPUs) + meta.MAC, meta.NetworkEnabled = meta.MacOS.MAC, true + meta.SkipGuestAgent = true + meta.StopGraceful = true + require.NoError(t, m.saveMetadata(meta)) + require.NoError(t, os.WriteFile(filepath.Join(meta.DataDir, macOSAuxStorageFile), []byte("aux-at-checkpoint"), 0600)) + return m, meta +} + +func writeMacOSMemorySnapshotPayload(t *testing.T, m *manager, guestDir string, stored *StoredMetadata) shimconfig.SnapshotManifest { + t.Helper() + snapshotDir := filepath.Join(guestDir, "snapshots", "snapshot-latest") + require.NoError(t, os.MkdirAll(snapshotDir, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(snapshotDir, shimconfig.SnapshotMachineStateFile), []byte("saved-memory"), 0600)) + manifest := shimconfig.SnapshotManifest{Hypervisor: "vz", MachineStateFile: shimconfig.SnapshotMachineStateFile, ShimConfig: shimconfig.ShimConfig{ + VCPUs: stored.Vcpus, MemoryBytes: stored.Size, + MacHardwareModelData: base64.StdEncoding.EncodeToString(stored.MacOS.HardwareModel), MacMachineIdentifierData: base64.StdEncoding.EncodeToString(stored.MacOS.MachineIdentifier), + MacAuxStoragePath: filepath.Join(m.paths.InstanceDir(stored.Id), macOSAuxStorageFile), + Disks: []shimconfig.DiskConfig{{Path: m.paths.InstanceOverlay(stored.Id)}}, + }} + if stored.NetworkEnabled { + manifest.ShimConfig.Networks = []shimconfig.NetworkConfig{{MAC: stored.MAC}} + } + writeMacOSSnapshotManifest(t, snapshotDir, manifest) + return manifest +} + +func writeMacOSSnapshotManifest(t *testing.T, snapshotDir string, manifest shimconfig.SnapshotManifest) { + t.Helper() + data, err := json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(snapshotDir, shimconfig.SnapshotManifestFile), data, 0600)) +} + +func TestMacOSMemorySnapshotCaptureRestoreAndIsolation(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + writeMacOSMemorySnapshotPayload(t, m, meta.DataDir, &meta.StoredMetadata) + meta.StopGraceful = false + bootTime := time.Now().Add(-2 * time.Hour) + checkpointTime := bootTime.Add(time.Hour) + meta.StartedAt = &bootTime + meta.Phases.Record(phasetracking.PhaseRunning, bootTime) + meta.Phases.Record(phasetracking.PhaseStandby, checkpointTime) + require.NoError(t, m.saveMetadata(meta)) + // Linux compression defaults must not start a codec job for VZ machine state. + m.snapshotDefaults.Compression = &snapshotstore.SnapshotCompressionConfig{Enabled: true, Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd} + ctx := context.Background() + snapshot, err := m.CreateSnapshot(ctx, meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby}) + require.NoError(t, err) + require.Equal(t, SnapshotKindStandby, snapshot.Kind) + require.Equal(t, snapshotstore.SnapshotCompressionStateNone, snapshot.CompressionState) + source, err := m.GetInstance(ctx, meta.Id) + require.NoError(t, err) + require.Equal(t, StateStandby, source.State) + _, err = m.ForkSnapshot(ctx, snapshot.Id, ForkSnapshotRequest{Name: "unsafe-warm-fork"}) + require.ErrorIs(t, err, ErrNotSupported) + current, err := m.loadMetadata(meta.Id) + require.NoError(t, err) + current.Phases.Record(phasetracking.PhaseRunning, checkpointTime.Add(10*time.Minute)) + current.Phases.Record(phasetracking.PhaseStandby, checkpointTime.Add(20*time.Minute)) + expiresAt := time.Now().Add(3 * time.Hour) + current.ExpiresAt = &expiresAt + require.NoError(t, m.saveMetadata(current)) + + require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(meta.Id), []byte("after-checkpoint"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(meta.DataDir, macOSAuxStorageFile), []byte("aux-after"), 0600)) + restored, err := m.RestoreSnapshot(ctx, meta.Id, snapshot.Id, RestoreSnapshotRequest{TargetState: StateStandby}) + require.NoError(t, err) + require.Equal(t, StateStandby, restored.State) + require.Equal(t, meta.MacOS.MachineIdentifier, restored.MacOS.MachineIdentifier) + require.Equal(t, meta.MAC, restored.MAC) + require.NotNil(t, restored.StartedAt) + require.True(t, bootTime.Equal(*restored.StartedAt), "a memory restore resumes the checkpoint's boot") + require.Equal(t, current.Phases.Cumulative[phasetracking.PhaseRunning], restored.Phases.Cumulative[phasetracking.PhaseRunning], "same-instance restore must not rewind host phase accounting") + require.NotNil(t, restored.ExpiresAt) + require.True(t, expiresAt.Equal(*restored.ExpiresAt)) + for name, want := range map[string]string{"overlay.raw": "overlay", macOSAuxStorageFile: "aux-at-checkpoint", "snapshots/snapshot-latest/" + shimconfig.SnapshotMachineStateFile: "saved-memory"} { + data, err := os.ReadFile(filepath.Join(meta.DataDir, name)) + require.NoError(t, err) + require.Equal(t, want, string(data)) + } + require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(meta.Id), []byte("restored-write"), 0600)) + data, err := os.ReadFile(filepath.Join(m.paths.SnapshotGuestDir(snapshot.Id), "overlay.raw")) + require.NoError(t, err) + require.Equal(t, "overlay", string(data), "restoring must not alias the persistent snapshot") + + restored, err = m.RestoreSnapshot(ctx, meta.Id, snapshot.Id, RestoreSnapshotRequest{TargetState: StateStopped}) + require.NoError(t, err) + require.Equal(t, StateStopped, restored.State) + require.False(t, restored.StopGraceful, "discarding memory does not establish a clean shutdown receipt") + require.Equal(t, restartpolicy.BlockedReasonManualStop, restored.RestartStatus.BlockedReason, "an explicit stopped target must not be automatically restarted") + _, err = m.CreateSnapshot(ctx, meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStopped}) + require.ErrorIs(t, err, ErrInvalidState) +} + +func TestMacOSMemorySnapshotRejectsInvalidPayloadBeforeStorageSwap(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*shimconfig.SnapshotManifest) + }{ + {"cpu", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.VCPUs++ }}, + {"ram", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.MemoryBytes++ }}, + {"identity", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.MacMachineIdentifierData = "different" }}, + {"hardware", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.MacHardwareModelData = "different" }}, + {"disk", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.Disks[0].Path = "/wrong-disk" }}, + {"aux", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.MacAuxStoragePath = "/wrong-aux" }}, + {"network", func(m *shimconfig.SnapshotManifest) { m.ShimConfig.Networks[0].MAC = "02:00:00:00:00:02" }}, + {"state-path", func(m *shimconfig.SnapshotManifest) { m.MachineStateFile = "../../other-state" }}, + } { + t.Run(test.name, func(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + writeMacOSMemorySnapshotPayload(t, m, meta.DataDir, &meta.StoredMetadata) + snapshot, err := m.CreateSnapshot(context.Background(), meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby}) + require.NoError(t, err) + snapshotDir := filepath.Join(m.paths.SnapshotGuestDir(snapshot.Id), "snapshots", "snapshot-latest") + rec, err := m.loadSnapshotRecord(snapshot.Id) + require.NoError(t, err) + manifest := writeMacOSMemorySnapshotPayload(t, m, m.paths.SnapshotGuestDir(snapshot.Id), &rec.StoredMetadata) + test.mutate(&manifest) + writeMacOSSnapshotManifest(t, snapshotDir, manifest) + require.NoError(t, os.WriteFile(m.paths.InstanceOverlay(meta.Id), []byte("current-disk"), 0600)) + _, err = m.RestoreSnapshot(context.Background(), meta.Id, snapshot.Id, RestoreSnapshotRequest{}) + require.ErrorIs(t, err, ErrInvalidRequest) + data, err := os.ReadFile(m.paths.InstanceOverlay(meta.Id)) + require.NoError(t, err) + require.Equal(t, "current-disk", string(data)) + }) + } +} + +type macOSSnapshotProbeTestHypervisor struct { + lifecycleNoopHypervisor + supportErr error + probeCalls int + pauseCalls int + resumeCalls int +} + +func (h *macOSSnapshotProbeTestHypervisor) ValidateSnapshotSupport(context.Context) error { + h.probeCalls++ + return h.supportErr +} + +func (h *macOSSnapshotProbeTestHypervisor) Pause(context.Context) error { + h.pauseCalls++ + return nil +} + +func (h *macOSSnapshotProbeTestHypervisor) Capabilities() hypervisor.Capabilities { + return hypervisor.Capabilities{SupportsSnapshot: true} +} + +func (h *macOSSnapshotProbeTestHypervisor) Resume(ctx context.Context) error { + h.resumeCalls++ + return ctx.Err() +} + +func TestMacOSSnapshotConfigurationProbeAndPauseRecovery(t *testing.T) { + stored := macOSGuestMetadata(2) + hv := &macOSSnapshotProbeTestHypervisor{supportErr: hypervisor.ErrNotSupported} + require.ErrorIs(t, validateMacOSSaveRestoreSupport(context.Background(), stored, hv), ErrNotSupported) + require.Equal(t, 1, hv.probeCalls) + require.NoError(t, validateMacOSSaveRestoreSupport(context.Background(), &StoredMetadata{}, hv)) + require.Equal(t, 1, hv.probeCalls, "Linux must not call the macOS configuration probe") + require.ErrorIs(t, validateMacOSSaveRestoreSupport(context.Background(), stored, lifecycleNoopHypervisor{}), ErrNotSupported) + hv.supportErr = nil + require.NoError(t, validateMacOSSaveRestoreSupport(context.Background(), stored, hv)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.NoError(t, resumeSnapshotAfterError(ctx, stored, hv)) + require.Equal(t, 1, hv.resumeCalls) +} + +func TestMacOSLiveConfigurationRejectedBeforePauseAndSchedulePersistence(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + const hvType hypervisor.Type = "macos-live-snapshot-probe-test" + hv := &macOSSnapshotProbeTestHypervisor{lifecycleNoopHypervisor: lifecycleNoopHypervisor{state: hypervisor.StateRunning}, supportErr: hypervisor.ErrNotSupported} + hypervisor.RegisterCapabilities(hvType, hypervisor.Capabilities{SupportsMacOSBoot: true, SupportsSnapshot: true}) + hypervisor.RegisterClientFactory(hvType, func(string) (hypervisor.Hypervisor, error) { return hv, nil }) + meta.HypervisorType = hvType + require.NoError(t, m.saveMetadata(meta)) + require.NoError(t, os.WriteFile(meta.SocketPath, nil, 0600)) + ctx := context.Background() + _, err := m.SetSnapshotSchedule(ctx, meta.Id, SetSnapshotScheduleRequest{Interval: time.Hour, NamePrefix: "unsupported", Retention: SnapshotScheduleRetention{MaxCount: 1}}) + require.ErrorIs(t, err, ErrNotSupported) + _, err = m.GetSnapshotSchedule(ctx, meta.Id) + require.ErrorIs(t, err, ErrSnapshotScheduleNotFound) + _, err = m.CreateSnapshot(ctx, meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby}) + require.ErrorIs(t, err, ErrNotSupported) + require.Equal(t, 2, hv.probeCalls, "configuration validation must pass through the tracing wrapper") + require.Equal(t, 0, hv.pauseCalls, "unsupported configurations are rejected before pausing") + inst, err := m.GetInstance(ctx, meta.Id) + require.NoError(t, err) + require.Equal(t, StateRunning, inst.State) +} + +type macOSCancelledSnapshotTestHypervisor struct { + macOSSnapshotProbeTestHypervisor + cancel context.CancelFunc + failStage string + writePayload func() +} + +func (h *macOSCancelledSnapshotTestHypervisor) Snapshot(ctx context.Context, _ string) error { + if h.failStage == "snapshot" { + h.cancel() + return ctx.Err() + } + h.writePayload() + return nil +} + +func (h *macOSCancelledSnapshotTestHypervisor) Capabilities() hypervisor.Capabilities { + return hypervisor.Capabilities{SupportsSnapshot: true, SupportsGracefulVMMShutdown: true} +} + +func (h *macOSCancelledSnapshotTestHypervisor) Shutdown(ctx context.Context) error { + h.cancel() + return ctx.Err() +} + +func TestMacOSSnapshotCancellationRecoversSource(t *testing.T) { + for _, stage := range []string{"snapshot", "shutdown"} { + t.Run(stage, func(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + hvType := hypervisor.Type("macos-cancelled-" + stage + "-test") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hv := &macOSCancelledSnapshotTestHypervisor{macOSSnapshotProbeTestHypervisor: macOSSnapshotProbeTestHypervisor{lifecycleNoopHypervisor: lifecycleNoopHypervisor{state: hypervisor.StateRunning}}, cancel: cancel, failStage: stage} + hypervisor.RegisterCapabilities(hvType, hypervisor.Capabilities{SupportsMacOSBoot: true, SupportsSnapshot: true}) + hypervisor.RegisterClientFactory(hvType, func(string) (hypervisor.Hypervisor, error) { return hv, nil }) + meta.HypervisorType = hvType + hv.writePayload = func() { writeMacOSMemorySnapshotPayload(t, m, meta.DataDir, &meta.StoredMetadata) } + m.vmStarters[hvType] = macOSSnapshotRestoreContextStarter{} + require.NoError(t, m.saveMetadata(meta)) + require.NoError(t, os.WriteFile(meta.SocketPath, nil, 0600)) + _, err := m.CreateSnapshot(ctx, meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby}) + if stage == "snapshot" { + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, 1, hv.resumeCalls) + } else { + require.NoError(t, err, "confirmed shutdown must recover the source despite cancellation") + } + require.Equal(t, 1, hv.pauseCalls) + inst, err := m.GetInstance(context.Background(), meta.Id) + require.NoError(t, err) + require.Equal(t, StateRunning, inst.State) + }) + } +} + +type macOSSnapshotRestoreContextStarter struct { + concurrentForkPrepareTestStarter +} + +func (s macOSSnapshotRestoreContextStarter) RestoreVM(ctx context.Context, p *paths.Paths, version, socket, snapshot string, options hypervisor.RestoreOptions) (int, hypervisor.Hypervisor, error) { + if err := ctx.Err(); err != nil { + return 0, nil, err + } + return s.concurrentForkPrepareTestStarter.RestoreVM(ctx, p, version, socket, snapshot, options) +} + +func TestMacOSSnapshotRestoresSourceAfterRequestCancellation(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + writeMacOSMemorySnapshotPayload(t, m, meta.DataDir, &meta.StoredMetadata) + m.vmStarters[macOSMemoryTestHypervisor] = macOSSnapshotRestoreContextStarter{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.NoError(t, m.restoreSnapshotSource(ctx, &meta.StoredMetadata)) + inst, err := m.GetInstance(context.Background(), meta.Id) + require.NoError(t, err) + require.Equal(t, StateRunning, inst.State) +} + +func TestMacOSMemorySnapshotRequiresPayloadAndRejectsCompression(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + _, err := m.CreateSnapshot(context.Background(), meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby, Compression: &snapshotstore.SnapshotCompressionConfig{Enabled: true}}) + require.ErrorIs(t, err, ErrInvalidRequest) + require.NoError(t, os.MkdirAll(m.paths.InstanceSnapshotLatest(meta.Id), 0700)) + require.NoError(t, os.WriteFile(filepath.Join(m.paths.InstanceSnapshotLatest(meta.Id), "incomplete"), []byte("bad-state"), 0600)) + _, err = m.CreateSnapshot(context.Background(), meta.Id, CreateSnapshotRequest{Kind: SnapshotKindStandby}) + require.ErrorIs(t, err, ErrInvalidRequest) +} + +func TestMacOSSnapshotRestoreRefusesLiveStates(t *testing.T) { + for _, state := range []State{StateCreated, StateInitializing, StateRunning, StatePaused, StateShutdown} { + require.ErrorIs(t, validateMacOSSnapshotRestoreState(&Instance{StoredMetadata: *macOSGuestMetadata(2), State: state}), ErrInvalidState) + } + for _, state := range []State{StateStopped, StateStandby} { + require.NoError(t, validateMacOSSnapshotRestoreState(&Instance{StoredMetadata: *macOSGuestMetadata(2), State: state})) + } + require.NoError(t, validateMacOSSnapshotRestoreState(&Instance{State: StatePaused}), "Linux admission is unchanged") +} + +func TestMacOSSnapshotSchedulesCaptureStoppedAndStandbyAndRetain(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + clock := time.Now().UTC() + m.now = func() time.Time { return clock } + ctx := context.Background() + _, err := m.SetSnapshotSchedule(ctx, meta.Id, SetSnapshotScheduleRequest{Interval: time.Hour, NamePrefix: "mac-backup", Retention: SnapshotScheduleRetention{MaxCount: 1}}) + require.NoError(t, err) + markSnapshotScheduleDue(t, m, meta.Id) + require.NoError(t, m.RunSnapshotSchedules(ctx)) + first, err := m.GetSnapshotSchedule(ctx, meta.Id) + require.NoError(t, err) + require.NotNil(t, first.LastSnapshotID) + stopped, err := m.GetSnapshot(ctx, *first.LastSnapshotID) + require.NoError(t, err) + require.Equal(t, SnapshotKindStopped, stopped.Kind) + + writeMacOSMemorySnapshotPayload(t, m, meta.DataDir, &meta.StoredMetadata) + meta.StopGraceful = false + require.NoError(t, m.saveMetadata(meta)) + clock = clock.Add(time.Hour) + markSnapshotScheduleDue(t, m, meta.Id) + require.NoError(t, m.RunSnapshotSchedules(ctx)) + second, err := m.GetSnapshotSchedule(ctx, meta.Id) + require.NoError(t, err) + require.NotNil(t, second.LastSnapshotID) + require.Nil(t, second.LastError) + memory, err := m.GetSnapshot(ctx, *second.LastSnapshotID) + require.NoError(t, err) + require.Equal(t, SnapshotKindStandby, memory.Kind) + _, err = m.GetSnapshot(ctx, stopped.Id) + require.ErrorIs(t, err, ErrSnapshotNotFound) + _, err = m.GetSnapshot(ctx, memory.Id) + require.NoError(t, err) + require.NoError(t, m.DeleteSnapshotSchedule(ctx, meta.Id)) +} + +func TestMacOSSnapshotScheduleRecordsFailureAndAdvancesInterval(t *testing.T) { + m, meta := macOSMemorySnapshotFixture(t) + meta.StopGraceful = false + require.NoError(t, m.saveMetadata(meta)) + ctx := context.Background() + _, err := m.SetSnapshotSchedule(ctx, meta.Id, SetSnapshotScheduleRequest{Interval: time.Hour, NamePrefix: "mac-backup", Retention: SnapshotScheduleRetention{MaxCount: 1}}) + require.NoError(t, err) + markSnapshotScheduleDue(t, m, meta.Id) + require.ErrorIs(t, m.RunSnapshotSchedules(ctx), ErrInvalidState) + schedule, err := m.GetSnapshotSchedule(ctx, meta.Id) + require.NoError(t, err) + require.NotNil(t, schedule.LastError) + require.Nil(t, schedule.LastSnapshotID) + require.True(t, schedule.NextRunAt.After(m.now().UTC())) +} diff --git a/lib/instances/macos_shim_gone_test.go b/lib/instances/macos_shim_gone_test.go index eb7ce75e3..dedcc967f 100644 --- a/lib/instances/macos_shim_gone_test.go +++ b/lib/instances/macos_shim_gone_test.go @@ -7,9 +7,11 @@ import ( "path/filepath" "syscall" "testing" + "time" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/paths" "github.com/kernel/hypeman/lib/resources" "github.com/stretchr/testify/require" @@ -83,3 +85,25 @@ func TestMacOSShimGoneRequiresRefusedConnection(t *testing.T) { require.False(t, macOSShimGone(stored, context.DeadlineExceeded)) require.False(t, macOSShimGone(stored, nil)) } + +func TestFinishMacOSResumeRemovesConsumedSnapshot(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + id := "resumed" + require.NoError(t, m.ensureDirectories(id)) + image := &images.MacOSImage{GuestAgent: true} + ready := time.Now().UTC().Add(-time.Minute) + meta := &metadata{StoredMetadata: StoredMetadata{Id: id, DataDir: p.InstanceDir(id), MacOS: image, GuestAgentReadyAt: &ready}} + meta.Phases.Record(phasetracking.PhaseStandby, ready) + require.NoError(t, m.saveMetadata(meta)) + snapshot := p.InstanceSnapshotLatest(id) + require.NoError(t, os.MkdirAll(snapshot, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(snapshot, "state"), []byte("x"), 0644)) + + require.NoError(t, m.finishMacOSResume(id)) + + require.NoDirExists(t, snapshot, "a consumed memory snapshot must not survive the resume") + saved, err := m.loadMetadata(id) + require.NoError(t, err) + require.Equal(t, phasetracking.PhaseRunning, saved.Phases.Current) +} diff --git a/lib/instances/macos_snapshot_plan_test.go b/lib/instances/macos_snapshot_plan_test.go new file mode 100644 index 000000000..a181fce74 --- /dev/null +++ b/lib/instances/macos_snapshot_plan_test.go @@ -0,0 +1,55 @@ +package instances + +import ( + "context" + "os" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestMacOSSnapshotTargetUsesMacOSBootConfig(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + meta := *macOSGuestMetadata(2) + meta.Id, meta.DataDir = "mac-snapshot", p.InstanceDir("mac-snapshot") + meta.Size, meta.Vcpus = 4<<30, 3 + for _, kind := range []SnapshotKind{SnapshotKindStopped, SnapshotKindStandby} { + config := m.plannedStoredVMConfig(kind, meta) + require.Equal(t, hypervisor.BootModeMacOS, config.BootMode) + require.Equal(t, meta.Size, config.MemoryBytes) + require.Equal(t, meta.Vcpus, config.VCPUs) + require.Len(t, config.Disks, 1) + require.Empty(t, config.KernelPath) + require.False(t, config.GuestMemory.EnableBalloon) + require.NoError(t, hypervisor.ValidateBootConfig(config)) + require.Error(t, hypervisor.ValidateDirectRawConfig("linux-backend", config)) + } +} + +func TestMacOSSnapshotForkRejectsLinuxBackendBeforeCopy(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p, macOSOnly: true, vmStarters: map[hypervisor.Type]hypervisor.VMStarter{hypervisor.TypeFirecracker: macOSSnapshotLinuxStarter{}}} + require.NoError(t, os.MkdirAll(p.GuestsDir(), 0700)) + source := *macOSGuestMetadata(2) + source.Id = "mac-source" + source.HypervisorType = hypervisor.TypeVZ + source.StopGraceful = true + source.MacOS.GuestAgent = true + require.NoError(t, m.saveSnapshotRecord(&snapshotRecord{Snapshot: Snapshot{Id: "mac-snapshot", Kind: SnapshotKindStopped, SourceInstanceID: source.Id, SourceHypervisor: hypervisor.TypeVZ}, StoredMetadata: source})) + _, err := m.ForkSnapshot(context.Background(), "mac-snapshot", ForkSnapshotRequest{Name: "linux-target", TargetHypervisor: hypervisor.TypeFirecracker}) + require.ErrorIs(t, err, ErrInvalidRequest) + entries, err := os.ReadDir(p.GuestsDir()) + require.NoError(t, err) + require.Empty(t, entries, "backend admission rejects the fork before copying files") +} + +type macOSSnapshotLinuxStarter struct { + hypervisor.VMStarter +} + +func (macOSSnapshotLinuxStarter) ValidateConfig(config hypervisor.VMConfig) error { + return hypervisor.ValidateDirectRawConfig("linux-test", config) +} diff --git a/lib/instances/macos_test.go b/lib/instances/macos_test.go index 7e9605d4d..1e61f3992 100644 --- a/lib/instances/macos_test.go +++ b/lib/instances/macos_test.go @@ -2,6 +2,7 @@ package instances import ( "context" + "encoding/json" "errors" "os" "path/filepath" @@ -161,6 +162,64 @@ func TestMacOSIdentityAdmissionSkipsDirectoriesWithoutMetadata(t *testing.T) { require.Error(t, m.checkMacOSIdentityAvailable(context.Background(), &stored)) } +func TestMacOSIdentityScanErrors(t *testing.T) { + for _, tc := range []struct { + name string + prefix string + check func(*manager, *StoredMetadata) error + }{ + {"stored MACs", "", func(m *manager, stored *StoredMetadata) error { + macs, err := m.storedMACs(stored.Id) + require.Nil(t, macs) + return err + }}, + {"MAC admission", "check macOS MAC availability: ", func(m *manager, stored *StoredMetadata) error { + return m.checkMacOSMACAvailable(context.Background(), stored) + }}, + {"identity admission", "check Mac identity admission: ", func(m *manager, stored *StoredMetadata) error { + return m.checkMacOSIdentityAvailable(context.Background(), stored) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + stored := StoredMetadata{Id: "self", MAC: "02:00:00:00:00:01", MacOS: testMacImage().MacOS} + _, readErr := os.ReadDir(p.GuestsDir()) + require.EqualError(t, tc.check(m, &stored), readErr.Error()) + + require.NoError(t, m.ensureDirectories("a-valid")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{Id: "a-valid", MAC: "02:00:00:00:00:02"}})) + require.NoError(t, m.ensureDirectories("b-invalid")) + require.NoError(t, os.WriteFile(p.InstanceMetadata("b-invalid"), []byte("{not json"), 0600)) + _, loadErr := m.loadMetadata("b-invalid") + err := tc.check(m, &stored) + require.EqualError(t, err, tc.prefix+loadErr.Error()) + var syntaxErr *json.SyntaxError + require.ErrorAs(t, err, &syntaxErr) + }) + } +} + +func TestMacOSStoredMACsScan(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + for _, id := range []string{"self", "orphan", "a", "b", "empty"} { + require.NoError(t, m.ensureDirectories(id)) + } + require.NoError(t, os.WriteFile(p.InstanceMetadata("self"), []byte("{not json"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(p.GuestsDir(), "file"), nil, 0600)) + for _, stored := range []StoredMetadata{ + {Id: "a", MAC: "02:00:00:00:00:01"}, + {Id: "b", MAC: "02:00:00:00:00:02", MacOS: testMacImage().MacOS}, + {Id: "empty"}, + } { + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: stored})) + } + macs, err := m.storedMACs("self") + require.NoError(t, err) + require.Equal(t, []string{"02:00:00:00:00:01", "02:00:00:00:00:02"}, macs) +} + func TestMacOSPreservedIdentityAdmission(t *testing.T) { p := paths.New(t.TempDir()) m := &manager{paths: p} @@ -174,10 +233,16 @@ func TestMacOSPreservedIdentityAdmission(t *testing.T) { other.Id = "other" err := m.checkMacOSIdentityAvailable(context.Background(), &other) require.True(t, errors.Is(err, ErrInsufficientResources)) + stored.MAC = "02:00:00:00:00:01" + other.MAC = stored.MAC + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: stored})) + require.EqualError(t, m.checkMacOSMACAvailable(context.Background(), &other), ErrInsufficientResources.Error()+": MAC "+stored.MAC+" is already in use by instance active; stop it first") + require.NoError(t, m.checkMacOSMACAvailable(context.Background(), &stored)) require.NoError(t, m.checkMacOSIdentityAvailable(context.Background(), &stored)) require.NoError(t, os.Remove(socket)) require.NoError(t, m.checkMacOSIdentityAvailable(context.Background(), &other)) - require.ErrorIs(t, (&manager{}).admitGuestOperation(&stored, "snapshot schedules"), ErrInvalidRequest) + require.NoError(t, m.checkMacOSMACAvailable(context.Background(), &other)) + require.NoError(t, (&manager{}).admitGuestOperation(&stored, "snapshot schedules")) } func TestMacOSRunningPhaseFollowsReadiness(t *testing.T) { diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 0b39814f0..2323bf407 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -329,6 +329,11 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa if err := m.recoverPendingStandbyCompressionJobs(context.Background()); err != nil { logger.FromContext(context.Background()).WarnContext(context.Background(), "failed to recover pending standby compression jobs", "error", err) } + // An interrupted macOS restore can leave the previous disk parked in staging. Failure + // is logged here; each restore retries recovery for its own instance before swapping. + if err := m.recoverMacOSRestoreStaging(""); err != nil { + logger.FromContext(context.Background()).ErrorContext(context.Background(), "macOS restore staging recovery failed", "error", err) + } return m, nil } @@ -648,6 +653,11 @@ func (m *manager) RestoreInstance(ctx context.Context, id string) (*Instance, er return nil, fmt.Errorf("resume paused vm: %w", err) } m.storeCachedHypervisorState(id, hypervisor.StateRunning) + if current.MacOS != nil { + if err := m.finishMacOSResume(id); err != nil { + return nil, fmt.Errorf("finish resumed restore: %w", err) + } + } current, err = m.currentInstanceWithoutHydration(ctx, id) if err != nil { return nil, err diff --git a/lib/instances/restore.go b/lib/instances/restore.go index bdbdb3b2a..2fd9637c3 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -45,6 +45,13 @@ func (m *manager) restoreInstance( if err := m.admitGuestOperation(&meta.StoredMetadata, "restore"); err != nil { return nil, err } + if meta.MacOS != nil { + m.macOSBootMu.Lock() + defer m.macOSBootMu.Unlock() + if err := m.checkMacOSBootAvailable(ctx, &meta.StoredMetadata); err != nil { + return nil, err + } + } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata @@ -138,8 +145,9 @@ func (m *manager) restoreInstance( } } - // 4. Recreate or allocate network if network enabled - if stored.NetworkEnabled { + // 4. Recreate or allocate network if network enabled. A macOS guest keeps the MAC + // in its saved configuration, and vmnet supplies the address, so no TAP is allocated. + if stored.NetworkEnabled && stored.MacOS == nil { networkCtx, networkSpanEnd := m.startLifecycleStep(ctx, "restore_network", attribute.String("instance_id", id), attribute.String("hypervisor", string(stored.HypervisorType)), diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 6e783ce7e..7d41b8bf4 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -66,6 +66,9 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if err := m.admitGuestOperation(&meta.StoredMetadata, "snapshot"); err != nil { return nil, err } + if err := validateMacOSSnapshotCreate(meta, req); err != nil { + return nil, err + } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata @@ -124,13 +127,16 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps return nil, fmt.Errorf("%w: standby snapshot requires source in %s or %s, got %s", ErrInvalidState, StateRunning, StateStandby, inst.State) } - copyErr := m.copySnapshotPayload(id, snapshotGuestDir) + copyErr := m.validateMacOSMemorySnapshotPayload(m.paths.InstanceDir(id), stored) + if copyErr == nil { + copyErr = m.copySnapshotPayload(id, snapshotGuestDir) + } if copyErr == nil { meta, copyErr = m.loadMetadata(id) } if restoreSource { - _, restoreErr := m.restoreInstance(ctx, id) + restoreErr := m.restoreSnapshotSource(ctx, stored) if restoreErr != nil { if copyErr != nil { return nil, fmt.Errorf("snapshot copy failed: %v; additionally failed to restore source: %w", copyErr, restoreErr) @@ -262,10 +268,16 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if err != nil { return nil, err } + if err := validateMacOSSnapshotRestore(sourceMeta, rec); err != nil { + return nil, err + } if err := m.admitGuestOperation(&sourceMeta.StoredMetadata, "restore snapshot"); err != nil { return nil, err } sourceInst := m.toInstance(ctx, sourceMeta) + if err := validateMacOSSnapshotRestoreState(&sourceInst); err != nil { + return nil, err + } if sourceInst.State == StateRunning { return nil, fmt.Errorf("%w: cannot restore snapshot while source is %s", ErrInvalidState, sourceInst.State) } @@ -297,17 +309,15 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, target.Target) } - if err := m.replaceInstanceWithSnapshotPayload(snapshotID, id); err != nil { - return nil, err - } - restored := cloneStoredMetadataWithoutPendingStandbyCompression(rec.StoredMetadata) restored.Id = sourceMeta.Id restored.Name = sourceMeta.Name restored.ExpiresAt = sourceMeta.ExpiresAt restored.DataDir = m.paths.InstanceDir(id) restored.HypervisorProcessIdentity.Clear() - restored.StartedAt = nil + if restored.MacOS == nil || rec.Snapshot.Kind == SnapshotKindStopped { + restored.StartedAt = nil + } restored.StoppedAt = nil restored.ExitCode = nil restored.ExitMessage = "" @@ -330,6 +340,20 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str restored.VsockCID = generateVsockCID(id) } + if restored.MacOS != nil { + if rec.Snapshot.Kind == SnapshotKindStandby { + if err := m.validateMacOSMemorySnapshotPayload(m.paths.SnapshotGuestDir(snapshotID), &rec.StoredMetadata); err != nil { + return nil, err + } + } + prepareMacOSSnapshotRestoreMetadata(&restored, &sourceMeta.StoredMetadata, rec.Snapshot.Kind, targetState) + if err := m.replaceMacOSInstanceWithSnapshotPayload(snapshotID, &metadata{StoredMetadata: restored}); err != nil { + return nil, err + } + } else if err := m.replaceInstanceWithSnapshotPayload(snapshotID, id); err != nil { + return nil, err + } + if err := m.saveMetadata(&metadata{StoredMetadata: restored}); err != nil { return nil, fmt.Errorf("save restored metadata: %w", err) } @@ -387,11 +411,14 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if err := m.admitGuestOperation(&rec.StoredMetadata, "fork snapshot"); err != nil { return nil, err } + if err := validateMacOSSnapshotFork(rec); err != nil { + return nil, err + } if err := validateForkVolumeSafety(rec.StoredMetadata.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) } - if err := m.ensureInstanceNameAvailableForSnapshotFork(ctx, req.Name, rec.StoredMetadata.NetworkEnabled); err != nil { + if err := m.ensureInstanceNameAvailableForSnapshotFork(ctx, req.Name, rec.StoredMetadata.NetworkEnabled && rec.StoredMetadata.MacOS == nil); err != nil { return nil, err } @@ -497,7 +524,12 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS } } - if err := m.saveMetadata(&metadata{StoredMetadata: forkMeta}); err != nil { + newMeta := &metadata{StoredMetadata: forkMeta} + if forkMeta.MacOS != nil { + if err := m.saveMacOSSnapshotFork(ctx, newMeta); err != nil { + return nil, err + } + } else if err := m.saveMetadata(newMeta); err != nil { return nil, fmt.Errorf("save fork metadata: %w", err) } diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 31752aa5e..75eabbfa9 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -210,6 +210,9 @@ func normalizeCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) (s } func (m *manager) resolveSnapshotCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { + if stored != nil && stored.MacOS != nil { + return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil + } cfg, err := m.resolveConfiguredCompressionPolicy(stored, override) if err != nil { return snapshotstore.SnapshotCompressionConfig{}, err diff --git a/lib/instances/snapshot_macos.go b/lib/instances/snapshot_macos.go new file mode 100644 index 000000000..f96593047 --- /dev/null +++ b/lib/instances/snapshot_macos.go @@ -0,0 +1,354 @@ +package instances + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/kernel/hypeman/lib/instances/phasetracking" + restartpolicy "github.com/kernel/hypeman/lib/restart-policy" +) + +func validateMacOSSnapshotCreate(meta *metadata, req CreateSnapshotRequest) error { + if meta.MacOS == nil { + return nil + } + if req.Kind == SnapshotKindStandby { + if req.Compression != nil && req.Compression.Enabled { + return fmt.Errorf("%w: macOS memory snapshots do not support compression", ErrInvalidRequest) + } + return validateMacOSMemorySnapshotSupport(&meta.StoredMetadata) + } + if req.Kind != SnapshotKindStopped { + return fmt.Errorf("%w: unsupported macOS snapshot kind %q", ErrInvalidRequest, req.Kind) + } + if !meta.StopGraceful { + return fmt.Errorf("%w: macOS stopped snapshots require a graceful stop", ErrInvalidState) + } + return nil +} + +func validateMacOSMemorySnapshotSupport(stored *StoredMetadata) error { + if stored.MacOS == nil { + return nil + } + caps, ok := hypervisor.CapabilitiesForType(stored.HypervisorType) + if !ok || !caps.SupportsMacOSBoot || !caps.SupportsSnapshot { + return fmt.Errorf("%w: macOS memory snapshots require a host with VZ save/restore support", ErrNotSupported) + } + return nil +} + +func validateMacOSSaveRestoreSupport(ctx context.Context, stored *StoredMetadata, hv hypervisor.Hypervisor) error { + if stored.MacOS == nil { + return nil + } + validator, ok := hv.(hypervisor.SnapshotSupportValidator) + if !ok { + return fmt.Errorf("%w: macOS snapshot configuration validation is unavailable", ErrNotSupported) + } + if err := validator.ValidateSnapshotSupport(ctx); err != nil { + if errors.Is(err, hypervisor.ErrNotSupported) { + return fmt.Errorf("%w: %w", ErrNotSupported, err) + } + return fmt.Errorf("validate macOS snapshot configuration: %w", err) + } + return nil +} + +func (m *manager) restoreSnapshotSource(ctx context.Context, stored *StoredMetadata) error { + if stored.MacOS != nil { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) + defer cancel() + } + _, err := m.restoreInstance(ctx, stored.Id) + return err +} + +func resumeSnapshotAfterError(ctx context.Context, stored *StoredMetadata, hv hypervisor.Hypervisor) error { + if stored.MacOS != nil { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + } + return hv.Resume(ctx) +} + +func (m *manager) validateMacOSSnapshotSchedule(ctx context.Context, meta *metadata) error { + if meta.MacOS == nil { + return nil + } + if err := validateMacOSMemorySnapshotSupport(&meta.StoredMetadata); err != nil { + return err + } + inst := m.toInstance(ctx, meta) + switch inst.State { + case StateRunning, StateInitializing, StateCreated, StatePaused: + hv, err := m.getHypervisor(inst.SocketPath, inst.HypervisorType) + if err != nil { + return err + } + return validateMacOSSaveRestoreSupport(ctx, &meta.StoredMetadata, hv) + } + return nil +} + +func prepareMacOSSnapshotRestoreMetadata(restored *StoredMetadata, source *StoredMetadata, kind SnapshotKind, target State) { + restored.Phases = source.Phases + phase := phasetracking.PhaseStopped + if kind == SnapshotKindStandby { + restored.StopGraceful = false + if target != StateStopped { + phase = phasetracking.PhaseStandby + } + } + restored.Phases.Record(phase, time.Now().UTC()) + if target == StateStopped { + restored.RestartStatus = restartpolicy.Status{BlockedReason: restartpolicy.BlockedReasonManualStop} + } +} + +func validateMacOSSnapshotRestoreState(inst *Instance) error { + if inst.MacOS != nil && inst.State != StateStopped && inst.State != StateStandby { + return fmt.Errorf("%w: macOS snapshot restore requires a stopped or standby instance, got %s", ErrInvalidState, inst.State) + } + return nil +} + +// The manifest is used by VZ rather than rebuilt from instance metadata. Validate +// the saved configuration and its disk/aux/memory bundle before replacing storage. +func (m *manager) validateMacOSMemorySnapshotPayload(guestDir string, stored *StoredMetadata) error { + if stored.MacOS == nil { + return nil + } + snapshotDir := filepath.Join(guestDir, "snapshots", "snapshot-latest") + data, err := os.ReadFile(filepath.Join(snapshotDir, shimconfig.SnapshotManifestFile)) + if err != nil { + return fmt.Errorf("%w: read macOS snapshot manifest: %v", ErrInvalidRequest, err) + } + var manifest shimconfig.SnapshotManifest + if err := json.Unmarshal(data, &manifest); err != nil { + return fmt.Errorf("%w: decode macOS snapshot manifest: %v", ErrInvalidRequest, err) + } + cfg := manifest.ShimConfig + origin := m.paths.InstanceDir(stored.Id) + if manifest.Hypervisor != string(hypervisor.TypeVZ) || (manifest.MachineStateFile != "" && manifest.MachineStateFile != shimconfig.SnapshotMachineStateFile) { + return fmt.Errorf("%w: invalid macOS snapshot format", ErrInvalidRequest) + } + if cfg.VCPUs != stored.Vcpus || cfg.MemoryBytes != stored.Size || cfg.MacHardwareModelData != base64.StdEncoding.EncodeToString(stored.MacOS.HardwareModel) || cfg.MacMachineIdentifierData != base64.StdEncoding.EncodeToString(stored.MacOS.MachineIdentifier) { + return fmt.Errorf("%w: macOS snapshot configuration does not match its metadata", ErrInvalidRequest) + } + if len(cfg.Disks) != 1 || cfg.Disks[0].Readonly || cfg.Disks[0].Path != m.paths.InstanceOverlay(stored.Id) || cfg.MacAuxStoragePath != filepath.Join(origin, macOSAuxStorageFile) { + return fmt.Errorf("%w: macOS snapshot storage configuration does not match its instance", ErrInvalidRequest) + } + if cfg.KernelPath != "" || cfg.InitrdPath != "" || cfg.EnableMemoryBalloon || cfg.RequireMemoryBalloon || cfg.EnableRosetta { + return fmt.Errorf("%w: unsupported macOS snapshot devices", ErrInvalidRequest) + } + if stored.NetworkEnabled { + if len(cfg.Networks) != 1 || !strings.EqualFold(cfg.Networks[0].MAC, stored.MAC) || !strings.EqualFold(stored.MAC, stored.MacOS.MAC) { + return fmt.Errorf("%w: macOS snapshot network identity does not match its metadata", ErrInvalidRequest) + } + } else if len(cfg.Networks) != 0 { + return fmt.Errorf("%w: macOS snapshot unexpectedly enables networking", ErrInvalidRequest) + } + for _, path := range []string{filepath.Join(guestDir, "overlay.raw"), filepath.Join(guestDir, macOSAuxStorageFile), filepath.Join(snapshotDir, shimconfig.SnapshotMachineStateFile)} { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("%w: missing or invalid macOS snapshot payload %s", ErrInvalidRequest, filepath.Base(path)) + } + } + return nil +} + +func validateMacOSSnapshotRestore(source *metadata, rec *snapshotRecord) error { + if source.MacOS == nil && rec.StoredMetadata.MacOS == nil { + return nil + } + if source.MacOS == nil || rec.StoredMetadata.MacOS == nil { + return fmt.Errorf("%w: snapshot and instance guest types differ", ErrInvalidRequest) + } + if rec.Snapshot.Kind != SnapshotKindStopped && rec.Snapshot.Kind != SnapshotKindStandby { + return fmt.Errorf("%w: unsupported macOS snapshot kind %q", ErrInvalidRequest, rec.Snapshot.Kind) + } + if rec.Snapshot.Kind == SnapshotKindStandby { + if err := validateMacOSMemorySnapshotSupport(&rec.StoredMetadata); err != nil { + return err + } + } + want, got := source.MacOS, rec.StoredMetadata.MacOS + if !bytes.Equal(want.MachineIdentifier, got.MachineIdentifier) || !bytes.Equal(want.HardwareModel, got.HardwareModel) || want.MAC != got.MAC { + return fmt.Errorf("%w: snapshot identity does not match this instance", ErrInvalidRequest) + } + return nil +} + +func validateMacOSSnapshotFork(rec *snapshotRecord) error { + if rec.StoredMetadata.MacOS == nil { + return nil + } + if rec.Snapshot.Kind != SnapshotKindStopped { + return fmt.Errorf("%w: macOS forks require stopped snapshots", ErrNotSupported) + } + if !rec.StoredMetadata.StopGraceful { + return fmt.Errorf("%w: macOS snapshot forks require a graceful stop receipt", ErrInvalidState) + } + if !rec.StoredMetadata.GuestAgentEnabled() { + return fmt.Errorf("%w: macOS snapshot forks require an enabled guest agent for identity rekey", ErrInvalidRequest) + } + return nil +} + +func (m *manager) saveMacOSSnapshotFork(ctx context.Context, meta *metadata) error { + used, err := m.storedMACs(meta.Id) + if err != nil { + return fmt.Errorf("list MACs for macOS snapshot clone: %w", err) + } + if err := prepareMacOSClone(&meta.StoredMetadata, used); err != nil { + return err + } + meta.ProgramStartedAt = nil + meta.GuestAgentReadyAt = nil + meta.Phases.Reset() + meta.Phases.Record(phasetracking.PhaseStopped, meta.CreatedAt) + return m.saveForkMetadata(ctx, meta) +} + +// replaceMacOSInstanceWithSnapshotPayload stages the snapshot and restored metadata +// beside the instance, then swaps both in. A failure before the swap leaves the +// previous directory intact; a failed swap is rolled back. The staging root lives outside the guests directory, +// so instance and snapshot listings never see a half-restored copy. +func (m *manager) replaceMacOSInstanceWithSnapshotPayload(snapshotID string, restored *metadata) error { + instanceID := restored.Id + if err := m.recoverMacOSRestoreStaging(instanceID); err != nil { + return fmt.Errorf("recover before restore: %w", err) + } + instanceDir := m.paths.InstanceDir(instanceID) + root := filepath.Join(m.paths.DataDir(), "restore-staging", instanceID) + staged := filepath.Join(root, "staged") + previous := filepath.Join(root, "previous") + fail := func(err error) error { + _ = os.RemoveAll(root) + return err + } + if err := os.RemoveAll(root); err != nil { + return fmt.Errorf("clear restore staging: %w", err) + } + if err := os.MkdirAll(root, 0700); err != nil { + return fmt.Errorf("create restore staging: %w", err) + } + if err := forkvm.CopyGuestDirectory(m.paths.SnapshotGuestDir(snapshotID), staged); err != nil { + if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { + return fail(fmt.Errorf("restore requires sparse-capable filesystem (SEEK_DATA/SEEK_HOLE unsupported): %w", err)) + } + return fail(fmt.Errorf("stage snapshot payload: %w", err)) + } + if restored.Phases.Current == phasetracking.PhaseStopped { + if err := os.RemoveAll(filepath.Join(staged, "snapshots", "snapshot-latest")); err != nil { + return fail(fmt.Errorf("discard staged memory checkpoint: %w", err)) + } + } + data, err := json.MarshalIndent(restored, "", " ") + if err != nil { + return fail(fmt.Errorf("marshal restored metadata: %w", err)) + } + if err := os.WriteFile(filepath.Join(staged, filepath.Base(m.paths.InstanceMetadata(instanceID))), data, 0644); err != nil { + return fail(fmt.Errorf("stage restored metadata: %w", err)) + } + if err := os.Rename(instanceDir, previous); err != nil { + return fail(fmt.Errorf("retire instance directory: %w", err)) + } + if err := os.Rename(staged, instanceDir); err != nil { + if rollbackErr := os.Rename(previous, instanceDir); rollbackErr != nil { + return fmt.Errorf("install snapshot payload: %w (rollback failed: %v; previous state kept at %s)", err, rollbackErr, previous) + } + return fail(fmt.Errorf("install snapshot payload: %w", err)) + } + _ = os.RemoveAll(root) + return nil +} + +// recoverMacOSRestoreStaging resolves a swap interrupted by a crash. With the instance +// directory missing and the previous directory parked, the previous state is put back. +// If the instance directory is present, a completed swap leaves stale staging, which is +// removed. Staging with no previous directory is an unfinished copy and is removed. +// An empty instanceID recovers every instance. +func (m *manager) recoverMacOSRestoreStaging(instanceID string) error { + root := filepath.Join(m.paths.DataDir(), "restore-staging") + ids := []string{instanceID} + if instanceID == "" { + entries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + ids = ids[:0] + for _, entry := range entries { + ids = append(ids, entry.Name()) + } + } + for _, id := range ids { + stagingRoot := filepath.Join(root, id) + previous := filepath.Join(stagingRoot, "previous") + instanceDir := m.paths.InstanceDir(id) + _, previousErr := os.Stat(previous) + if previousErr != nil && !errors.Is(previousErr, os.ErrNotExist) { + return fmt.Errorf("inspect restore staging for %s: %w", id, previousErr) + } + if previousErr == nil { + _, instanceErr := os.Stat(instanceDir) + if instanceErr != nil && !errors.Is(instanceErr, os.ErrNotExist) { + return fmt.Errorf("inspect instance directory for %s: %w", id, instanceErr) + } + if errors.Is(instanceErr, os.ErrNotExist) { + if err := os.MkdirAll(filepath.Dir(instanceDir), 0755); err != nil { + return fmt.Errorf("prepare rollback for %s: %w", id, err) + } + if err := os.Rename(previous, instanceDir); err != nil { + return fmt.Errorf("roll back restore for %s: %w", id, err) + } + } + } + if err := os.RemoveAll(stagingRoot); err != nil { + return fmt.Errorf("clear restore staging for %s: %w", id, err) + } + } + return nil +} + +// finishMacOSResume completes a restore whose API process died after the shim had +// loaded the VM, once resuming the Paused guest succeeds. It records the shim, and +// removes the consumed memory snapshot so a later crash cannot restore stale memory +// onto the guest's newer disk. +func (m *manager) finishMacOSResume(id string) error { + meta, err := m.loadMetadata(id) + if err != nil { + return err + } + if meta.HypervisorPID == nil || !ProcessExists(*meta.HypervisorPID) { + pid, err := vzShimOwner(meta.SocketPath) + if err != nil { + return err + } + if pid > 0 { + meta.HypervisorProcessIdentity.Set(pid) + } + } + if err := os.RemoveAll(m.paths.InstanceSnapshotLatest(id)); err != nil { + return fmt.Errorf("remove consumed snapshot: %w", err) + } + meta.Phases.Record(initialBootPhase(&meta.StoredMetadata), time.Now().UTC()) + return m.saveMetadata(meta) +} diff --git a/lib/instances/snapshot_macos_darwin_test.go b/lib/instances/snapshot_macos_darwin_test.go new file mode 100644 index 000000000..f14a47c88 --- /dev/null +++ b/lib/instances/snapshot_macos_darwin_test.go @@ -0,0 +1,72 @@ +//go:build darwin && arm64 + +package instances + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/vz" + "github.com/kernel/hypeman/lib/instances/phasetracking" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestMacOSStoppedSnapshotForkRebindsIdentity(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p, macOSOnly: true, lifecycleEvents: newLifecycleSubscribers(), vmStarters: map[hypervisor.Type]hypervisor.VMStarter{hypervisor.TypeVZ: concurrentForkPrepareTestStarter{}}} + require.NoError(t, os.MkdirAll(p.GuestsDir(), 0700)) + const snapshotID = "mac-fork-snapshot" + source := *macOSGuestMetadata(2) + source.Id, source.Name = "removed-source", "removed-source" + source.DataDir = p.InstanceDir(source.Id) + source.HypervisorType = hypervisor.TypeVZ + source.Size, source.Vcpus = 8<<30, 4 + source.NetworkEnabled = true + source.MAC = source.MacOS.MAC + source.MacOS.GuestAgent = true + source.StopGraceful = true + now := time.Now() + source.GuestAgentReadyAt = &now + source.Phases.Record(phasetracking.PhaseRunning, now.Add(-time.Hour)) + source.Phases.Record(phasetracking.PhaseStopped, now) + require.NoError(t, vz.NewStarter().ValidateConfig(m.plannedStoredVMConfig(SnapshotKindStopped, source))) + require.NoError(t, m.saveSnapshotRecord(&snapshotRecord{Snapshot: Snapshot{Id: snapshotID, Kind: SnapshotKindStopped, SourceInstanceID: source.Id, SourceName: source.Name, SourceHypervisor: source.HypervisorType, CreatedAt: now}, StoredMetadata: source})) + writeGuestFiles(t, p.SnapshotGuestDir(snapshotID), map[string]string{"overlay.raw": "snapshot-disk", macOSAuxStorageFile: "snapshot-aux"}) + + first, err := m.ForkSnapshot(context.Background(), snapshotID, ForkSnapshotRequest{Name: "mac-snapshot-clone-1"}) + require.NoError(t, err) + second, err := m.ForkSnapshot(context.Background(), snapshotID, ForkSnapshotRequest{Name: "mac-snapshot-clone-2"}) + require.NoError(t, err) + for _, clone := range []*Instance{first, second} { + require.Equal(t, StateStopped, clone.State) + require.True(t, clone.IdentityRekeyPending) + require.NotEqual(t, source.MacOS.MachineIdentifier, clone.MacOS.MachineIdentifier) + require.NotEqual(t, source.MAC, clone.MAC) + require.Equal(t, clone.MAC, clone.MacOS.MAC) + require.Empty(t, clone.IP) + require.Nil(t, clone.GuestAgentReadyAt) + require.Nil(t, clone.HealthCheckRuntime) + require.Empty(t, clone.Phases.Cumulative) + require.Equal(t, phasetracking.PhaseStopped, clone.Phases.Current) + require.Equal(t, clone.CreatedAt, clone.Phases.Since) + data, err := os.ReadFile(filepath.Join(clone.DataDir, macOSAuxStorageFile)) + require.NoError(t, err) + require.Equal(t, "snapshot-aux", string(data)) + } + require.NotEqual(t, first.MAC, second.MAC) + require.NotEqual(t, first.MacOS.MachineIdentifier, second.MacOS.MachineIdentifier) + _, err = m.ForkSnapshot(context.Background(), snapshotID, ForkSnapshotRequest{Name: first.Name}) + require.ErrorIs(t, err, ErrAlreadyExists) + persisted, err := m.loadSnapshotRecord(snapshotID) + require.NoError(t, err) + require.Equal(t, source.MacOS, persisted.StoredMetadata.MacOS) + require.NoError(t, os.WriteFile(p.InstanceOverlay(first.Id), []byte("clone-write"), 0600)) + data, err := os.ReadFile(filepath.Join(p.SnapshotGuestDir(snapshotID), "overlay.raw")) + require.NoError(t, err) + require.Equal(t, "snapshot-disk", string(data)) +} diff --git a/lib/instances/snapshot_macos_test.go b/lib/instances/snapshot_macos_test.go new file mode 100644 index 000000000..99cb542a5 --- /dev/null +++ b/lib/instances/snapshot_macos_test.go @@ -0,0 +1,164 @@ +package instances + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func macOSGuestMetadata(identity byte) *StoredMetadata { + return &StoredMetadata{MacOS: &images.MacOSImage{ + HardwareModel: []byte{1}, MachineIdentifier: []byte{identity}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, + }} +} + +func TestMacOSSnapshotCreateGuardsGracefulStopAndHostSupport(t *testing.T) { + require.NoError(t, validateMacOSSnapshotCreate(&metadata{StoredMetadata: StoredMetadata{}}, CreateSnapshotRequest{Kind: SnapshotKindStandby}), "Linux guests keep their rules") + + mac := &metadata{StoredMetadata: *macOSGuestMetadata(2)} + require.ErrorIs(t, validateMacOSSnapshotCreate(mac, CreateSnapshotRequest{Kind: SnapshotKindStandby}), ErrNotSupported) + require.ErrorIs(t, validateMacOSSnapshotCreate(mac, CreateSnapshotRequest{Kind: SnapshotKindStopped}), ErrInvalidState, "a forced stop is not a consistent disk") + + mac.StopGraceful = true + require.NoError(t, validateMacOSSnapshotCreate(mac, CreateSnapshotRequest{Kind: SnapshotKindStopped})) +} + +func TestMacOSSnapshotRestoreRequiresSameIdentityAndSupportedKind(t *testing.T) { + source := &metadata{StoredMetadata: *macOSGuestMetadata(2)} + same := &snapshotRecord{Snapshot: Snapshot{Kind: SnapshotKindStopped}, StoredMetadata: *macOSGuestMetadata(2)} + require.NoError(t, validateMacOSSnapshotRestore(source, same)) + + other := &snapshotRecord{Snapshot: Snapshot{Kind: SnapshotKindStopped}, StoredMetadata: *macOSGuestMetadata(9)} + require.ErrorIs(t, validateMacOSSnapshotRestore(source, other), ErrInvalidRequest, "a different machine identifier is another guest") + + standby := &snapshotRecord{Snapshot: Snapshot{Kind: SnapshotKindStandby}, StoredMetadata: *macOSGuestMetadata(2)} + require.ErrorIs(t, validateMacOSSnapshotRestore(source, standby), ErrNotSupported, "unsupported hosts cannot restore memory snapshots") + + linux := &snapshotRecord{Snapshot: Snapshot{Kind: SnapshotKindStopped}} + require.ErrorIs(t, validateMacOSSnapshotRestore(source, linux), ErrInvalidRequest, "guest types must match") +} + +func TestMacOSSnapshotForkRequiresGracefulStoppedSnapshotAndAgent(t *testing.T) { + rec := &snapshotRecord{Snapshot: Snapshot{Kind: SnapshotKindStopped}, StoredMetadata: *macOSGuestMetadata(2)} + require.ErrorIs(t, validateMacOSSnapshotFork(rec), ErrInvalidState) + rec.StoredMetadata.StopGraceful = true + require.ErrorIs(t, validateMacOSSnapshotFork(rec), ErrInvalidRequest) + rec.StoredMetadata.MacOS.GuestAgent = true + require.NoError(t, validateMacOSSnapshotFork(rec)) + rec.StoredMetadata.SkipGuestAgent = true + require.ErrorIs(t, validateMacOSSnapshotFork(rec), ErrInvalidRequest) + rec.StoredMetadata.SkipGuestAgent = false + rec.Snapshot.Kind = SnapshotKindStandby + require.ErrorIs(t, validateMacOSSnapshotFork(rec), ErrNotSupported) + require.NoError(t, validateMacOSSnapshotFork(&snapshotRecord{}), "Linux admission is unchanged") +} + +func writeGuestFiles(t *testing.T, dir string, files map[string]string) { + t.Helper() + require.NoError(t, os.MkdirAll(dir, 0700)) + for name, body := range files { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0600)) + } +} + +func TestMacOSRestoreSwapsPayloadAndKeepsPreviousOnStagingFailure(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + const id, snap = "mac-restore", "snap-1" + writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{"overlay.raw": "after", "mac-aux.img": "aux-after"}) + writeGuestFiles(t, m.paths.SnapshotGuestDir(snap), map[string]string{"overlay.raw": "before", "mac-aux.img": "aux-before"}) + + require.NoError(t, m.replaceMacOSInstanceWithSnapshotPayload(snap, &metadata{StoredMetadata: StoredMetadata{Id: id}})) + data, err := os.ReadFile(filepath.Join(m.paths.InstanceDir(id), "overlay.raw")) + require.NoError(t, err) + require.Equal(t, "before", string(data)) + _, err = os.Stat(filepath.Join(m.paths.DataDir(), "restore-staging", id)) + require.ErrorIs(t, err, os.ErrNotExist, "staging is removed after a successful swap") + + // A missing snapshot payload fails before the instance directory is retired. + writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{"overlay.raw": "current"}) + require.Error(t, m.replaceMacOSInstanceWithSnapshotPayload("missing-snapshot", &metadata{StoredMetadata: StoredMetadata{Id: id}})) + data, err = os.ReadFile(filepath.Join(m.paths.InstanceDir(id), "overlay.raw")) + require.NoError(t, err) + require.Equal(t, "current", string(data), "a failed staging leaves the previous disk untouched") +} + +func TestMacOSRestoreStagesMetadataWithPayload(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + const id, snapshotID = "atomic-mac-restore", "atomic-checkpoint" + writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{"overlay.raw": "current", macOSAuxStorageFile: "current-aux"}) + writeGuestFiles(t, m.paths.SnapshotGuestDir(snapshotID), map[string]string{"overlay.raw": "checkpoint", macOSAuxStorageFile: "checkpoint-aux", "metadata.json": "stale-metadata"}) + expiresAt := time.Now().Add(time.Hour) + restored := &metadata{StoredMetadata: StoredMetadata{Id: id, Name: "current-name", ExpiresAt: &expiresAt, Vcpus: 4, Size: 8 << 30}} + require.NoError(t, m.replaceMacOSInstanceWithSnapshotPayload(snapshotID, restored)) + // Simulate restart before the caller's subsequent saveMetadata: the installed + // directory must already contain the metadata corresponding to the restore. + require.NoError(t, m.recoverMacOSRestoreStaging(id)) + got, err := m.loadMetadata(id) + require.NoError(t, err) + require.Equal(t, restored.Name, got.Name) + require.Equal(t, restored.Vcpus, got.Vcpus) + require.NotNil(t, got.ExpiresAt) + require.True(t, expiresAt.Equal(*got.ExpiresAt)) +} + +func TestMacOSRestoreRecoveryRollsBackAndCleans(t *testing.T) { + const id = "mac-crash" + newManager := func(t *testing.T) (*manager, string) { + m := &manager{paths: paths.New(t.TempDir())} + return m, filepath.Join(m.paths.DataDir(), "restore-staging", id) + } + + // Crash after retiring the instance directory, before the swap: roll the previous state back. + m, root := newManager(t) + writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{"overlay.raw": "previous"}) + require.NoError(t, m.recoverMacOSRestoreStaging(id)) + data, err := os.ReadFile(filepath.Join(m.paths.InstanceDir(id), "overlay.raw")) + require.NoError(t, err) + require.Equal(t, "previous", string(data)) + _, err = os.Stat(root) + require.ErrorIs(t, err, os.ErrNotExist) + + // Crash after a completed swap: the instance directory is current, so stale staging goes. + m, root = newManager(t) + writeGuestFiles(t, m.paths.InstanceDir(id), map[string]string{"overlay.raw": "restored"}) + writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{"overlay.raw": "old"}) + require.NoError(t, m.recoverMacOSRestoreStaging(id)) + data, err = os.ReadFile(filepath.Join(m.paths.InstanceDir(id), "overlay.raw")) + require.NoError(t, err) + require.Equal(t, "restored", string(data)) + _, err = os.Stat(root) + require.ErrorIs(t, err, os.ErrNotExist) + + // Crash during staging copy: no previous directory, so the partial copy is removed. + m, root = newManager(t) + writeGuestFiles(t, filepath.Join(root, "staged"), map[string]string{"overlay.raw": "partial"}) + require.NoError(t, m.recoverMacOSRestoreStaging("")) + _, err = os.Stat(root) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestMacOSRestoreRecoveryKeepsBackupWhenInstanceStateIsUnknown(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores directory permissions") + } + const id = "mac-unknown" + m := &manager{paths: paths.New(t.TempDir())} + root := filepath.Join(m.paths.DataDir(), "restore-staging", id) + writeGuestFiles(t, filepath.Join(root, "previous"), map[string]string{"overlay.raw": "previous"}) + guests := filepath.Dir(m.paths.InstanceDir(id)) + require.NoError(t, os.MkdirAll(guests, 0755)) + // An unreadable guests directory makes the instance directory's state unknown. + require.NoError(t, os.Chmod(guests, 0)) + t.Cleanup(func() { _ = os.Chmod(guests, 0755) }) + + require.Error(t, m.recoverMacOSRestoreStaging(id)) + require.NoError(t, os.Chmod(guests, 0755)) + data, err := os.ReadFile(filepath.Join(root, "previous", "overlay.raw")) + require.NoError(t, err, "the parked backup must survive an inconclusive check") + require.Equal(t, "previous", string(data)) +} diff --git a/lib/instances/snapshot_schedule.go b/lib/instances/snapshot_schedule.go index 4eccc4983..4c01a9037 100644 --- a/lib/instances/snapshot_schedule.go +++ b/lib/instances/snapshot_schedule.go @@ -40,10 +40,12 @@ func (m *manager) SetSnapshotSchedule(ctx context.Context, instanceID string, re if err != nil { return nil, err } - // Schedules run the snapshot path, which macOS guests reject; refuse up front. if err := m.admitGuestOperation(&meta.StoredMetadata, "snapshot schedules"); err != nil { return nil, err } + if err := m.validateMacOSSnapshotSchedule(ctx, meta); err != nil { + return nil, err + } now := m.now().UTC() createdAt := now diff --git a/lib/instances/snapshot_schedule_test.go b/lib/instances/snapshot_schedule_test.go index d208b9abf..1578a4ff0 100644 --- a/lib/instances/snapshot_schedule_test.go +++ b/lib/instances/snapshot_schedule_test.go @@ -658,7 +658,7 @@ func markSnapshotScheduleDue(t *testing.T, mgr *manager, instanceID string) { require.NoError(t, mgr.saveSnapshotScheduleUnlocked(schedule)) } -func TestSnapshotScheduleRejectedForMacOSGuests(t *testing.T) { +func TestSnapshotScheduleRejectsMacOSWithoutSaveRestoreSupport(t *testing.T) { t.Parallel() mgr, _ := setupTestManager(t) ctx := context.Background() @@ -674,6 +674,6 @@ func TestSnapshotScheduleRejectedForMacOSGuests(t *testing.T) { NamePrefix: "nightly", Retention: SnapshotScheduleRetention{MaxCount: 1}, }) - require.ErrorIs(t, err, ErrInvalidRequest) - require.Contains(t, err.Error(), "snapshot schedules") + require.ErrorIs(t, err, ErrNotSupported) + require.Contains(t, err.Error(), "save/restore support") } diff --git a/lib/instances/standby.go b/lib/instances/standby.go index f521b251e..215a04975 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -45,6 +45,11 @@ func (m *manager) standbyInstance( if err := m.admitGuestOperation(&meta.StoredMetadata, "standby"); err != nil { return nil, err } + if meta.MacOS != nil { + // macOS saved state is not compressed. The framework already omits zero pages, + // and compression would add time inside the paused window. + skipCompression = true + } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata @@ -91,7 +96,7 @@ func (m *manager) standbyInstance( // 3. Get network allocation BEFORE killing VMM (while we can still query it) // This is needed to delete the TAP device after VMM shuts down var networkAlloc *network.Allocation - if inst.NetworkEnabled { + if inst.NetworkEnabled && inst.MacOS == nil { log.DebugContext(ctx, "getting network allocation", "instance_id", id) networkAlloc, err = m.networkManager.GetAllocation(ctx, id) if err != nil { @@ -112,6 +117,10 @@ func (m *manager) standbyInstance( return nil, fmt.Errorf("hypervisor %s does not support standby (snapshots)", stored.HypervisorType) } + if err := validateMacOSSaveRestoreSupport(ctx, stored, hv); err != nil { + return nil, err + } + // 6. Transition: Running → Paused log.DebugContext(ctx, "pausing VM", "instance_id", id) pauseCtx, pauseSpanEnd := m.startLifecycleStep(ctx, "pause_vm", @@ -122,6 +131,11 @@ func (m *manager) standbyInstance( if err := hv.Pause(pauseCtx); err != nil { pauseSpanEnd(err) log.ErrorContext(ctx, "failed to pause VM", "instance_id", id, "error", err) + if stored.MacOS != nil { + if resumeErr := resumeSnapshotAfterError(ctx, stored, hv); resumeErr != nil { + log.ErrorContext(ctx, "failed to resume VM after pause error", "instance_id", id, "error", resumeErr) + } + } return nil, fmt.Errorf("pause vm failed: %w", err) } pauseSpanEnd(nil) @@ -135,7 +149,7 @@ func (m *manager) standbyInstance( var err error promotedExistingBase, err = prepareRetainedSnapshotTarget(snapshotDir, retainedBaseDir) if err != nil { - if resumeErr := hv.Resume(ctx); resumeErr != nil { + if resumeErr := resumeSnapshotAfterError(ctx, stored, hv); resumeErr != nil { log.ErrorContext(ctx, "failed to resume VM after retained snapshot target preparation error", "instance_id", id, "error", resumeErr) } return nil, fmt.Errorf("prepare retained snapshot target: %w", err) @@ -144,7 +158,7 @@ func (m *manager) standbyInstance( // place; if fanout forks still hardlink its inode, replace it with a // private copy first so their memory is never mutated. if err := ensureExclusiveSnapshotMemoryOwnership(ctx, snapshotDir); err != nil { - if resumeErr := hv.Resume(ctx); resumeErr != nil { + if resumeErr := resumeSnapshotAfterError(ctx, stored, hv); resumeErr != nil { log.ErrorContext(ctx, "failed to resume VM after snapshot memory unshare error", "instance_id", id, "error", resumeErr) } if promotedExistingBase { @@ -166,7 +180,7 @@ func (m *manager) standbyInstance( snapshotSpanEnd(err) // Snapshot failed - try to resume VM log.ErrorContext(ctx, "snapshot failed, attempting to resume VM", "instance_id", id, "error", err) - if resumeErr := hv.Resume(ctx); resumeErr != nil { + if resumeErr := resumeSnapshotAfterError(ctx, stored, hv); resumeErr != nil { log.ErrorContext(ctx, "failed to resume VM after snapshot error", "instance_id", id, "error", resumeErr) } if promotedExistingBase { @@ -190,7 +204,7 @@ func (m *manager) standbyInstance( // The hypervisor may still be running: releasing its TAP or clearing // its identity now would orphan a live VMM. The snapshot on disk is // harmless and a retried standby redoes it. - if resumeErr := hv.Resume(ctx); resumeErr != nil { + if resumeErr := resumeSnapshotAfterError(ctx, stored, hv); resumeErr != nil { log.ErrorContext(ctx, "failed to resume VM after shutdown error", "instance_id", id, "error", resumeErr) } return nil, fmt.Errorf("shutdown hypervisor: %w", err) @@ -215,7 +229,7 @@ func (m *manager) standbyInstance( // 9. Release network allocation (delete TAP device) // TAP devices with explicit Owner/Group fields do NOT auto-delete when VMM exits // They must be explicitly deleted - if inst.NetworkEnabled { + if inst.NetworkEnabled && inst.MacOS == nil { m.unregisterEgressProxyInstance(ctx, id) log.DebugContext(ctx, "releasing network", "instance_id", id, "network", "default") releaseNetworkCtx, releaseNetworkSpanEnd := m.startLifecycleStep(ctx, "release_network", diff --git a/lib/instances/start.go b/lib/instances/start.go index 763ed44c1..dca067781 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -167,6 +167,14 @@ func (m *manager) startInstance( }) } } + if stored.MacOS != nil { + if err := m.checkMacOSMACAvailable(ctx, stored); err != nil { + return nil, err + } + if err := m.clearMacOSStopReceipt(stored); err != nil { + return nil, err + } + } // 4b. Recreate the vGPU if this instance had a GPU profile // Note: GPU availability was already validated in step 2b @@ -240,6 +248,10 @@ func (m *manager) startInstance( // Success - release cleanup stack (prevent cleanup) cu.Release() + if err := m.rekeyMacOSCloneOnStart(ctx, stored); err != nil { + return nil, err + } + // 7. Update metadata (set PID, StartedAt). Boot markers were cleared at // the top of this function, so we are in Initializing until they hydrate. stored.Phases.Record(initialBootPhase(stored), time.Now().UTC()) diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 2007e54fb..7ecc75927 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -169,6 +169,7 @@ func (m *manager) stopInstance( attribute.String("operation", "graceful_guest_shutdown"), attribute.Int("stop_timeout_seconds", stopTimeout), ) + macOSVMMLive := inst.MacOS != nil && hypervisorProcessLive(&inst) gracefulShutdown := m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout) if gracefulShutdown { gracefulSpanEnd(nil) @@ -285,6 +286,10 @@ func (m *manager) stopInstance( // 10. Update metadata (clear PID, set StoppedAt) now := time.Now().UTC() stored.StoppedAt = &now + if stored.MacOS != nil { + // A VMM that was already gone exits "gracefully" without a guest shutdown. + stored.StopGraceful = gracefulShutdown && macOSVMMLive + } stored.HypervisorProcessIdentity.Clear() // Boot markers are per-boot-run and must not carry across stop/restore/start. stored.ProgramStartedAt = nil diff --git a/lib/instances/types.go b/lib/instances/types.go index 638dbf2b9..1e98a864a 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -94,6 +94,9 @@ type StoredMetadata struct { // Read-only; echoed on the instance API. Platform string MacOS *images.MacOSImage `json:"MacOS,omitempty"` // Non-nil for experimental local macOS disk guests. + // IdentityRekeyPending marks a macOS clone whose SSH host keys still come from its + // source. The next start rekeys them and clears the flag. + IdentityRekeyPending bool `json:"identity_rekey_pending,omitempty"` // Resources (matching Cloud Hypervisor terminology) Size int64 // Base memory in bytes @@ -121,6 +124,9 @@ type StoredMetadata struct { ExpiresAt *time.Time // Absolute expiration time; nil disables expiration StartedAt *time.Time // Boot epoch start time (set on create/start; preserved across standby restore) StoppedAt *time.Time // Last time VM was stopped + // StopGraceful records whether the last stop was an acknowledged guest shutdown with + // owned VMM exit. macOS stopped snapshots require it; a forced stop is not consistent. + StopGraceful bool `json:"stop_graceful,omitempty"` // Boot progress markers (derived from guest serial log sentinels and persisted) ProgramStartedAt *time.Time // Set when guest program handoff/start boundary is reached diff --git a/lib/instances/vm_config_validation.go b/lib/instances/vm_config_validation.go index 61639e9b4..8a04b7f41 100644 --- a/lib/instances/vm_config_validation.go +++ b/lib/instances/vm_config_validation.go @@ -42,6 +42,9 @@ func (m *manager) validateStoredVMConfig(starter hypervisor.VMStarter, snapshotK } func (m *manager) plannedStoredVMConfig(snapshotKind SnapshotKind, meta StoredMetadata) hypervisor.VMConfig { + if meta.MacOS != nil { + return m.macOSVMConfig(&Instance{StoredMetadata: meta}) + } hasVGPU := storedVGPUDevicePath(&meta) != "" || meta.GPUProfile != "" config := m.plannedVMConfig( meta.HotplugSize, diff --git a/lib/network/allocate.go b/lib/network/allocate.go index 112ee151b..02d9005bb 100644 --- a/lib/network/allocate.go +++ b/lib/network/allocate.go @@ -690,3 +690,14 @@ func GenerateTAPName(instanceID string) string { } return TAPPrefix + strings.ToLower(shortID) } + +// NewUniqueMAC returns a locally administered MAC that is not in used. Callers that +// assign MACs outside the network allocator (macOS vmnet) use it so both paths share +// one uniqueness rule. +func NewUniqueMAC(used []string) (string, error) { + usedMACs := make(map[string]bool, len(used)) + for _, mac := range used { + usedMACs[strings.ToLower(strings.TrimSpace(mac))] = true + } + return allocateUniqueMACFromSet(usedMACs, generateMAC) +}