macOS guests 4/6: snapshots, schedules and stopped forks

cmd/vz-shim/server.go

@@ -59,6 +59,7 @@ func (s *ShimServer) Handler() http.Handler {
 	mux.HandleFunc("PUT /api/v1/vm.resume", s.handleResume)
 	mux.HandleFunc("PUT /api/v1/vm.shutdown", s.handleShutdown)
 	mux.HandleFunc("PUT /api/v1/vm.snapshot", s.handleSnapshot)
+	mux.HandleFunc("GET /api/v1/vm.snapshot-support", s.handleSnapshotSupport)
 	mux.HandleFunc("GET /api/v1/vm.balloon", s.handleGetBalloon)
 	mux.HandleFunc("PUT /api/v1/vm.balloon", s.handleSetBalloon)
 	mux.HandleFunc("PUT /api/v1/vm.power-button", s.handlePowerButton)
@@ -138,6 +139,21 @@ func (s *ShimServer) handleShutdown(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
+func (s *ShimServer) handleSnapshotSupport(w http.ResponseWriter, r *http.Request) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	response := struct {
+		Supported bool   `json:"supported"`
+		Reason    string `json:"reason,omitempty"`
+	}{Supported: true}
+	if err := validateSaveRestoreSupport(s.vmConfig); err != nil {
+		response.Supported = false
+		response.Reason = err.Error()
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(response)
+}
+
 func (s *ShimServer) handleSnapshot(w http.ResponseWriter, r *http.Request) {
 	s.mu.Lock()
 	defer s.mu.Unlock()

docs/macos-snapshots.md

@@ -0,0 +1,122 @@
+# macOS snapshots, schedules and restore (PR 4, draft)
+
+Supported experimentally: stopped disk snapshots, same-instance memory checkpoints,
+snapshot schedules, and cold forks from gracefully stopped instances or snapshots.
+Independent warm memory forks remain unsupported. Native validation of the new
+memory-snapshot and schedule paths is still required.
+
+## Stopped snapshots and cold forks
+
+- `kind: Stopped` requires a stopped source and a graceful shutdown receipt:
+  acknowledged guest shutdown with confirmed owned VMM exit. A forced stop is
+  refused with 409.
+- Restore targets are Stopped or Running. The source instance, hardware model,
+  machine identifier and MAC must match the snapshot.
+- A stopped-snapshot fork requires a graceful receipt and an enabled guest agent.
+  Disk and auxiliary storage are copied, the clone gets a fresh machine identifier
+  and MAC, and SSH host keys are rekeyed on first boot. The default target is
+  Stopped; Running cold-boots and completes rekeying before returning.
+- Forking a stopped instance uses the same identity/rekey contract. Hostnames,
+  account credentials and application-specific identities are not rekeyed.
+- Failed rekeying stops the clone and leaves rekey pending for a later retry.
+
+## Same-instance memory snapshots
+
+Use `POST /instances/{id}/snapshots` with `{"kind":"Standby"}`.
+
+- A Running source goes through pause, VZ machine-state save and confirmed VMM
+  shutdown. The disk, auxiliary storage and machine state are copied together
+  while the VMM is down; then the source resumes from its own checkpoint.
+- A Standby source is copied without being resumed. Stopped and other source
+  states are rejected with 409.
+- Before pausing, the shim validates the exact VM configuration using Apple's
+  `validateSaveRestoreSupport`. Host/architecture capability checks alone are
+  not proof that the guest's configured devices support save/restore.
+- Pause waits for raw Paused state, not transitional Pausing state, before save.
+  Error-path resume and source recovery use bounded cleanup contexts that survive
+  cancellation of the snapshot request. If recovery fails, the API reports the
+  failure; the source may need explicit recovery.
+- VZ machine-state files stay uncompressed. Explicit enabled compression is
+  rejected before capture; Linux server compression defaults do not apply.
+
+Restore using `POST /instances/{id}/snapshots/{snapshot_id}/restore`.
+
+- This is a same-instance operation, not a clone. Restore requires an inactive
+  Stopped or Standby source; Running, Paused and other live states are refused.
+- The default target for a memory snapshot is Running. Explicit Standby installs
+  the checkpoint without resuming it. Stopped discards memory and does not create
+  a graceful-shutdown receipt; a subsequent clean boot/stop is required before
+  making a stopped snapshot or cold fork.
+- Preflight checks saved CPU/RAM, hardware model, machine identifier, NIC MAC,
+  disk/auxiliary paths and the complete memory/disk/auxiliary bundle before
+  replacing source storage. Invalid payloads leave current storage untouched.
+- Restored metadata is staged with the payload, including the current instance
+  name and expiration. Directory-swap recovery therefore commits or rolls back
+  the matching metadata and storage together. Host phase accounting is preserved,
+  rather than replayed from the checkpoint; memory restore retains its boot time.
+  An explicit Stopped target suppresses automatic restart.
+- Restoring never mutates the persistent snapshot. Framework restore failure
+  after installation leaves the restored checkpoint available in Standby.
+- If the API dies during a restore, the shim it started keeps loading the VM.
+  Until it opens its socket the instance reads Standby, and start or restore
+  return 409 while that shim runs. Once loaded it reads Paused: `POST /restore`
+  resumes it and completes the restore (records the shim, removes the consumed
+  checkpoint), and DELETE kills it. As on Linux, stop does not accept Paused.
+- Saved state is not portable: use the originating Mac and OS account, with a
+  compatible framework version and VM configuration. API availability on macOS
+  14+ does not guarantee support for a particular macOS guest configuration.
+
+## Schedules
+
+The existing snapshot-schedule API, interval handling and retention policies are
+shared with Linux. macOS schedules require host save/restore support; admission
+also probes the exact configuration when a live VM is available.
+
+- A gracefully Stopped source produces a stopped snapshot.
+- A Running source produces a memory snapshot and is resumed afterwards.
+- A Standby source produces a memory snapshot and remains in Standby.
+- Ineligible state, absent graceful receipt, configuration or capture failure is
+  recorded in `last_error`; no successful snapshot ID is reported for that run.
+- Failed runs consume the interval, matching existing scheduler behavior. The
+  next run follows the configured cadence rather than tight retries.
+- Retention and schedule deletion use the existing controller. Scheduling does
+  not stop a running guest to manufacture a graceful disk snapshot.
+
+## Warm memory forks: feasibility gate
+
+`ForkSnapshot` still rejects macOS memory snapshots with 501, and `ForkInstance`
+still requires a gracefully stopped macOS source. These guards must not be
+removed merely because same-instance restore works.
+
+VZ restore requires compatible saved device/platform configuration. The current
+VZ fork preparation preserves NIC identity, while independent macOS guests need
+unique machine identifiers, auxiliary storage and network identities. Hypeman
+also rejects concurrently live duplicate machine identifiers or MACs.
+
+A native experiment must establish a supported identity-transition mechanism
+before independent warm forks can be enabled. Reusing the source identity or
+rekeying only SSH host keys is not an independent warm fork.
+
+References:
+- [Apple: macOS virtual machines](https://developer.apple.com/documentation/virtualization/running-macos-in-a-virtual-machine-on-apple-silicon)
+- [Apple: Mac platform configuration](https://developer.apple.com/documentation/virtualization/vzmacplatformconfiguration)
+- [Apple: restoring machine state](https://developer.apple.com/documentation/virtualization/vzvirtualmachine/restoremachinestatefrom(url:completionhandler:))
+
+## Evidence and remaining native QA
+
+- Linux-runnable filesystem/metadata tests: memory-bundle capture from Standby,
+  same-instance restore, disk/aux/memory isolation, manifest mismatch rejection,
+  preserved expiration/boot time/phase accounting, stopped-target restart
+  suppression, schedules, retention, failed-run cadence and crash-swap metadata.
+- Mock tests cover configuration rejection before pause/schedule persistence and
+  recovery contexts after request cancellation. HTTP fixtures cover VZ pause
+  completion, cancellation and configuration probing. These are not native VMs.
+- A Darwin metadata test covers two independent stopped-snapshot forks with fresh
+  identifiers/MACs, snapshot isolation and duplicate-name rejection. It does not
+  boot a guest.
+- Earlier revisions were tested live for stopped snapshot restore and stopped
+  instance clones with distinct UUID, MAC, address and SSH keys. Those runs do
+  not validate the new memory-snapshot/schedule implementation.
+- Still required on macOS: exact-configuration probe, Running capture/resume,
+  same-instance memory rollback, repeated restore, scheduled captures/retention,
+  cancellation after pause/save, and restart during staged restore.

docs/macos-support.md

@@ -15,14 +15,18 @@ owns the behavior.
 | Memory reclaim | Unsupported | Reported per guest as `unsupported`. No balloon change. | PR 1 |
 | Guest port forwarding | Agent-gated | HTTP and WebSocket to a guest loopback port at `/instances/{id}/ports/{port}/*`. The image starts its own services. See [macos-guest-ports.md](macos-guest-ports.md). | PR 6 |
 | Builds | Partial | Opt-in `macos_builds`. Restricted recipe. Linux Dockerfile builds unsupported. | PR 5 |
-| Stopped snapshots, same-instance restore | Partial | Graceful stop required. Running restore has an open readiness issue. | PR 4 |
-| Standby, memory snapshot, memory fork | Unsupported | Rejected with a stable error. | — |
-| Stopped clones, fork from snapshot | Unsupported | Needs identity rebinding and guest rekeying. | PR 4 (later) |
+| Stopped snapshots, same-instance restore | Experimental | Graceful stop required. Stopped and Running targets; restored metadata and storage are staged together. | PR 4 |
+| Standby and restore (same instance) | Experimental | Saves memory, restores the same instance. Restore can fail (about 2% observed, cause unknown); a failed restore leaves the instance in Standby. Recover by deleting, as on Linux. Standby auto-policies rejected: no host-visible connection tracking on NAT. | PR 4 |
+| Same-instance memory snapshots | Experimental | Running/Standby capture, with exact VZ configuration validation and an uncompressed disk/aux/memory bundle. Same-instance restore only; new path needs native QA. | PR 4 |
+| Independent warm memory forks | Unsupported | Remain rejected: saved-state compatibility and independent machine/NIC identity need native proof. | — |
+| Stopped clones (fork from stopped) | Experimental | Rebinds machine identifier and MAC; rekeys SSH host keys on first start. Hostname and credentials not yet checked. | PR 4 |
+| Fork from stopped snapshot | Experimental | Graceful receipt and enabled agent required. Fresh machine identifier/MAC; SSH rekey on first cold boot. Stopped and Running targets supported. | PR 4 |
 | Update (PATCH), VM restart policy | Partial | Expiration/TTL and `restart_policy: always` or `never` are supported. Manual stop suppresses automatic restart. A shim that dies without removing its socket (for example, SIGKILL) reads as Stopped, so `always` restarts it. `on_failure` is rejected: no workload exit status is available. | PR 1 |
 | Exec health checks | Agent-gated | Supported at create and PATCH; `type: none` disables them. HTTP/TCP checks remain unsupported. | PR 3 |
 | Volumes, env and commands, credentials and auto-standby policies, passthrough, shaping | Unsupported | Rejected at create, start, or update. | — |
 | Ingress | Unsupported | Refused in macOS-only mode and for macOS targets. | — |
-| Snapshot schedules | Unsupported | Refused at admission. | — |
+| Snapshot schedules | Experimental | State-based stopped or memory capture, existing cadence/retention, host save/restore gate and live configuration probe. New path needs native QA. | PR 4 |
 
-Limits that apply everywhere: preserved machine identity (one active instance per
-identifier, no rekeying), no clean-base sanitation audit, and no external review verdict.
+Limits that apply everywhere: one active VM per machine identifier, no clean-base
+sanitation audit, and no external review verdict. Same-instance memory restore
+preserves identity; stopped clones get fresh machine identifiers and SSH host keys.

lib/hypervisor/snapshot_support.go

@@ -0,0 +1,8 @@
+package hypervisor
+
+import "context"
+
+// SnapshotSupportValidator checks configuration-specific support before pausing.
+type SnapshotSupportValidator interface {
+	ValidateSnapshotSupport(context.Context) error
+}

lib/hypervisor/snapshot_support_test.go

@@ -0,0 +1,34 @@
+package hypervisor
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type fakeSnapshotSupportValidator struct {
+	fakeHypervisor
+	err    error
+	called bool
+}
+
+func (h *fakeSnapshotSupportValidator) ValidateSnapshotSupport(context.Context) error {
+	h.called = true
+	return h.err
+}
+
+func TestTracingPreservesSnapshotSupportValidation(t *testing.T) {
+	probeErr := errors.New("configuration probe failed")
+	hv := &fakeSnapshotSupportValidator{err: probeErr}
+	wrapped := WrapHypervisor(TypeVZ, hv)
+	validator, ok := wrapped.(SnapshotSupportValidator)
+	require.True(t, ok)
+	require.ErrorIs(t, validator.ValidateSnapshotSupport(context.Background()), probeErr)
+	require.True(t, hv.called)
+	hv.err = nil
+	require.NoError(t, validator.ValidateSnapshotSupport(context.Background()))
+	unsupported := WrapHypervisor(TypeCloudHypervisor, fakeHypervisor{}).(SnapshotSupportValidator)
+	require.ErrorIs(t, unsupported.ValidateSnapshotSupport(context.Background()), ErrNotSupported)
+}

lib/hypervisor/tracing.go

@@ -219,6 +219,18 @@ func (h *tracingHypervisor) Resume(ctx context.Context) (err error) {
 	return h.next.Resume(ctx)
 }
 
+func (h *tracingHypervisor) ValidateSnapshotSupport(ctx context.Context) (err error) {
+	validator, ok := h.next.(SnapshotSupportValidator)
+	if !ok {
+		return ErrNotSupported
+	}
+	ctx, span := startTraceSpan(ctx, h.tracer, "hypervisor.validate_snapshot_support",
+		h.spanAttrs(attribute.String("operation", "validate_snapshot_support"))...,
+	)
+	defer func() { finishTraceSpan(span, err) }()
+	return validator.ValidateSnapshotSupport(ctx)
+}
+
 func (h *tracingHypervisor) Snapshot(ctx context.Context, destPath string) (err error) {
 	ctx, span := startTraceSpan(ctx, h.tracer, "hypervisor.snapshot",
 		h.spanAttrs(

lib/hypervisor/vz/client.go

@@ -262,7 +262,28 @@ func (c *Client) GetVMInfo(ctx context.Context) (*hypervisor.VMInfo, error) {
 }
 
 func (c *Client) Pause(ctx context.Context) error {
-	return c.doPut(ctx, "/api/v1/vm.pause", nil)
+	if err := c.doPut(ctx, "/api/v1/vm.pause", nil); err != nil {
+		return err
+	}
+	return c.waitForRawState(ctx, "Paused")
+}
+
+func (c *Client) ValidateSnapshotSupport(ctx context.Context) error {
+	body, err := c.doGet(ctx, "/api/v1/vm.snapshot-support")
+	if err != nil {
+		return err
+	}
+	var response struct {
+		Supported bool   `json:"supported"`
+		Reason    string `json:"reason"`
+	}
+	if err := json.Unmarshal(body, &response); err != nil {
+		return fmt.Errorf("decode snapshot support: %w", err)
+	}
+	if !response.Supported {
+		return fmt.Errorf("%w: VZ configuration cannot be saved/restored: %s", hypervisor.ErrNotSupported, response.Reason)
+	}
+	return nil
 }
 
 func (c *Client) Resume(ctx context.Context) error {
@@ -279,16 +300,16 @@ func (c *Client) Resume(ctx context.Context) error {
 }
 
 const (
-	resumeReadyTimeout      = 10 * time.Second
-	resumeReadyPollInterval = 25 * time.Millisecond
+	stateReadyTimeout      = 10 * time.Second
+	stateReadyPollInterval = 25 * time.Millisecond
 )
 
 // waitForRawState polls vm.info until the shim's raw state string equals want or
-// resumeReadyTimeout elapses. It checks the raw state (not the hypervisor.VMState
+// stateReadyTimeout elapses. It checks the raw state (not the hypervisor.VMState
 // mapping, which collapses "Resuming" into Running) so the post-condition is the
 // guest actually being in the requested state.
 func (c *Client) waitForRawState(ctx context.Context, want string) error {
-	deadline := time.Now().Add(resumeReadyTimeout)
+	deadline := time.Now().Add(stateReadyTimeout)
 	for {
 		state, err := c.rawVMState(ctx)
 		if err == nil && state == want {
@@ -298,12 +319,12 @@ func (c *Client) waitForRawState(ctx context.Context, want string) error {
 			if err != nil {
 				return fmt.Errorf("wait for vm state %q: %w", want, err)
 			}
-			return fmt.Errorf("vm did not reach state %q within %s (last state %q)", want, resumeReadyTimeout, state)
+			return fmt.Errorf("vm did not reach state %q within %s (last state %q)", want, stateReadyTimeout, state)
 		}
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
-		case <-time.After(resumeReadyPollInterval):
+		case <-time.After(stateReadyPollInterval):
 		}
 	}
 }