feat(sandbox): add SmolVM local sandbox provider

README.md

@@ -62,6 +62,26 @@ scores = await episode.result()  # every score the run recorded, by step id
 
 The scores are the ones `agent-env run` reports, so making a reward of them is yours. `trainer_agents=` names the agents you play when the task prompts several, `turn_timeout` is each turn's deadline in seconds (the prompt's own `timeout_seconds` otherwise), `episode.abandon(reason)` stops a run early, and `from_task` names any step an outside player can't stand behind. [Running tasks](https://www.agentenvframework.com/docs/tasks/running) covers the rest.
 
+## Run agents in local microVMs
+
+Install `agentenv-framework[smol]` and select the `smol_vm` sandbox provider in
+`.agentenv/config.toml`:
+
+```toml
+[sandbox]
+default = "smol_vm"
+agent_default = "smol_vm"
+```
+
+Each agent and environment gets its own local SmolVM machine with a Docker daemon,
+so workloads are isolated from the host Docker daemon. This needs a supported
+Apple Silicon macOS or Linux host; on Linux, `/dev/kvm` must be available. On Apple Silicon,
+build images for `linux/arm64` (or provide their build contexts so SmolVM can rebuild them);
+this provider does not install amd64 emulation in the guest. AgentEnv can still
+use its default local object and image stores: guest containers can fetch local
+object grants, and the VM forwards pulls from the host's loopback registry.
+Cloud image and object stores continue to work through the same provider.
+
 ## Plugins
 
 A package of your own can add env types, providers, task steps and stores to agent-env, and `agent-env plugin` lists, checks, adds and removes installed plugins. The [plugin docs](https://www.agentenvframework.com/docs/plugins) show how.

pyproject.toml

@@ -81,7 +81,7 @@ tenki = [
     "tenki[async]~=1.4.1",  # AsyncClient/AsyncSandbox exec, stdin, ports and egress APIs validated against the 1.4 line
 ]
 dev = [
-    "agentenv-framework[aws,explorer,gcp,sail,tensorlake,tenki]",  # so the AWS, explorer, GCP backend, Sail, Tensorlake and Tenki provider tests run in CI
+    "agentenv-framework[aws,explorer,gcp,sail,tensorlake,tenki]",  # exercise portable optional providers and stores in CI
     "fastmcp>=3.4,<4",  # the standalone-fastmcp mount path of the protocol; <4 keeps mcp 1.x
     "griffelib==2.3.0",  # the plugin API check; exact, since what it reports as a break changes between releases
     "moto>=5.0.0",
@@ -93,6 +93,9 @@ dev = [
     "pytest-timeout>=2.3.0",
     "pytest-xdist>=3.0.0",
 ]
+smol = [
+    "smolmachines>=1.27.0",
+]
 
 [project.scripts]
 agent-env = "agent_env.cli:cli"

src/agent_env/providers/sandbox_providers/chained_sandbox_provider.py

@@ -66,6 +66,17 @@ async def create_sandbox(
         attribution: Attribution | None = None,
         **kwargs,
     ) -> Sandbox:
+        return await self._create("create_sandbox", attribution=attribution, **kwargs)
+
+    async def create_container(
+        self,
+        *,
+        attribution: Attribution | None = None,
+        **kwargs,
+    ) -> Sandbox:
+        return await self._create("create_container", attribution=attribution, **kwargs)
+
+    async def _create(self, method: str, *, attribution: Attribution | None = None, **kwargs) -> Sandbox:
         sandbox_providers = self.filter_sandbox_providers(self._providers, kwargs.get("network_policy"))
         image = self._images.get(kwargs.get("image_name"))
         errors: list[tuple[str, Exception]] = []
@@ -76,9 +87,9 @@ async def create_sandbox(
                     # outside the deadline, which bounds creating the sandbox, not a build
                     await p.prepare_image(image, attribution=attribution)
                 async with asyncio.timeout(_PROVISION_DEADLINE_SECONDS):
-                    sandbox = await p.create_sandbox(attribution=attribution, **kwargs)
+                    sandbox = await getattr(p, method)(attribution=attribution, **kwargs)
             except Exception as e:
-                logger.warning(f"{name}.create_sandbox failed: {e}; trying next provider")
+                logger.warning(f"{name}.{method} failed: {e}; trying next provider")
                 errors.append((name, e))
                 continue
             # A VM refuses an image it can't get or run, and a container runs only what its provider's containers do;

src/agent_env/providers/sandbox_providers/sandbox_provider.py

@@ -108,12 +108,19 @@ async def create_container(
         Caller still owns readiness polling for the container's own endpoints. ``private_network`` puts it on a
         network this provider's other such containers share, at its ``private_host``.
 
-        Default implementation: create_vm, authenticate Docker to ECR if applicable, docker pull,
-        docker run. Backends that bake the image in at sandbox-creation time (Modal) override.
+        Default implementation: a container-only backend uses create_sandbox; a VM backend creates
+        a VM, pulls the image, and starts it. Backends that bake the image in at creation override.
         """
         if private_network:
             raise ValueError(f"{type(self).__name__} runs each container in a VM of its own, so it can't put "
                              "containers on a private network they share")
+        if (type(self).create_vm is SandboxProvider.create_vm
+                and type(self).create_sandbox is not SandboxProvider.create_sandbox):
+            return await self.create_sandbox(
+                image_name=image_name, port=port, env=env, cpu=cpu, memory=memory,
+                disk_size_gb=disk_size_gb, timeout=timeout, attribution=attribution,
+                network_policy=network_policy,
+            )
         sandbox = await self.create_vm(
             cpu=cpu, memory=memory, disk_size_gb=disk_size_gb,
             exposed_ports=[port], timeout=timeout,
@@ -295,6 +302,7 @@ def effective_network_policy(cls, policy: Optional[NetworkPolicy]) -> NetworkPol
     "modal_vm": "agent_env.providers.sandbox_providers.modal_vm_sandbox:ModalVmSandboxProvider",
     "e2b": "agent_env.providers.sandbox_providers.e2b:E2BSandboxProvider",
     "sail_vm": "agent_env.providers.sandbox_providers.sail_vm.provider:SailVmSandboxProvider",
+    "smol_vm": "agent_env.providers.sandbox_providers.smol_vm.provider:SmolVmSandboxProvider",
     "tensorlake": "agent_env.providers.sandbox_providers.tensorlake.provider:TensorlakeSandboxProvider",
     "tenki": "agent_env.providers.sandbox_providers.tenki.provider:TenkiSandboxProvider",
     "local": "agent_env.providers.sandbox_providers.local_sandbox:LocalSandboxProvider",

src/agent_env/providers/sandbox_providers/smol_vm/__init__.py

@@ -0,0 +1 @@
+"""Local SmolVM sandbox provider (imported only when the optional extra is selected)."""

src/agent_env/providers/sandbox_providers/smol_vm/provider.py

@@ -0,0 +1,211 @@
+"""Run AgentEnv workloads in local SmolVM machines."""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+import logging
+import math
+import shlex
+import socket
+import uuid
+from urllib.parse import urlsplit
+from typing import ClassVar
+
+try:
+    from smol import ConnectOptions, Machine, MachineConfig, PortSpec, ResourceSpec
+except ModuleNotFoundError as exc:
+    if exc.name != "smol":
+        raise
+    raise ImportError("SmolVM requires pip install 'agentenv-framework[smol]'") from exc
+
+from agent_env.attribution import Attribution
+from agent_env.config import get_config
+from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore
+from agent_env.store.routing import LocalRunImageStore
+from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy
+from agent_env.providers.sandbox_providers.local_sandbox import LOCAL_TRUST_ENV, LocalSandboxProvider, local_grant_trust, start_trusting
+from agent_env.providers.sandbox_providers.sandbox_provider import (
+    Accepts, SandboxProvider, refuse_unenforceable_policy,
+)
+from agent_env.providers.sandbox_providers.smol_vm.sandbox import SmolVmSandbox
+
+_DUMMY_PORT = 60832
+logger = logging.getLogger(__name__)
+_reapers: set[asyncio.Task] = set()
+
+
+async def _delete_machine(machine: Machine) -> None:
+    try:
+        await asyncio.to_thread(machine.delete)
+    except Exception:
+        logger.exception("Could not delete SmolVM machine %s after failed creation", machine.id)
+
+
+async def _delete_when_created(creation: asyncio.Task) -> None:
+    try:
+        machine = await asyncio.shield(creation)
+    except Exception:  # native creation failed and left no machine to delete
+        return
+    await _delete_machine(machine)
+
+
+def _start_reaper(coroutine) -> asyncio.Task:
+    # Retain the task until cleanup ends, even if a caller is cancelled again.
+    task = asyncio.create_task(coroutine)
+    _reapers.add(task)
+    task.add_done_callback(_reapers.discard)
+    return task
+
+
+def _host_port() -> int:
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+class SmolVmSandboxProvider(SandboxProvider):
+    _DOCKER_READY_ATTEMPTS: ClassVar[int] = 40
+    _LOCAL_REGISTRY_DEFAULT_PORT: ClassVar[int] = 5000
+    ON_THIS_MACHINE: ClassVar[bool] = True
+    REACHES_HOST_AT: ClassVar[str] = "127.0.0.1"
+    CREATES_VMS: ClassVar[bool] = True
+    SANDBOX_ACCEPTS: ClassVar[Accepts] = Accepts.LOADABLE
+    CONTAINER_ACCEPTS: ClassVar[Accepts] = Accepts.NAME
+
+    @classmethod
+    def shares_network_with(cls, sandbox_type: str | None) -> bool:
+        return False
+
+    @classmethod
+    def get_external_url(cls, url: str) -> str:
+        return LocalSandboxProvider.get_external_url(url)
+
+    def url_from_sandbox(self, url: str) -> str:
+        return LocalSandboxProvider.get_external_url(url)
+
+    async def create_sandbox(self, *, image_name: str, port: int, env: dict[str, str],
+                             cpu: float = 1.0, memory: int = 8192, disk_size_gb: float = 10,
+                             timeout: int = 7200, attribution: Attribution | None = None,
+                             network_policy: NetworkPolicy | None = None) -> SmolVmSandbox:
+        return await self.create_vm(
+            exposed_ports=[port], cpu=cpu, memory=memory, disk_size_gb=disk_size_gb,
+            timeout=timeout, attribution=attribution, network_policy=network_policy,
+        )
+
+    async def create_vm(self, *, image: str | None = None, boot_mode: str | None = None,
+                        cpu: float = 1.0, memory: int = 8192, disk_size_gb: float = 10,
+                        timeout: int = 7200, exposed_ports: list[int] | None = None,
+                        setup_for_gateway: bool = True, attribution: Attribution | None = None,
+                        network_policy: NetworkPolicy | None = None) -> SmolVmSandbox:
+        refuse_unenforceable_policy(self, network_policy)
+        if image is not None or boot_mode is not None:
+            raise ValueError("smol_vm provisions its own Docker-capable guest; image and boot_mode overrides are unsupported")
+        if cpu < 1 or memory <= 0 or disk_size_gb <= 0:
+            raise ValueError("CPU, memory, and storage must be positive")
+        name = f"agentenv-{uuid.uuid4().hex[:12]}"
+        ports = {port: _host_port() for port in dict.fromkeys(exposed_ports or [])}
+        # Even a portless VM needs a published mapping to select routed virtio-net for nested Docker.
+        published = ports or {_DUMMY_PORT: _host_port()}
+        config = MachineConfig(
+            name=name, persistent=True, wait_for_ports=False, ttl_seconds=timeout,
+            resources=ResourceSpec(cpus=math.ceil(cpu), memory_mb=memory, storage_gb=math.ceil(disk_size_gb), network=True),
+            ports=[PortSpec(host=host, guest=guest) for guest, host in published.items()],
+        )
+        creation = asyncio.create_task(asyncio.to_thread(Machine.create, config, ConnectOptions(target="local")))
+        try:
+            machine = await asyncio.shield(creation)
+        except asyncio.CancelledError:
+            # Native creation continues after cancellation. The independent reaper also
+            # survives a second cancellation while waiting for the machine to appear.
+            await asyncio.shield(_start_reaper(_delete_when_created(creation)))
+            raise
+        try:
+            sandbox = SmolVmSandbox(machine, ports)
+            await self._setup_docker(sandbox)
+            await sandbox.detect_platform()
+            await self._configure_host_access(sandbox)
+            await sandbox.persist_ports()
+            sandbox.network_policy = self.effective_network_policy(network_policy)
+            return sandbox
+        except BaseException:
+            await asyncio.shield(_start_reaper(_delete_machine(machine)))
+            raise
+
+    async def _setup_docker(self, sandbox: SmolVmSandbox) -> None:
+        code, stdout, stderr = await sandbox.exec_with_output(
+            "sh", "-c", "command -v bash >/dev/null && command -v docker >/dev/null && "
+            "command -v curl >/dev/null && command -v python3 >/dev/null && command -v unzip >/dev/null && command -v socat >/dev/null "
+            "|| apk add --no-cache docker docker-cli-compose bash curl python3 unzip socat",
+        )
+        if code:
+            raise RuntimeError(f"Could not install guest Docker dependencies ({code}): {stdout[-500:]} {stderr[-500:]}")
+        await sandbox.exec_script(f"""
+set -e
+mkdir -p /storage/docker /storage/containerd /var/lib/docker /var/lib/containerd
+mountpoint -q /var/lib/docker || mount --bind /storage/docker /var/lib/docker
+mountpoint -q /var/lib/containerd || mount --bind /storage/containerd /var/lib/containerd
+if ! docker info >/dev/null 2>&1; then
+    rm -f /var/run/docker.pid /run/containerd/containerd.pid
+    dockerd --storage-driver=overlay2 >/tmp/agentenv-dockerd.log 2>&1 &
+    for i in $(seq 1 {self._DOCKER_READY_ATTEMPTS}); do docker info >/dev/null 2>&1 && exit 0; sleep 1; done
+    tail -80 /tmp/agentenv-dockerd.log
+    exit 1
+fi
+""")
+        await self._ensure_local_registry_proxy(sandbox)
+
+    async def _ensure_local_registry_proxy(self, sandbox: SmolVmSandbox) -> None:
+        store = get_config().get_image_store()
+        stores = (store.configured, store.local) if isinstance(store, LocalRunImageStore) else (store,)
+        ports = set()
+        for candidate in stores:
+            if isinstance(candidate, LocalRegistryImageStore):
+                registry = urlsplit(f"//{candidate.registry_host}")
+                if registry.hostname in ("localhost", "127.0.0.1"):
+                    ports.add(registry.port or self._LOCAL_REGISTRY_DEFAULT_PORT)
+        for port in sorted(ports):
+            pid_file = f"/run/agentenv-registry-proxy-{port}.pid"
+            await sandbox.exec_script(f"""
+if ! test -f {pid_file} || ! kill -0 "$(cat {pid_file})" 2>/dev/null; then
+    socat TCP-LISTEN:{port},bind=127.0.0.1,reuseaddr,fork TCP:host.smolvm.internal:{port} \
+        >/tmp/agentenv-registry-proxy-{port}.log 2>&1 &
+    echo $! > {pid_file}
+fi
+sleep 0.1
+kill -0 "$(cat {pid_file})"
+""")
+
+    async def get_sandbox(self, sandbox_id: str) -> SmolVmSandbox:
+        sandbox = await SmolVmSandbox.reconnect(sandbox_id)
+        await self._setup_docker(sandbox)
+        await self._configure_host_access(sandbox)
+        sandbox.network_policy = self.effective_network_policy(None)
+        return sandbox
+
+    async def _configure_host_access(self, sandbox: SmolVmSandbox) -> None:
+        code, stdout, stderr = await sandbox.exec_with_output("getent", "ahostsv4", "host.smolvm.internal")
+        if code or not stdout.strip():
+            raise RuntimeError(f"Cannot resolve SmolVM host gateway: {stderr[-500:]}")
+        gateway = str(ipaddress.IPv4Address(stdout.split()[0]))
+        # VM-level downloads use the same host name as the containers and local grants.
+        # Keep the entry current across reconnects without changing the host's trust CA.
+        await sandbox.exec_script(
+            "sed -i -e 's/[[:space:]]host\\.docker\\.internal\\([[:space:]]\\|$\\)/\\1/g' "
+            "-e '/^[[:space:]]*[[:xdigit:].:]\\+[[:space:]]*$/d' /etc/hosts && "
+            f"printf '%s\\n' {shlex.quote(gateway + ' host.docker.internal')} >> /etc/hosts"
+        )
+        sandbox.extra_hosts = (f"host.docker.internal:{gateway}",)
+
+    def _container_args(self, sandbox: SmolVmSandbox, *, image_name: str, port: int, env: dict[str, str]) -> str:
+        args = super()._container_args(sandbox, image_name=image_name, port=port, env=env)
+        return " ".join(f"--add-host {shlex.quote(entry)}" for entry in sandbox.extra_hosts) + " " + args
+
+    async def _start_container(self, sandbox: SmolVmSandbox, *, image_name: str, port: int, env: dict[str, str]) -> None:
+        trust_dir = await asyncio.to_thread(local_grant_trust)
+        if trust_dir is None:
+            await super()._start_container(sandbox, image_name=image_name, port=port, env=env)
+            return
+        args = self._container_args(sandbox, image_name=image_name, port=port, env={**LOCAL_TRUST_ENV, **env})
+        await sandbox.exec_script(f"docker create {args} > /dev/null")
+        await start_trusting(sandbox, sandbox.container_name, trust_dir)