From 04fdabb656e8a192957ced5333ce2ca2d263b717 Mon Sep 17 00:00:00 2001 From: Elisabeth Rulke Date: Fri, 9 Oct 2026 12:25:37 -0700 Subject: [PATCH 1/4] feat(sandbox): add Vercel Sandbox provider --- .agentenv/config.example.toml | 10 + README.md | 32 ++ pyproject.toml | 7 +- .../sandbox_providers/sandbox_provider.py | 1 + .../sandbox_providers/vercel/__init__.py | 6 + .../sandbox_providers/vercel/provider.py | 415 ++++++++++++++ .../sandbox_providers/vercel/sandbox.py | 351 ++++++++++++ .../vercel_sandbox_smoke_test.py | 121 ++++ .../sandbox_providers/egress_hosts_test.py | 4 +- .../network_policy_providers_test.py | 5 +- .../network_policy_stamping_test.py | 15 + .../vercel_sandbox_provider_test.py | 307 ++++++++++ .../sandbox_providers/vercel_sandbox_test.py | 540 ++++++++++++++++++ tst/unit/test_capabilities.py | 41 ++ tst/util/capabilities.py | 26 +- uv.lock | 123 +++- 16 files changed, 1992 insertions(+), 12 deletions(-) create mode 100644 src/agent_env/providers/sandbox_providers/vercel/__init__.py create mode 100644 src/agent_env/providers/sandbox_providers/vercel/provider.py create mode 100644 src/agent_env/providers/sandbox_providers/vercel/sandbox.py create mode 100644 tst/integration/providers/sandbox_providers/vercel_sandbox_smoke_test.py create mode 100644 tst/unit/providers/sandbox_providers/vercel_sandbox_provider_test.py create mode 100644 tst/unit/providers/sandbox_providers/vercel_sandbox_test.py diff --git a/.agentenv/config.example.toml b/.agentenv/config.example.toml index 30903567..3286a8d2 100644 --- a/.agentenv/config.example.toml +++ b/.agentenv/config.example.toml @@ -14,6 +14,16 @@ secret = "local" # process env vars, or an optional flat YAML file default = "local" # envs run as containers on the host Docker daemon agent_default = "local" # agents too +# Optional Vercel Sandboxes (install the 'vercel' extra): +# [sandbox.providers.vercel.config] +# image = "vercel/sandbox/universal" +# region = "iad1" +# failover_regions = ["cle1", "sfo1"] +# network_id = "secure-compute-network-id" +# token = "secret:vercel_token" # with team_id and project_id; omit all three for SDK OIDC +# team_id = "your-team" +# project_id = "your-project" + [agents] default_a2a_agent_id = "a2a-default" # what a deploy_agent step that names no agent deploys diff --git a/README.md b/README.md index 67aa102e..7b017428 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,38 @@ A plain install runs on local stores and needs no cloud SDK. The cloud store bac The distribution is named `agentenv-framework`, the import package is `agent_env` and the command is `agent-env`. It depends on `agentenv-framework-protocol`, whose import package is `agentenv_protocol`, and installs it too. +### Sandbox providers + +The default provider runs environments and agents as containers on the local Docker daemon. The `vercel` provider runs each environment or agent in an ephemeral, Docker-capable Vercel Sandbox and exposes declared ports through public `*.vercel.run` routes. Install it with the optional extra and select it in `.agentenv/config.toml`: + +```bash +uv tool install 'agentenv-framework[vercel]' +``` + +```toml +[sandbox] +default = "vercel" +agent_default = "vercel" + +[sandbox.providers.vercel.config] +# Optional. The default managed image installs Docker and Compose v2 when apt is available; +# a prepared image avoids that download under a restrictive network policy. +image = "vercel/sandbox/universal" +# region = "iad1" +# failover_regions = ["cle1", "sfo1"] +# network_id = "your-secure-compute-network" +# Omit all three credential fields to use the SDK's default OIDC credentials. +# token = "secret:vercel_token" +# team_id = "your-team" +# project_id = "your-project" +``` + +Credentials are resolved by AgentEnv's `env:` and `secret:` references before the provider builds its scoped Vercel SDK client; the provider never changes the process environment or uses an ambient SDK session. Each sandbox is ephemeral, its execution time follows AgentEnv's timeout, and teardown destroys the sandbox and its orphan snapshots. Vercel currently provisions 64 GB of ephemeral disk: requests above 64 GB are rejected rather than silently ignored. Resource sizing uses the smallest supported even vCPU count (or one vCPU) that covers both the requested CPU and memory (2048 MB per vCPU), up to 32 vCPUs and 64 GB at the provider-wide ceiling; the account's plan may impose lower limits. + +The provider treats CPU and memory requests as minimums. If a request does not already match a supported shape, AgentEnv logs both the requested and allocated CPU and memory before provisioning. For example, `cpu = 0.5` and `memory = 5000` allocate four vCPUs and 8192 MB. A disk request below 64 GB is also ignored: every sandbox receives the fixed 64 GB disk, and the provider logs the ignored request. Configure all three of `token`, `team_id`, and `project_id` for an explicit scoped token, or omit all three to continue using the SDK's OIDC credentials. + +Outbound policy supports AgentEnv's allow-all and allowlist forms, including hostname and CIDR entries. Vercel's custom transforms, forwarding rules, and denied CIDR exceptions are not reconstructed as ordinary allowlists: reconnect leaves such a policy unknown and image loading fails closed. At creation, public Vercel routes are added to a restricted workload's platform egress floor so it can reach another Vercel sandbox's declared route. Image and file downloads add their exact signed-URL hostnames to the current allowlist; these hosts remain allowed until the policy is replaced. A CIDR-only Vercel policy allows unrestricted DNS resolution; use domain rules to restrict DNS, or deny-all to block it. + To work on agent-env itself, install from a clone with [uv](https://docs.astral.sh/uv/): ```bash diff --git a/pyproject.toml b/pyproject.toml index f385f4c6..0819d83b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,8 +72,13 @@ gcp = [ sail = [ "sail~=0.12.8", # Sailbox exec/fs/listener/egress APIs validated against the 0.12 line ] +vercel = [ + "anyio>=4,<5", + "httpx2>=2.13.1,<3", + "vercel-sandbox>=0.7.0,<0.8", # provides the vercel.sandbox namespace without the full vercel bundle +] dev = [ - "agentenv-framework[aws,explorer,gcp,sail]", # so the AWS, explorer, GCP backend and Sail provider tests run in CI + "agentenv-framework[aws,explorer,gcp,sail,vercel]", # so the optional backends and their tests run in CI "griffelib==2.3.0", # the plugin API check; exact, since what it reports as a break changes between releases "moto>=5.0.0", "psycopg2-binary>=2.9.0", # the gateway server module's driver; outside its container only the tests import it diff --git a/src/agent_env/providers/sandbox_providers/sandbox_provider.py b/src/agent_env/providers/sandbox_providers/sandbox_provider.py index d1d5355b..8632990b 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox_provider.py +++ b/src/agent_env/providers/sandbox_providers/sandbox_provider.py @@ -207,6 +207,7 @@ def effective_network_policy(cls, policy: Optional[NetworkPolicy]) -> NetworkPol "modal_vm": "agent_env.providers.sandbox_providers.modal_vm_sandbox:ModalVmSandboxProvider", "e2b": "agent_env.providers.sandbox_providers.e2b:E2BSandboxProvider", "sail_vm": "agent_env.providers.sandbox_providers.sail_vm.provider:SailVmSandboxProvider", + "vercel": "agent_env.providers.sandbox_providers.vercel.provider:VercelSandboxProvider", "local": "agent_env.providers.sandbox_providers.local_sandbox:LocalSandboxProvider", } diff --git a/src/agent_env/providers/sandbox_providers/vercel/__init__.py b/src/agent_env/providers/sandbox_providers/vercel/__init__.py new file mode 100644 index 00000000..2b56753b --- /dev/null +++ b/src/agent_env/providers/sandbox_providers/vercel/__init__.py @@ -0,0 +1,6 @@ +"""Vercel Sandbox provider.""" + +from agent_env.providers.sandbox_providers.vercel.provider import VercelSandboxProvider +from agent_env.providers.sandbox_providers.vercel.sandbox import VercelSandbox + +__all__ = ["VercelSandbox", "VercelSandboxProvider"] diff --git a/src/agent_env/providers/sandbox_providers/vercel/provider.py b/src/agent_env/providers/sandbox_providers/vercel/provider.py new file mode 100644 index 00000000..98a71e64 --- /dev/null +++ b/src/agent_env/providers/sandbox_providers/vercel/provider.py @@ -0,0 +1,415 @@ +"""Vercel Sandbox provider.""" + +from __future__ import annotations + +import asyncio +import logging +import math +import uuid +from typing import Any, Callable, ClassVar, Self + +from agent_env.attribution import Attribution +from agent_env.config.errors import ConfigError +from agent_env.providers.sandbox_providers.sandbox import ( + NetworkPolicy, + NetworkPolicyUnsupportedError, +) +from agent_env.providers.sandbox_providers.sandbox_provider import ( + SandboxProvider, + apply_default_attribution, +) +from agent_env.providers.sandbox_providers.vercel.sandbox import ( + VercelSandbox, + network_policy_from_vercel, + vercel_network_policy, +) + +logger = logging.getLogger(__name__) + +_MAX_VCPUS = 32 +_MAX_MEMORY_MB = 64 * 1024 +_MAX_DISK_GB = 64 +_MAX_PORTS = 15 +_MAX_TAGS = 5 +_MAX_TIMEOUT_SECONDS = 24 * 60 * 60 +_MEMORY_PER_VCPU_MB = 2048 +_CLEANUP_ATTEMPTS = 3 + + +def resource_shape(cpu: float, memory: int) -> tuple[int, int]: + if isinstance(cpu, bool) or not isinstance(cpu, (int, float)) or not math.isfinite(cpu) or cpu <= 0: + raise ValueError("cpu must be a finite number greater than zero") + if isinstance(memory, bool) or not isinstance(memory, int) or memory <= 0 or memory > _MAX_MEMORY_MB: + raise ValueError(f"memory must be an integer from 1 to {_MAX_MEMORY_MB} MB") + vcpus = max(math.ceil(cpu), math.ceil(memory / _MEMORY_PER_VCPU_MB), 1) + if vcpus != 1 and vcpus % 2: + vcpus += 1 + if vcpus > _MAX_VCPUS: + raise ValueError( + f"no Vercel resource shape fits cpu={cpu}, memory={memory}MB " + f"(largest supported provider shape is {_MAX_VCPUS} vCPUs)" + ) + return vcpus, vcpus * _MEMORY_PER_VCPU_MB + + +def _sdk_resources(vcpus: int, memory: int) -> Any: + try: + from vercel.sandbox import SandboxResources + except ImportError as error: + raise RuntimeError( + "The vercel sandbox provider requires vercel-sandbox; " + "install agentenv-framework[vercel]" + ) from error + return SandboxResources(vcpus=vcpus, memory=memory) + + +def _is_missing_resource(error: BaseException) -> bool: + return getattr(error, "status_code", None) == 404 + + +async def _destroy_named(client: Any, name: str) -> None: + try: + sandbox = await client.get_sandbox(name=name) + await sandbox.destroy(delete_orphan_snapshots=True) + except BaseException as error: + if not _is_missing_resource(error): + raise + + +async def _reap(client: Any, name: str) -> None: + for attempt in range(_CLEANUP_ATTEMPTS): + try: + await _destroy_named(client, name) + logger.info("Destroyed Vercel sandbox %s left by an interrupted create", name) + return + except Exception as error: + logger.warning( + "Destroying Vercel sandbox %s failed (attempt %s): %s", + name, + attempt + 1, + error, + ) + await asyncio.sleep(2**attempt) + logger.error("Vercel sandbox %s may remain after %s destroy attempts", name, _CLEANUP_ATTEMPTS) + + +async def _create_or_reclaim( + client: Any, create: Callable[[], Any], name: str, pending: set[asyncio.Task] +) -> Any: + task = asyncio.ensure_future(create()) + pending.add(task) + task.add_done_callback(pending.discard) + + async def reclaim() -> None: + try: + await task + except BaseException: + pass + await _reap(client, name) + + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + reaper = asyncio.create_task(reclaim()) + pending.add(reaper) + reaper.add_done_callback(pending.discard) + raise + + +class VercelSandboxProvider(SandboxProvider): + """Ephemeral Docker-capable Vercel Sandboxes.""" + + EGRESS_HOSTS: ClassVar[tuple[str, ...]] = ("*.vercel.run",) + + def __init__( + self, + *, + image: str = "vercel/sandbox/universal", + region: str | None = None, + failover_regions: list[str] | None = None, + network_id: str | None = None, + token: str | None = None, + team_id: str | None = None, + project_id: str | None = None, + client_factory: Callable[[], Any] | None = None, + ): + self._image = image + self._region = region + self._failover_regions = failover_regions + self._network_id = network_id + self._token = token + self._team_id = team_id + self._project_id = project_id + self._client_factory = client_factory or self._default_client_factory + self._client: Any | None = None + self._pending: set[asyncio.Task] = set() + + @classmethod + def from_config(cls, **config: Any) -> Self: + section = "[sandbox.providers.vercel.config]" + for key in ("image", "region", "network_id"): + value = config.get(key) + if value is not None and (not isinstance(value, str) or not value.strip()): + raise ConfigError(f"{section} '{key}' must be a non-empty string when set") + credentials = {key: config.get(key) for key in ("token", "team_id", "project_id")} + if any(value is not None for value in credentials.values()) and not all( + isinstance(value, str) and value.strip() for value in credentials.values() + ): + raise ConfigError( + f"{section} requires token, team_id and project_id together; omit all three for SDK OIDC credentials" + ) + failover = config.get("failover_regions") + if failover is not None: + if not isinstance(failover, list) or not all( + isinstance(value, str) and value.strip() for value in failover + ): + raise ConfigError(f"{section} 'failover_regions' must be a list of non-empty strings") + if len(failover) != len(set(failover)): + raise ConfigError(f"{section} 'failover_regions' cannot contain duplicates") + if config.get("region") in failover: + raise ConfigError(f"{section} 'failover_regions' cannot include region") + unknown = set(config) - { + "image", + "region", + "failover_regions", + "network_id", + "token", + "team_id", + "project_id", + } + if unknown: + raise ConfigError(f"{section} has unknown key(s): {sorted(unknown)}") + return cls(**config) + + def _default_client_factory(self) -> Any: + try: + from vercel.sandbox import SandboxClient, SandboxServiceOptions + except ImportError as error: + raise RuntimeError( + "The vercel sandbox provider requires vercel-sandbox; " + "install agentenv-framework[vercel]" + ) from error + + options_kwargs: dict[str, Any] = {"region": self._region} + if self._token is None: + return SandboxClient.create(options=SandboxServiceOptions(**options_kwargs)) + from vercel.sandbox import SandboxCredentials + + async def credentials() -> Any: + return SandboxCredentials( + token=self._token, + team_id=self._team_id, + project_id=self._project_id, + ) + + options_kwargs["credentials_factory"] = credentials + return SandboxClient.create(options=SandboxServiceOptions(**options_kwargs)) + + def _get_client(self) -> Any: + if self._client is None: + self._client = self._client_factory() + return self._client + + @classmethod + def supports_network_policy(cls, policy: NetworkPolicy) -> bool: + return True + + @classmethod + def shares_network_with(cls, sandbox_type: str | None) -> bool: + return False + + @staticmethod + def _validated_ports(exposed_ports: list[int] | None) -> list[int]: + ports = list(dict.fromkeys(exposed_ports or [])) + if len(ports) > _MAX_PORTS: + raise ValueError(f"Vercel exposes at most {_MAX_PORTS} distinct ports") + if any(isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535 for port in ports): + raise ValueError("ports must be integers from 1 to 65535") + return ports + + @staticmethod + def _validated_timeout(timeout: int) -> int: + if isinstance(timeout, bool) or not isinstance(timeout, int) or not 0 < timeout <= _MAX_TIMEOUT_SECONDS: + raise ValueError(f"timeout must be an integer from 1 to {_MAX_TIMEOUT_SECONDS} seconds") + return timeout + + @staticmethod + def _validated_disk(disk_size_gb: float) -> None: + if ( + isinstance(disk_size_gb, bool) + or not isinstance(disk_size_gb, (int, float)) + or not math.isfinite(disk_size_gb) + or not 0 < disk_size_gb <= _MAX_DISK_GB + ): + raise ValueError(f"disk_size_gb must be a finite number greater than zero and at most {_MAX_DISK_GB}") + + @staticmethod + def _tags(attribution: Attribution | None) -> dict[str, str]: + tags = { + key: str(value) + for key, value in apply_default_attribution(dict(attribution or {})).items() + if value is not None + } + if len(tags) > _MAX_TAGS: + raise ValueError(f"Vercel accepts at most {_MAX_TAGS} attribution tags") + if any(not key or not value for key, value in tags.items()): + raise ValueError("attribution keys and values must be non-empty") + return tags + + async def create_vm( + self, + *, + image: str | None = None, + boot_mode: str | None = None, + cpu: float = 1.0, + memory: int = 8192, + disk_size_gb: float = 10, + timeout: int = 3600 * 2, + exposed_ports: list[int] | None = None, + setup_for_gateway: bool = True, + attribution: Attribution | None = None, + network_policy: NetworkPolicy | None = None, + ) -> VercelSandbox: + if image is not None: + raise ValueError("the Vercel outer image is configured by the provider; image overrides are unsupported") + if boot_mode is not None: + raise ValueError("Vercel custom kernel boot modes are unsupported") + vcpus, memory_mb = resource_shape(cpu, memory) + self._validated_disk(disk_size_gb) + if vcpus != cpu or memory_mb != memory: + logger.info( + "Vercel resource shape adjusted: requested_cpu=%s requested_memory_mb=%s " + "allocated_vcpus=%s allocated_memory_mb=%s", + cpu, + memory, + vcpus, + memory_mb, + ) + if disk_size_gb != _MAX_DISK_GB: + logger.warning( + "Vercel sandbox disk size is fixed: requested_disk_gb=%s fixed_disk_gb=%s", + disk_size_gb, + _MAX_DISK_GB, + ) + timeout = self._validated_timeout(timeout) + ports = self._validated_ports(exposed_ports) + tags = self._tags(attribution) + effective_policy = self.effective_network_policy(network_policy) + if not self.supports_network_policy(effective_policy): + raise NetworkPolicyUnsupportedError( + f"Vercel cannot enforce network policy {effective_policy.to_dict()}" + ) + + client = self._get_client() + resources = _sdk_resources(vcpus, memory_mb) + name = f"agentenv-{uuid.uuid4().hex[:12]}" + + async def create() -> Any: + return await client.create_sandbox( + name=name, + image=self._image, + ports=ports, + execution_time_limit=timeout, + resources=resources, + persistent=False, + network_policy=vercel_network_policy(effective_policy), + network_id=self._network_id, + region=self._region, + tags=tags, + failover_regions=self._failover_regions, + ) + + try: + raw = await _create_or_reclaim(client, create, name, self._pending) + except asyncio.CancelledError: + raise + except BaseException: + await _reap(client, name) + raise + + applied = network_policy_from_vercel(raw.network_policy) + if applied is None: + await _destroy_named(client, name) + raise RuntimeError( + f"Vercel sandbox {name} returned a network policy agent-env cannot represent" + ) + sandbox = VercelSandbox( + raw, + client=client, + tunnel_urls={route.port: route.url for route in raw.routes if route.port in ports}, + network_policy=applied, + ) + missing = [port for port in ports if port not in sandbox.tunnel_urls] + if missing: + await sandbox.terminate() + raise RuntimeError(f"Vercel sandbox {name} has no public route for port(s) {missing}") + try: + if setup_for_gateway: + await sandbox.setup_vm_for_gateway(ports) + logger.info( + "Vercel sandbox started: name=%s image=%s vcpus=%s memory=%sMB ports=%s attribution=%s", + name, + self._image, + vcpus, + memory_mb, + ports, + tags, + ) + return sandbox + except BaseException: + try: + await sandbox.terminate() + except Exception as cleanup_error: + logger.warning( + "Failed to destroy Vercel sandbox %s after setup failure: %s", + name, + cleanup_error, + ) + raise + + async def create_sandbox( + self, + *, + image_name: str, + port: int, + env: dict[str, str], + cpu: float = 1.0, + memory: int = 8192, + disk_size_gb: float = 10, + timeout: int = 3600 * 2, + attribution: Attribution | None = None, + network_policy: NetworkPolicy | None = None, + ) -> VercelSandbox: + del image_name, env + return await self.create_vm( + cpu=cpu, + memory=memory, + disk_size_gb=disk_size_gb, + timeout=timeout, + exposed_ports=[port], + attribution=attribution, + network_policy=network_policy, + ) + + async def get_sandbox(self, sandbox_id: str) -> VercelSandbox: + raw = await self._get_client().get_sandbox(name=sandbox_id) + applied = network_policy_from_vercel(raw.network_policy) + if applied is None: + logger.warning( + "Vercel sandbox %s has a network policy agent-env can't represent; image loading will fail closed", + sandbox_id, + ) + return VercelSandbox( + raw, + client=self._get_client(), + tunnel_urls={route.port: route.url for route in raw.routes}, + network_policy=applied, + ) + + async def close(self) -> None: + while self._pending: + await asyncio.gather(*tuple(self._pending), return_exceptions=True) + client = self._client + self._client = None + if client is not None: + await client.aclose() diff --git a/src/agent_env/providers/sandbox_providers/vercel/sandbox.py b/src/agent_env/providers/sandbox_providers/vercel/sandbox.py new file mode 100644 index 00000000..3232c4dd --- /dev/null +++ b/src/agent_env/providers/sandbox_providers/vercel/sandbox.py @@ -0,0 +1,351 @@ +"""AgentEnv adapter for an asynchronous Vercel Sandbox.""" + +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import io +import shlex +import ssl +import uuid +import logging +import weakref +from ipaddress import ip_network +from typing import Any +from urllib.parse import urlparse + +from agent_env.config import get_config +from agent_env.providers.sandbox_providers.sandbox import ( + CURL_RETRY_FLAGS, + NetworkMode, + NetworkPolicy, + VmSandbox, +) +from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM + +logger = logging.getLogger(__name__) + +_STREAM_CHUNK_BYTES = 2 * 1024 * 1024 +_DOCKER_SETUP_TIMEOUT = 300 +_policy_locks: weakref.WeakValueDictionary[ + tuple[asyncio.AbstractEventLoop, str], asyncio.Lock +] = weakref.WeakValueDictionary() +_MISSING_FINAL_METADATA = "Sandbox process response is missing final metadata" +_MISSING_RETURN_CODE = ( + "Sandbox process response final metadata is missing a return code" +) + + +def network_policy_from_vercel(policy: Any) -> NetworkPolicy | None: + """Translate a Vercel policy, or return None for one agent-env cannot represent.""" + mode = getattr(policy, "mode", None) + if mode == "allow-all": + return NetworkPolicy() + if mode == "deny-all": + return NetworkPolicy(mode=NetworkMode.ALLOWLIST) + if mode != "custom": + return None + + allow = getattr(policy, "allow", None) + if not hasattr(allow, "items"): + return None + hosts = [] + for domain, rules in allow.items(): + if not isinstance(domain, str) or not domain or tuple(rules or ()): + return None + hosts.append(domain) + + subnets = getattr(policy, "subnets", None) + cidrs: list[str] = [] + if subnets is not None: + allowed = getattr(subnets, "allow", None) + denied = getattr(subnets, "deny", None) + if denied: + return None + if allowed is not None: + for entry in allowed: + if not isinstance(entry, str) or not entry: + return None + try: + ip_network(entry, strict=False) + except ValueError: + return None + cidrs.append(entry) + return NetworkPolicy( + mode=NetworkMode.ALLOWLIST, + allow_hosts=tuple(hosts), + allow_cidrs=tuple(cidrs), + ) + + +def vercel_network_policy(policy: NetworkPolicy) -> Any: + """Translate an agent-env policy into the pinned Vercel SDK models.""" + from vercel.sandbox import NetworkPolicy as VercelNetworkPolicy, NetworkPolicySubnets + + if policy.mode is NetworkMode.ALLOW_ALL: + return VercelNetworkPolicy.allow_all() + if not policy.allow_hosts and not policy.allow_cidrs: + return VercelNetworkPolicy.deny_all() + return VercelNetworkPolicy.custom( + allow={host: () for host in policy.allow_hosts}, + subnets=NetworkPolicySubnets(allow=policy.allow_cidrs) + if policy.allow_cidrs + else None, + ) + + +def _is_missing_resource(error: BaseException) -> bool: + return getattr(error, "status_code", None) == 404 + + +def _is_lost_process_transport(error: BaseException) -> bool: + if isinstance(error, (ConnectionError, ssl.SSLEOFError)): + return True + try: + from anyio import BrokenResourceError, EndOfStream + from httpx2 import ReadError, WriteError, RemoteProtocolError, ReadTimeout, WriteTimeout + from vercel.sandbox import SandboxResponseError + except ImportError: + return False + if isinstance(error, (BrokenResourceError, EndOfStream, ReadError, WriteError, RemoteProtocolError, ReadTimeout, WriteTimeout)): + return True + return isinstance(error, SandboxResponseError) and str(error) in { + _MISSING_FINAL_METADATA, + _MISSING_RETURN_CODE, + } + + +def _returncode(result: Any) -> int: + returncode = getattr(result, "returncode", None) + if returncode is None: + logger.warning("Vercel process result is missing a return code; mapping to exit -1") + return -1 + if isinstance(returncode, bool) or not isinstance(returncode, int): + raise TypeError(f"Vercel process returned a malformed exit code: {returncode!r}") + return returncode + + +class _OutputSink: + def __init__(self, reader: asyncio.StreamReader): + self.reader = reader + + def write(self, text: str) -> int: + self.reader.feed_data(text.encode()) + return len(text) + + def flush(self) -> None: + pass + + def writable(self) -> bool: + return True + + +class _VercelProcess: + """Route one native command log stream into AgentEnv's two byte readers.""" + + def __init__(self, operation: Any, tasks: set[asyncio.Task]): + self.stdout = asyncio.StreamReader() + self.stderr = asyncio.StreamReader() + + async def run() -> int: + try: + result = await operation(stdout=_OutputSink(self.stdout), stderr=_OutputSink(self.stderr)) + return _returncode(result) + except asyncio.CancelledError: + raise + except BaseException as error: + if _is_lost_process_transport(error): + logger.warning( + "Vercel command transport ended without a final process result; " + "mapping to exit -1: %s", + error, + ) + return -1 + self.stdout.set_exception(error) + self.stderr.set_exception(error) + raise + finally: + self.stdout.feed_eof() + self.stderr.feed_eof() + + self._task = asyncio.create_task(run()) + tasks.add(self._task) + self._task.add_done_callback(tasks.discard) + self._task.add_done_callback(lambda task: None if task.cancelled() else task.exception()) + + async def wait(self) -> int: + return await asyncio.shield(self._task) + + +class VercelSandbox(VmSandbox): + type = "vercel" + + def __init__( + self, + sandbox: Any, + *, + client: Any, + tunnel_urls: dict[int, str], + network_policy: NetworkPolicy | None, + ): + self._sandbox = sandbox + self._client = client + self._processes: set[asyncio.Task] = set() + self.sandbox_id = sandbox.name + self.tunnel_urls = tunnel_urls + self.vnc_url = None + self.mode = SANDBOX_MODE_VM + self.network_policy = network_policy + + async def terminate(self) -> None: + try: + await self._sandbox.destroy(delete_orphan_snapshots=True) + except BaseException as error: + if not _is_missing_resource(error): + raise + logger.info("Vercel sandbox %s was already gone at terminate", self.sandbox_id) + + async def exec(self, *command: str) -> _VercelProcess: + return await self._run(*command) + + async def _run(self, *command: str, timeout: float | None = None) -> _VercelProcess: + argv = list(command) + sudo = argv[:1] == ["sudo"] + if sudo: + argv = argv[1:] + if not argv: + raise ValueError("a Vercel exec requires a command") + async def run(**output: Any) -> Any: + return await self._sandbox.run_process( + argv[0], argv[1:], sudo=sudo, kill_after=timeout, **output + ) + + return _VercelProcess(run, self._processes) + + async def _exec_with_output( + self, *command: str, timeout: float | None = None + ) -> tuple[int, str, str]: + process = await self._run(*command, timeout=timeout) + stdout, stderr = await asyncio.gather( + process.stdout.read(), + process.stderr.read(), + ) + return await process.wait(), stdout.decode(errors="replace"), stderr.decode(errors="replace") + + async def wait_for_vm(self) -> None: + script = """ +if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + exit 0 +fi +if ! command -v docker >/dev/null 2>&1 || ! docker compose version >/dev/null 2>&1; then +if ! command -v apt-get >/dev/null 2>&1; then + echo 'Vercel image has neither Docker Compose v2 nor apt-get; use a prepared Docker-capable image' >&2 + exit 2 +fi +DEBIAN_FRONTEND=noninteractive apt-get update -qq && + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker.io docker-compose-v2 || + { echo 'Docker installation failed; use a prepared Docker-capable image for restricted egress' >&2; exit 3; } +fi +docker info >/dev/null 2>&1 || (nohup dockerd >/var/log/agentenv-dockerd.log 2>&1 /dev/null 2>&1 && docker compose version >/dev/null 2>&1 && exit 0 + sleep 1 +done +tail -40 /var/log/agentenv-dockerd.log >&2 +exit 1 +""" + exit_code, _, stderr = await self._exec_with_output( + "sudo", "bash", "-c", script, timeout=_DOCKER_SETUP_TIMEOUT + ) + if exit_code != 0: + raise RuntimeError( + f"Docker and Compose are unavailable in Vercel sandbox {self.sandbox_id} " + f"(exit {exit_code}): {stderr[-1000:]}" + ) + logger.info("Docker and Compose ready in Vercel sandbox %s", self.sandbox_id) + + async def setup_vm_for_gateway(self, exposed_ports: list[int] | None = None) -> None: + await self.wait_for_vm() + + async def _write_bytes_to_vm_path(self, data: bytes, vm_path: str) -> None: + if len(data) <= _STREAM_CHUNK_BYTES: + await self._sandbox.fs.write_bytes(vm_path, data) + else: + await self._write_stream_to_vm(io.BytesIO(data), vm_path) + + async def _write_unsigned_object(self, object_store: Any, object_url: str, vm_path: str) -> None: + with contextlib.closing(await asyncio.to_thread(object_store.open, object_url)) as source: + await self._write_stream_to_vm(source, vm_path) + + async def _write_stream_to_vm(self, source: Any, vm_path: str) -> None: + """Assemble bounded SDK uploads and verify contents before replacing the destination.""" + staged = f"/tmp/agentenv-upload-{uuid.uuid4().hex}" + chunk_path = staged + ".part" + digest = hashlib.sha256() + try: + await self.exec_script(f": > {shlex.quote(staged)}") + while chunk := await asyncio.to_thread(source.read, _STREAM_CHUNK_BYTES): + digest.update(chunk) + for attempt in range(3): + try: + await self._sandbox.fs.write_bytes(chunk_path, chunk) + break + except Exception: + # SDK transport failures have several types; only the replaceable part is retried. + if attempt == 2: + raise + await asyncio.sleep(0.25 * 2**attempt) + await self.exec_script(f"cat {shlex.quote(chunk_path)} >> {shlex.quote(staged)}") + await self.exec_script( + f"printf '%s %s\n' {digest.hexdigest()} {shlex.quote(staged)} | sha256sum -c - >/dev/null " + f"&& mv -- {shlex.quote(staged)} {shlex.quote(vm_path)}" + ) + finally: + await self._remove_vm_temp_file(staged, chunk_path) + + def _policy_lock(self) -> asyncio.Lock: + return _policy_locks.setdefault( + (asyncio.get_running_loop(), self.sandbox_id), asyncio.Lock() + ) + + async def apply_network_policy(self, policy: NetworkPolicy) -> None: + async with self._policy_lock(): + await self._sandbox.update_network_policy(vercel_network_policy(policy)) + self.network_policy = policy + + async def _allow_download_hosts(self, urls: list[str | None]) -> None: + hosts = {parsed.hostname for url in urls if url and (parsed := urlparse(url)).hostname} + async with self._policy_lock(): + current = await self._client.get_sandbox(name=self.sandbox_id) + self.network_policy = network_policy_from_vercel(current.network_policy) + policy = self.network_policy + if policy is None: + raise RuntimeError( + f"Cannot download artifacts in Vercel sandbox {self.sandbox_id}: " + "its applied network policy is unknown, so signed download hosts cannot be added safely" + ) + if not policy.restricts_egress or not hosts: + return + widened = policy.with_hosts(sorted(hosts)) + if widened != policy: + await self._sandbox.update_network_policy(vercel_network_policy(widened)) + self.network_policy = widened + + async def load_docker_images(self, artifacts: list) -> None: + if not artifacts: + return + signed_urls = await self._signed_image_urls(artifacts) + await self._allow_download_hosts(signed_urls) + await self._load_docker_images(artifacts, signed_urls) + + async def _download_object_to_vm(self, object_url: str, vm_path: str) -> None: + object_store = get_config().get_object_store_at(object_url) + signed = await asyncio.to_thread(object_store.signed_get_url, object_url) + if signed is None: + await self._write_unsigned_object(object_store, object_url, vm_path) + return + await self._allow_download_hosts([signed]) + await self.exec_script( + f"curl -fsSL {CURL_RETRY_FLAGS} {shlex.quote(signed)} -o {shlex.quote(vm_path)}" + ) diff --git a/tst/integration/providers/sandbox_providers/vercel_sandbox_smoke_test.py b/tst/integration/providers/sandbox_providers/vercel_sandbox_smoke_test.py new file mode 100644 index 00000000..ff6bbbad --- /dev/null +++ b/tst/integration/providers/sandbox_providers/vercel_sandbox_smoke_test.py @@ -0,0 +1,121 @@ +"""Vercel Sandbox remote smoke test. + +Runs when the resolved config can build the ``vercel`` sandbox provider, the +``remote_sandbox`` capability, and skips with the declared reason otherwise. The +test never reads or prints resolved credentials. +""" + +from __future__ import annotations + +import asyncio +import hashlib + +import httpx +import pytest +import pytest_asyncio + +from agent_env.providers.sandbox_providers.sandbox import NetworkMode, NetworkPolicy +from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider +from tst.util.capabilities import skip_without_remote_sandbox + +_PORT = 8080 +_COMPOSE = f"""services: + web: + image: python:3.12-alpine + container_name: agentenv-vercel-smoke + command: ["python", "-m", "http.server", "{_PORT}", "--bind", "0.0.0.0"] + ports: ["{_PORT}:{_PORT}"] +""" + +pytestmark = [ + pytest.mark.integration, + pytest.mark.int_test_slow, + pytest.mark.asyncio(loop_scope="module"), + skip_without_remote_sandbox("vercel"), +] + + +@pytest.fixture(scope="module") +def vercel_provider(): + return build_sandbox_provider("vercel") + + +@pytest_asyncio.fixture(scope="module") +async def vercel_sandbox(vercel_provider): + sandbox = None + try: + sandbox = await vercel_provider.create_vm( + cpu=1, + memory=2048, + exposed_ports=[_PORT], + timeout=900, + ) + yield sandbox + finally: + if sandbox is not None: + try: + await sandbox.terminate() + finally: + await vercel_provider.close() + else: + await vercel_provider.close() + + +async def test_vercel_runs_docker_serves_its_port_and_reconnects(vercel_provider, vercel_sandbox): + assert (vercel_sandbox.type, vercel_sandbox.mode) == ("vercel", "vm") + + await vercel_sandbox.write_host_file(_COMPOSE.encode(), "/tmp/agentenv-vercel-smoke/compose.yaml") + await vercel_sandbox.exec_script("docker compose -f /tmp/agentenv-vercel-smoke/compose.yaml up -d") + url = vercel_sandbox.tunnel_urls[_PORT] + assert url.startswith("https://") and ".vercel.run" in url + async with httpx.AsyncClient(timeout=30) as client: + response: httpx.Response | None = None + for _ in range(30): + try: + response = await client.get(url) + if response.status_code == 200: + break + except httpx.TransportError: + pass + await asyncio.sleep(1) + assert response is not None and response.status_code == 200 + + data = bytes(range(256)) * 4096 + await vercel_sandbox.write_host_file(data, "/tmp/agentenv-vercel-smoke/blob.bin") + exit_code, digest, stderr = await vercel_sandbox.exec_with_output( + "sha256sum", "/tmp/agentenv-vercel-smoke/blob.bin" + ) + assert exit_code == 0, stderr + assert digest.split()[0] == hashlib.sha256(data).hexdigest() + + policy = NetworkPolicy(mode=NetworkMode.ALLOWLIST, allow_hosts=("pypi.org",)) + await vercel_sandbox.apply_network_policy(policy) + probe = ( + "import urllib.request\n" + "print(urllib.request.urlopen('https://pypi.org/simple/', timeout=20).status)\n" + "try:\n" + " urllib.request.urlopen('https://example.com', timeout=10)\n" + "except Exception:\n" + " print('example-blocked')\n" + "else:\n" + " raise SystemExit('unexpected access')" + ) + for command in ( + ("python3", "-c", probe), + ("sudo", "docker", "exec", "agentenv-vercel-smoke", "python", "-c", probe), + ): + exit_code, output, stderr = await vercel_sandbox.exec_with_output(*command) + assert exit_code == 0, stderr + assert output.split() == ["200", "example-blocked"] + + fresh_provider = build_sandbox_provider("vercel") + try: + fresh_sandbox = await fresh_provider.get_sandbox(vercel_sandbox.sandbox_id) + assert fresh_sandbox.sandbox_id == vercel_sandbox.sandbox_id + assert fresh_sandbox.tunnel_urls == vercel_sandbox.tunnel_urls + assert fresh_sandbox.network_policy == vercel_sandbox.network_policy + assert fresh_sandbox.network_policy == policy + await fresh_sandbox.terminate() + await fresh_sandbox.terminate() + finally: + await fresh_provider.close() diff --git a/tst/unit/providers/sandbox_providers/egress_hosts_test.py b/tst/unit/providers/sandbox_providers/egress_hosts_test.py index 7024b69d..006f7ef9 100644 --- a/tst/unit/providers/sandbox_providers/egress_hosts_test.py +++ b/tst/unit/providers/sandbox_providers/egress_hosts_test.py @@ -95,4 +95,6 @@ def test_the_builtin_floor_is_pinned(builtin_registry): """Every restricted sandbox gets these, so an empty allowlist is not total denial. Pinned rather than counted: a new entry should be argued for in review. Platforms registered from config add theirs on top.""" - assert set(all_sandbox_egress_hosts()) == {"*.modal.host", "*.w.modal.host", "*.e2b.app", "*.sail.box"} + assert set(all_sandbox_egress_hosts()) == { + "*.modal.host", "*.w.modal.host", "*.e2b.app", "*.sail.box", "*.vercel.run" + } diff --git a/tst/unit/providers/sandbox_providers/network_policy_providers_test.py b/tst/unit/providers/sandbox_providers/network_policy_providers_test.py index c58a2cd7..86fa1b7e 100644 --- a/tst/unit/providers/sandbox_providers/network_policy_providers_test.py +++ b/tst/unit/providers/sandbox_providers/network_policy_providers_test.py @@ -13,6 +13,7 @@ NetworkPolicyUnsupportedError, Sandbox, ) +from agent_env.providers.sandbox_providers.vercel.provider import VercelSandboxProvider from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider RESTRICTIVE = NetworkPolicy(mode=NetworkMode.ALLOWLIST, allow_hosts=("only-this.example.com",)) @@ -41,7 +42,9 @@ def test_backends_without_enforcement_refuse_restrictive_policies(provider_cls, assert provider_cls.supports_network_policy(ALLOW_ALL) is True -@pytest.mark.parametrize("provider_cls", [ModalSandboxProvider, ModalVmSandboxProvider]) +@pytest.mark.parametrize( + "provider_cls", [ModalSandboxProvider, ModalVmSandboxProvider, VercelSandboxProvider] +) @pytest.mark.parametrize("policy", [RESTRICTIVE, ALLOWLIST, ALLOW_ALL]) def test_modal_backends_advertise_full_support(provider_cls, policy): assert provider_cls.supports_network_policy(policy) is True diff --git a/tst/unit/providers/sandbox_providers/network_policy_stamping_test.py b/tst/unit/providers/sandbox_providers/network_policy_stamping_test.py index 0b801a53..a445b25e 100644 --- a/tst/unit/providers/sandbox_providers/network_policy_stamping_test.py +++ b/tst/unit/providers/sandbox_providers/network_policy_stamping_test.py @@ -21,6 +21,7 @@ from agent_env.providers.sandbox_providers.sail_vm.provider import SailVmSandboxProvider from agent_env.providers.sandbox_providers.sandbox import Sandbox from agent_env.providers.sandbox_providers.sandbox_provider import _BUILTIN_SANDBOX_PROVIDERS +from agent_env.providers.sandbox_providers.vercel.provider import VercelSandboxProvider def _modal_provider(cls): @@ -86,11 +87,25 @@ async def _sail() -> Sandbox: return await provider.create_vm(exposed_ports=[], setup_for_gateway=False) +async def _vercel() -> Sandbox: + raw = MagicMock(name="vercel-test") + raw.name = "vercel-test" + raw.routes = () + raw.network_policy = SimpleNamespace(mode="allow-all", allow={}) + client = MagicMock() + client.create_sandbox = AsyncMock(return_value=raw) + provider = VercelSandboxProvider(client_factory=lambda: client) + sandbox = await provider.create_vm(exposed_ports=[], setup_for_gateway=False) + assert sandbox._client is client + return sandbox + + FACTORIES = { "modal": _modal, "modal_vm": _modal_vm, "e2b": _e2b, "sail_vm": _sail, + "vercel": _vercel, "local": _local, } diff --git a/tst/unit/providers/sandbox_providers/vercel_sandbox_provider_test.py b/tst/unit/providers/sandbox_providers/vercel_sandbox_provider_test.py new file mode 100644 index 00000000..ca62b5e7 --- /dev/null +++ b/tst/unit/providers/sandbox_providers/vercel_sandbox_provider_test.py @@ -0,0 +1,307 @@ +"""Unit tests for the Vercel provider; all SDK boundaries are fakes.""" + +import asyncio +import logging +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from agent_env.config.errors import ConfigError +from agent_env.providers.sandbox_providers import vercel as vercel_module +from agent_env.providers.sandbox_providers.vercel import sandbox as vercel_sandbox_module +from agent_env.providers.sandbox_providers.sandbox import NetworkMode, NetworkPolicy +from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider +from agent_env.providers.sandbox_providers.vercel.provider import ( + VercelSandboxProvider, + resource_shape, +) +from agent_env.providers.sandbox_providers.vercel import provider as provider_module + + +def _raw(routes=((8080, "https://sb-1.vercel.run"),), policy_mode="allow-all"): + raw = MagicMock(name="sb-1") + raw.name = "sb-1" + raw.routes = [SimpleNamespace(port=port, url=url) for port, url in routes] + raw.network_policy = SimpleNamespace( + mode=policy_mode, + allow={}, + subnets=SimpleNamespace(allow=(), deny=None), + ) + raw.destroy = AsyncMock() + return raw + + +def _client(raw): + client = MagicMock() + client.create_sandbox = AsyncMock(return_value=raw) + client.get_sandbox = AsyncMock(return_value=raw) + client.aclose = AsyncMock() + return client + + +def _provider(raw=None, **config): + raw = raw or _raw() + client = _client(raw) + provider = VercelSandboxProvider(client_factory=lambda: client, **config) + provider._client = client + return provider + + +@pytest.mark.parametrize( + ("cpu", "memory", "shape"), + [(1, 2048, (1, 2048)), (1, 8192, (4, 8192)), (3, 2048, (4, 8192)), (0.5, 5000, (4, 8192)), (1, 1, (1, 2048))], +) +def test_resource_shape_is_the_smallest_supported_cpu_count(cpu, memory, shape): + assert resource_shape(cpu, memory) == shape + + +@pytest.mark.parametrize( + ("cpu", "memory"), + [(True, 2048), (0, 2048), (float("nan"), 2048), (1, True), (1, 0), (1, 64 * 1024 + 1), (33, 8192)], +) +def test_invalid_resources_are_refused(cpu, memory): + with pytest.raises(ValueError): + resource_shape(cpu, memory) + + +@pytest.mark.parametrize( + ("config", "message"), + [ + ({"image": ""}, "'image' must be a non-empty string"), + ({"token": "t"}, "token, team_id and project_id together"), + ({"failover_regions": "iad1"}, "'failover_regions' must be a list"), + ({"failover_regions": ["iad1", "iad1"]}, "cannot contain duplicates"), + ({"region": "iad1", "failover_regions": ["iad1"]}, "cannot include region"), + ({"unknown": 1}, "unknown key"), + ], +) +def test_from_config_rejects_invalid_config(config, message): + with pytest.raises(ConfigError, match=message): + VercelSandboxProvider.from_config(**config) + + +def test_provider_is_registered_and_does_not_claim_a_private_shared_network(): + provider = build_sandbox_provider("vercel") + assert isinstance(provider, VercelSandboxProvider) + assert provider.shares_network_with("vercel") is False + + +@pytest.mark.asyncio +async def test_create_vm_sends_the_validated_provider_configuration(monkeypatch): + setup = AsyncMock() + monkeypatch.setattr(vercel_module.VercelSandbox, "setup_vm_for_gateway", setup) + monkeypatch.setattr(provider_module, "vercel_network_policy", lambda policy: policy) + raw = _raw(routes=((8080, "https://sb-1.vercel.run"), (9000, "https://sb-2.vercel.run"))) + provider = _provider( + raw, + image="prepared-image:tag", + region="sfo1", + failover_regions=["cle1"], + network_id="network-1", + ) + policy = NetworkPolicy(mode=NetworkMode.ALLOWLIST, allow_hosts=("pypi.org",)) + + sandbox = await provider.create_vm( + cpu=1, + memory=8192, + disk_size_gb=20, + timeout=900, + exposed_ports=[8080, 9000, 8080], + attribution={"run_id": "inst-1"}, + network_policy=policy, + ) + + client = provider._client + kwargs = client.create_sandbox.await_args.kwargs + assert kwargs["name"].startswith("agentenv-") + assert kwargs["image"] == "prepared-image:tag" + assert kwargs["ports"] == [8080, 9000] + assert kwargs["execution_time_limit"] == 900 + assert (kwargs["resources"].vcpus, kwargs["resources"].memory) == (4, 8192) + assert kwargs["persistent"] is False + assert kwargs["network_id"] == "network-1" + assert kwargs["region"] == "sfo1" + assert kwargs["failover_regions"] == ["cle1"] + assert kwargs["tags"] == {"run_id": "inst-1"} + assert "*.vercel.run" in kwargs["network_policy"].allow_hosts + assert (sandbox.sandbox_id, sandbox.mode) == ("sb-1", "vm") + assert sandbox.tunnel_urls == { + 8080: "https://sb-1.vercel.run", + 9000: "https://sb-2.vercel.run", + } + setup.assert_awaited_once_with([8080, 9000]) + + +@pytest.mark.asyncio +async def test_a_setup_failure_destroys_the_created_sandbox(monkeypatch): + setup = AsyncMock(side_effect=RuntimeError("Docker never came up")) + monkeypatch.setattr(vercel_module.VercelSandbox, "setup_vm_for_gateway", setup) + monkeypatch.setattr(vercel_sandbox_module, "vercel_network_policy", lambda policy: policy) + raw = _raw() + provider = _provider(raw) + with pytest.raises(RuntimeError, match="Docker never came up"): + await provider.create_vm(exposed_ports=[]) + raw.destroy.assert_awaited_once_with(delete_orphan_snapshots=True) + + +@pytest.mark.asyncio +async def test_create_vm_logs_requested_and_allocated_resources_and_fixed_disk(monkeypatch, caplog): + monkeypatch.setattr(vercel_module.VercelSandbox, "setup_vm_for_gateway", AsyncMock()) + monkeypatch.setattr(vercel_sandbox_module, "vercel_network_policy", lambda policy: policy) + provider = _provider() + + with caplog.at_level(logging.INFO, logger=provider_module.logger.name): + await provider.create_vm( + cpu=0.5, + memory=5000, + disk_size_gb=10, + exposed_ports=[], + setup_for_gateway=False, + ) + + assert "requested_cpu=0.5 requested_memory_mb=5000 allocated_vcpus=4 allocated_memory_mb=8192" in caplog.text + assert "requested_disk_gb=10 fixed_disk_gb=64" in caplog.text + resources = provider._client.create_sandbox.await_args.kwargs["resources"] + assert (resources.vcpus, resources.memory) == (4, 8192) + + +@pytest.mark.asyncio +async def test_create_vm_does_not_log_an_exact_resource_or_disk_match(monkeypatch, caplog): + monkeypatch.setattr(vercel_module.VercelSandbox, "setup_vm_for_gateway", AsyncMock()) + monkeypatch.setattr(vercel_sandbox_module, "vercel_network_policy", lambda policy: policy) + provider = _provider() + + with caplog.at_level(logging.INFO, logger=provider_module.logger.name): + await provider.create_vm( + cpu=1, + memory=2048, + disk_size_gb=64, + exposed_ports=[], + setup_for_gateway=False, + ) + + assert "resource shape adjusted" not in caplog.text + assert "disk size is fixed" not in caplog.text + + +@pytest.mark.asyncio +async def test_a_missing_route_destroys_the_sandbox(monkeypatch): + monkeypatch.setattr(vercel_module.VercelSandbox, "setup_vm_for_gateway", AsyncMock()) + monkeypatch.setattr(vercel_sandbox_module, "vercel_network_policy", lambda policy: policy) + raw = _raw(routes=()) + provider = _provider(raw) + with pytest.raises(RuntimeError, match="no public route for port\\(s\\) \\[8080\\]"): + await provider.create_vm(exposed_ports=[8080]) + raw.destroy.assert_awaited_once_with(delete_orphan_snapshots=True) + + +@pytest.mark.asyncio +async def test_a_cancelled_create_reaps_the_sandbox_it_produces(monkeypatch): + monkeypatch.setattr(vercel_sandbox_module, "vercel_network_policy", lambda policy: policy) + raw = _raw() + client = _client(raw) + created = asyncio.Event() + + async def slow_create(**_kwargs): + await created.wait() + return raw + + client.create_sandbox = slow_create + provider = VercelSandboxProvider(client_factory=lambda: client) + caller = asyncio.ensure_future(provider.create_vm(exposed_ports=[])) + await asyncio.sleep(0) + caller.cancel() + with pytest.raises(asyncio.CancelledError): + await caller + created.set() + for _ in range(20): + if raw.destroy.await_count: + break + await asyncio.sleep(0) + raw.destroy.assert_awaited_once_with(delete_orphan_snapshots=True) + + +@pytest.mark.asyncio +async def test_get_sandbox_restores_routes_and_representable_policy(): + raw = _raw() + raw.network_policy = SimpleNamespace( + mode="custom", + allow={"pypi.org": ()}, + subnets=SimpleNamespace(allow=("10.0.0.0/8",), deny=None), + ) + provider = _provider(raw) + sandbox = await provider.get_sandbox("sb-1") + provider._client.get_sandbox.assert_awaited_once_with(name="sb-1") + assert sandbox.tunnel_urls == {8080: "https://sb-1.vercel.run"} + assert sandbox.network_policy == NetworkPolicy( + mode=NetworkMode.ALLOWLIST, allow_hosts=("pypi.org",), allow_cidrs=("10.0.0.0/8",) + ) + + +@pytest.mark.asyncio +async def test_close_closes_the_long_lived_sdk_client(): + provider = _provider() + client = provider._client + await provider.create_vm(exposed_ports=[], setup_for_gateway=False) + await provider.close() + client.aclose.assert_awaited_once_with() + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"image": "ubuntu"}, "image overrides are unsupported"), + ({"boot_mode": "custom"}, "boot modes are unsupported"), + ({"disk_size_gb": 65}, "disk_size_gb"), + ({"timeout": 0}, "timeout"), + ({"exposed_ports": list(range(1, 17))}, "at most 15"), + ({"exposed_ports": [0]}, "ports must be integers"), + ({"attribution": {f"k{i}": "v" for i in range(6)}}, "at most 5"), + ], +) +@pytest.mark.asyncio +async def test_invalid_create_arguments_fail_before_provisioning(kwargs, message): + provider = _provider() + with pytest.raises((ValueError,), match=message): + await provider.create_vm(**kwargs) + provider._client.create_sandbox.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("lost_response", [False, True]) +async def test_close_waits_for_cancelled_creation_and_reclaims_even_a_lost_response(lost_response): + raw = _raw() + client = _client(raw) + started, release = asyncio.Event(), asyncio.Event() + order = [] + + async def create(**kwargs): + started.set() + await release.wait() + if lost_response: + raise RuntimeError("response lost after allocation") + return raw + + async def destroy(**kwargs): + order.append("destroy") + + async def close(): + order.append("close") + + client.create_sandbox = create + raw.destroy.side_effect = destroy + client.aclose.side_effect = close + provider = VercelSandboxProvider(client_factory=lambda: client) + caller = asyncio.create_task(provider.create_vm(setup_for_gateway=False)) + await started.wait() + caller.cancel() + with pytest.raises(asyncio.CancelledError): + await caller + closing = asyncio.create_task(provider.close()) + await asyncio.sleep(0) + assert not closing.done() + assert order == [] + release.set() + await asyncio.wait_for(closing, timeout=1) + assert order == ["destroy", "close"] diff --git a/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py b/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py new file mode 100644 index 00000000..19bd946b --- /dev/null +++ b/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py @@ -0,0 +1,540 @@ +"""Unit tests for the Vercel Sandbox adapter over a fake SDK handle.""" + +import asyncio +import io +import subprocess +import ssl +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from anyio import BrokenResourceError, EndOfStream +import httpx +import httpx2 +from vercel.sandbox import ( + SandboxApiError, + SandboxCredentialsError, + SandboxResponseError, + SandboxStreamError, +) + +from agent_env.providers.sandbox_providers import vercel as vercel_module +from agent_env.providers.sandbox_providers.vercel import sandbox as vercel_sandbox_module +from agent_env.providers.sandbox_providers.sandbox import NetworkMode, NetworkPolicy +from agent_env.providers.sandbox_providers.vercel.sandbox import ( + VercelSandbox, + network_policy_from_vercel, +) + + +class _Reader: + def __init__(self, value: str): + self._value = value + + async def read(self, size: int = -1) -> str: + value = self._value if size < 0 else self._value[:size] + self._value = self._value[len(value):] + return value + + async def readline(self) -> str: + length = self._value.find("\n") + 1 + if length == 0: + length = len(self._value) + value, self._value = self._value[:length], self._value[length:] + return value + + async def aclose(self): + pass + + def __aiter__(self): + return self + + async def __anext__(self) -> str: + if self._value: + value, self._value = self._value, "" + return value + raise StopAsyncIteration + + +def _process(stdout="", stderr="", exit_code=0): + process = MagicMock() + process.stdout = _Reader(stdout) + process.stderr = _Reader(stderr) + process.wait = AsyncMock(return_value=exit_code) + return process + + +def _raw(policy=None, routes=()): + raw = MagicMock(name="sb-1") + raw.name = "sb-1" + raw.routes = [SimpleNamespace(port=port, url=url) for port, url in routes] + raw.network_policy = policy + raw.create_process = AsyncMock(return_value=_process()) + raw.get_process = AsyncMock(side_effect=lambda _id: _process( + stdout=raw.create_process.return_value.stdout._value, + stderr=raw.create_process.return_value.stderr._value, + )) + async def run_process(command, args, **kwargs): + process = raw.create_process.return_value + kwargs["stdout"].write(await process.stdout.read()) + kwargs["stderr"].write(await process.stderr.read()) + return SimpleNamespace(returncode=await process.wait()) + + raw.run_process = AsyncMock(side_effect=run_process) + raw.destroy = AsyncMock() + raw.update_network_policy = AsyncMock() + raw.fs.write_bytes = AsyncMock() + return raw + + +def _sandbox(raw=None, policy=NetworkPolicy()): + raw = raw or _raw() + if raw.network_policy is None and policy is not None: + raw.network_policy = vercel_sandbox_module.vercel_network_policy(policy) + client = MagicMock(get_sandbox=AsyncMock(return_value=raw)) + return VercelSandbox(raw, client=client, tunnel_urls={}, network_policy=policy) + + +@pytest.mark.asyncio +async def test_exec_returns_exact_unicode_bytes_and_nonzero_exit(): + raw = _raw() + raw.create_process = AsyncMock( + return_value=_process(stdout="hello ☃", stderr="warn", exit_code=7) + ) + sandbox = _sandbox(raw) + + process = await sandbox.exec("sudo", "bash", "-c", "echo test") + + assert (await process.stdout.read(), await process.stderr.read(), await process.wait()) == ( + b"hello \xe2\x98\x83", b"warn", 7 + ) + assert raw.run_process.await_args.args == ("bash", ["-c", "echo test"]) + assert raw.run_process.await_args.kwargs["sudo"] is True + + +@pytest.mark.asyncio +async def test_stdout_supports_async_iteration(): + raw = _raw() + raw.create_process = AsyncMock(return_value=_process(stdout="one\ntwo\n")) + process = await _sandbox(raw).exec("cat", "log") + assert [chunk async for chunk in process.stdout] == [b"one\n", b"two\n"] + + +@pytest.mark.asyncio +async def test_terminate_treats_only_missing_resources_as_idempotent(): + raw = _raw() + missing = type("Missing", (Exception,), {"status_code": 404})() + raw.destroy = AsyncMock(side_effect=missing) + await _sandbox(raw).terminate() + + genuine = type("Conflict", (Exception,), {"status_code": 409})() + raw.destroy = AsyncMock(side_effect=genuine) + with pytest.raises(BaseException): + await _sandbox(raw).terminate() + + +@pytest.mark.asyncio +async def test_host_files_use_the_native_filesystem_api(): + raw = _raw() + await _sandbox(raw)._write_bytes_to_vm_path(b"\x00\x01", "/opt/data/blob.bin") + raw.fs.write_bytes.assert_awaited_once_with("/opt/data/blob.bin", b"\x00\x01") + + +@pytest.mark.asyncio +async def test_apply_network_policy_replaces_the_current_session_policy(monkeypatch): + monkeypatch.setattr(vercel_sandbox_module, "vercel_network_policy", lambda policy: policy) + raw = _raw() + policy = NetworkPolicy(mode=NetworkMode.ALLOWLIST, allow_hosts=("pypi.org",)) + sandbox = _sandbox(raw) + + await sandbox.apply_network_policy(policy) + + raw.update_network_policy.assert_awaited_once_with(policy) + assert sandbox.network_policy == policy + + +@pytest.mark.asyncio +async def test_restrictive_image_loading_adds_only_signed_download_hosts(monkeypatch): + raw = _raw() + policy = NetworkPolicy(mode=NetworkMode.ALLOWLIST, allow_hosts=("pypi.org",)) + sandbox = _sandbox(raw, policy=policy) + monkeypatch.setattr( + VercelSandbox, + "_signed_image_urls", + AsyncMock(return_value=["https://bucket.example/a?sig=1", None]), + ) + monkeypatch.setattr(VercelSandbox, "_load_docker_images", AsyncMock()) + monkeypatch.setattr( + vercel_sandbox_module, + "vercel_network_policy", + lambda policy: policy.with_hosts(["bucket.example"]), + ) + + await sandbox.load_docker_images(["artifact"]) + + raw.update_network_policy.assert_awaited_once() + applied = raw.update_network_policy.await_args.args[0] + assert applied.allow_hosts == ("pypi.org", "bucket.example") + assert sandbox.network_policy.allow_hosts == applied.allow_hosts + + +@pytest.mark.asyncio +async def test_unknown_reconnected_policy_stays_unknown_for_image_loading(): + sandbox = _sandbox(policy=None) + sandbox._signed_image_urls = AsyncMock(return_value=["https://bucket.example/image"]) + with pytest.raises(RuntimeError, match="applied network policy is unknown"): + await sandbox.load_docker_images(["artifact"]) + + +@pytest.mark.asyncio +async def test_docker_setup_is_bounded_and_requires_compose(monkeypatch): + exec_with_output = AsyncMock(return_value=(0, "", "")) + monkeypatch.setattr(VercelSandbox, "_exec_with_output", exec_with_output) + await _sandbox().wait_for_vm() + + command = exec_with_output.await_args.args + assert command[:3] == ("sudo", "bash", "-c") + assert "apt-get install -y -qq docker.io docker-compose-v2" in command[3] + assert "docker compose version" in command[3] + assert exec_with_output.await_args.kwargs["timeout"] == 300 + + +def test_vercel_policies_round_trip_through_agent_env(): + custom = SimpleNamespace( + mode="custom", + allow={"pypi.org": (), "example.com": ()}, + subnets=SimpleNamespace(allow=("10.0.0.0/8",), deny=None), + ) + assert network_policy_from_vercel(custom) == NetworkPolicy( + mode=NetworkMode.ALLOWLIST, + allow_hosts=("pypi.org", "example.com"), + allow_cidrs=("10.0.0.0/8",), + ) + assert network_policy_from_vercel(SimpleNamespace(mode="deny-all", allow={})) == NetworkPolicy( + mode=NetworkMode.ALLOWLIST + ) + + +@pytest.mark.parametrize( + "policy", + [ + SimpleNamespace(mode="custom", allow={"pypi.org": (object(),)}, subnets=None), + SimpleNamespace(mode="custom", allow={}, subnets=SimpleNamespace(allow=None, deny=["10.0.0.0/8"])), + SimpleNamespace(mode="unknown", allow={}), + ], +) +def test_vercel_rules_agent_env_cannot_represent_are_unknown(policy): + assert network_policy_from_vercel(policy) is None + + +@pytest.mark.asyncio +async def test_byte_reads_and_lines_preserve_partial_utf8_and_eof(): + raw = _raw() + raw.create_process.return_value = _process(stdout="☃a\nz") + process = await _sandbox(raw).exec("printf", "test") + reader = process.stdout + assert await reader.read(0) == b"" + assert await reader.readexactly(1) == b"\xe2" + assert await reader.readexactly(2) == b"\x98\x83" + assert await reader.readline() == b"a\n" + assert await reader.read(1) == b"z" + assert await reader.read() == b"" + assert await reader.readline() == b"" + + +@pytest.mark.asyncio +async def test_command_streams_route_both_outputs_through_one_native_operation(): + raw = _raw() + raw.create_process.return_value = _process(stdout="one\ntwo ☃\n", stderr="warning") + process = await _sandbox(raw).exec("printf", "test") + stdout, stderr = await asyncio.gather(process.stdout.read(), process.stderr.read()) + assert (stdout, stderr) == ("one\ntwo ☃\n".encode(), b"warning") + raw.run_process.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_output_is_readable_before_the_command_finishes(): + raw = _raw() + release = asyncio.Event() + + async def run(command, args, **kwargs): + kwargs["stdout"].write("ready\n") + await release.wait() + kwargs["stderr"].write("finished\n") + return SimpleNamespace(returncode=7) + + raw.run_process.side_effect = run + process = await _sandbox(raw).exec("command") + assert await asyncio.wait_for(process.stdout.readline(), timeout=1) == b"ready\n" + waiting = asyncio.create_task(process.wait()) + await asyncio.sleep(0) + assert not waiting.done() + release.set() + assert await waiting == 7 + assert await process.stderr.read() == b"finished\n" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "error", + [ + ConnectionError("stream disconnected"), + httpx2.ReadError("connection reset"), + httpx2.WriteError("connection reset"), + httpx2.RemoteProtocolError("incomplete response body"), + httpx2.ReadTimeout("read timeout"), + httpx2.WriteTimeout("write timeout"), + BrokenResourceError(), + EndOfStream(), + ssl.SSLEOFError("TLS EOF"), + SandboxResponseError("Sandbox process response is missing final metadata"), + SandboxResponseError( + "Sandbox process response final metadata is missing a return code" + ), + ], +) +async def test_lost_process_transport_maps_to_minus_one_with_partial_output_and_eof(error): + raw = _raw() + + async def run(command, args, **kwargs): + kwargs["stdout"].write("partial ☃\n") + kwargs["stderr"].write("partial error\n") + raise error + + raw.run_process.side_effect = run + process = await _sandbox(raw).exec("command") + stdout, stderr = await asyncio.gather(process.stdout.read(), process.stderr.read()) + assert (stdout, stderr, await process.wait()) == ( + "partial ☃\n".encode(), + b"partial error\n", + -1, + ) + assert await process.stdout.read() == b"" + assert await process.stderr.read() == b"" + + +@pytest.mark.asyncio +async def test_a_missing_returncode_result_maps_to_minus_one(): + raw = _raw() + + async def run(command, args, **kwargs): + kwargs["stdout"].write("partial") + return SimpleNamespace(returncode=None) + + raw.run_process.side_effect = run + process = await _sandbox(raw).exec("command") + assert await process.stdout.read() == b"partial" + assert await process.wait() == -1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "error", + [ + SandboxResponseError("Sandbox process response included an unexpected NDJSON record"), + SandboxStreamError("server rejected the command", code="command_refused"), + SandboxApiError( + httpx.Response(401, request=httpx.Request("POST", "https://vercel.example")), + "unauthorized", + ), + SandboxCredentialsError("credentials are not configured"), + OSError("unrelated filesystem failure"), + ssl.SSLCertVerificationError("certificate verification failed"), + httpx2.ConnectError("certificate verification failed"), + httpx2.LocalProtocolError("invalid request headers"), + ], +) +async def test_non_transport_sdk_and_os_errors_propagate(error): + raw = _raw() + + async def run(command, args, **kwargs): + kwargs["stdout"].write("partial") + raise error + + raw.run_process.side_effect = run + process = await _sandbox(raw).exec("command") + with pytest.raises(type(error)) as raised: + await asyncio.gather(process.stdout.read(), process.wait()) + assert raised.value is error + + +@pytest.mark.asyncio +async def test_malformed_returncodes_are_not_concealed(): + raw = _raw() + raw.run_process = AsyncMock(return_value=SimpleNamespace(returncode="7")) + process = await _sandbox(raw).exec("command") + with pytest.raises(TypeError, match="malformed exit code"): + await process.wait() + + +@pytest.mark.asyncio +async def test_cancellation_propagates_and_preserves_partial_output_with_eof(): + raw = _raw() + released = asyncio.Event() + + async def run(command, args, **kwargs): + kwargs["stdout"].write("partial\n") + await released.wait() + return SimpleNamespace(returncode=0) + + raw.run_process.side_effect = run + process = await _sandbox(raw).exec("command") + await asyncio.sleep(0) + process._task.cancel() + with pytest.raises(asyncio.CancelledError): + await process.wait() + assert await process.stdout.read() == b"partial\n" + assert await process.stdout.read() == b"" + released.set() + + +@pytest.mark.asyncio +async def test_inherited_retry_is_opt_in_and_arbitrary_commands_do_not_retry(): + raw = _raw() + sandbox = _sandbox(raw) + + async def run(command, args, **kwargs): + if raw.run_process.await_count != 2: + raise httpx2.ReadError("incomplete response") + kwargs["stdout"].write("done") + return SimpleNamespace(returncode=0) + + raw.run_process.side_effect = run + assert await sandbox.exec_script("idempotent", max_retries=1) == "done" + assert raw.run_process.await_count == 2 + with pytest.raises(RuntimeError, match="exit -1"): + await sandbox.exec_script("arbitrary") + assert raw.run_process.await_count == 3 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("corrupt", [False, True]) +async def test_large_object_upload_checks_contents_before_replacing_destination(tmp_path, corrupt): + raw = _raw() + sandbox = _sandbox(raw) + parts = [] + payload = bytes(range(256)) * 20000 + destination = tmp_path / "existing 'file.bin" + destination.write_bytes(b"previous") + + async def write_bytes(path, data): + parts.append((path, len(data))) + Path(path).write_bytes(b"corrupted" if corrupt else data) + + async def execute(script): + result = await asyncio.to_thread(subprocess.run, ["bash", "-c", script], capture_output=True, text=True) + if result.returncode: + raise RuntimeError(result.stderr) + return result.stdout + + raw.fs.write_bytes.side_effect = write_bytes + sandbox.exec_script = execute + if corrupt: + with pytest.raises(RuntimeError, match="checksum"): + await sandbox._write_stream_to_vm(io.BytesIO(payload), str(destination)) + assert destination.read_bytes() == b"previous" + else: + await sandbox._write_stream_to_vm(io.BytesIO(payload), str(destination)) + assert destination.read_bytes() == payload + assert len(parts) == 3 and max(size for _, size in parts) <= 2 * 1024 * 1024 + assert all(not Path(path).exists() and not Path(path.removesuffix(".part")).exists() for path, _ in parts) + + +@pytest.mark.asyncio +async def test_signed_object_download_adds_host_and_preserves_current_policy(monkeypatch): + raw = _raw() + sandbox = _sandbox(raw, policy=NetworkPolicy(mode=NetworkMode.ALLOWLIST, allow_hosts=("stale.example",))) + current = SimpleNamespace(mode="custom", allow={"current.example": ()}, subnets=None) + sandbox._client = MagicMock(get_sandbox=AsyncMock(return_value=SimpleNamespace(network_policy=current))) + store = MagicMock(signed_get_url=MagicMock(return_value="https://bucket.example/file?signature=abc")) + config = MagicMock(get_object_store_at=MagicMock(return_value=store)) + monkeypatch.setattr(vercel_sandbox_module, "get_config", lambda: config, raising=False) + monkeypatch.setattr("agent_env.providers.sandbox_providers.sandbox.get_config", lambda: config) + script = AsyncMock() + sandbox.exec_script = script + + await sandbox.load_object_file("s3://bucket/file", "/tmp/a file") + + applied = raw.update_network_policy.await_args.args[0] + assert set(applied.allow) == {"current.example", "bucket.example"} + assert "stale.example" not in sandbox.network_policy.allow_hosts + assert "https://bucket.example/file?signature=abc" in script.await_args.args[0] + assert "'/tmp/a file'" in script.await_args.args[0] + config.get_object_store_at.assert_called_once_with("s3://bucket/file") + + +@pytest.mark.asyncio +async def test_signed_download_refuses_an_unknown_current_policy(monkeypatch): + sandbox = _sandbox(policy=NetworkPolicy(mode=NetworkMode.ALLOWLIST)) + custom = SimpleNamespace(mode="custom", allow={"service.example": (object(),)}, subnets=None) + sandbox._client = MagicMock(get_sandbox=AsyncMock(return_value=SimpleNamespace(network_policy=custom))) + store = MagicMock(signed_get_url=MagicMock(return_value="https://bucket.example/file")) + config = MagicMock(get_object_store_at=MagicMock(return_value=store)) + monkeypatch.setattr(vercel_sandbox_module, "get_config", lambda: config, raising=False) + monkeypatch.setattr("agent_env.providers.sandbox_providers.sandbox.get_config", lambda: config) + sandbox.exec_script = AsyncMock() + + with pytest.raises(RuntimeError, match="applied network policy is unknown"): + await sandbox.load_object_file("s3://bucket/file", "/tmp/file") + + sandbox._sandbox.update_network_policy.assert_not_awaited() + sandbox.exec_script.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_image_fallback_can_download_an_unsigned_object_without_deadlock(monkeypatch): + sandbox = _sandbox(policy=NetworkPolicy(mode=NetworkMode.ALLOWLIST)) + current = SimpleNamespace(mode="deny-all", allow={}, subnets=None) + sandbox._client = MagicMock(get_sandbox=AsyncMock(return_value=SimpleNamespace(network_policy=current))) + store = MagicMock(signed_get_url=MagicMock(return_value=None)) + config = MagicMock(get_object_store_at=MagicMock(return_value=store)) + monkeypatch.setattr(vercel_sandbox_module, "get_config", lambda: config, raising=False) + monkeypatch.setattr("agent_env.providers.sandbox_providers.sandbox.get_config", lambda: config) + sandbox._signed_image_urls = AsyncMock(return_value=[None]) + sandbox._write_unsigned_object = AsyncMock() + async def load(artifacts, urls): + await sandbox._download_object_to_vm("file:///store/image", "/tmp/image") + sandbox._load_docker_images = load + + await asyncio.wait_for(sandbox.load_docker_images(["image"]), 1) + + sandbox._write_unsigned_object.assert_awaited_once_with(store, "file:///store/image", "/tmp/image") + sandbox._sandbox.update_network_policy.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failures", [1, 3]) +@pytest.mark.parametrize("error_type", [BrokenResourceError, ssl.SSLError]) +async def test_interrupted_chunk_upload_retries_without_appending_partial_bytes(tmp_path, failures, error_type): + raw = _raw() + sandbox = _sandbox(raw) + payload = b"binary\x00" * 400000 + destination = tmp_path / "destination" + destination.write_bytes(b"previous") + uploads = [] + + async def write_bytes(path, data): + uploads.append(path) + Path(path).write_bytes(data[:100] if len(uploads) <= failures else data) + if len(uploads) <= failures: + raise error_type + + async def execute(script): + result = await asyncio.to_thread(subprocess.run, ["bash", "-c", script], capture_output=True, text=True) + if result.returncode: + raise RuntimeError(result.stderr) + return result.stdout + + raw.fs.write_bytes.side_effect = write_bytes + sandbox.exec_script = execute + if failures == 3: + with pytest.raises(error_type): + await sandbox._write_stream_to_vm(io.BytesIO(payload), str(destination)) + assert len(uploads) == 3 + assert destination.read_bytes() == b"previous" + else: + await sandbox._write_stream_to_vm(io.BytesIO(payload), str(destination)) + assert len(uploads) == 3 + assert destination.read_bytes() == payload + assert all(not Path(path).exists() for path in uploads) diff --git a/tst/unit/test_capabilities.py b/tst/unit/test_capabilities.py index 3aecc9d8..87aeb7f8 100644 --- a/tst/unit/test_capabilities.py +++ b/tst/unit/test_capabilities.py @@ -180,3 +180,44 @@ def build(spec): monkeypatch.setattr("agent_env.providers.sandbox_providers.sandbox_provider.build_sandbox_provider", build) assert capabilities.remote_sandbox_is_available("sail_vm") is available + + +def _fake_vercel_config(monkeypatch, config): + monkeypatch.setattr( + "agent_env.config.get_config", + lambda: type("Cfg", (), {"section": lambda self, _name: {"providers": {"vercel": {"config": config}}}})(), + ) + + +def test_vercel_is_absent_without_configured_or_ambient_credentials(monkeypatch): + monkeypatch.setattr( + "agent_env.providers.sandbox_providers.sandbox_provider.build_sandbox_provider", + lambda spec: object(), + ) + for name in ("VERCEL_TOKEN", "VERCEL_TEAM_ID", "VERCEL_PROJECT_ID"): + monkeypatch.delenv(name, raising=False) + _fake_vercel_config(monkeypatch, {}) + assert capabilities.remote_sandbox_is_available("vercel") is False + + +def test_vercel_is_present_with_complete_config_credentials(monkeypatch): + monkeypatch.setattr( + "agent_env.providers.sandbox_providers.sandbox_provider.build_sandbox_provider", + lambda spec: object(), + ) + _fake_vercel_config( + monkeypatch, + {"token": "secret:vercel_token", "team_id": "team", "project_id": "project"}, + ) + assert capabilities.remote_sandbox_is_available("vercel") is True + + +def test_vercel_is_present_with_all_ambient_credential_names(monkeypatch): + monkeypatch.setattr( + "agent_env.providers.sandbox_providers.sandbox_provider.build_sandbox_provider", + lambda spec: object(), + ) + for name in ("VERCEL_TOKEN", "VERCEL_TEAM_ID", "VERCEL_PROJECT_ID"): + monkeypatch.setenv(name, "configured") + _fake_vercel_config(monkeypatch, {}) + assert capabilities.remote_sandbox_is_available("vercel") is True diff --git a/tst/util/capabilities.py b/tst/util/capabilities.py index ef86c8f6..d4165a0e 100644 --- a/tst/util/capabilities.py +++ b/tst/util/capabilities.py @@ -10,6 +10,7 @@ from __future__ import annotations +import os import subprocess import pytest @@ -19,7 +20,7 @@ #: Only the agent-driving paths need a model endpoint. MODEL_ENDPOINT = "model_endpoint_configured" -#: The remote sandbox providers (``modal``, ``modal_vm``, ``e2b``, ``sail_vm``) need credentials the resolved +#: The remote sandbox providers (``modal``, ``modal_vm``, ``e2b``, ``sail_vm``, ``vercel``) need credentials the resolved #: config may not carry; the local default never does. REMOTE_SANDBOX = "remote_sandbox" @@ -75,16 +76,16 @@ def skip_without_model_endpoint() -> pytest.MarkDecorator: def remote_sandbox_is_available(provider: str) -> bool: - """Whether the resolved config can build the ``modal`` / ``modal_vm`` / ``e2b`` / ``sail_vm`` sandbox + """Whether the resolved config can build the ``modal`` / ``modal_vm`` / ``e2b`` / ``sail_vm`` / ``vercel`` sandbox provider, credentials included. - Modal: any failure to resolve the credentials answers False. E2B and Sail: building the provider + Modal: any failure to resolve the credentials answers False. E2B, Sail and Vercel: building the provider resolves ``[sandbox.providers..config]`` and its ``secret:`` references; only a ``ConfigError`` (absent or incomplete config) answers False, any other failure answers True and lets the test fail on the real problem, as ``model_endpoint_is_configured`` does.""" - if provider not in ("modal", "modal_vm", "e2b", "sail_vm"): + if provider not in ("modal", "modal_vm", "e2b", "sail_vm", "vercel"): raise ValueError(f"unknown remote sandbox provider {provider!r}") - if provider in ("e2b", "sail_vm"): + if provider in ("e2b", "sail_vm", "vercel"): from agent_env.config.errors import ConfigError from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider @@ -94,7 +95,20 @@ def remote_sandbox_is_available(provider: str) -> bool: return False except Exception: return True - return True + if provider != "vercel": + return True + from agent_env.config import get_config + + vercel = get_config().section("sandbox").get("providers", {}).get("vercel", {}) + config = vercel.get("config", {}) if isinstance(vercel, dict) else {} + if isinstance(config, dict) and all( + key in config for key in ("token", "team_id", "project_id") + ): + return True + return bool(os.getenv("VERCEL_OIDC_TOKEN")) or all( + os.getenv(name) + for name in ("VERCEL_TOKEN", "VERCEL_TEAM_ID", "VERCEL_PROJECT_ID") + ) from agent_env.config import get_config try: diff --git a/uv.lock b/uv.lock index aaa3c246..8669313e 100644 --- a/uv.lock +++ b/uv.lock @@ -7,9 +7,14 @@ resolution-markers = [ "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version < '3.13' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform != 'win32'", - "python_full_version < '3.13' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "(python_full_version < '3.12' and sys_platform == 'emscripten') or (python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32')", ] +[options] +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer-span = "P2D" + [manifest] members = [ "agentenv-framework", @@ -69,6 +74,7 @@ aws = [ { name = "botocore" }, ] dev = [ + { name = "anyio" }, { name = "boto3" }, { name = "botocore" }, { name = "fastapi" }, @@ -77,6 +83,7 @@ dev = [ { name = "google-cloud-secret-manager" }, { name = "google-cloud-storage" }, { name = "griffelib" }, + { name = "httpx2" }, { name = "moto" }, { name = "psycopg2-binary" }, { name = "pytest" }, @@ -88,6 +95,7 @@ dev = [ { name = "requests" }, { name = "sail" }, { name = "uvicorn" }, + { name = "vercel-sandbox" }, ] explorer = [ { name = "fastapi" }, @@ -103,12 +111,18 @@ gcp = [ sail = [ { name = "sail" }, ] +vercel = [ + { name = "anyio" }, + { name = "httpx2" }, + { name = "vercel-sandbox" }, +] [package.metadata] requires-dist = [ { name = "a2a-sdk", specifier = "==0.3.26" }, - { name = "agentenv-framework", extras = ["aws", "explorer", "gcp", "sail"], marker = "extra == 'dev'" }, + { name = "agentenv-framework", extras = ["aws", "explorer", "gcp", "sail", "vercel"], marker = "extra == 'dev'" }, { name = "agentenv-framework-protocol", editable = "packages/agentenv-protocol" }, + { name = "anyio", marker = "extra == 'vercel'", specifier = ">=4,<5" }, { name = "boto3", marker = "extra == 'aws'", specifier = ">=1.35.0" }, { name = "botocore", marker = "extra == 'aws'", specifier = ">=1.35.0" }, { name = "certifi", specifier = ">=2024.2.2" }, @@ -122,6 +136,7 @@ requires-dist = [ { name = "google-cloud-storage", marker = "extra == 'gcp'", specifier = ">=3.0" }, { name = "griffelib", marker = "extra == 'dev'", specifier = "==2.3.0" }, { name = "httpx", specifier = ">=0.27.0" }, + { name = "httpx2", marker = "extra == 'vercel'", specifier = ">=2.13.1,<3" }, { name = "litellm", specifier = ">=1.96.0,<1.97" }, { name = "mcp", specifier = ">=1.0.0,<2" }, { name = "modal", specifier = ">=1.5.2.dev22" }, @@ -143,8 +158,9 @@ requires-dist = [ { name = "starlette", specifier = ">=0.40" }, { name = "uvicorn", specifier = ">=0.47" }, { name = "uvicorn", marker = "extra == 'explorer'", specifier = ">=0.29" }, + { name = "vercel-sandbox", marker = "extra == 'vercel'", specifier = ">=0.7.0,<0.8" }, ] -provides-extras = ["aws", "explorer", "gcp", "sail", "dev"] +provides-extras = ["aws", "explorer", "gcp", "sail", "vercel", "dev"] [[package]] name = "agentenv-framework-protocol" @@ -1300,6 +1316,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/f3/1db7aa2bc2524062192bb0e0323969492d1883152a232fe36eea65f4e35c/httpcore2-2.13.1.tar.gz", hash = "sha256:e0aa977abe17e69a3b820a24542a6fa88702676d83880b8d194dcd18408e5103", size = 68071, upload-time = "2026-09-23T07:47:22.372Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/ba/a4568248771ce81957bfb7cc600264a40fbcda092391ee1c415c50be4bea/httpcore2-2.13.1-py3-none-any.whl", hash = "sha256:e1e05d4f25f7d7d496bfb96748f6f4b67657b03da069b3a68c36069f3db73d0a", size = 83423, upload-time = "2026-09-23T07:47:19.365Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -1324,6 +1353,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, ] +[[package]] +name = "httpx2" +version = "2.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "python_full_version >= '3.12' and sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/44/474bef2a0e9d90f1715d32cb98b0738695ca17ba324095fb2497ed7fbd59/httpx2-2.13.1.tar.gz", hash = "sha256:e48744a19e3af5ee48313d0ce5fe941d5422fae5705ea922a4aabf94d7800dfa", size = 100405, upload-time = "2026-09-23T07:47:23.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/9c/6fe8931fd9f381042a9e4c7d5a7b4cbf7016b252bec0c99a49fce42c3326/httpx2-2.13.1-py3-none-any.whl", hash = "sha256:6dff50fabc270ee5fd25d845d0b078ed20564579744d6d962850975996d2f9a4", size = 95597, upload-time = "2026-09-23T07:47:20.995Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + [[package]] name = "huggingface-hub" version = "1.16.1" @@ -3095,6 +3150,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl", hash = "sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03", size = 78337, upload-time = "2026-06-17T07:36:50.132Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "typer" version = "0.26.8" @@ -3171,6 +3235,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/fa/e1388bbcf24ef3274f45c0c1c7b501fd14971037c1b6ee23610553307497/uvicorn-0.49.0-py3-none-any.whl", hash = "sha256:ba3d14c3ee7e41c6c654c46c9eb489d33213cdd30aa1696eab1374337c13f68f", size = 71376, upload-time = "2026-06-03T22:01:29.037Z" }, ] +[[package]] +name = "vercel-headers" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/0c/241e47d351413c108f0b411df087feeb15cf32f422000414ce24d3791537/vercel_headers-0.7.2.tar.gz", hash = "sha256:f5255da014032b357a3f18769bd19ff675b0ee3e4fc355c6bbd574dfc674222d", size = 6148, upload-time = "2026-09-08T17:09:14.589Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/64/12d53f70253dfd00f1a394bc930b5d55bad84ac17c9fc77b5ca0525ccd18/vercel_headers-0.7.2-py3-none-any.whl", hash = "sha256:26dcb0016b5c553e638385e532cb8557f071e247b46f2575d5737e48718dc8c6", size = 3922, upload-time = "2026-09-08T17:09:13.585Z" }, +] + +[[package]] +name = "vercel-internal-core" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx2" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/7e/3bda29611761732de929be27bb001c7d5ac396d821461301aa24503b03d7/vercel_internal_core-0.2.1.tar.gz", hash = "sha256:e2aac2eba7104783e809c390d8376b9ac25dd7097358d2320a293877c12d3838", size = 20245, upload-time = "2026-10-06T18:37:57.65Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/50/db31539ce88a597d95029e5b33d86897dca5fad950ac764bede54447ea39/vercel_internal_core-0.2.1-py3-none-any.whl", hash = "sha256:6d79198c7c4fd0ea44e8ca09c06dd4a6653fd86b4cab7a28148162d2689b1408", size = 24763, upload-time = "2026-10-06T18:37:56.79Z" }, +] + +[[package]] +name = "vercel-oidc" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx2" }, + { name = "vercel-headers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/74/d2dfab8dccf6df6db894df598edf1f3f3bc41406efc7e9c1d464b8e2a211/vercel_oidc-0.10.0.tar.gz", hash = "sha256:24310f5ce5db05455bcac82ff6cb0cb254b28af93dacb526cc375a9403aca6ee", size = 16591, upload-time = "2026-10-06T18:38:02.295Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/ab/dab581dd1c0e1340048c1d95ca7862e007a0f84302ae8bfe3fac6f97be0f/vercel_oidc-0.10.0-py3-none-any.whl", hash = "sha256:abc8317317482e542834ab35883e8c4a0f9a1baea554898e3f041ca037f90bec", size = 16554, upload-time = "2026-10-06T18:38:01.422Z" }, +] + +[[package]] +name = "vercel-sandbox" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx2" }, + { name = "pydantic" }, + { name = "vercel-internal-core" }, + { name = "vercel-oidc" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/f7/3427e75b63056f77ed3fd5986818a1b521d86c2c8704d2e929cedcbf70aa/vercel_sandbox-0.7.0.tar.gz", hash = "sha256:07d703790d0141dc75e0962baf35d666a6961bc1de913749137267a2550c81e2", size = 80326, upload-time = "2026-09-23T16:46:22.354Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/6d/372055569df9db7c1f5663bf128b914e70ab96f3b480fd9cf9c1be2c5aff/vercel_sandbox-0.7.0-py3-none-any.whl", hash = "sha256:f7eb897a1a5092fe8c38319ce803033b1a9ced8f414564a43a668432f91ffcd9", size = 91791, upload-time = "2026-09-23T16:46:21.169Z" }, +] + [[package]] name = "watchfiles" version = "1.2.0" From 53090f5a5fda885ff57dc4711bfd213276823c04 Mon Sep 17 00:00:00 2001 From: Elisabeth Rulke Date: Fri, 9 Oct 2026 12:51:18 -0700 Subject: [PATCH 2/4] fix(vercel): preserve image loading modes and add evaluation demo --- README.md | 10 +- examples/vercel-sandbox/README.md | 66 +++++ examples/vercel-sandbox/RESULTS.md | 23 ++ examples/vercel-sandbox/artifact_server.py | 42 +++ examples/vercel-sandbox/build_images.sh | 40 +++ examples/vercel-sandbox/config.toml | 3 + examples/vercel-sandbox/fixture_agent.py | 48 ++++ examples/vercel-sandbox/run_e2e.py | 242 ++++++++++++++++++ examples/vercel-sandbox/with_vercel_login.py | 27 ++ .../sandbox_providers/vercel/sandbox.py | 4 +- .../sandbox_providers/vercel_sandbox_test.py | 54 +++- uv.lock | 4 - 12 files changed, 549 insertions(+), 14 deletions(-) create mode 100644 examples/vercel-sandbox/README.md create mode 100644 examples/vercel-sandbox/RESULTS.md create mode 100644 examples/vercel-sandbox/artifact_server.py create mode 100644 examples/vercel-sandbox/build_images.sh create mode 100644 examples/vercel-sandbox/config.toml create mode 100644 examples/vercel-sandbox/fixture_agent.py create mode 100644 examples/vercel-sandbox/run_e2e.py create mode 100644 examples/vercel-sandbox/with_vercel_login.py diff --git a/README.md b/README.md index 7b017428..adee5ec6 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,12 @@ The distribution is named `agentenv-framework`, the import package is `agent_env ### Sandbox providers -The default provider runs environments and agents as containers on the local Docker daemon. The `vercel` provider runs each environment or agent in an ephemeral, Docker-capable Vercel Sandbox and exposes declared ports through public `*.vercel.run` routes. Install it with the optional extra and select it in `.agentenv/config.toml`: +For a complete evaluation using your existing Vercel login, follow the [Vercel Sandbox demo](examples/vercel-sandbox/README.md). + +The default provider runs environments and agents as containers on the local Docker daemon. The `vercel` provider runs each environment or agent in an ephemeral, Docker-capable Vercel Sandbox and exposes declared ports through public `*.vercel.run` routes. From this source checkout, install the optional extra and select it in `.agentenv/config.toml`. This provider is not yet in a published PyPI release: ```bash -uv tool install 'agentenv-framework[vercel]' +uv sync --frozen --extra vercel ``` ```toml @@ -62,9 +64,9 @@ image = "vercel/sandbox/universal" # project_id = "your-project" ``` -Credentials are resolved by AgentEnv's `env:` and `secret:` references before the provider builds its scoped Vercel SDK client; the provider never changes the process environment or uses an ambient SDK session. Each sandbox is ephemeral, its execution time follows AgentEnv's timeout, and teardown destroys the sandbox and its orphan snapshots. Vercel currently provisions 64 GB of ephemeral disk: requests above 64 GB are rejected rather than silently ignored. Resource sizing uses the smallest supported even vCPU count (or one vCPU) that covers both the requested CPU and memory (2048 MB per vCPU), up to 32 vCPUs and 64 GB at the provider-wide ceiling; the account's plan may impose lower limits. +Credentials are resolved by AgentEnv's `env:` and `secret:` references before the provider builds its scoped Vercel SDK client; the provider never changes the process environment or makes its SDK session the active one. Each sandbox is ephemeral, its execution time follows AgentEnv's timeout, and teardown destroys the sandbox and its orphan snapshots. Vercel currently provisions 64 GB of ephemeral disk: requests above 64 GB are rejected rather than silently ignored. Resource sizing uses the smallest supported even vCPU count (or one vCPU) that covers both the requested CPU and memory (2048 MB per vCPU), up to 32 vCPUs and 64 GB at the provider-wide ceiling; the account's plan may impose lower limits. -The provider treats CPU and memory requests as minimums. If a request does not already match a supported shape, AgentEnv logs both the requested and allocated CPU and memory before provisioning. For example, `cpu = 0.5` and `memory = 5000` allocate four vCPUs and 8192 MB. A disk request below 64 GB is also ignored: every sandbox receives the fixed 64 GB disk, and the provider logs the ignored request. Configure all three of `token`, `team_id`, and `project_id` for an explicit scoped token, or omit all three to continue using the SDK's OIDC credentials. +See the [Vercel resource limits](https://vercel.com/docs/sandbox/pricing#resource-limits) and [firewall policy](https://vercel.com/docs/sandbox/concepts/firewall) documentation. The provider treats CPU and memory requests as minimums. If a request does not already match a supported shape, AgentEnv logs both the requested and allocated CPU and memory before provisioning. For example, `cpu = 0.5` and `memory = 5000` allocate four vCPUs and 8192 MB. A disk request below 64 GB is also ignored: every sandbox receives the fixed 64 GB disk, and the provider logs the ignored request. Configure all three of `token`, `team_id`, and `project_id` for an explicit scoped token, or omit all three to continue using the SDK's OIDC credentials. Outbound policy supports AgentEnv's allow-all and allowlist forms, including hostname and CIDR entries. Vercel's custom transforms, forwarding rules, and denied CIDR exceptions are not reconstructed as ordinary allowlists: reconnect leaves such a policy unknown and image loading fails closed. At creation, public Vercel routes are added to a restricted workload's platform egress floor so it can reach another Vercel sandbox's declared route. Image and file downloads add their exact signed-URL hostnames to the current allowlist; these hosts remain allowed until the policy is replaced. A CIDR-only Vercel policy allows unrestricted DNS resolution; use domain rules to restrict DNS, or deny-all to block it. diff --git a/examples/vercel-sandbox/README.md b/examples/vercel-sandbox/README.md new file mode 100644 index 00000000..bacb46c9 --- /dev/null +++ b/examples/vercel-sandbox/README.md @@ -0,0 +1,66 @@ +# AgentEnv on Vercel Sandbox + +This candidate adds the `vercel` sandbox provider to AgentEnv. AgentEnv runs the task, connects the agent to its environment, collects the trajectory and scores the result. Vercel provides the separate sandboxes running the environment and agent. + +The included demo exercises one complete evaluation: deploy an items environment behind a gateway with its database services, load `starting-item`, deploy a deterministic agent, ask it to add `vercel-review-complete` through MCP, and grade its response. The controller independently reads the final state, verifies the stored task and trajectory, reconnects, and tears down the successful run. The agent performs real MCP calls; it does not call a language model. This validates the integration, not model quality. + +## Setup + +Prerequisites: Git, Python 3.11.4 or newer, uv, Node.js and Vercel CLI 62.5.0 or newer (the demo was tested with 62.5.0). Use a Vercel project where you can create 4-vCPU/8-GB sandboxes for the build and environment, plus a 2-vCPU/4-GB agent. The demo builds Linux images in a fresh Vercel sandbox, so local Docker is not needed. Network access to the package and image registries referenced by the repository Dockerfiles is required. + +Clone this branch to use the unreleased provider. The published PyPI package does not contain this candidate. + +```bash +git clone --branch feat/vercel-sandbox https://github.com/vercel-labs/agentenv-framework.git +cd agentenv-framework +uv sync --frozen --extra vercel +export AGENT_ENV_CONFIG="$PWD/examples/vercel-sandbox/config.toml" +``` + +The dedicated config selects Vercel for environments and agents. The demo overrides the document and object stores with local stores in its output directory. + +## Run with an existing Vercel login + +If needed, run `vercel login`. Choose an existing project and team that you have access to. The wrapper uses the public `vercel project token` command and passes its short-lived OIDC token only to the child process. It does not print the token or write it to disk. + +Use a fresh output directory for each attempt; the demo refuses to overwrite an existing one. Keep the dedicated configuration selected above so unrelated settings cannot affect the demo. Do not add explicit `token`, `team_id` or `project_id` fields for this OIDC route. + +```bash +.venv/bin/python examples/vercel-sandbox/with_vercel_login.py \ + --scope YOUR_TEAM --project YOUR_PROJECT -- \ + .venv/bin/python examples/vercel-sandbox/run_e2e.py \ + --repo . --output /tmp/agentenv-vercel-e2e-run-1 +``` + +The demo creates three temporary sandboxes: an image builder, an environment and an agent. Successful runs remove all three and verify subsequent lookups return 404. Failed runs stop and retain tracked sandboxes and record their identifiers for diagnosis. The runner also records handles reaching its termination guard during setup rollback. Failures before a handle reaches the runner can still follow the provider’s own creation-cleanup behavior. Retained resources are intentionally left for diagnosis; this demo does not automatically reap them. + +The image-download fixture serves only disposable test images. Its fixed signing key is public test data; this fixture is not a production artifact store. The demo uses local SQLite and filesystem stores for run records and artifacts, and applies a restricted policy allowing the test package host and Vercel routes to the workload sandboxes. Scale's actual stores and network requirements need their own acceptance run. + +## What passing means + +Run with normal Python execution; the runner refuses `python -O` because its assertions are acceptance checks. + +Expected final output: + +```text +END-TO-END EVALUATION PASSED: score 1 +``` + +The output folder retains: + +- `success.json`: final pass, score, run identifier and cleanup assertions. +- `task.json`, `task-instance.json` and `context.json`: the five task steps and recorded result. +- `environment-state.json`: exactly `starting-item` and `vercel-review-complete`. +- `objects/`: collected task artifacts, including the agent trajectory. +- `resources.json`, `images.json` and `source-hashes.json`: resource, image and source provenance. +- `build.stdout`, `build.stderr` and `build-command.json`: image-build evidence. + +The stdout/stderr stream can also be redirected to a local log. Output folders contain deployment coordinates and private local paths; review them before sharing. Keep these generated outputs outside the checkout. + +## Review scope + +The integration remains in-tree as requested, with optional Python SDK dependencies. CPU and memory are minimum requests rounded to supported shapes; fixed disk sizing is logged. OIDC and explicit configuration credentials are supported. This POC covers ephemeral runs and active reconnect. Persistent resume, credential brokering and model-quality benchmarking are separate work. + +Recorded follow-ups include cleanup of commands still running during teardown, upload retry diagnostics, resource log severity and readiness behavior. Production workload acceptance, actual stores, CPU/memory semantics, optional OIDC and maintenance ownership remain for Scale to confirm. + +See [RESULTS.md](RESULTS.md) for the validation scope and remaining acceptance work. diff --git a/examples/vercel-sandbox/RESULTS.md b/examples/vercel-sandbox/RESULTS.md new file mode 100644 index 00000000..d118589b --- /dev/null +++ b/examples/vercel-sandbox/RESULTS.md @@ -0,0 +1,23 @@ +# End-to-end result, October 9, 2026 + +The reviewed provider completed a real AgentEnv evaluation using project OIDC obtained from an existing Vercel CLI login. No manually created API token or model key was needed. + +| Step | Observed result | +| --- | --- | +| Deploy environment | Gateway, items MCP server and database services running in Vercel Sandbox | +| Load data | Initial state contains `starting-item` | +| Deploy agent | A separate Vercel Sandbox runs the deterministic review agent | +| Prompt agent | Agent calls `items_add_item` and `list_items` through the environment's public MCP route | +| Grade response | Task status `completed`, all five steps successful, score **1.0** | + +The controller independently read the final environment state as `["starting-item", "vercel-review-complete"]`. AgentEnv collected the native trajectory into its local object store and retained the task-instance result. Fresh-provider reconnect restored routes and policy. Teardown removed the agent, environment and image builder; all three subsequent lookups returned 404. Repeated teardown succeeded. + +The initial attempt exposed a missing MCP client dependency in the new test-agent image. The dependency was added and checked at build time; two subsequent full runs passed. The final runner also passed an injected setup-failure test that verifies recording and retention before create returns. The initial diagnostic resources were stopped and retained. No provider implementation changes were needed for this evaluation. + +The publication candidate is based on upstream commit `c70011524ae65fbd35eb2b8292c9560eb81967cc`. Upstream changes and dependency metadata were incorporated before publication. The provider now inherits the current VM image-loading contract, preserving registry pulls, builds from stored contexts and early refusal of unavailable images. The five demo scripts are unchanged. The complete evaluation above passed again on this base using a fresh installation with only the `vercel` extra. After the compatibility correction, 171 focused provider, policy, capability and VM tests passed. A separate live Vercel probe pulled and ran a registry image, built a context image and checked its contents, rejected an unavailable image, and verified cleanup with a 404 lookup. The plugin API check reported no break. + +Earlier broad verification on base `6e20760bebacd73ebb74e8add0bd69407b24baa4`: 6,810 unit/protocol passes with 9 declared skips, 83 provider-specific passes, all 60 installer cases passed across two runs, build and clean installation passed. These historical results do not establish the outcome of the broader suite on the newer base. See the upstream PR for current validation and CI status. + +This is a deterministic integration evaluation. It does not benchmark a language model or establish acceptance of Scale's production workloads, image/object stores, mixed-provider setup, persistence or credential brokering. Maintainers should review the documented CPU/memory minimum semantics, OIDC option, operational ownership and recorded provider follow-ups, particularly commands still running during teardown and upload retry diagnostics. + +Private run logs, deployment coordinates, local state and diagnostic resource identifiers are excluded from the repository. The demo scripts produce those records for each reviewer's own run. diff --git a/examples/vercel-sandbox/artifact_server.py b/examples/vercel-sandbox/artifact_server.py new file mode 100644 index 00000000..1c906601 --- /dev/null +++ b/examples/vercel-sandbox/artifact_server.py @@ -0,0 +1,42 @@ +"""Serve only disposable review fixture image archives, with short-lived signatures.""" + +import hashlib +import hmac +import shutil +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +ROOT = Path("/tmp/poc-artifacts") +KEY = b"agentenv-poc-public-image-fixture" +ALLOWED = {"manifest.json", *(name + ".tar.gz" for name in ( + "poc-gateway", "poc-db", "poc-db-web", "poc-db-mcp", "poc-items", "poc-agent"))} + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + request = urlsplit(self.path) + name = request.path.removeprefix("/") + query = parse_qs(request.query) + expires = query.get("expires", ["0"])[0] + signature = query.get("signature", [""])[0] + expected = hmac.new(KEY, (name + ":" + expires).encode(), hashlib.sha256).hexdigest() + if name not in ALLOWED: + self.send_error(404) + return + if not expires.isdigit() or int(expires) < time.time() or not hmac.compare_digest(signature, expected): + self.send_error(403) + return + path = ROOT / name + if not path.is_file(): + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Length", str(path.stat().st_size)) + self.end_headers() + with path.open("rb") as source: + shutil.copyfileobj(source, self.wfile) + + +ThreadingHTTPServer(("0.0.0.0", 8080), Handler).serve_forever() diff --git a/examples/vercel-sandbox/build_images.sh b/examples/vercel-sandbox/build_images.sh new file mode 100644 index 00000000..0a206bcf --- /dev/null +++ b/examples/vercel-sandbox/build_images.sh @@ -0,0 +1,40 @@ +set -euo pipefail +mkdir -p /tmp/agentenv /tmp/poc-artifacts /tmp/poc-logs +cd /tmp/agentenv +tar -xzf /tmp/source.tar.gz +apt-get update -qq +DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker-buildx +build_one() { + docker build -f "$1" -t "$3" "$2" > "/tmp/poc-logs/$3-build.log" 2>&1 + docker save "$3" | gzip -1 > "/tmp/poc-artifacts/$3.tar.gz" + docker image inspect "$3" --format '{{.Id}}' > "/tmp/poc-artifacts/$3.image-id" + echo "BUILT $3" +} +build_one src/agent_env/env/gateway/Dockerfile src/agent_env/env poc-gateway +build_one src/agent_env/env/envs/service_db/Dockerfile src/agent_env/env/envs/service_db poc-db +build_one src/agent_env/env/envs/service_db/Dockerfile.db-web src/agent_env/env/envs/service_db poc-db-web +build_one src/agent_env/env/envs/service_db/Dockerfile.db-mcp src/agent_env/env/envs/service_db poc-db-mcp +mkdir -p /tmp/items-context /tmp/agent-context +cp -R packages/agentenv-protocol/src/agentenv_protocol /tmp/items-context/agentenv_protocol +cp tst/data/agentenv_mcp/Dockerfile tst/data/agentenv_mcp/server.py tst/data/agentenv_mcp/seed.json /tmp/items-context/ +build_one /tmp/items-context/Dockerfile /tmp/items-context poc-items +cp -R packages/agentenv-protocol /tmp/agent-context/agentenv-protocol +cp tst/data/a2a_agent/Dockerfile /tmp/agent-context/Dockerfile +cp /tmp/fixture_agent.py /tmp/agent-context/agent.py +python3 - <<'PYDOCKER' +from pathlib import Path +p=Path('/tmp/agent-context/Dockerfile') +s=p.read_text() +s=s.replace('USER agent', 'RUN pip install --no-cache-dir mcp==1.28.1\nRUN python -c "from mcp.client.streamable_http import streamable_http_client; import agent"\nUSER agent') +p.write_text(s) +PYDOCKER +build_one /tmp/agent-context/Dockerfile /tmp/agent-context poc-agent +python3 - <<'PY' +from pathlib import Path +import json,hashlib +p=Path('/tmp/poc-artifacts') +manifest={f.stem.removesuffix('.tar'):{'sha256':hashlib.sha256(f.read_bytes()).hexdigest(),'bytes':f.stat().st_size,'image_id':f.with_name(f.stem.removesuffix('.tar')+'.image-id').read_text().strip()} for f in p.glob('*.tar.gz')} +(p/'manifest.json').write_text(json.dumps(manifest,indent=2)) +assert len(manifest)==6 +print('ALL SIX IMAGE BUILDS PASSED') +PY diff --git a/examples/vercel-sandbox/config.toml b/examples/vercel-sandbox/config.toml new file mode 100644 index 00000000..49333b95 --- /dev/null +++ b/examples/vercel-sandbox/config.toml @@ -0,0 +1,3 @@ +[sandbox] +default = "vercel" +agent_default = "vercel" diff --git a/examples/vercel-sandbox/fixture_agent.py b/examples/vercel-sandbox/fixture_agent.py new file mode 100644 index 00000000..12a7ea4c --- /dev/null +++ b/examples/vercel-sandbox/fixture_agent.py @@ -0,0 +1,48 @@ +"""Deterministic review agent: add one item through the configured MCP server.""" + +import json + +import httpx +from mcp import ClientSession +from mcp.client.streamable_http import streamable_http_client +from agentenv_protocol.a2a_agent import ( + MCP_CONFIG_V1, TRAJECTORY_V1, AgentConfig, AgentEnvAgent, + AgentIdentity, TaskRequest, TaskResult, TextPart, Usage, a2a_agent, +) + + +@a2a_agent( + identity=AgentIdentity(name="vercel-review-agent", version="1.0.0", + description="Adds the requested item through MCP and reports stored state."), + config=AgentConfig, + extensions=(MCP_CONFIG_V1, TRAJECTORY_V1), +) +class ReviewAgent(AgentEnvAgent): + async def run(self, request: TaskRequest[AgentConfig]) -> TaskResult: + item = "".join(part.text for part in request.parts if isinstance(part, TextPart)).strip() + if len(request.mcp_servers) != 1 or not item: + raise ValueError("The review task requires one MCP server and an item name") + registration = next(iter(request.mcp_servers.values())) + async with httpx.AsyncClient(headers=registration.get("headers") or {}, timeout=60) as client: + async with streamable_http_client(registration["url"], http_client=client) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + tools = await session.list_tools() + names = [tool.name for tool in tools.tools] + assert "items_add_item" in names and "list_items" in names, names + added = await session.call_tool("items_add_item", {"item": item}) + assert not added.isError, added + listed = await session.call_tool("list_items", {}) + assert not listed.isError, listed + text = "\n".join(c.text for c in listed.content if c.type == "text") + reply = "Stored items: " + text + return (TaskResult.builder().succeeded().parts([TextPart(text=reply)]) + .usage(Usage(tool_call_count=2, input_tokens=0, output_tokens=0)) + .native_trajectory(format="vercel-review/v1", payload=[ + {"tool": "items_add_item", "arguments": {"item": item}}, + {"tool": "list_items", "response": json.loads(text)}, + ]).build()) + + +if __name__ == "__main__": + ReviewAgent().serve() diff --git a/examples/vercel-sandbox/run_e2e.py b/examples/vercel-sandbox/run_e2e.py new file mode 100644 index 00000000..d9bec9c5 --- /dev/null +++ b/examples/vercel-sandbox/run_e2e.py @@ -0,0 +1,242 @@ +"""Build fixtures and run a scored AgentEnv task on Vercel; retain any failures.""" + +import argparse +import asyncio +import hashlib +import hmac +import json +import logging +import subprocess +import tarfile +import time +from dataclasses import asdict +from pathlib import Path +from urllib.parse import urlencode + +import httpx +from agentenv_protocol import client as protocol_v1 +from agent_env.a2a_agent import A2AAgent +from agent_env.artifact import DockerImageArtifact, EnvironmentArtifact, FileArtifact +from agent_env.config import configure +from agent_env.env import Env, GatewayEnv, legacy_protocol +from agent_env.env.envs.mcp_server import MCPServerEnv +from agent_env.env.envs.service_db import ServiceDBEnv +from agent_env.providers.sandbox_providers.sandbox import NetworkMode, NetworkPolicy +from agent_env.providers.sandbox_providers.vercel import VercelSandbox, VercelSandboxProvider +from agent_env.store.document_store import LocalSqliteDocumentStore +from agent_env.store.object_store import LocalFilesystemObjectStore +from agent_env.task import Task +from agent_env.task.teardown import teardown_run +from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep +from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep +from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import AgentPromptResponseVerifierTaskStep + +HERE = Path(__file__).resolve().parent +KEY = b"agentenv-poc-public-image-fixture" + + +async def main(): + if not __debug__: + raise SystemExit("Run without Python -O; this validation requires assertions") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + repo, output = args.repo.resolve(), args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + logging.basicConfig(level=logging.INFO) + logging.getLogger("httpx").setLevel(logging.WARNING) + def save(name, value): + (output / name).write_text(json.dumps(value, indent=2, default=str) + "\n") + + tracked = subprocess.check_output( + ["git", "-C", str(repo), "ls-files", "-z", "--cached", "--others", "--exclude-standard"] + ).decode().split("\0") + tracked = [name for name in tracked if name and (repo / name).is_file() + and not any(part.startswith(".env") for part in Path(name).parts)] + save("source-hashes.json", {name: hashlib.sha256((repo / name).read_bytes()).hexdigest() for name in tracked}) + with tarfile.open(output / "source.tar.gz", "w:gz") as archive: + for name in tracked: + archive.add(repo / name, arcname=name, recursive=False) + + resources, owned, clients = [], [], [] + phase = "build" + cleanup = False + completed = False + create_vm = VercelSandboxProvider.create_vm + terminate = VercelSandbox.terminate + + def record_resource(box, *, setup_failed=False): + if not any(known.sandbox_id == box.sandbox_id for known in owned): + owned.append(box) + resources.append({"name": box.sandbox_id, "session_id": box._sandbox.current_session_id, + "phase": phase, "routes": box.tunnel_urls, "created_at": time.time(), + "setup_failed": setup_failed}) + save("resources.json", resources) + + async def tracked_create(provider, **kwargs): + if provider not in clients: + clients.append(provider) + box = await create_vm(provider, **kwargs) + record_resource(box) + if phase == "task": + await box.apply_network_policy(NetworkPolicy( + mode=NetworkMode.ALLOWLIST, allow_hosts=("pypi.org", "*.vercel.run"))) + print("CREATED", phase, box.sandbox_id, flush=True) + return box + + async def guarded_terminate(box): + if cleanup: + return await terminate(box) + record_resource(box, setup_failed=True) + print("RETAINING until assertions pass:", box.sandbox_id, flush=True) + + VercelSandboxProvider.create_vm = tracked_create + VercelSandbox.terminate = guarded_terminate + controller = VercelSandboxProvider.from_config() + clients.append(controller) + try: + builder = await controller.create_vm(cpu=4, memory=8192, exposed_ports=[8080], timeout=2400) + for source, target in [(output / "source.tar.gz", "/tmp/source.tar.gz"), + (HERE / "fixture_agent.py", "/tmp/fixture_agent.py"), + (HERE / "artifact_server.py", "/tmp/artifact_server.py")]: + await builder.write_host_file(source.read_bytes(), target) + with (output / "build.stdout").open("w") as stdout, (output / "build.stderr").open("w") as stderr: + result = await builder._sandbox.run_process( + "bash", ["-lc", (HERE / "build_images.sh").read_text()], + sudo=True, kill_after=1800, stdout=stdout, stderr=stderr) + save("build-command.json", {"command_id": result.id, "session_id": result.session_id, + "exit_code": result.returncode}) + assert result.returncode == 0, "Image build failed; see build logs and retained builder" + manifest = json.loads(await builder.exec_script("cat /tmp/poc-artifacts/manifest.json")) + assert len(manifest) == 6 + save("images.json", manifest) + await builder.exec_script("nohup python3 /tmp/artifact_server.py >/tmp/artifact-server.log 2>&1 None: await self._sandbox.update_network_policy(vercel_network_policy(widened)) self.network_policy = widened - async def load_docker_images(self, artifacts: list) -> None: - if not artifacts: - return + async def _load_tarballs(self, artifacts: list) -> None: signed_urls = await self._signed_image_urls(artifacts) await self._allow_download_hosts(signed_urls) await self._load_docker_images(artifacts, signed_urls) diff --git a/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py b/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py index 19bd946b..09109537 100644 --- a/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py @@ -19,6 +19,7 @@ SandboxStreamError, ) +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.providers.sandbox_providers import vercel as vercel_module from agent_env.providers.sandbox_providers.vercel import sandbox as vercel_sandbox_module from agent_env.providers.sandbox_providers.sandbox import NetworkMode, NetworkPolicy @@ -96,6 +97,11 @@ def _sandbox(raw=None, policy=NetworkPolicy()): return VercelSandbox(raw, client=client, tunnel_urls={}, network_policy=policy) +def _tarball(): + return DockerImageArtifact(id="image", description="", image_name="image:v1", + tar_gz_object_url="file:///store/image.tar.gz") + + @pytest.mark.asyncio async def test_exec_returns_exact_unicode_bytes_and_nonzero_exit(): raw = _raw() @@ -171,7 +177,7 @@ async def test_restrictive_image_loading_adds_only_signed_download_hosts(monkeyp lambda policy: policy.with_hosts(["bucket.example"]), ) - await sandbox.load_docker_images(["artifact"]) + await sandbox.load_docker_images([_tarball()]) raw.update_network_policy.assert_awaited_once() applied = raw.update_network_policy.await_args.args[0] @@ -184,7 +190,7 @@ async def test_unknown_reconnected_policy_stays_unknown_for_image_loading(): sandbox = _sandbox(policy=None) sandbox._signed_image_urls = AsyncMock(return_value=["https://bucket.example/image"]) with pytest.raises(RuntimeError, match="applied network policy is unknown"): - await sandbox.load_docker_images(["artifact"]) + await sandbox.load_docker_images([_tarball()]) @pytest.mark.asyncio @@ -497,7 +503,7 @@ async def load(artifacts, urls): await sandbox._download_object_to_vm("file:///store/image", "/tmp/image") sandbox._load_docker_images = load - await asyncio.wait_for(sandbox.load_docker_images(["image"]), 1) + await asyncio.wait_for(sandbox.load_docker_images([_tarball()]), 1) sandbox._write_unsigned_object.assert_awaited_once_with(store, "file:///store/image", "/tmp/image") sandbox._sandbox.update_network_policy.assert_not_awaited() @@ -538,3 +544,45 @@ async def execute(script): assert len(uploads) == 3 assert destination.read_bytes() == payload assert all(not Path(path).exists() for path in uploads) + + +@pytest.mark.asyncio +async def test_registry_and_context_images_use_the_vm_loading_contract(monkeypatch): + sandbox = _sandbox(policy=NetworkPolicy(mode=NetworkMode.ALLOWLIST, allow_hosts=("registry.example",))) + sandbox.exec_script = AsyncMock(return_value="") + sandbox._signed_image_urls = AsyncMock(side_effect=AssertionError("non-tarball image was signed")) + store = MagicMock(signed_get_url=MagicMock(return_value="https://bucket.example/context.tar.gz")) + registry = MagicMock(auth=MagicMock(return_value=None)) + config = MagicMock(get_object_store_at=MagicMock(return_value=store), + get_image_store=MagicMock(return_value=registry)) + monkeypatch.setattr(vercel_sandbox_module, "get_config", lambda: config) + monkeypatch.setattr("agent_env.providers.sandbox_providers.sandbox.get_config", lambda: config) + reference = DockerImageArtifact(id="ref", description="", image_name="registry.example/tool:v1") + context = DockerImageArtifact(id="context", description="", image_name="context:v1", + build_context_object_url="file:///store/context.tar.gz") + + await sandbox.load_docker_images([reference, context]) + + scripts = [call.args[0] for call in sandbox.exec_script.await_args_list] + assert "docker pull registry.example/tool:v1" in scripts + assert any("docker build --platform linux/amd64 -f Dockerfile -t context:v1 ." in script for script in scripts) + assert any("https://bucket.example/context.tar.gz" in script and "curl -fsSL" in script for script in scripts) + store.signed_get_url.assert_called_once_with("file:///store/context.tar.gz") + sandbox._signed_image_urls.assert_not_awaited() + assert sandbox.network_policy.allow_hosts == ("registry.example", "bucket.example") + + +@pytest.mark.asyncio +async def test_unobtainable_image_is_refused_before_loading_any_image(): + sandbox = _sandbox() + sandbox._signed_image_urls = AsyncMock(side_effect=AssertionError("validation was bypassed")) + sandbox.exec_script = AsyncMock() + tarball = DockerImageArtifact(id="tar", description="", image_name="tar:v1", + tar_gz_object_url="file:///store/image.tar.gz") + missing = DockerImageArtifact(id="missing", description="", image_name="missing:v1") + + with pytest.raises(RuntimeError, match="Can't load images: 'missing'"): + await sandbox.load_docker_images([tarball, missing]) + + sandbox._signed_image_urls.assert_not_awaited() + sandbox.exec_script.assert_not_awaited() diff --git a/uv.lock b/uv.lock index 8669313e..b8e960fb 100644 --- a/uv.lock +++ b/uv.lock @@ -11,10 +11,6 @@ resolution-markers = [ "(python_full_version < '3.12' and sys_platform == 'emscripten') or (python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32')", ] -[options] -exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. -exclude-newer-span = "P2D" - [manifest] members = [ "agentenv-framework", From 62cce9965d42fbcab1787e052b8be0945160c0fa Mon Sep 17 00:00:00 2001 From: Elisabeth Rulke Date: Mon, 12 Oct 2026 21:57:32 -0700 Subject: [PATCH 3/4] fix(vercel): address provider review feedback --- README.md | 6 +- examples/vercel-sandbox/README.md | 66 ----- examples/vercel-sandbox/RESULTS.md | 23 -- examples/vercel-sandbox/artifact_server.py | 42 --- examples/vercel-sandbox/build_images.sh | 40 --- examples/vercel-sandbox/config.toml | 3 - examples/vercel-sandbox/fixture_agent.py | 48 ---- examples/vercel-sandbox/run_e2e.py | 242 ------------------ examples/vercel-sandbox/with_vercel_login.py | 27 -- .../sandbox_providers/sandbox_provider.py | 4 + .../sandbox_providers/vercel/provider.py | 21 +- .../sandbox_providers/vercel/sandbox.py | 8 +- src/agent_env/task/teardown.py | 7 +- .../task_steps/teardown_sandboxes.py | 6 +- .../vercel_sandbox_provider_test.py | 94 ++++++- .../sandbox_providers/vercel_sandbox_test.py | 19 ++ tst/unit/test_capabilities.py | 2 +- 17 files changed, 153 insertions(+), 505 deletions(-) delete mode 100644 examples/vercel-sandbox/README.md delete mode 100644 examples/vercel-sandbox/RESULTS.md delete mode 100644 examples/vercel-sandbox/artifact_server.py delete mode 100644 examples/vercel-sandbox/build_images.sh delete mode 100644 examples/vercel-sandbox/config.toml delete mode 100644 examples/vercel-sandbox/fixture_agent.py delete mode 100644 examples/vercel-sandbox/run_e2e.py delete mode 100644 examples/vercel-sandbox/with_vercel_login.py diff --git a/README.md b/README.md index adee5ec6..16cc633c 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ The distribution is named `agentenv-framework`, the import package is `agent_env ### Sandbox providers -For a complete evaluation using your existing Vercel login, follow the [Vercel Sandbox demo](examples/vercel-sandbox/README.md). +For a complete evaluation using your existing Vercel login, follow the [Vercel Sandbox demo](https://github.com/vercel-labs/agentenv-framework/tree/vercel-demo/examples/vercel-sandbox). The default provider runs environments and agents as containers on the local Docker daemon. The `vercel` provider runs each environment or agent in an ephemeral, Docker-capable Vercel Sandbox and exposes declared ports through public `*.vercel.run` routes. From this source checkout, install the optional extra and select it in `.agentenv/config.toml`. This provider is not yet in a published PyPI release: @@ -68,6 +68,10 @@ Credentials are resolved by AgentEnv's `env:` and `secret:` references before th See the [Vercel resource limits](https://vercel.com/docs/sandbox/pricing#resource-limits) and [firewall policy](https://vercel.com/docs/sandbox/concepts/firewall) documentation. The provider treats CPU and memory requests as minimums. If a request does not already match a supported shape, AgentEnv logs both the requested and allocated CPU and memory before provisioning. For example, `cpu = 0.5` and `memory = 5000` allocate four vCPUs and 8192 MB. A disk request below 64 GB is also ignored: every sandbox receives the fixed 64 GB disk, and the provider logs the ignored request. Configure all three of `token`, `team_id`, and `project_id` for an explicit scoped token, or omit all three to continue using the SDK's OIDC credentials. +Attribution is limited to five tags. When more are supplied, the provider keeps `pipeline_step` and `run_id` first, then fills the remaining slots by key name in alphabetical order. It warns with the omitted key names without logging their values. Invalid empty keys or values are still rejected. + +A missing sandbox raises `SandboxNotFoundError` when reconnecting. Run cleanup and the `teardown_sandboxes` step treat this condition as already cleaned up; authentication and transport errors still fail cleanup. + Outbound policy supports AgentEnv's allow-all and allowlist forms, including hostname and CIDR entries. Vercel's custom transforms, forwarding rules, and denied CIDR exceptions are not reconstructed as ordinary allowlists: reconnect leaves such a policy unknown and image loading fails closed. At creation, public Vercel routes are added to a restricted workload's platform egress floor so it can reach another Vercel sandbox's declared route. Image and file downloads add their exact signed-URL hostnames to the current allowlist; these hosts remain allowed until the policy is replaced. A CIDR-only Vercel policy allows unrestricted DNS resolution; use domain rules to restrict DNS, or deny-all to block it. To work on agent-env itself, install from a clone with [uv](https://docs.astral.sh/uv/): diff --git a/examples/vercel-sandbox/README.md b/examples/vercel-sandbox/README.md deleted file mode 100644 index bacb46c9..00000000 --- a/examples/vercel-sandbox/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# AgentEnv on Vercel Sandbox - -This candidate adds the `vercel` sandbox provider to AgentEnv. AgentEnv runs the task, connects the agent to its environment, collects the trajectory and scores the result. Vercel provides the separate sandboxes running the environment and agent. - -The included demo exercises one complete evaluation: deploy an items environment behind a gateway with its database services, load `starting-item`, deploy a deterministic agent, ask it to add `vercel-review-complete` through MCP, and grade its response. The controller independently reads the final state, verifies the stored task and trajectory, reconnects, and tears down the successful run. The agent performs real MCP calls; it does not call a language model. This validates the integration, not model quality. - -## Setup - -Prerequisites: Git, Python 3.11.4 or newer, uv, Node.js and Vercel CLI 62.5.0 or newer (the demo was tested with 62.5.0). Use a Vercel project where you can create 4-vCPU/8-GB sandboxes for the build and environment, plus a 2-vCPU/4-GB agent. The demo builds Linux images in a fresh Vercel sandbox, so local Docker is not needed. Network access to the package and image registries referenced by the repository Dockerfiles is required. - -Clone this branch to use the unreleased provider. The published PyPI package does not contain this candidate. - -```bash -git clone --branch feat/vercel-sandbox https://github.com/vercel-labs/agentenv-framework.git -cd agentenv-framework -uv sync --frozen --extra vercel -export AGENT_ENV_CONFIG="$PWD/examples/vercel-sandbox/config.toml" -``` - -The dedicated config selects Vercel for environments and agents. The demo overrides the document and object stores with local stores in its output directory. - -## Run with an existing Vercel login - -If needed, run `vercel login`. Choose an existing project and team that you have access to. The wrapper uses the public `vercel project token` command and passes its short-lived OIDC token only to the child process. It does not print the token or write it to disk. - -Use a fresh output directory for each attempt; the demo refuses to overwrite an existing one. Keep the dedicated configuration selected above so unrelated settings cannot affect the demo. Do not add explicit `token`, `team_id` or `project_id` fields for this OIDC route. - -```bash -.venv/bin/python examples/vercel-sandbox/with_vercel_login.py \ - --scope YOUR_TEAM --project YOUR_PROJECT -- \ - .venv/bin/python examples/vercel-sandbox/run_e2e.py \ - --repo . --output /tmp/agentenv-vercel-e2e-run-1 -``` - -The demo creates three temporary sandboxes: an image builder, an environment and an agent. Successful runs remove all three and verify subsequent lookups return 404. Failed runs stop and retain tracked sandboxes and record their identifiers for diagnosis. The runner also records handles reaching its termination guard during setup rollback. Failures before a handle reaches the runner can still follow the provider’s own creation-cleanup behavior. Retained resources are intentionally left for diagnosis; this demo does not automatically reap them. - -The image-download fixture serves only disposable test images. Its fixed signing key is public test data; this fixture is not a production artifact store. The demo uses local SQLite and filesystem stores for run records and artifacts, and applies a restricted policy allowing the test package host and Vercel routes to the workload sandboxes. Scale's actual stores and network requirements need their own acceptance run. - -## What passing means - -Run with normal Python execution; the runner refuses `python -O` because its assertions are acceptance checks. - -Expected final output: - -```text -END-TO-END EVALUATION PASSED: score 1 -``` - -The output folder retains: - -- `success.json`: final pass, score, run identifier and cleanup assertions. -- `task.json`, `task-instance.json` and `context.json`: the five task steps and recorded result. -- `environment-state.json`: exactly `starting-item` and `vercel-review-complete`. -- `objects/`: collected task artifacts, including the agent trajectory. -- `resources.json`, `images.json` and `source-hashes.json`: resource, image and source provenance. -- `build.stdout`, `build.stderr` and `build-command.json`: image-build evidence. - -The stdout/stderr stream can also be redirected to a local log. Output folders contain deployment coordinates and private local paths; review them before sharing. Keep these generated outputs outside the checkout. - -## Review scope - -The integration remains in-tree as requested, with optional Python SDK dependencies. CPU and memory are minimum requests rounded to supported shapes; fixed disk sizing is logged. OIDC and explicit configuration credentials are supported. This POC covers ephemeral runs and active reconnect. Persistent resume, credential brokering and model-quality benchmarking are separate work. - -Recorded follow-ups include cleanup of commands still running during teardown, upload retry diagnostics, resource log severity and readiness behavior. Production workload acceptance, actual stores, CPU/memory semantics, optional OIDC and maintenance ownership remain for Scale to confirm. - -See [RESULTS.md](RESULTS.md) for the validation scope and remaining acceptance work. diff --git a/examples/vercel-sandbox/RESULTS.md b/examples/vercel-sandbox/RESULTS.md deleted file mode 100644 index d118589b..00000000 --- a/examples/vercel-sandbox/RESULTS.md +++ /dev/null @@ -1,23 +0,0 @@ -# End-to-end result, October 9, 2026 - -The reviewed provider completed a real AgentEnv evaluation using project OIDC obtained from an existing Vercel CLI login. No manually created API token or model key was needed. - -| Step | Observed result | -| --- | --- | -| Deploy environment | Gateway, items MCP server and database services running in Vercel Sandbox | -| Load data | Initial state contains `starting-item` | -| Deploy agent | A separate Vercel Sandbox runs the deterministic review agent | -| Prompt agent | Agent calls `items_add_item` and `list_items` through the environment's public MCP route | -| Grade response | Task status `completed`, all five steps successful, score **1.0** | - -The controller independently read the final environment state as `["starting-item", "vercel-review-complete"]`. AgentEnv collected the native trajectory into its local object store and retained the task-instance result. Fresh-provider reconnect restored routes and policy. Teardown removed the agent, environment and image builder; all three subsequent lookups returned 404. Repeated teardown succeeded. - -The initial attempt exposed a missing MCP client dependency in the new test-agent image. The dependency was added and checked at build time; two subsequent full runs passed. The final runner also passed an injected setup-failure test that verifies recording and retention before create returns. The initial diagnostic resources were stopped and retained. No provider implementation changes were needed for this evaluation. - -The publication candidate is based on upstream commit `c70011524ae65fbd35eb2b8292c9560eb81967cc`. Upstream changes and dependency metadata were incorporated before publication. The provider now inherits the current VM image-loading contract, preserving registry pulls, builds from stored contexts and early refusal of unavailable images. The five demo scripts are unchanged. The complete evaluation above passed again on this base using a fresh installation with only the `vercel` extra. After the compatibility correction, 171 focused provider, policy, capability and VM tests passed. A separate live Vercel probe pulled and ran a registry image, built a context image and checked its contents, rejected an unavailable image, and verified cleanup with a 404 lookup. The plugin API check reported no break. - -Earlier broad verification on base `6e20760bebacd73ebb74e8add0bd69407b24baa4`: 6,810 unit/protocol passes with 9 declared skips, 83 provider-specific passes, all 60 installer cases passed across two runs, build and clean installation passed. These historical results do not establish the outcome of the broader suite on the newer base. See the upstream PR for current validation and CI status. - -This is a deterministic integration evaluation. It does not benchmark a language model or establish acceptance of Scale's production workloads, image/object stores, mixed-provider setup, persistence or credential brokering. Maintainers should review the documented CPU/memory minimum semantics, OIDC option, operational ownership and recorded provider follow-ups, particularly commands still running during teardown and upload retry diagnostics. - -Private run logs, deployment coordinates, local state and diagnostic resource identifiers are excluded from the repository. The demo scripts produce those records for each reviewer's own run. diff --git a/examples/vercel-sandbox/artifact_server.py b/examples/vercel-sandbox/artifact_server.py deleted file mode 100644 index 1c906601..00000000 --- a/examples/vercel-sandbox/artifact_server.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Serve only disposable review fixture image archives, with short-lived signatures.""" - -import hashlib -import hmac -import shutil -import time -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from urllib.parse import parse_qs, urlsplit - -ROOT = Path("/tmp/poc-artifacts") -KEY = b"agentenv-poc-public-image-fixture" -ALLOWED = {"manifest.json", *(name + ".tar.gz" for name in ( - "poc-gateway", "poc-db", "poc-db-web", "poc-db-mcp", "poc-items", "poc-agent"))} - - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - request = urlsplit(self.path) - name = request.path.removeprefix("/") - query = parse_qs(request.query) - expires = query.get("expires", ["0"])[0] - signature = query.get("signature", [""])[0] - expected = hmac.new(KEY, (name + ":" + expires).encode(), hashlib.sha256).hexdigest() - if name not in ALLOWED: - self.send_error(404) - return - if not expires.isdigit() or int(expires) < time.time() or not hmac.compare_digest(signature, expected): - self.send_error(403) - return - path = ROOT / name - if not path.is_file(): - self.send_error(404) - return - self.send_response(200) - self.send_header("Content-Length", str(path.stat().st_size)) - self.end_headers() - with path.open("rb") as source: - shutil.copyfileobj(source, self.wfile) - - -ThreadingHTTPServer(("0.0.0.0", 8080), Handler).serve_forever() diff --git a/examples/vercel-sandbox/build_images.sh b/examples/vercel-sandbox/build_images.sh deleted file mode 100644 index 0a206bcf..00000000 --- a/examples/vercel-sandbox/build_images.sh +++ /dev/null @@ -1,40 +0,0 @@ -set -euo pipefail -mkdir -p /tmp/agentenv /tmp/poc-artifacts /tmp/poc-logs -cd /tmp/agentenv -tar -xzf /tmp/source.tar.gz -apt-get update -qq -DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker-buildx -build_one() { - docker build -f "$1" -t "$3" "$2" > "/tmp/poc-logs/$3-build.log" 2>&1 - docker save "$3" | gzip -1 > "/tmp/poc-artifacts/$3.tar.gz" - docker image inspect "$3" --format '{{.Id}}' > "/tmp/poc-artifacts/$3.image-id" - echo "BUILT $3" -} -build_one src/agent_env/env/gateway/Dockerfile src/agent_env/env poc-gateway -build_one src/agent_env/env/envs/service_db/Dockerfile src/agent_env/env/envs/service_db poc-db -build_one src/agent_env/env/envs/service_db/Dockerfile.db-web src/agent_env/env/envs/service_db poc-db-web -build_one src/agent_env/env/envs/service_db/Dockerfile.db-mcp src/agent_env/env/envs/service_db poc-db-mcp -mkdir -p /tmp/items-context /tmp/agent-context -cp -R packages/agentenv-protocol/src/agentenv_protocol /tmp/items-context/agentenv_protocol -cp tst/data/agentenv_mcp/Dockerfile tst/data/agentenv_mcp/server.py tst/data/agentenv_mcp/seed.json /tmp/items-context/ -build_one /tmp/items-context/Dockerfile /tmp/items-context poc-items -cp -R packages/agentenv-protocol /tmp/agent-context/agentenv-protocol -cp tst/data/a2a_agent/Dockerfile /tmp/agent-context/Dockerfile -cp /tmp/fixture_agent.py /tmp/agent-context/agent.py -python3 - <<'PYDOCKER' -from pathlib import Path -p=Path('/tmp/agent-context/Dockerfile') -s=p.read_text() -s=s.replace('USER agent', 'RUN pip install --no-cache-dir mcp==1.28.1\nRUN python -c "from mcp.client.streamable_http import streamable_http_client; import agent"\nUSER agent') -p.write_text(s) -PYDOCKER -build_one /tmp/agent-context/Dockerfile /tmp/agent-context poc-agent -python3 - <<'PY' -from pathlib import Path -import json,hashlib -p=Path('/tmp/poc-artifacts') -manifest={f.stem.removesuffix('.tar'):{'sha256':hashlib.sha256(f.read_bytes()).hexdigest(),'bytes':f.stat().st_size,'image_id':f.with_name(f.stem.removesuffix('.tar')+'.image-id').read_text().strip()} for f in p.glob('*.tar.gz')} -(p/'manifest.json').write_text(json.dumps(manifest,indent=2)) -assert len(manifest)==6 -print('ALL SIX IMAGE BUILDS PASSED') -PY diff --git a/examples/vercel-sandbox/config.toml b/examples/vercel-sandbox/config.toml deleted file mode 100644 index 49333b95..00000000 --- a/examples/vercel-sandbox/config.toml +++ /dev/null @@ -1,3 +0,0 @@ -[sandbox] -default = "vercel" -agent_default = "vercel" diff --git a/examples/vercel-sandbox/fixture_agent.py b/examples/vercel-sandbox/fixture_agent.py deleted file mode 100644 index 12a7ea4c..00000000 --- a/examples/vercel-sandbox/fixture_agent.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Deterministic review agent: add one item through the configured MCP server.""" - -import json - -import httpx -from mcp import ClientSession -from mcp.client.streamable_http import streamable_http_client -from agentenv_protocol.a2a_agent import ( - MCP_CONFIG_V1, TRAJECTORY_V1, AgentConfig, AgentEnvAgent, - AgentIdentity, TaskRequest, TaskResult, TextPart, Usage, a2a_agent, -) - - -@a2a_agent( - identity=AgentIdentity(name="vercel-review-agent", version="1.0.0", - description="Adds the requested item through MCP and reports stored state."), - config=AgentConfig, - extensions=(MCP_CONFIG_V1, TRAJECTORY_V1), -) -class ReviewAgent(AgentEnvAgent): - async def run(self, request: TaskRequest[AgentConfig]) -> TaskResult: - item = "".join(part.text for part in request.parts if isinstance(part, TextPart)).strip() - if len(request.mcp_servers) != 1 or not item: - raise ValueError("The review task requires one MCP server and an item name") - registration = next(iter(request.mcp_servers.values())) - async with httpx.AsyncClient(headers=registration.get("headers") or {}, timeout=60) as client: - async with streamable_http_client(registration["url"], http_client=client) as (read, write, _): - async with ClientSession(read, write) as session: - await session.initialize() - tools = await session.list_tools() - names = [tool.name for tool in tools.tools] - assert "items_add_item" in names and "list_items" in names, names - added = await session.call_tool("items_add_item", {"item": item}) - assert not added.isError, added - listed = await session.call_tool("list_items", {}) - assert not listed.isError, listed - text = "\n".join(c.text for c in listed.content if c.type == "text") - reply = "Stored items: " + text - return (TaskResult.builder().succeeded().parts([TextPart(text=reply)]) - .usage(Usage(tool_call_count=2, input_tokens=0, output_tokens=0)) - .native_trajectory(format="vercel-review/v1", payload=[ - {"tool": "items_add_item", "arguments": {"item": item}}, - {"tool": "list_items", "response": json.loads(text)}, - ]).build()) - - -if __name__ == "__main__": - ReviewAgent().serve() diff --git a/examples/vercel-sandbox/run_e2e.py b/examples/vercel-sandbox/run_e2e.py deleted file mode 100644 index d9bec9c5..00000000 --- a/examples/vercel-sandbox/run_e2e.py +++ /dev/null @@ -1,242 +0,0 @@ -"""Build fixtures and run a scored AgentEnv task on Vercel; retain any failures.""" - -import argparse -import asyncio -import hashlib -import hmac -import json -import logging -import subprocess -import tarfile -import time -from dataclasses import asdict -from pathlib import Path -from urllib.parse import urlencode - -import httpx -from agentenv_protocol import client as protocol_v1 -from agent_env.a2a_agent import A2AAgent -from agent_env.artifact import DockerImageArtifact, EnvironmentArtifact, FileArtifact -from agent_env.config import configure -from agent_env.env import Env, GatewayEnv, legacy_protocol -from agent_env.env.envs.mcp_server import MCPServerEnv -from agent_env.env.envs.service_db import ServiceDBEnv -from agent_env.providers.sandbox_providers.sandbox import NetworkMode, NetworkPolicy -from agent_env.providers.sandbox_providers.vercel import VercelSandbox, VercelSandboxProvider -from agent_env.store.document_store import LocalSqliteDocumentStore -from agent_env.store.object_store import LocalFilesystemObjectStore -from agent_env.task import Task -from agent_env.task.teardown import teardown_run -from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep -from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep -from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep -from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep -from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import AgentPromptResponseVerifierTaskStep - -HERE = Path(__file__).resolve().parent -KEY = b"agentenv-poc-public-image-fixture" - - -async def main(): - if not __debug__: - raise SystemExit("Run without Python -O; this validation requires assertions") - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repo", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - repo, output = args.repo.resolve(), args.output.resolve() - output.mkdir(parents=True, exist_ok=False) - logging.basicConfig(level=logging.INFO) - logging.getLogger("httpx").setLevel(logging.WARNING) - def save(name, value): - (output / name).write_text(json.dumps(value, indent=2, default=str) + "\n") - - tracked = subprocess.check_output( - ["git", "-C", str(repo), "ls-files", "-z", "--cached", "--others", "--exclude-standard"] - ).decode().split("\0") - tracked = [name for name in tracked if name and (repo / name).is_file() - and not any(part.startswith(".env") for part in Path(name).parts)] - save("source-hashes.json", {name: hashlib.sha256((repo / name).read_bytes()).hexdigest() for name in tracked}) - with tarfile.open(output / "source.tar.gz", "w:gz") as archive: - for name in tracked: - archive.add(repo / name, arcname=name, recursive=False) - - resources, owned, clients = [], [], [] - phase = "build" - cleanup = False - completed = False - create_vm = VercelSandboxProvider.create_vm - terminate = VercelSandbox.terminate - - def record_resource(box, *, setup_failed=False): - if not any(known.sandbox_id == box.sandbox_id for known in owned): - owned.append(box) - resources.append({"name": box.sandbox_id, "session_id": box._sandbox.current_session_id, - "phase": phase, "routes": box.tunnel_urls, "created_at": time.time(), - "setup_failed": setup_failed}) - save("resources.json", resources) - - async def tracked_create(provider, **kwargs): - if provider not in clients: - clients.append(provider) - box = await create_vm(provider, **kwargs) - record_resource(box) - if phase == "task": - await box.apply_network_policy(NetworkPolicy( - mode=NetworkMode.ALLOWLIST, allow_hosts=("pypi.org", "*.vercel.run"))) - print("CREATED", phase, box.sandbox_id, flush=True) - return box - - async def guarded_terminate(box): - if cleanup: - return await terminate(box) - record_resource(box, setup_failed=True) - print("RETAINING until assertions pass:", box.sandbox_id, flush=True) - - VercelSandboxProvider.create_vm = tracked_create - VercelSandbox.terminate = guarded_terminate - controller = VercelSandboxProvider.from_config() - clients.append(controller) - try: - builder = await controller.create_vm(cpu=4, memory=8192, exposed_ports=[8080], timeout=2400) - for source, target in [(output / "source.tar.gz", "/tmp/source.tar.gz"), - (HERE / "fixture_agent.py", "/tmp/fixture_agent.py"), - (HERE / "artifact_server.py", "/tmp/artifact_server.py")]: - await builder.write_host_file(source.read_bytes(), target) - with (output / "build.stdout").open("w") as stdout, (output / "build.stderr").open("w") as stderr: - result = await builder._sandbox.run_process( - "bash", ["-lc", (HERE / "build_images.sh").read_text()], - sudo=True, kill_after=1800, stdout=stdout, stderr=stderr) - save("build-command.json", {"command_id": result.id, "session_id": result.session_id, - "exit_code": result.returncode}) - assert result.returncode == 0, "Image build failed; see build logs and retained builder" - manifest = json.loads(await builder.exec_script("cat /tmp/poc-artifacts/manifest.json")) - assert len(manifest) == 6 - save("images.json", manifest) - await builder.exec_script("nohup python3 /tmp/artifact_server.py >/tmp/artifact-server.log 2>&1 ]`` key.""" +class SandboxNotFoundError(LookupError): + """The requested sandbox no longer exists; cleanup may treat it as already done.""" + + class SandboxProvider(ABC): """Compute backend that provisions sandboxes. Selected via [sandbox] default / build_sandbox_provider(); swap via set_sandbox_provider().""" diff --git a/src/agent_env/providers/sandbox_providers/vercel/provider.py b/src/agent_env/providers/sandbox_providers/vercel/provider.py index 98a71e64..6fca2cf0 100644 --- a/src/agent_env/providers/sandbox_providers/vercel/provider.py +++ b/src/agent_env/providers/sandbox_providers/vercel/provider.py @@ -8,13 +8,14 @@ import uuid from typing import Any, Callable, ClassVar, Self -from agent_env.attribution import Attribution +from agent_env.attribution import Attribution, PIPELINE_STEP_KEY, RUN_ID_KEY from agent_env.config.errors import ConfigError from agent_env.providers.sandbox_providers.sandbox import ( NetworkPolicy, NetworkPolicyUnsupportedError, ) from agent_env.providers.sandbox_providers.sandbox_provider import ( + SandboxNotFoundError, SandboxProvider, apply_default_attribution, ) @@ -250,10 +251,17 @@ def _tags(attribution: Attribution | None) -> dict[str, str]: for key, value in apply_default_attribution(dict(attribution or {})).items() if value is not None } - if len(tags) > _MAX_TAGS: - raise ValueError(f"Vercel accepts at most {_MAX_TAGS} attribution tags") if any(not key or not value for key, value in tags.items()): raise ValueError("attribution keys and values must be non-empty") + if len(tags) > _MAX_TAGS: + preferred = [key for key in (PIPELINE_STEP_KEY, RUN_ID_KEY) if key in tags] + keys = preferred + sorted(key for key in tags if key not in preferred) + logger.warning( + "Vercel accepts at most %s attribution tags; omitting keys: %s", + _MAX_TAGS, + ", ".join(keys[_MAX_TAGS:]), + ) + tags = {key: tags[key] for key in keys[:_MAX_TAGS]} return tags async def create_vm( @@ -392,7 +400,12 @@ async def create_sandbox( ) async def get_sandbox(self, sandbox_id: str) -> VercelSandbox: - raw = await self._get_client().get_sandbox(name=sandbox_id) + try: + raw = await self._get_client().get_sandbox(name=sandbox_id) + except Exception as error: + if _is_missing_resource(error): + raise SandboxNotFoundError(f"Vercel sandbox {sandbox_id!r} was not found") from error + raise applied = network_policy_from_vercel(raw.network_policy) if applied is None: logger.warning( diff --git a/src/agent_env/providers/sandbox_providers/vercel/sandbox.py b/src/agent_env/providers/sandbox_providers/vercel/sandbox.py index 74ef0716..573e7f9c 100644 --- a/src/agent_env/providers/sandbox_providers/vercel/sandbox.py +++ b/src/agent_env/providers/sandbox_providers/vercel/sandbox.py @@ -27,6 +27,8 @@ logger = logging.getLogger(__name__) _STREAM_CHUNK_BYTES = 2 * 1024 * 1024 +_UPLOAD_ATTEMPTS = 3 +_UPLOAD_RETRY_DELAY_SECONDS = 0.25 _DOCKER_SETUP_TIMEOUT = 300 _policy_locks: weakref.WeakValueDictionary[ tuple[asyncio.AbstractEventLoop, str], asyncio.Lock @@ -287,15 +289,15 @@ async def _write_stream_to_vm(self, source: Any, vm_path: str) -> None: await self.exec_script(f": > {shlex.quote(staged)}") while chunk := await asyncio.to_thread(source.read, _STREAM_CHUNK_BYTES): digest.update(chunk) - for attempt in range(3): + for attempt in range(_UPLOAD_ATTEMPTS): try: await self._sandbox.fs.write_bytes(chunk_path, chunk) break except Exception: # SDK transport failures have several types; only the replaceable part is retried. - if attempt == 2: + if attempt + 1 == _UPLOAD_ATTEMPTS: raise - await asyncio.sleep(0.25 * 2**attempt) + await asyncio.sleep(_UPLOAD_RETRY_DELAY_SECONDS * 2**attempt) await self.exec_script(f"cat {shlex.quote(chunk_path)} >> {shlex.quote(staged)}") await self.exec_script( f"printf '%s %s\n' {digest.hexdigest()} {shlex.quote(staged)} | sha256sum -c - >/dev/null " diff --git a/src/agent_env/task/teardown.py b/src/agent_env/task/teardown.py index 4e6ea006..5652f1e5 100644 --- a/src/agent_env/task/teardown.py +++ b/src/agent_env/task/teardown.py @@ -17,7 +17,7 @@ from agent_env.a2a_agent.staging import drain_all, staged_changelogs from agent_env.env.env import DeployedEnv from agent_env.providers.sandbox_providers.local_sandbox import LocalSandbox, remove_local_work_dir -from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider +from agent_env.providers.sandbox_providers.sandbox_provider import SandboxNotFoundError, build_sandbox_provider from agent_env.task.interrupts import tearing_down from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.teardown_sandboxes import _DEFAULT_PROVIDERS, TORN_DOWN_KEY, _env_sandbox_ids @@ -136,7 +136,10 @@ async def _remove_folder(sandbox: RecordedSandbox) -> str | None: async def _terminate(sandbox: RecordedSandbox): provider = (build_sandbox_provider(sandbox.sandbox_type) if sandbox.sandbox_type else _DEFAULT_PROVIDERS[sandbox.slot]()) - handle = await provider.get_sandbox(sandbox.sandbox_id) + try: + handle = await provider.get_sandbox(sandbox.sandbox_id) + except SandboxNotFoundError: + return None await handle.terminate() return handle diff --git a/src/agent_env/task_step/task_steps/teardown_sandboxes.py b/src/agent_env/task_step/task_steps/teardown_sandboxes.py index 7f8f1539..be2755c2 100644 --- a/src/agent_env/task_step/task_steps/teardown_sandboxes.py +++ b/src/agent_env/task_step/task_steps/teardown_sandboxes.py @@ -18,6 +18,7 @@ from agent_env.entity_refs import EntityRef from agent_env.env.env import DeployedSandboxEnv from agent_env.providers.sandbox_providers.sandbox_provider import ( + SandboxNotFoundError, SandboxProvider, build_sandbox_provider, get_agent_sandbox_provider, @@ -169,5 +170,8 @@ async def _terminate(sandbox_id: str, sandbox_type: Optional[str], default_slot: provider: SandboxProvider = ( build_sandbox_provider(sandbox_type) if sandbox_type else _DEFAULT_PROVIDERS[default_slot]() ) - sandbox = await provider.get_sandbox(sandbox_id) + try: + sandbox = await provider.get_sandbox(sandbox_id) + except SandboxNotFoundError: + return await sandbox.terminate() diff --git a/tst/unit/providers/sandbox_providers/vercel_sandbox_provider_test.py b/tst/unit/providers/sandbox_providers/vercel_sandbox_provider_test.py index ca62b5e7..382e3612 100644 --- a/tst/unit/providers/sandbox_providers/vercel_sandbox_provider_test.py +++ b/tst/unit/providers/sandbox_providers/vercel_sandbox_provider_test.py @@ -8,10 +8,13 @@ import pytest from agent_env.config.errors import ConfigError +from agent_env.task import teardown +from agent_env.task_step.context import DeployedSandbox, TaskStepContext +from agent_env.task_step.task_steps import teardown_sandboxes from agent_env.providers.sandbox_providers import vercel as vercel_module from agent_env.providers.sandbox_providers.vercel import sandbox as vercel_sandbox_module from agent_env.providers.sandbox_providers.sandbox import NetworkMode, NetworkPolicy -from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider +from agent_env.providers.sandbox_providers.sandbox_provider import SandboxNotFoundError, build_sandbox_provider from agent_env.providers.sandbox_providers.vercel.provider import ( VercelSandboxProvider, resource_shape, @@ -257,7 +260,6 @@ async def test_close_closes_the_long_lived_sdk_client(): ({"timeout": 0}, "timeout"), ({"exposed_ports": list(range(1, 17))}, "at most 15"), ({"exposed_ports": [0]}, "ports must be integers"), - ({"attribution": {f"k{i}": "v" for i in range(6)}}, "at most 5"), ], ) @pytest.mark.asyncio @@ -305,3 +307,91 @@ async def close(): release.set() await asyncio.wait_for(closing, timeout=1) assert order == ["destroy", "close"] + + +def test_excess_tags_keep_run_and_step_then_sorted_keys_and_warn_without_values(caplog): + original = {"z": "private-value", "run_id": "run", "d": "4", "pipeline_step": "step", "c": "3", "b": "2", "a": "1"} + with caplog.at_level(logging.WARNING): + tags = VercelSandboxProvider._tags(original) + reordered = VercelSandboxProvider._tags(dict(reversed(list(original.items())))) + assert tags == reordered == {"pipeline_step": "step", "run_id": "run", "a": "1", "b": "2", "c": "3"} + assert len(original) == 7 + assert "d, z" in caplog.text + assert "private-value" not in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("entry", ["run", "step"]) +async def test_cleanup_records_an_already_deleted_sandbox_as_torn_down(monkeypatch, entry): + provider = _provider() + provider._client.get_sandbox.side_effect = type("Missing", (Exception,), {"status_code": 404})("gone") + monkeypatch.setattr(teardown, "build_sandbox_provider", lambda _: provider) + monkeypatch.setattr(teardown_sandboxes, "build_sandbox_provider", lambda _: provider) + context = TaskStepContext(deployed_sandboxes=[DeployedSandbox( + sandbox_name="box", sandbox_id="gone", sandbox_mode="vm", sandbox_type="vercel", + )]) + if entry == "run": + report = await teardown.teardown_run(context) + assert not report.failed and not report.still_up + assert [box.sandbox_id for box in report.terminated] == ["gone"] + else: + await teardown_sandboxes.TeardownSandboxesTaskStep( + id="cleanup", version=1, sandbox_names=["box"], + ).execute(context) + assert context.metadata[teardown_sandboxes.TORN_DOWN_KEY] == ["gone"] + provider._client.get_sandbox.assert_awaited_once_with(name="gone") + + +@pytest.mark.asyncio +async def test_missing_lookup_still_raises_and_preserves_the_sdk_error(): + provider = _provider() + missing = type("Missing", (Exception,), {"status_code": 404})("gone") + provider._client.get_sandbox.side_effect = missing + with pytest.raises(SandboxNotFoundError) as caught: + await provider.get_sandbox("gone") + assert caught.value.__cause__ is missing + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [ + type("Unauthorized", (Exception,), {"status_code": 401})("unauthorized"), + type("Forbidden", (Exception,), {"status_code": 403})("forbidden"), + ConnectionError("offline"), + asyncio.CancelledError(), +]) +async def test_lookup_preserves_auth_transport_and_cancellation_errors(error): + provider = _provider() + provider._client.get_sandbox.side_effect = error + with pytest.raises(type(error)) as caught: + await provider.get_sandbox("box") + assert caught.value is error + + +@pytest.mark.asyncio +@pytest.mark.parametrize("entry", ["run", "step"]) +async def test_cleanup_does_not_mark_permission_failure_as_torn_down(monkeypatch, entry): + provider = _provider() + provider._client.get_sandbox.side_effect = type("Forbidden", (Exception,), {"status_code": 403})("forbidden") + monkeypatch.setattr(teardown, "build_sandbox_provider", lambda _: provider) + monkeypatch.setattr(teardown_sandboxes, "build_sandbox_provider", lambda _: provider) + context = TaskStepContext(deployed_sandboxes=[DeployedSandbox( + sandbox_name="box", sandbox_id="box", sandbox_mode="vm", sandbox_type="vercel", + )]) + if entry == "run": + report = await teardown.teardown_run(context) + assert len(report.failed) == 1 and report.still_up + else: + await teardown_sandboxes.TeardownSandboxesTaskStep(id="cleanup", version=1, sandbox_names=["box"]).execute(context) + assert not context.metadata.get(teardown_sandboxes.TORN_DOWN_KEY) + + +@pytest.mark.parametrize("tags", [{}, {"a": "1"}, {str(i): str(i) for i in range(5)}]) +def test_supported_tags_are_preserved_without_warning(tags, caplog): + with caplog.at_level(logging.WARNING): + assert VercelSandboxProvider._tags(tags) == tags + assert not caplog.records + + +def test_invalid_tag_is_not_hidden_by_overflow(): + with pytest.raises(ValueError, match="non-empty"): + VercelSandboxProvider._tags({**{str(i): "v" for i in range(6)}, "z": ""}) diff --git a/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py b/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py index 09109537..89f47b49 100644 --- a/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py @@ -586,3 +586,22 @@ async def test_unobtainable_image_is_refused_before_loading_any_image(): sandbox._signed_image_urls.assert_not_awaited() sandbox.exec_script.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cached", [None, NetworkPolicy()]) +@pytest.mark.parametrize("current_mode", ["deny-all", "unknown"]) +async def test_signed_download_refreshes_even_an_unrestricted_or_unknown_cached_policy(cached, current_mode): + sandbox = _sandbox(policy=cached) + sandbox._client.get_sandbox.return_value = SimpleNamespace( + network_policy=SimpleNamespace(mode=current_mode, allow={}, subnets=None), + ) + if current_mode == "unknown": + with pytest.raises(RuntimeError, match="applied network policy is unknown"): + await sandbox._allow_download_hosts(["https://bucket.example/artifact"]) + sandbox._sandbox.update_network_policy.assert_not_awaited() + else: + await sandbox._allow_download_hosts(["https://bucket.example/artifact"]) + assert sandbox.network_policy == NetworkPolicy(mode=NetworkMode.ALLOWLIST, allow_hosts=("bucket.example",)) + sandbox._sandbox.update_network_policy.assert_awaited_once() + sandbox._client.get_sandbox.assert_awaited_once_with(name=sandbox.sandbox_id) diff --git a/tst/unit/test_capabilities.py b/tst/unit/test_capabilities.py index 87aeb7f8..762569a1 100644 --- a/tst/unit/test_capabilities.py +++ b/tst/unit/test_capabilities.py @@ -194,7 +194,7 @@ def test_vercel_is_absent_without_configured_or_ambient_credentials(monkeypatch) "agent_env.providers.sandbox_providers.sandbox_provider.build_sandbox_provider", lambda spec: object(), ) - for name in ("VERCEL_TOKEN", "VERCEL_TEAM_ID", "VERCEL_PROJECT_ID"): + for name in ("VERCEL_TOKEN", "VERCEL_TEAM_ID", "VERCEL_PROJECT_ID", "VERCEL_OIDC_TOKEN"): monkeypatch.delenv(name, raising=False) _fake_vercel_config(monkeypatch, {}) assert capabilities.remote_sandbox_is_available("vercel") is False From 3d5a64b545ebeb1683ab0747ec233672dc4185c7 Mon Sep 17 00:00:00 2001 From: Pratyush Singhal Date: Wed, 14 Oct 2026 17:38:14 -0700 Subject: [PATCH 4/4] fix(vercel): address greptile review findings - Name the Docker readiness attempts, delay, dockerd log tail and setup error tail used by wait_for_vm's script. - Drop an unused import in vercel_sandbox_test.py that fails ruff. Co-Authored-By: Claude Opus 5.5 --- .../sandbox_providers/vercel/sandbox.py | 16 ++++++++++------ .../sandbox_providers/vercel_sandbox_test.py | 1 - 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/agent_env/providers/sandbox_providers/vercel/sandbox.py b/src/agent_env/providers/sandbox_providers/vercel/sandbox.py index 573e7f9c..c564f6ea 100644 --- a/src/agent_env/providers/sandbox_providers/vercel/sandbox.py +++ b/src/agent_env/providers/sandbox_providers/vercel/sandbox.py @@ -30,6 +30,10 @@ _UPLOAD_ATTEMPTS = 3 _UPLOAD_RETRY_DELAY_SECONDS = 0.25 _DOCKER_SETUP_TIMEOUT = 300 +_DOCKER_READY_ATTEMPTS = 30 +_DOCKER_READY_DELAY_SECONDS = 1 +_DOCKERD_LOG_TAIL_LINES = 40 +_SETUP_ERROR_TAIL_CHARS = 1000 _policy_locks: weakref.WeakValueDictionary[ tuple[asyncio.AbstractEventLoop, str], asyncio.Lock ] = weakref.WeakValueDictionary() @@ -236,7 +240,7 @@ async def _exec_with_output( return await process.wait(), stdout.decode(errors="replace"), stderr.decode(errors="replace") async def wait_for_vm(self) -> None: - script = """ + script = f""" if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then exit 0 fi @@ -247,14 +251,14 @@ async def wait_for_vm(self) -> None: fi DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker.io docker-compose-v2 || - { echo 'Docker installation failed; use a prepared Docker-capable image for restricted egress' >&2; exit 3; } + {{ echo 'Docker installation failed; use a prepared Docker-capable image for restricted egress' >&2; exit 3; }} fi docker info >/dev/null 2>&1 || (nohup dockerd >/var/log/agentenv-dockerd.log 2>&1 /dev/null 2>&1 && docker compose version >/dev/null 2>&1 && exit 0 - sleep 1 + sleep {_DOCKER_READY_DELAY_SECONDS} done -tail -40 /var/log/agentenv-dockerd.log >&2 +tail -{_DOCKERD_LOG_TAIL_LINES} /var/log/agentenv-dockerd.log >&2 exit 1 """ exit_code, _, stderr = await self._exec_with_output( @@ -263,7 +267,7 @@ async def wait_for_vm(self) -> None: if exit_code != 0: raise RuntimeError( f"Docker and Compose are unavailable in Vercel sandbox {self.sandbox_id} " - f"(exit {exit_code}): {stderr[-1000:]}" + f"(exit {exit_code}): {stderr[-_SETUP_ERROR_TAIL_CHARS:]}" ) logger.info("Docker and Compose ready in Vercel sandbox %s", self.sandbox_id) diff --git a/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py b/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py index 2ba8fe36..ac7ef82c 100644 --- a/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/vercel_sandbox_test.py @@ -20,7 +20,6 @@ ) from agent_env.artifact.artifacts.docker_image import DockerImageArtifact -from agent_env.providers.sandbox_providers import vercel as vercel_module from agent_env.providers.sandbox_providers.vercel import sandbox as vercel_sandbox_module from agent_env.providers.sandbox_providers.sandbox import NetworkMode, NetworkPolicy from agent_env.providers.sandbox_providers.vercel.sandbox import (