From a727a6617eb2ea6197c535299b7c06c1a3657530 Mon Sep 17 00:00:00 2001 From: Alexander Schwartzman Date: Wed, 14 Oct 2026 00:51:23 -0400 Subject: [PATCH 1/3] ci: run the gateway integration tests in the slow tier --- .github/workflows/local-backends.yml | 16 ++++++---------- tst/integration/env/gateway/gateway_test.py | 8 +++++--- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 3dcd03ef..8ad7637f 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -94,13 +94,10 @@ jobs: uv venv .venv uv pip install --python .venv/bin/python -e ./packages/agentenv-protocol -e '.[dev]' echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - # The whole suite minus gateway_test (needs an x86 Chromium build, and MCP server sources from - # outside this repository; `make int-test-slow` runs it). Marker selection splits fast from slow. - name: Integration tier on the local defaults run: | .venv/bin/python -m pytest -m 'not int_test_slow' -rA --timeout=900 --junitxml=junit-integration.xml \ - tst/integration \ - --ignore=tst/integration/env/gateway/gateway_test.py + tst/integration - name: Skips must be declared capability gaps if: always() run: | @@ -109,11 +106,11 @@ jobs: --no-skips-under tst/integration/store/ # The slow tier: builds the env images and the echo A2A agent (tst/data/a2a_agent) and deploys - # them on the runner's Docker. The tests that need a real model, a remote sandbox or a registered - # default A2A agent skip with the capability reason. + # them on the runner's Docker. The tests that need a real model, a remote sandbox, a registered + # default A2A agent or MCP server sources from outside this repository skip with the capability reason. integration-local-slow: runs-on: ubuntu-latest - timeout-minutes: 40 + timeout-minutes: 120 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 @@ -161,13 +158,12 @@ jobs: - name: Slow integration tier on the local defaults run: | .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ - tst/integration \ - --ignore=tst/integration/env/gateway/gateway_test.py + tst/integration - name: Skips must be declared capability gaps if: always() run: | .venv/bin/python .github/scripts/check_skip_policy.py junit-integration-slow.xml \ - --allow model_endpoint_configured remote_sandbox default_a2a_agent + --allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources # Whether this change touches the code the installer tier covers; the tier costs minutes, so # it runs only then. A skipped job reports as passed to a required check. diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index 4b326a12..07f43701 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -2816,6 +2816,11 @@ async def test_gateway_queued_trigger_is_not_reported_settled(sandbox_provider, @pytest.mark.asyncio @pytest.mark.integration @pytest.mark.int_test_slow +@pytest.mark.parametrize( + "sandbox_provider", + [pytest.param("modal_vm", id="modal_vm", marks=skip_without_remote_sandbox("modal_vm"))], + indirect=True, +) async def test_gateway_triggers_nl_executor(sandbox_provider, mcp_server_envs, email_service_artifact): """Deployed e2e for the nl fire-action: a real claude-code A2A executor agent realizes an NL event over A2A, gated by the engine's typed verify. The one trigger type the hermetic test can't cover @@ -2828,9 +2833,6 @@ async def test_gateway_triggers_nl_executor(sandbox_provider, mcp_server_envs, e from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.register_env_triggers import RegisterEnvTriggersStep - if sandbox_provider != "modal_vm": - pytest.skip("nl-executor e2e runs on modal_vm only (VM-mode agent deploy)") - email_ref = mcp_server_envs["email"] email_env = Env.get(email_ref.id, version=email_ref.version) From 38644ca8a0616d0a84c06de61f8899f3281f442f Mon Sep 17 00:00:00 2001 From: Alexander Schwartzman Date: Wed, 14 Oct 2026 11:16:31 -0400 Subject: [PATCH 2/3] ci: run the gateway tests as their own slow-tier check, and fix the two that went stale --- .github/workflows/local-backends.yml | 14 ++++++++++++-- AGENTS.md | 5 +++-- CONTRIBUTING.md | 15 +++++++-------- tst/integration/env/gateway/gateway_test.py | 19 ++++++++++++++++--- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 8ad7637f..8ca8aefd 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -108,9 +108,19 @@ jobs: # The slow tier: builds the env images and the echo A2A agent (tst/data/a2a_agent) and deploys # them on the runner's Docker. The tests that need a real model, a remote sandbox, a registered # default A2A agent or MCP server sources from outside this repository skip with the capability reason. + # The gateway tests take as long as the rest of the tier, so they run beside it as a check of their own. integration-local-slow: + name: ${{ matrix.check }} + strategy: + fail-fast: false + matrix: + include: + - check: integration-local-slow + tests: tst/integration --ignore=tst/integration/env/gateway/gateway_test.py + - check: integration-local-gateway + tests: tst/integration/env/gateway/gateway_test.py runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 40 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 @@ -158,7 +168,7 @@ jobs: - name: Slow integration tier on the local defaults run: | .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ - tst/integration + ${{ matrix.tests }} - name: Skips must be declared capability gaps if: always() run: | diff --git a/AGENTS.md b/AGENTS.md index 42d3574e..d81e6a39 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -183,8 +183,9 @@ orchestrators call it the same way. What callers rely on: ## CI The required checks on pull requests and `main` are `unit`, `integration-local`, -`integration-local-slow`, `installer`, `plugin-api` and `clean-install`. `.github/workflows/local-backends.yml` runs -the first four, all installed from public PyPI with no secrets and no external services (a `registry:2` service container for the integration jobs). The +`integration-local-slow`, `integration-local-gateway`, `installer`, `plugin-api` and `clean-install`. +`.github/workflows/local-backends.yml` runs the first five (`integration-local-gateway` is the slow tier's +`gateway_test.py`, run beside the rest of it), all installed from public PyPI with no secrets and no external services (a `registry:2` service container for the integration jobs). The `unit` job also fails if `uv.lock` resolves anything from a registry other than PyPI or if `agentenv-framework-protocol` is not the editable workspace member. The integration jobs run `.github/scripts/check_skip_policy.py` over the JUnit report: only `agentenv-capability-missing` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f54e2c3d..3b5e205c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,15 +43,14 @@ This is the same install CI performs. `make install` is equivalent. | `make clean-install-test` | both distributions built as the release builds them, installed into a fresh venv from public PyPI, and `agent-env run hello` run twice by name | Python 3.11, uv and git | | `make installer-test` | `plugin add` and `remove` through the real pip, uv and pipx, offline against wheels built from the checkout, and the new-user journey in containers with no network | uv, pipx and Docker; the first run downloads the dependencies once | -CI runs the same tiers as the `unit`, `integration-local` and `integration-local-slow` jobs; -all three must pass, and so must `plugin-api` (see "Making changes"). The `installer` job runs +CI runs the same tiers as the `unit`, `integration-local`, `integration-local-slow` and +`integration-local-gateway` jobs (the last runs the slow tier's gateway tests beside the rest); +all four must pass, and so must `plugin-api` (see "Making changes"). The `installer` job runs `make installer-test` too, on pull requests that touch the plugin code, the installer tier or -the lockfile, and must pass when it runs. One exception: the public jobs exclude -`tst/integration/env/gateway/gateway_test.py`, which needs an x86 Chromium build that the -hosted runners do not have; its virtual-clock tests also build MCP servers from sources outside -this repository, found through `AGENT_ENV_TEST_MCP_SERVERS_DIR` (one `/Dockerfile` per -server), and skip without them. `make int-test-slow` does run it, so run that locally when your -change touches the gateway, and say so in the pull request. A test may skip only when a capability is genuinely unavailable, and the skip reason +the lockfile, and must pass when it runs. The gateway's virtual-clock tests build MCP servers +from sources outside this repository, found through `AGENT_ENV_TEST_MCP_SERVERS_DIR` (one +`/Dockerfile` per server), and skip without them; when your change touches the virtual +clock, run `make int-test-slow` with it set, and say so in the pull request. A test may skip only when a capability is genuinely unavailable, and the skip reason must name it (`agentenv-capability-missing: `, see `tst/util/capabilities.py`); the `check_skip_policy` step fails the run otherwise. The `clean-install` job runs the same script as `make clean-install-test` on every pull request, and is one of the required checks. diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index 07f43701..59211e74 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -36,7 +36,13 @@ ) from agent_env.config import get_config from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep -from tst.util.capabilities import MCP_SERVER_SOURCES, missing_capability_reason, skip_without_remote_sandbox +from tst.util.capabilities import ( + MCP_SERVER_SOURCES, + missing_capability_reason, + skip_without_default_a2a_agent, + skip_without_model_endpoint, + skip_without_remote_sandbox, +) logger = logging.getLogger(__name__) @@ -1609,7 +1615,7 @@ async def test_snapshot_cli(multi_env, environment_universe_artifact): """Test env snapshot: deploy MultiEnv, load universe, snapshot, verify.""" from agent_env.artifact import Artifact from agent_env.cli.env.snapshot import run_snapshot - from agent_env.env.snapshot_store import get_env_snapshot_store + from agent_env.env.snapshot_store import compute_env_fingerprint, get_env_snapshot_store multi_env_ref = multi_env multi_env = Env.get(multi_env_ref.id, version=multi_env_ref.version) @@ -1637,7 +1643,12 @@ async def test_snapshot_cli(multi_env, environment_universe_artifact): logger.info(f"Snapshot record: {snapshot}") # Verify get_clean returns the same snapshot - clean_snapshot = get_env_snapshot_store().get_clean(multi_env.id, environment_universe_artifact.id) + clean_snapshot = get_env_snapshot_store().get_clean( + multi_env.id, + environment_universe_artifact.id, + environment_universe_artifact.version, + env_fingerprint=compute_env_fingerprint(multi_env), + ) assert clean_snapshot is not None, "Expected clean snapshot via get_clean" assert clean_snapshot.instance_id == result.instance_id @@ -1758,6 +1769,8 @@ async def assert_loaded_via_step(client, gateway_url, tools, tool_names): @pytest.mark.asyncio @pytest.mark.integration +@skip_without_model_endpoint() +@skip_without_default_a2a_agent() async def test_env_validator_task(mcp_server_envs): """Test MCPServerEnv.validate(): lazily creates task, runs validation, persists metadata.""" from agent_env.task import Task From c2ca3a6092002fea411395416f5a3a230dc3b75b Mon Sep 17 00:00:00 2001 From: Alexander Schwartzman Date: Wed, 14 Oct 2026 12:00:17 -0400 Subject: [PATCH 3/3] Import compute_env_fingerprint at the top of the gateway test module --- tst/integration/env/gateway/gateway_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index 59211e74..92f08d63 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -34,6 +34,7 @@ WEBSITE_BROWSER_IMAGE_TAG, WEBSITE_BROWSER_ENVIRONMENT_NAME, ) +from agent_env.env.snapshot_store import compute_env_fingerprint from agent_env.config import get_config from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from tst.util.capabilities import ( @@ -1615,7 +1616,7 @@ async def test_snapshot_cli(multi_env, environment_universe_artifact): """Test env snapshot: deploy MultiEnv, load universe, snapshot, verify.""" from agent_env.artifact import Artifact from agent_env.cli.env.snapshot import run_snapshot - from agent_env.env.snapshot_store import compute_env_fingerprint, get_env_snapshot_store + from agent_env.env.snapshot_store import get_env_snapshot_store multi_env_ref = multi_env multi_env = Env.get(multi_env_ref.id, version=multi_env_ref.version)