diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 12cd6f15..2ca4e146 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -92,13 +92,10 @@ jobs: python -m pip install --quiet uv uv sync --locked --extra dev --python "$(command -v python)" echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - # The whole suite minus gateway_test (needs an x86 Chromium build, and MCP server sources from - # outside this repository; `make int-test-slow` runs it). Marker selection splits fast from slow. - name: Integration tier on the local defaults run: | .venv/bin/python -m pytest -m 'not int_test_slow' -rA --timeout=900 --junitxml=junit-integration.xml \ - tst/integration \ - --ignore=tst/integration/env/gateway/gateway_test.py + tst/integration - name: Skips must be declared capability gaps if: always() run: | @@ -107,9 +104,19 @@ jobs: --no-skips-under tst/integration/store/ # The slow tier: builds the env images and the echo A2A agent (tst/data/a2a_agent) and deploys - # them on the runner's Docker. The tests that need a real model, a remote sandbox or a registered - # default A2A agent skip with the capability reason. + # them on the runner's Docker. The tests that need a real model, a remote sandbox, a registered + # default A2A agent or MCP server sources from outside this repository skip with the capability reason. + # The gateway tests take as long as the rest of the tier, so they run beside it as a check of their own. integration-local-slow: + name: ${{ matrix.check }} + strategy: + fail-fast: false + matrix: + include: + - check: integration-local-slow + tests: tst/integration --ignore=tst/integration/env/gateway/gateway_test.py + - check: integration-local-gateway + tests: tst/integration/env/gateway/gateway_test.py runs-on: ubuntu-latest timeout-minutes: 40 steps: @@ -158,13 +165,12 @@ jobs: - name: Slow integration tier on the local defaults run: | .venv/bin/python -m pytest -m int_test_slow -rA --timeout=1200 --junitxml=junit-integration-slow.xml \ - tst/integration \ - --ignore=tst/integration/env/gateway/gateway_test.py + ${{ matrix.tests }} - name: Skips must be declared capability gaps if: always() run: | .venv/bin/python .github/scripts/check_skip_policy.py junit-integration-slow.xml \ - --allow model_endpoint_configured remote_sandbox default_a2a_agent + --allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources # Whether this change touches the code the installer tier covers; the tier costs minutes, so # it runs only then. A skipped job reports as passed to a required check. diff --git a/AGENTS.md b/AGENTS.md index 2ec621ba..99b44d10 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -192,8 +192,9 @@ orchestrators call it the same way. What callers rely on: ## CI The required checks on pull requests and `main` are `unit`, `integration-local`, -`integration-local-slow`, `installer`, `plugin-api` and `clean-install`. `.github/workflows/local-backends.yml` runs -the first four, all installed from `uv.lock` (public PyPI only) with no secrets and no external services (a `registry:2` service container for the integration jobs). The +`integration-local-slow`, `integration-local-gateway`, `installer`, `plugin-api` and `clean-install`. +`.github/workflows/local-backends.yml` runs the first five (`integration-local-gateway` is the slow tier's +`gateway_test.py`, run beside the rest of it), all installed from `uv.lock` (public PyPI only) with no secrets and no external services (a `registry:2` service container for the integration jobs). The `unit` job also fails if `uv.lock` resolves anything from a registry other than PyPI or if `agentenv-framework-protocol` is not the editable workspace member. The integration jobs run `.github/scripts/check_skip_policy.py` over the JUnit report: only `agentenv-capability-missing` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 27cb6dd0..aa48bd6b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,15 +43,14 @@ reinstalling. | `make clean-install-test` | both distributions built as the release builds them, installed into a fresh venv from public PyPI, and `agent-env run hello` run twice by name | Python 3.11, uv and git | | `make installer-test` | `plugin add` and `remove` through the real pip, uv and pipx, offline against wheels built from the checkout, and the new-user journey in containers with no network | uv, pipx and Docker; the first run downloads the dependencies once | -CI runs the same tiers as the `unit`, `integration-local` and `integration-local-slow` jobs; -all three must pass, and so must `plugin-api` (see "Making changes"). The `installer` job runs +CI runs the same tiers as the `unit`, `integration-local`, `integration-local-slow` and +`integration-local-gateway` jobs (the last runs the slow tier's gateway tests beside the rest); +all four must pass, and so must `plugin-api` (see "Making changes"). The `installer` job runs `make installer-test` too, on pull requests that touch the plugin code, the installer tier or -the lockfile, and must pass when it runs. One exception: the public jobs exclude -`tst/integration/env/gateway/gateway_test.py`, which needs an x86 Chromium build that the -hosted runners do not have; its virtual-clock tests also build MCP servers from sources outside -this repository, found through `AGENT_ENV_TEST_MCP_SERVERS_DIR` (one `/Dockerfile` per -server), and skip without them. `make int-test-slow` does run it, so run that locally when your -change touches the gateway, and say so in the pull request. A test may skip only when a capability is genuinely unavailable, and the skip reason +the lockfile, and must pass when it runs. The gateway's virtual-clock tests build MCP servers +from sources outside this repository, found through `AGENT_ENV_TEST_MCP_SERVERS_DIR` (one +`/Dockerfile` per server), and skip without them; when your change touches the virtual +clock, run `make int-test-slow` with it set, and say so in the pull request. A test may skip only when a capability is genuinely unavailable, and the skip reason must name it (`agentenv-capability-missing: `, see `tst/util/capabilities.py`); the `check_skip_policy` step fails the run otherwise. The `clean-install` job runs the same script as `make clean-install-test` on every pull request, and is one of the required checks. diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index 4b326a12..92f08d63 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -34,9 +34,16 @@ WEBSITE_BROWSER_IMAGE_TAG, WEBSITE_BROWSER_ENVIRONMENT_NAME, ) +from agent_env.env.snapshot_store import compute_env_fingerprint from agent_env.config import get_config from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep -from tst.util.capabilities import MCP_SERVER_SOURCES, missing_capability_reason, skip_without_remote_sandbox +from tst.util.capabilities import ( + MCP_SERVER_SOURCES, + missing_capability_reason, + skip_without_default_a2a_agent, + skip_without_model_endpoint, + skip_without_remote_sandbox, +) logger = logging.getLogger(__name__) @@ -1637,7 +1644,12 @@ async def test_snapshot_cli(multi_env, environment_universe_artifact): logger.info(f"Snapshot record: {snapshot}") # Verify get_clean returns the same snapshot - clean_snapshot = get_env_snapshot_store().get_clean(multi_env.id, environment_universe_artifact.id) + clean_snapshot = get_env_snapshot_store().get_clean( + multi_env.id, + environment_universe_artifact.id, + environment_universe_artifact.version, + env_fingerprint=compute_env_fingerprint(multi_env), + ) assert clean_snapshot is not None, "Expected clean snapshot via get_clean" assert clean_snapshot.instance_id == result.instance_id @@ -1758,6 +1770,8 @@ async def assert_loaded_via_step(client, gateway_url, tools, tool_names): @pytest.mark.asyncio @pytest.mark.integration +@skip_without_model_endpoint() +@skip_without_default_a2a_agent() async def test_env_validator_task(mcp_server_envs): """Test MCPServerEnv.validate(): lazily creates task, runs validation, persists metadata.""" from agent_env.task import Task @@ -2816,6 +2830,11 @@ async def test_gateway_queued_trigger_is_not_reported_settled(sandbox_provider, @pytest.mark.asyncio @pytest.mark.integration @pytest.mark.int_test_slow +@pytest.mark.parametrize( + "sandbox_provider", + [pytest.param("modal_vm", id="modal_vm", marks=skip_without_remote_sandbox("modal_vm"))], + indirect=True, +) async def test_gateway_triggers_nl_executor(sandbox_provider, mcp_server_envs, email_service_artifact): """Deployed e2e for the nl fire-action: a real claude-code A2A executor agent realizes an NL event over A2A, gated by the engine's typed verify. The one trigger type the hermetic test can't cover @@ -2828,9 +2847,6 @@ async def test_gateway_triggers_nl_executor(sandbox_provider, mcp_server_envs, e from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.register_env_triggers import RegisterEnvTriggersStep - if sandbox_provider != "modal_vm": - pytest.skip("nl-executor e2e runs on modal_vm only (VM-mode agent deploy)") - email_ref = mcp_server_envs["email"] email_env = Env.get(email_ref.id, version=email_ref.version)