diff --git a/.github/scripts/wait_for_service_ready.sh b/.github/scripts/wait_for_service_ready.sh new file mode 100755 index 000000000..6c0a28885 --- /dev/null +++ b/.github/scripts/wait_for_service_ready.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Poll a freshly deployed service's /readiness-check until it answers 200. +# +# The staging services are scale-to-zero, so a candidate revision is cold on its +# first request and must import the tax-benefit system (~200s for the US system) +# before it can serve a calculate. Run as a gate job ahead of the smoke suites, +# this makes the tests hit a warm instance instead of racing the cold-start +# import (which otherwise times out and blocks the production deploy). + +url="${SERVICE_URL:?SERVICE_URL is required}" +url="${url%/}" +deadline_seconds="${READINESS_DEADLINE_SECONDS:-480}" +poll_interval_seconds="${READINESS_POLL_INTERVAL_SECONDS:-5}" +request_timeout_seconds="${READINESS_REQUEST_TIMEOUT_SECONDS:-30}" + +deadline=$(( SECONDS + deadline_seconds )) +attempt=0 + +while true; do + attempt=$(( attempt + 1 )) + status="$(curl -s -o /dev/null -w '%{http_code}' \ + --max-time "${request_timeout_seconds}" "${url}/readiness-check" || true)" + + if [[ "${status}" == "200" ]]; then + echo "Ready after ${attempt} attempt(s): ${url}/readiness-check -> 200" + exit 0 + fi + + if (( SECONDS >= deadline )); then + echo "::error::${url} did not become ready within ${deadline_seconds}s (last status: ${status:-none})" >&2 + exit 1 + fi + + echo "Attempt ${attempt}: ${url}/readiness-check -> ${status:-no response}; retrying in ${poll_interval_seconds}s" + sleep "${poll_interval_seconds}" +done diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 513417d1f..d6770ca46 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -323,10 +323,27 @@ jobs: env: APP_ENGINE_VERSION: ${{ needs.deploy-staging.outputs.version }} + wait-for-cloud-run-staging-ready: + name: Wait for staging Cloud Run readiness + runs-on: ubuntu-latest + needs: deploy-cloud-run-staging + if: | + (github.repository == 'PolicyEngine/policyengine-api') + && (github.event.head_commit.message == 'Update PolicyEngine API') + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Wait for /readiness-check to return 200 + run: bash .github/scripts/wait_for_service_ready.sh + env: + SERVICE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }} + integration-tests-staging-cloud-run: name: Run Cloud Run staging integration tests runs-on: ubuntu-latest - needs: deploy-cloud-run-staging + needs: + - deploy-cloud-run-staging + - wait-for-cloud-run-staging-ready if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') diff --git a/changelog.d/staging-smoke-readiness-gate.fixed.md b/changelog.d/staging-smoke-readiness-gate.fixed.md new file mode 100644 index 000000000..7ea01981e --- /dev/null +++ b/changelog.d/staging-smoke-readiness-gate.fixed.md @@ -0,0 +1 @@ +Add a dedicated readiness-gate job that polls /readiness-check until the freshly deployed staging Cloud Run revision is warm before the smoke suite runs, so the first calculate no longer races the cold-start import and intermittently fails (which skips the production Cloud Run deploy). diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 34d8b3e76..2a99493ee 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -644,6 +644,25 @@ def test_push_workflow_tests_app_engine_and_cloud_run_staging_tracks(): assert "bash .github/scripts/get_cloud_run_service_url.sh" in cloud_run_promotion +def test_cloud_run_staging_tests_gate_on_a_readiness_job(): + workflow = _push_workflow() + readiness_gate = _workflow_job_block(workflow, "wait-for-cloud-run-staging-ready") + cloud_run_tests = _workflow_job_block( + workflow, "integration-tests-staging-cloud-run" + ) + + # A dedicated job proves the deployed candidate answers /readiness-check + # before any smoke test runs, so the first calculate does not race the + # cold-start import. + assert "bash .github/scripts/wait_for_service_ready.sh" in readiness_gate + assert ( + "SERVICE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}" + in readiness_gate + ) + # The Cloud Run smoke suite must depend on that gate. + assert "- wait-for-cloud-run-staging-ready" in cloud_run_tests + + def test_push_workflow_deploys_app_engine_production_candidate_before_staging_gate(): workflow = _push_workflow() app_engine_candidate = _workflow_job_block(workflow, "deploy-production-candidate")