From 81bcfa0513da86df127db9d19a93ba3fde8eafa9 Mon Sep 17 00:00:00 2001 From: Ram Muthiah Date: Sat, 22 Aug 2026 22:43:15 +0100 Subject: [PATCH 1/2] ci: Modernize target array parsing with mapfile in e2e test runners Replace unquoted command substitutions and string splitting with `mapfile -t targets < <(...)` and quoted expansion (`"${targets[@]}"`) for safe, modern line-based array parsing without word-splitting bugs. TAG=agy CONV=cee1b9c6-d222-4154-8c11-924d01e88f39 --- .../actions/run-cw-sharded-e2e-test/action.yaml | 5 +++-- .github/workflows/presubmit.yaml | 17 +++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/actions/run-cw-sharded-e2e-test/action.yaml b/.github/actions/run-cw-sharded-e2e-test/action.yaml index 3cd34e5accf..d1c050294f9 100644 --- a/.github/actions/run-cw-sharded-e2e-test/action.yaml +++ b/.github/actions/run-cw-sharded-e2e-test/action.yaml @@ -70,12 +70,13 @@ runs: echo "${runner_tests}" # Run tests - targets=( $(echo ${runner_tests}) ) + mapfile -t targets < <(echo "${runner_tests}") for t in "${targets[@]}"; do + [ -z "$t" ] && continue echo "running test: ${t}" sudo podman rm -f tester sudo podman run --name tester -d --privileged --pids-limit=8192 -v /tmp/cw_bazel:/tmp/cw_bazel -v .:/src/workspace -w /src/workspace/e2etests android-cuttlefish-e2etest:latest - sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output test --sandbox_writable_path=/home/testrunner $t + sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output test --sandbox_writable_path=/home/testrunner "$t" done - name: Upload test logs if: always() diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index a5510c6616d..aded050c171 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -407,22 +407,23 @@ jobs: # Flaky tests would be executed in clean container everytime, rather than using `cvd reset` # in a tainted container. sudo podman run --name tester -d --privileged --pids-limit=8192 -v /tmp/cw_bazel:/tmp/cw_bazel -v .:/src/workspace -w /src/workspace/e2etests android-cuttlefish-e2etest:latest - targets=$(sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output query --noshow_progress 'attr(flaky, 1, orchestration/...)' | sort) - echo "FLAKY TARGETS:" - echo "${targets}" + mapfile -t targets < <(sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output query --noshow_progress 'attr(flaky, 1, orchestration/...)' | sort) + echo "FLAKY TARGETS:" + printf '%s\n' "${targets[@]}" echo "" readonly ATTEMPTS=3 - for t in ${targets}; do + for t in "${targets[@]}"; do + [ -z "$t" ] && continue echo "Executing target: ${t}" attempt=1 - while [[ ${attempt} -le ${ATTEMPTS} ]]; do + while [[ ${attempt} -le ${ATTEMPTS} ]]; do echo "Attempt: ${attempt} of ${ATTEMPTS}" sudo podman rm -f tester sudo podman run --name tester -d --privileged --pids-limit=8192 -v /tmp/cw_bazel:/tmp/cw_bazel -v .:/src/workspace -w /src/workspace/e2etests android-cuttlefish-e2etest:latest - sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output test --test_timeout 600 --sandbox_writable_path=/home/testrunner --flaky_test_attempts=1 ${t} && break || true - attempt=$((attempt+1)) + sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output test --test_timeout 600 --sandbox_writable_path=/home/testrunner --flaky_test_attempts=1 "${t}" && break || true + attempt=$((attempt+1)) done - if [[ ${attempt} -gt ${ATTEMPTS} ]]; then + if [[ ${attempt} -gt ${ATTEMPTS} ]]; then # Every previous attempt failed. exit 1 fi From 5aa0e3e298f82cd0a5c3a81a59bd421f8dbc09dd Mon Sep 17 00:00:00 2001 From: Ram Muthiah Date: Sat, 22 Aug 2026 22:28:49 +0100 Subject: [PATCH 2/2] ci: Shard special e2e tests into parallel runners and composite action Extract e2e-tests-orchestration-runner-special into a dedicated composite action (.github/actions/run-cw-special-e2e-test) and shard the workload across two parallel runner jobs: - runner-special-1: Executes flaky test retry loops in clean containers. - runner-special-2: Executes environment and credential tests. - runner-special: Lightweight aggregator job maintaining compatibility with GitHub branch protection rules. Also improvements: - Replace static sleep 30s with active systemd service readiness checks. - Use wildcard paths for resilient testlog artifact collection. TAG=agy CONV=cee1b9c6-d222-4154-8c11-924d01e88f39 --- .../run-cw-special-e2e-test/action.yaml | 101 +++++++++++++++++ .github/workflows/presubmit.yaml | 102 ++++++------------ 2 files changed, 136 insertions(+), 67 deletions(-) create mode 100644 .github/actions/run-cw-special-e2e-test/action.yaml diff --git a/.github/actions/run-cw-special-e2e-test/action.yaml b/.github/actions/run-cw-special-e2e-test/action.yaml new file mode 100644 index 00000000000..ec9ac8e9104 --- /dev/null +++ b/.github/actions/run-cw-special-e2e-test/action.yaml @@ -0,0 +1,101 @@ +name: 'Run special e2e tests' +description: 'Run flaky and configuration-specific e2e orchestration tests in isolated containers' +inputs: + runner-index: + description: 'Index of this runner shard (1-based)' + required: false + default: '1' + runners-total: + description: 'Total number of runner shards' + required: false + default: '1' +runs: + using: "composite" + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # aka v1.3.1 + with: + tool-cache: true + - name: Download image + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + with: + name: android-cuttlefish-e2etest-image-tar + github-token: ${{ github.token }} + - name: Upgrade crun + uses: ./.github/actions/upgrade-crun + - name: Run tests + shell: bash + env: + INPUT_RUNNER_INDEX: ${{ inputs.runner-index }} + INPUT_RUNNERS_TOTAL: ${{ inputs.runners-total }} + run: | + sudo podman info + sudo podman load --quiet -i android-cuttlefish-e2etest.tar && rm android-cuttlefish-e2etest.tar + mkdir -p -m 777 /tmp/cw_bazel + + wait_for_orchestrator() { + local container="$1" + for i in {1..30}; do + if sudo podman exec "${container}" systemctl is-active --quiet cuttlefish-host_orchestrator; then + return 0 + fi + sleep 1 + done + echo "Warning: cuttlefish-host_orchestrator service was not reported active within 30s" + } + + # Shard 1 (or single runner): Run flaky tests with clean container retries. + if [[ "${INPUT_RUNNER_INDEX}" -eq 1 || "${INPUT_RUNNERS_TOTAL}" -eq 1 ]]; then + sudo podman run --name tester -d --privileged --pids-limit=8192 -v /tmp/cw_bazel:/tmp/cw_bazel -v .:/src/workspace -w /src/workspace/e2etests android-cuttlefish-e2etest:latest + mapfile -t targets < <(sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output query --noshow_progress 'attr(flaky, 1, orchestration/...)' | sort) + echo "FLAKY TARGETS:" + printf '%s\n' "${targets[@]}" + echo "" + readonly ATTEMPTS=3 + for t in "${targets[@]}"; do + [ -z "$t" ] && continue + echo "Executing target: ${t}" + attempt=1 + while [[ ${attempt} -le ${ATTEMPTS} ]]; do + echo "Attempt: ${attempt} of ${ATTEMPTS}" + sudo podman rm -f tester + sudo podman run --name tester -d --privileged --pids-limit=8192 -v /tmp/cw_bazel:/tmp/cw_bazel -v .:/src/workspace -w /src/workspace/e2etests android-cuttlefish-e2etest:latest + sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output test --test_timeout 600 --sandbox_writable_path=/home/testrunner --flaky_test_attempts=1 "${t}" && break || true + attempt=$((attempt+1)) + done + if [[ ${attempt} -gt ${ATTEMPTS} ]]; then + # Every previous attempt failed. + exit 1 + fi + done + sudo podman rm -f tester + fi + + # Shard 2 (or single runner): Run environment/credential tests. + if [[ "${INPUT_RUNNER_INDEX}" -eq 2 || "${INPUT_RUNNERS_TOTAL}" -eq 1 ]]; then + # Run verify_access_token_test + sudo podman run --name tester -d --privileged --pids-limit=8192 -v /tmp/cw_bazel:/tmp/cw_bazel -v .:/src/workspace -w /src/workspace/e2etests android-cuttlefish-e2etest:latest + wait_for_orchestrator tester + sudo podman exec tester sh -c 'echo "orchestrator_android_build_url=http://localhost:8090" >> /etc/default/cuttlefish-host_orchestrator && service cuttlefish-host_orchestrator restart' + wait_for_orchestrator tester + sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output test //orchestration/verify_access_token_test:verify_access_token_test_test + sudo podman rm -f tester + + # Run create_with_gce_metadata_credentials_test + sudo podman run --name tester -d --privileged --pids-limit=8192 -v /tmp/cw_bazel:/tmp/cw_bazel -v .:/src/workspace -w /src/workspace/e2etests android-cuttlefish-e2etest:latest + wait_for_orchestrator tester + sudo podman exec tester sh -c 'echo "build_api_credentials_use_gce_metadata=true" >> /etc/default/cuttlefish-host_orchestrator && service cuttlefish-host_orchestrator restart' + wait_for_orchestrator tester + sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output test //orchestration/create_with_gce_metadata_credentials_test:create_with_gce_metadata_credentials_test_test + sudo podman rm -f tester + fi + - name: Upload test logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: "e2e-tests-orchestration-runner-special-${{ inputs.runner-index }}-testlogs" + path: /tmp/cw_bazel/output/*/execroot/_main/bazel-out/k8-fastbuild/testlogs + - name: Remaining disk space + shell: bash + run: | + df -h diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index aded050c171..fc21b3443b4 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -378,78 +378,46 @@ jobs: with: runner-index: 3 runners-total: 3 - e2e-tests-orchestration-runner-special: + e2e-tests-orchestration-runner-special-1: runs-on: ubuntu-24.04 needs: [e2e-tests-orchestration-build-image] steps: - - name: Free disk space - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # aka v1.3.1 - with: - tool-cache: true - - name: checkout repository - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - persist-credentials: false - - name: download image - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 - with: - name: android-cuttlefish-e2etest-image-tar - github-token: ${{ github.token }} - - name: Upgrade crun - uses: ./.github/actions/upgrade-crun - - name: Run tests - run: | - sudo podman info - sudo podman load --quiet -i android-cuttlefish-e2etest.tar && rm android-cuttlefish-e2etest.tar - mkdir -p -m 777 /tmp/cw_bazel - # Run flaky tests. - # - # Flaky tests would be executed in clean container everytime, rather than using `cvd reset` - # in a tainted container. - sudo podman run --name tester -d --privileged --pids-limit=8192 -v /tmp/cw_bazel:/tmp/cw_bazel -v .:/src/workspace -w /src/workspace/e2etests android-cuttlefish-e2etest:latest - mapfile -t targets < <(sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output query --noshow_progress 'attr(flaky, 1, orchestration/...)' | sort) - echo "FLAKY TARGETS:" - printf '%s\n' "${targets[@]}" - echo "" - readonly ATTEMPTS=3 - for t in "${targets[@]}"; do - [ -z "$t" ] && continue - echo "Executing target: ${t}" - attempt=1 - while [[ ${attempt} -le ${ATTEMPTS} ]]; do - echo "Attempt: ${attempt} of ${ATTEMPTS}" - sudo podman rm -f tester - sudo podman run --name tester -d --privileged --pids-limit=8192 -v /tmp/cw_bazel:/tmp/cw_bazel -v .:/src/workspace -w /src/workspace/e2etests android-cuttlefish-e2etest:latest - sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output test --test_timeout 600 --sandbox_writable_path=/home/testrunner --flaky_test_attempts=1 "${t}" && break || true - attempt=$((attempt+1)) - done - if [[ ${attempt} -gt ${ATTEMPTS} ]]; then - # Every previous attempt failed. + - name: checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + - name: Run special tests (shard 1 - flaky tests) + uses: ./.github/actions/run-cw-special-e2e-test + with: + runner-index: 1 + runners-total: 2 + e2e-tests-orchestration-runner-special-2: + runs-on: ubuntu-24.04 + needs: [e2e-tests-orchestration-build-image] + steps: + - name: checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + - name: Run special tests (shard 2 - config tests) + uses: ./.github/actions/run-cw-special-e2e-test + with: + runner-index: 2 + runners-total: 2 + e2e-tests-orchestration-runner-special: + runs-on: ubuntu-24.04 + needs: + - e2e-tests-orchestration-runner-special-1 + - e2e-tests-orchestration-runner-special-2 + if: always() + steps: + - name: Check special runner shards status + run: | + if [ "${{ needs.e2e-tests-orchestration-runner-special-1.result }}" != "success" ] || \ + [ "${{ needs.e2e-tests-orchestration-runner-special-2.result }}" != "success" ]; then + echo "One or more special runner shards failed." exit 1 fi - done - sudo podman rm -f tester - # Run verify_access_token_test - sudo podman run --name tester -d --privileged --pids-limit=8192 -v /tmp/cw_bazel:/tmp/cw_bazel -v .:/src/workspace -w /src/workspace/e2etests android-cuttlefish-e2etest:latest - sleep 30s # Add delay before restarting cuttlefish-host_orchestrator service. - sudo podman exec tester sh -c 'echo "orchestrator_android_build_url=http://localhost:8090" >> /etc/default/cuttlefish-host_orchestrator && service cuttlefish-host_orchestrator restart' - sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output test //orchestration/verify_access_token_test:verify_access_token_test_test - sudo podman rm -f tester - # Run create_with_gce_metadata_credentials_test - sudo podman run --name tester -d --privileged --pids-limit=8192 -v /tmp/cw_bazel:/tmp/cw_bazel -v .:/src/workspace -w /src/workspace/e2etests android-cuttlefish-e2etest:latest - sleep 30s # Add delay before restarting cuttlefish-host_orchestrator service. - sudo podman exec tester sh -c 'echo "build_api_credentials_use_gce_metadata=true" >> /etc/default/cuttlefish-host_orchestrator && service cuttlefish-host_orchestrator restart' - sudo podman exec --user=testrunner tester bazel --output_user_root=/tmp/cw_bazel/output test //orchestration/create_with_gce_metadata_credentials_test:create_with_gce_metadata_credentials_test_test - sudo podman rm -f tester - - name: Upload test logs - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: e2e-tests-orchestration-runner-special-testlogs - path: /tmp/cw_bazel/output/5d2d32753412f49aca3a92f1e1e5e35e/execroot/_main/bazel-out/k8-fastbuild/testlogs - - name: Used disk space - run: | - df -h docker-image-check: runs-on: ubuntu-22.04 needs: [build-docker-image-amd64]