From 6bcdc01f7b20de45361b4bba7e480e59fa4ed4aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 13:22:29 +0100 Subject: [PATCH 1/3] ci(ios): overlap device-free smoke setup as one parallel group The iOS smoke leg ran the simulator boot, the XCTest runner build, and the fixture app restore one after another, although none of the three needs a booted simulator. Move them to one job-level parallel group so the leg pays the slowest member instead of all three. A composite action cannot host a parallel group, and it cannot read the caller's step context, so the group lives in apple.yml and the boot, build, and restore results cross into smoke-ios-run as inputs. The host focus canary stays ahead of the group because the boot can raise the Simulator app, and readiness stays mandatory inside the phase so no step reaches a simulator that is still starting. Co-Authored-By: Claude Opus 5.5 --- .github/actions/smoke-ios-run/action.yml | 92 +++++++++------------- .github/workflows/apple.yml | 95 ++++++++++++++++++++++- scripts/__tests__/apple-ci-impact.test.ts | 56 ++++++++++--- 3 files changed, 175 insertions(+), 68 deletions(-) diff --git a/.github/actions/smoke-ios-run/action.yml b/.github/actions/smoke-ios-run/action.yml index da08a958c2..d3a6cd1019 100644 --- a/.github/actions/smoke-ios-run/action.yml +++ b/.github/actions/smoke-ios-run/action.yml @@ -1,19 +1,29 @@ name: Run iOS smoke tests description: Run iOS smoke tests + +# The caller boots the simulator, builds the runner, and restores the fixture app as one parallel +# group before it calls this action, because none of those three needs a booted simulator. Their +# outputs arrive as inputs: a composite cannot read the caller's step context. +inputs: + simulator-udid: + description: 'UDID of the iOS simulator the caller booted' + required: true + boot-timeout-seconds: + description: 'Boot wait budget the caller-owned readiness check must honor' + required: true + fixture-app-path: + description: 'Path to the restored fixture app bundle' + required: true + fixture-app-id: + description: 'Bundle identifier of the restored fixture app' + required: true + fixture-source: + description: 'Where the fixture binary came from: artifact or build' + required: true + runs: using: composite steps: - - name: Set fixture producer wait policy - id: fixture-producer - run: | - set -euo pipefail - WAIT_SECONDS=1800 - if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then - echo "Fork pull request cannot publish the shared fixture artifact; a cache miss will build inline." - WAIT_SECONDS=0 - fi - echo "wait-seconds=$WAIT_SECONDS" >> "$GITHUB_OUTPUT" - shell: bash - name: Select iOS runner XCTests id: xctest-impact env: @@ -21,38 +31,16 @@ runs: run: | node --experimental-strip-types scripts/apple-ci-impact.ts xctest shell: bash - - name: Establish host focus canary - id: host-focus-canary - run: | - set -euo pipefail - osascript -e 'tell application "Finder" to activate' - sleep 1 - FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')" - test "$FRONTMOST" = "com.apple.finder" - mkdir -p .tmp - echo "$FRONTMOST" > .tmp/ios-host-focus-before - shell: bash - - name: Boot iOS test simulator - id: ios-simulator - uses: ./.github/actions/boot-ios-test-simulator - with: - runtime-version: ${{ env.IOS_RUNTIME_VERSION }} - preferred-device-name: iPhone 17 Pro - wait-for-boot: 'false' - - name: Restore and build iOS XCTest runner - uses: ./.github/actions/setup-apple-runner-build - with: - derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} - cache-key-prefix: ios-runner-prebuilt - cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64 - gate: swift-runner-ios - xcuitest-platform: ios - xcuitest-destination: generic/platform=iOS Simulator + # The caller recorded the host focus canary before its boot started, so a simulator that takes + # focus during boot is still measured against the pre-boot frontmost app. + # The caller booted this simulator and built this runner alongside the fixture restore, so + # both are already in flight when this action starts. Readiness stays mandatory here: no step + # below may reach the device before it boots. - name: Wait for iOS test simulator run: | bash scripts/wait-for-ios-simulator.sh \ - "${{ steps.ios-simulator.outputs.simulator-udid }}" \ - "${{ steps.ios-simulator.outputs.boot-timeout-seconds }}" + "${{ inputs.simulator-udid }}" \ + "${{ inputs.boot-timeout-seconds }}" shell: bash - name: Run iOS runner platform checks id: ios-xctest @@ -70,7 +58,7 @@ runs: mkdir -p "$GITHUB_WORKSPACE/test/artifacts" xcodebuild test-without-building \ -xctestrun "$XCTESTRUN_PATH" \ - -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ + -destination "platform=iOS Simulator,id=${{ inputs.simulator-udid }}" \ -resultBundlePath "$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions.xcresult" \ "${IOS_PR_TEST_ARGS[@]}" 2>&1 | tee /tmp/agent-device-runner-regressions.log xcrun xcresulttool get test-results summary \ @@ -92,34 +80,28 @@ runs: const c = await ensureDaemon({ stateDir: process.env.AGENT_DEVICE_STATE_DIR }); console.log(JSON.stringify({ phase: "daemon_cold_startup", started: c.started, pid: c.daemon.pid, durationMs: Math.round(performance.now() - t0) })); ' - node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json + node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ inputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json shell: bash - name: Run iOS Settings replay smoke test run: | - node bin/agent-device.mjs test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml + node bin/agent-device.mjs test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ inputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml shell: bash - - name: Fetch current fixture app - id: fixture-app - uses: ./.github/actions/setup-fixture-app - with: - install: 'false' - wait-for-artifact-seconds: ${{ steps.fixture-producer.outputs.wait-seconds }} - name: Report fixture cache source run: | - echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" + echo "Fixture app source: ${{ inputs.fixture-source }}" >> "$GITHUB_STEP_SUMMARY" shell: bash - name: Run gesture pan-duration smoke replay run: | - xcrun simctl install "${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.fixture-app.outputs.app-path }}" - node bin/agent-device.mjs test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml + xcrun simctl install "${{ inputs.simulator-udid }}" "${{ inputs.fixture-app-path }}" + node bin/agent-device.mjs test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ inputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml shell: bash - name: Run fixture-backed iOS simulator E2E smoke env: - AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} - AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} + AGENT_DEVICE_FIXTURE_APP_ID: ${{ inputs.fixture-app-id }} + AGENT_DEVICE_FIXTURE_APP_PATH: ${{ inputs.fixture-app-path }} AGENT_DEVICE_IOS_E2E: '1' AGENT_DEVICE_IOS_E2E_TIER: smoke - AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + AGENT_DEVICE_IOS_UDID: ${{ inputs.simulator-udid }} run: | xcrun simctl spawn "$AGENT_DEVICE_IOS_UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index df10a2ef56..ba9801e329 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -37,6 +37,7 @@ jobs: timeout-minutes: 90 env: BASE_SHA: ${{ github.event.pull_request.base.sha }} + IOS_RUNTIME_VERSION: '26.2' IOS_STATE_DIR: ${{ github.workspace }}/.tmp/ios-agent-device-state MACOS_STATE_DIR: ${{ github.workspace }}/.tmp/macos-agent-device-state IOS_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived @@ -83,13 +84,95 @@ jobs: uses: ./.github/actions/run-gate with: gate: build + # The canary records the frontmost app before anything can take focus. It stays ahead of the + # group because one of its members starts the simulator boot, which can raise the Simulator + # app. When it fails, the baseline file is never written, so the phase's own focus assertion + # fails too rather than passing against a stale baseline. + - name: Establish host focus canary + id: host-focus-canary + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + run: | + set -euo pipefail + osascript -e 'tell application "Finder" to activate' + sleep 1 + FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')" + test "$FRONTMOST" = "com.apple.finder" + mkdir -p .tmp + echo "$FRONTMOST" > .tmp/ios-host-focus-before + shell: bash + # The fixture wait policy decides how long the restore polls for a concurrently produced + # artifact, and a background step's outputs reach only the steps after its group, so this + # cheap decision stays sequential ahead of the group. + - name: Set fixture producer wait policy + id: fixture-producer + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + run: | + set -euo pipefail + WAIT_SECONDS=1800 + if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then + echo "Fork pull request cannot publish the shared fixture artifact; a cache miss will build inline." + WAIT_SECONDS=0 + fi + echo "wait-seconds=$WAIT_SECONDS" >> "$GITHUB_OUTPUT" + shell: bash + # None of these needs a booted simulator, and the iOS phase cannot start XCTest, preflight, + # or the fixture replays until all three finish. The phase ran them in sequence, so the + # runner paid the boot, the native build, and the fixture restore one after another while the + # simulator sat idle. They share no outputs and write disjoint paths (the simulator, the + # runner DerivedData, and .tmp/fixture-app through examples/test-app/node_modules), so they + # run as one parallel group and the phase pays the slowest instead of all three. The boot + # keeps its deferred readiness so it still overlaps the build it now starts beside. + - parallel: + - name: Boot iOS test simulator + id: ios-simulator + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + uses: ./.github/actions/boot-ios-test-simulator + with: + runtime-version: ${{ env.IOS_RUNTIME_VERSION }} + preferred-device-name: iPhone 17 Pro + wait-for-boot: 'false' + + - name: Restore and build iOS XCTest runner + id: ios-runner-build + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + uses: ./.github/actions/setup-apple-runner-build + with: + derived-path: ${{ env.IOS_DERIVED_PATH }} + cache-key-prefix: ios-runner-prebuilt + cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64 + gate: swift-runner-ios + xcuitest-platform: ios + xcuitest-destination: generic/platform=iOS Simulator + # The build variant is part of the cache key, so these must stay exactly as the phase + # passed them while the build ran inside it. + env: + AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' + AGENT_DEVICE_XCUITEST_ARCHS: arm64 + + - name: Restore fixture app + id: fixture-app + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + uses: ./.github/actions/setup-fixture-app + with: + install: 'false' + wait-for-artifact-seconds: ${{ steps.fixture-producer.outputs.wait-seconds }} + + # A background step publishes its outputs only after the group's wait, which is why the + # phase reads them as inputs rather than as its own steps. It also takes each member's + # outcome: a member that failed would otherwise leave the phase skipped, and the phase + # guard would report an unexplained skip instead of the member that caused it. - name: iOS smoke tests id: ios uses: ./.github/actions/smoke-ios-run continue-on-error: true timeout-minutes: 80 + with: + simulator-udid: ${{ steps.ios-simulator.outputs.simulator-udid }} + boot-timeout-seconds: ${{ steps.ios-simulator.outputs.boot-timeout-seconds }} + fixture-app-path: ${{ steps.fixture-app.outputs.app-path }} + fixture-app-id: ${{ steps.fixture-app.outputs.app-id }} + fixture-source: ${{ steps.fixture-app.outputs.source }} env: - IOS_RUNTIME_VERSION: '26.2' IOS_UDID: ${{ vars.IOS_UDID }} AGENT_DEVICE_STATE_DIR: ${{ env.IOS_STATE_DIR }} AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ env.IOS_DERIVED_PATH }} @@ -97,7 +180,7 @@ jobs: AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' AGENT_DEVICE_XCUITEST_ARCHS: arm64 AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload} - if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-simulator.outcome == 'success' && steps.ios-runner-build.outcome == 'success' && steps.fixture-app.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} - name: Clean the iOS host id: ios-cleanup if: ${{ always() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} @@ -120,7 +203,9 @@ jobs: XCTEST_LANE: host if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.macos-impact.outputs.run != 'false' }} - name: Diagnose the Apple host after a phase failure - if: ${{ always() && (steps.ios.outcome == 'failure' || steps.macos.outcome == 'failure' || steps.ios-cleanup.outcome == 'failure') }} + # The boot, runner build, and fixture restore can now fail outside the phase, where the + # phase's own outcome reads as skipped rather than failed. + if: ${{ always() && (steps.ios.outcome == 'failure' || steps.macos.outcome == 'failure' || steps.ios-cleanup.outcome == 'failure' || steps.ios-simulator.outcome == 'failure' || steps.ios-runner-build.outcome == 'failure' || steps.fixture-app.outcome == 'failure') }} continue-on-error: true shell: bash run: | @@ -145,10 +230,14 @@ jobs: MACOS_OUTCOME: ${{ steps.macos.outcome }} IOS_CLEANUP_OUTCOME: ${{ steps.ios-cleanup.outcome }} MACOS_CLEANUP_OUTCOME: ${{ steps.macos-cleanup.outcome }} + IOS_SETUP_OUTCOMES: 'boot=${{ steps.ios-simulator.outcome }} runner=${{ steps.ios-runner-build.outcome }} fixture=${{ steps.fixture-app.outcome }}' shell: bash run: | set -euo pipefail printf "### Apple smoke phases\n\n| Phase | Outcome |\n| --- | --- |\n| iOS | %s |\n| macOS | %s |\n" "$IOS_OUTCOME" "$MACOS_OUTCOME" >> "$GITHUB_STEP_SUMMARY" + # The device-free setup steps run as background steps, so a member that failed shows up + # below only as a skipped phase. Name them so the skip has a cause. + printf 'iOS device-free setup: %s\n' "$IOS_SETUP_OUTCOMES" >> "$GITHUB_STEP_SUMMARY" if [ "$IOS_SELECTED" = false ]; then test "$IOS_OUTCOME" = skipped test "$IOS_CLEANUP_OUTCOME" = skipped diff --git a/scripts/__tests__/apple-ci-impact.test.ts b/scripts/__tests__/apple-ci-impact.test.ts index 49e0e1aa7c..3b948f3204 100644 --- a/scripts/__tests__/apple-ci-impact.test.ts +++ b/scripts/__tests__/apple-ci-impact.test.ts @@ -23,6 +23,7 @@ type AppleStep = { env?: Record; with?: Record; 'continue-on-error'?: boolean; + parallel?: AppleStep[]; }; function appleWorkflow() { @@ -323,6 +324,9 @@ test.each([ MACOS_OUTCOME: '${{ steps.macos.outcome }}', IOS_CLEANUP_OUTCOME: '${{ steps.ios-cleanup.outcome }}', MACOS_CLEANUP_OUTCOME: '${{ steps.macos-cleanup.outcome }}', + // A background step that fails leaves the phase skipped, so the summary names the members. + IOS_SETUP_OUTCOMES: + 'boot=${{ steps.ios-simulator.outcome }} runner=${{ steps.ios-runner-build.outcome }} fixture=${{ steps.fixture-app.outcome }}', }); const result = spawnSync('/bin/bash', ['-c', guard.run!], { env: { @@ -333,6 +337,7 @@ test.each([ MACOS_OUTCOME: 'success', IOS_CLEANUP_OUTCOME: 'success', MACOS_CLEANUP_OUTCOME: 'success', + IOS_SETUP_OUTCOMES: 'boot=success runner=success fixture=success', ...overrides, GITHUB_STEP_SUMMARY: path.join(mkdtempForTestSync('apple-smoke-status-'), 'summary.md'), }, @@ -666,28 +671,59 @@ test('a shallow PR merge still yields a known change set for all Apple selectors } }); -test('iOS boot overlaps the native build and readiness remains mandatory before XCTest', () => { +// None of these needs a booted simulator, so they overlap as one parallel group and the phase +// pays the slowest instead of all three. The readiness barrier stays inside the phase: overlapping +// the boot must never let a step reach a simulator that is still starting. +test('device-free iOS setup overlaps as one group and readiness stays mandatory before XCTest', () => { + const group = appleJob().steps.find((step) => step.parallel !== undefined)!; + const members = group.parallel!; + expect(members.map((step) => step.uses)).toEqual([ + './.github/actions/boot-ios-test-simulator', + './.github/actions/setup-apple-runner-build', + './.github/actions/setup-fixture-app', + ]); + expect( + members.find((step) => step.uses === './.github/actions/boot-ios-test-simulator')!.with?.[ + 'wait-for-boot' + ], + ).toBe('false'); + const document = parse( fs.readFileSync(path.join(repoRoot, '.github/actions/smoke-ios-run/action.yml'), 'utf8'), ) as { runs: { steps: AppleStep[] } }; const steps = document.runs.steps; - const boot = steps.findIndex((step) => step.id === 'ios-simulator'); - const build = steps.findIndex( - (step) => step.uses === './.github/actions/setup-apple-runner-build', + expect(steps.some((step) => step.uses === './.github/actions/boot-ios-test-simulator')).toBe( + false, ); + expect(steps.some((step) => step.uses === './.github/actions/setup-fixture-app')).toBe(false); const wait = steps.findIndex((step) => step.run?.includes('scripts/wait-for-ios-simulator.sh')); const tests = steps.findIndex((step) => step.id === 'ios-xctest'); - expect(boot).toBeGreaterThanOrEqual(0); - expect(build).toBeGreaterThan(boot); - expect(wait).toBeGreaterThan(build); + expect(wait).toBeGreaterThanOrEqual(0); expect(tests).toBeGreaterThan(wait); - expect(steps[boot]!.with?.['wait-for-boot']).toBe('false'); - expect(steps[wait]!.run).toContain('steps.ios-simulator.outputs.simulator-udid'); - expect(steps[wait]!.run).toContain('steps.ios-simulator.outputs.boot-timeout-seconds'); + expect(steps[wait]!.run).toContain('inputs.simulator-udid'); + expect(steps[wait]!.run).toContain('inputs.boot-timeout-seconds'); expect(steps[wait]!.if).toBeUndefined(); expect(steps[wait]!['continue-on-error']).not.toBe(true); }); +// The group's outputs cross into the phase as inputs, and a group member that failed would leave +// the phase running on empty paths: `!cancelled()` does not see a sibling's failure. +test('the iOS phase requires every device-free setup member to have succeeded', () => { + const ios = appleJob().steps.find((step) => step.id === 'ios')!; + expect(ios.with).toEqual( + expect.objectContaining({ + 'simulator-udid': '${{ steps.ios-simulator.outputs.simulator-udid }}', + 'boot-timeout-seconds': '${{ steps.ios-simulator.outputs.boot-timeout-seconds }}', + 'fixture-app-path': '${{ steps.fixture-app.outputs.app-path }}', + 'fixture-app-id': '${{ steps.fixture-app.outputs.app-id }}', + 'fixture-source': '${{ steps.fixture-app.outputs.source }}', + }), + ); + for (const member of ['ios-simulator', 'ios-runner-build', 'fixture-app']) { + expect(ios.if).toContain(`steps.${member}.outcome == 'success'`); + } +}); + function simulatorBootFixture(mode: 'ready' | 'failed' | 'stalled') { const directory = mkdtempForTestSync('ios-boot-wait-'); const calls = path.join(directory, 'calls.ndjson'); From 13fbcf563fb0e3d2485845e19245607936aa76b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 13:37:19 +0100 Subject: [PATCH 2/3] ci(apple): keep a failed focus canary from running the iOS phase The canary used to run inside the iOS phase, so its failure failed the phase. It now runs ahead of the device-free parallel group, and `!cancelled()` does not see a prior step's failure, so the phase would have run roughly 200 s of XCTest, replays, and E2E against a host that cannot hand focus to Finder, and reported a replay failure instead of the cause. Require the canary's outcome in the phase condition, and name it in the host diagnostic trigger and the phase summary, where a hoisted step that failed otherwise shows up only as a skipped phase. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/apple.yml | 20 ++++++------- scripts/__tests__/apple-ci-impact.test.ts | 34 ++++++++++++++++++----- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index ba9801e329..3ab628c0a5 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -86,8 +86,8 @@ jobs: gate: build # The canary records the frontmost app before anything can take focus. It stays ahead of the # group because one of its members starts the simulator boot, which can raise the Simulator - # app. When it fails, the baseline file is never written, so the phase's own focus assertion - # fails too rather than passing against a stale baseline. + # app. A host that cannot hand focus to Finder is not a usable automation host, so the iOS + # phase below requires its success: it no longer fails the phase by running inside it. - name: Establish host focus canary id: host-focus-canary if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} @@ -180,7 +180,7 @@ jobs: AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' AGENT_DEVICE_XCUITEST_ARCHS: arm64 AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload} - if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-simulator.outcome == 'success' && steps.ios-runner-build.outcome == 'success' && steps.fixture-app.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.host-focus-canary.outcome == 'success' && steps.ios-simulator.outcome == 'success' && steps.ios-runner-build.outcome == 'success' && steps.fixture-app.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} - name: Clean the iOS host id: ios-cleanup if: ${{ always() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} @@ -203,9 +203,9 @@ jobs: XCTEST_LANE: host if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.macos-impact.outputs.run != 'false' }} - name: Diagnose the Apple host after a phase failure - # The boot, runner build, and fixture restore can now fail outside the phase, where the - # phase's own outcome reads as skipped rather than failed. - if: ${{ always() && (steps.ios.outcome == 'failure' || steps.macos.outcome == 'failure' || steps.ios-cleanup.outcome == 'failure' || steps.ios-simulator.outcome == 'failure' || steps.ios-runner-build.outcome == 'failure' || steps.fixture-app.outcome == 'failure') }} + # The canary, boot, runner build, and fixture restore can now fail outside the phase, where + # the phase's own outcome reads as skipped rather than failed. + if: ${{ always() && (steps.ios.outcome == 'failure' || steps.macos.outcome == 'failure' || steps.ios-cleanup.outcome == 'failure' || steps.host-focus-canary.outcome == 'failure' || steps.ios-simulator.outcome == 'failure' || steps.ios-runner-build.outcome == 'failure' || steps.fixture-app.outcome == 'failure') }} continue-on-error: true shell: bash run: | @@ -230,14 +230,14 @@ jobs: MACOS_OUTCOME: ${{ steps.macos.outcome }} IOS_CLEANUP_OUTCOME: ${{ steps.ios-cleanup.outcome }} MACOS_CLEANUP_OUTCOME: ${{ steps.macos-cleanup.outcome }} - IOS_SETUP_OUTCOMES: 'boot=${{ steps.ios-simulator.outcome }} runner=${{ steps.ios-runner-build.outcome }} fixture=${{ steps.fixture-app.outcome }}' + IOS_SETUP_OUTCOMES: 'canary=${{ steps.host-focus-canary.outcome }} boot=${{ steps.ios-simulator.outcome }} runner=${{ steps.ios-runner-build.outcome }} fixture=${{ steps.fixture-app.outcome }}' shell: bash run: | set -euo pipefail printf "### Apple smoke phases\n\n| Phase | Outcome |\n| --- | --- |\n| iOS | %s |\n| macOS | %s |\n" "$IOS_OUTCOME" "$MACOS_OUTCOME" >> "$GITHUB_STEP_SUMMARY" - # The device-free setup steps run as background steps, so a member that failed shows up - # below only as a skipped phase. Name them so the skip has a cause. - printf 'iOS device-free setup: %s\n' "$IOS_SETUP_OUTCOMES" >> "$GITHUB_STEP_SUMMARY" + # A step hoisted out of the phase shows up below only as a skipped phase. Name them so the + # skip has a cause. + printf 'iOS setup outside the phase: %s\n' "$IOS_SETUP_OUTCOMES" >> "$GITHUB_STEP_SUMMARY" if [ "$IOS_SELECTED" = false ]; then test "$IOS_OUTCOME" = skipped test "$IOS_CLEANUP_OUTCOME" = skipped diff --git a/scripts/__tests__/apple-ci-impact.test.ts b/scripts/__tests__/apple-ci-impact.test.ts index 3b948f3204..65b2fd9998 100644 --- a/scripts/__tests__/apple-ci-impact.test.ts +++ b/scripts/__tests__/apple-ci-impact.test.ts @@ -324,9 +324,9 @@ test.each([ MACOS_OUTCOME: '${{ steps.macos.outcome }}', IOS_CLEANUP_OUTCOME: '${{ steps.ios-cleanup.outcome }}', MACOS_CLEANUP_OUTCOME: '${{ steps.macos-cleanup.outcome }}', - // A background step that fails leaves the phase skipped, so the summary names the members. + // A hoisted step that fails leaves the phase skipped, so the summary names them. IOS_SETUP_OUTCOMES: - 'boot=${{ steps.ios-simulator.outcome }} runner=${{ steps.ios-runner-build.outcome }} fixture=${{ steps.fixture-app.outcome }}', + 'canary=${{ steps.host-focus-canary.outcome }} boot=${{ steps.ios-simulator.outcome }} runner=${{ steps.ios-runner-build.outcome }} fixture=${{ steps.fixture-app.outcome }}', }); const result = spawnSync('/bin/bash', ['-c', guard.run!], { env: { @@ -337,7 +337,7 @@ test.each([ MACOS_OUTCOME: 'success', IOS_CLEANUP_OUTCOME: 'success', MACOS_CLEANUP_OUTCOME: 'success', - IOS_SETUP_OUTCOMES: 'boot=success runner=success fixture=success', + IOS_SETUP_OUTCOMES: 'canary=success boot=success runner=success fixture=success', ...overrides, GITHUB_STEP_SUMMARY: path.join(mkdtempForTestSync('apple-smoke-status-'), 'summary.md'), }, @@ -706,9 +706,10 @@ test('device-free iOS setup overlaps as one group and readiness stays mandatory expect(steps[wait]!['continue-on-error']).not.toBe(true); }); -// The group's outputs cross into the phase as inputs, and a group member that failed would leave -// the phase running on empty paths: `!cancelled()` does not see a sibling's failure. -test('the iOS phase requires every device-free setup member to have succeeded', () => { +// The group's outputs cross into the phase as inputs, and a setup step that failed would leave the +// phase running on empty paths: `!cancelled()` does not see a sibling's failure. The canary moved +// out of the phase too, so a host that cannot hand focus to Finder must stop the phase here. +test('the iOS phase requires every hoisted setup step to have succeeded', () => { const ios = appleJob().steps.find((step) => step.id === 'ios')!; expect(ios.with).toEqual( expect.objectContaining({ @@ -719,11 +720,30 @@ test('the iOS phase requires every device-free setup member to have succeeded', 'fixture-source': '${{ steps.fixture-app.outputs.source }}', }), ); - for (const member of ['ios-simulator', 'ios-runner-build', 'fixture-app']) { + for (const member of ['host-focus-canary', 'ios-simulator', 'ios-runner-build', 'fixture-app']) { expect(ios.if).toContain(`steps.${member}.outcome == 'success'`); } }); +// Each hoisted step can now fail where the phase reads as skipped, so the host diagnostic still has +// to fire for all of them. +test('a failed iOS setup step still captures the host diagnostic', () => { + const diagnose = appleJob().steps.find( + (step) => step.name === 'Diagnose the Apple host after a phase failure', + )!; + for (const member of [ + 'ios', + 'macos', + 'ios-cleanup', + 'host-focus-canary', + 'ios-simulator', + 'ios-runner-build', + 'fixture-app', + ]) { + expect(diagnose.if).toContain(`steps.${member}.outcome == 'failure'`); + } +}); + function simulatorBootFixture(mode: 'ready' | 'failed' | 'stalled') { const directory = mkdtempForTestSync('ios-boot-wait-'); const calls = path.join(directory, 'calls.ndjson'); From 8a2c399d8f1ad888449a69654fba11e238134a26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 13:43:46 +0100 Subject: [PATCH 3/3] test(apple): let the diagnostic guard read a step name The host-diagnostic guard looks the step up by name, so declare the field the workflow parser already returns. Co-Authored-By: Claude Opus 5.5 --- scripts/__tests__/apple-ci-impact.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/__tests__/apple-ci-impact.test.ts b/scripts/__tests__/apple-ci-impact.test.ts index 65b2fd9998..e2c2c4f080 100644 --- a/scripts/__tests__/apple-ci-impact.test.ts +++ b/scripts/__tests__/apple-ci-impact.test.ts @@ -16,6 +16,7 @@ import { selectChecks } from '../check-affected/model.ts'; const repoRoot = path.resolve(import.meta.dirname, '../..'); type AppleStep = { + name?: string; id?: string; uses?: string; if?: string;