diff --git a/.github/actions/smoke-ios-run/action.yml b/.github/actions/smoke-ios-run/action.yml index da08a958c2..d3a6cd1019 100644 --- a/.github/actions/smoke-ios-run/action.yml +++ b/.github/actions/smoke-ios-run/action.yml @@ -1,19 +1,29 @@ name: Run iOS smoke tests description: Run iOS smoke tests + +# The caller boots the simulator, builds the runner, and restores the fixture app as one parallel +# group before it calls this action, because none of those three needs a booted simulator. Their +# outputs arrive as inputs: a composite cannot read the caller's step context. +inputs: + simulator-udid: + description: 'UDID of the iOS simulator the caller booted' + required: true + boot-timeout-seconds: + description: 'Boot wait budget the caller-owned readiness check must honor' + required: true + fixture-app-path: + description: 'Path to the restored fixture app bundle' + required: true + fixture-app-id: + description: 'Bundle identifier of the restored fixture app' + required: true + fixture-source: + description: 'Where the fixture binary came from: artifact or build' + required: true + runs: using: composite steps: - - name: Set fixture producer wait policy - id: fixture-producer - run: | - set -euo pipefail - WAIT_SECONDS=1800 - if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then - echo "Fork pull request cannot publish the shared fixture artifact; a cache miss will build inline." - WAIT_SECONDS=0 - fi - echo "wait-seconds=$WAIT_SECONDS" >> "$GITHUB_OUTPUT" - shell: bash - name: Select iOS runner XCTests id: xctest-impact env: @@ -21,38 +31,16 @@ runs: run: | node --experimental-strip-types scripts/apple-ci-impact.ts xctest shell: bash - - name: Establish host focus canary - id: host-focus-canary - run: | - set -euo pipefail - osascript -e 'tell application "Finder" to activate' - sleep 1 - FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')" - test "$FRONTMOST" = "com.apple.finder" - mkdir -p .tmp - echo "$FRONTMOST" > .tmp/ios-host-focus-before - shell: bash - - name: Boot iOS test simulator - id: ios-simulator - uses: ./.github/actions/boot-ios-test-simulator - with: - runtime-version: ${{ env.IOS_RUNTIME_VERSION }} - preferred-device-name: iPhone 17 Pro - wait-for-boot: 'false' - - name: Restore and build iOS XCTest runner - uses: ./.github/actions/setup-apple-runner-build - with: - derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} - cache-key-prefix: ios-runner-prebuilt - cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64 - gate: swift-runner-ios - xcuitest-platform: ios - xcuitest-destination: generic/platform=iOS Simulator + # The caller recorded the host focus canary before its boot started, so a simulator that takes + # focus during boot is still measured against the pre-boot frontmost app. + # The caller booted this simulator and built this runner alongside the fixture restore, so + # both are already in flight when this action starts. Readiness stays mandatory here: no step + # below may reach the device before it boots. - name: Wait for iOS test simulator run: | bash scripts/wait-for-ios-simulator.sh \ - "${{ steps.ios-simulator.outputs.simulator-udid }}" \ - "${{ steps.ios-simulator.outputs.boot-timeout-seconds }}" + "${{ inputs.simulator-udid }}" \ + "${{ inputs.boot-timeout-seconds }}" shell: bash - name: Run iOS runner platform checks id: ios-xctest @@ -70,7 +58,7 @@ runs: mkdir -p "$GITHUB_WORKSPACE/test/artifacts" xcodebuild test-without-building \ -xctestrun "$XCTESTRUN_PATH" \ - -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ + -destination "platform=iOS Simulator,id=${{ inputs.simulator-udid }}" \ -resultBundlePath "$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions.xcresult" \ "${IOS_PR_TEST_ARGS[@]}" 2>&1 | tee /tmp/agent-device-runner-regressions.log xcrun xcresulttool get test-results summary \ @@ -92,34 +80,28 @@ runs: const c = await ensureDaemon({ stateDir: process.env.AGENT_DEVICE_STATE_DIR }); console.log(JSON.stringify({ phase: "daemon_cold_startup", started: c.started, pid: c.daemon.pid, durationMs: Math.round(performance.now() - t0) })); ' - node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json + node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ inputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json shell: bash - name: Run iOS Settings replay smoke test run: | - node bin/agent-device.mjs test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml + node bin/agent-device.mjs test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ inputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml shell: bash - - name: Fetch current fixture app - id: fixture-app - uses: ./.github/actions/setup-fixture-app - with: - install: 'false' - wait-for-artifact-seconds: ${{ steps.fixture-producer.outputs.wait-seconds }} - name: Report fixture cache source run: | - echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" + echo "Fixture app source: ${{ inputs.fixture-source }}" >> "$GITHUB_STEP_SUMMARY" shell: bash - name: Run gesture pan-duration smoke replay run: | - xcrun simctl install "${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.fixture-app.outputs.app-path }}" - node bin/agent-device.mjs test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml + xcrun simctl install "${{ inputs.simulator-udid }}" "${{ inputs.fixture-app-path }}" + node bin/agent-device.mjs test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ inputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml shell: bash - name: Run fixture-backed iOS simulator E2E smoke env: - AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} - AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} + AGENT_DEVICE_FIXTURE_APP_ID: ${{ inputs.fixture-app-id }} + AGENT_DEVICE_FIXTURE_APP_PATH: ${{ inputs.fixture-app-path }} AGENT_DEVICE_IOS_E2E: '1' AGENT_DEVICE_IOS_E2E_TIER: smoke - AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + AGENT_DEVICE_IOS_UDID: ${{ inputs.simulator-udid }} run: | xcrun simctl spawn "$AGENT_DEVICE_IOS_UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index df10a2ef56..3ab628c0a5 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -37,6 +37,7 @@ jobs: timeout-minutes: 90 env: BASE_SHA: ${{ github.event.pull_request.base.sha }} + IOS_RUNTIME_VERSION: '26.2' IOS_STATE_DIR: ${{ github.workspace }}/.tmp/ios-agent-device-state MACOS_STATE_DIR: ${{ github.workspace }}/.tmp/macos-agent-device-state IOS_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived @@ -83,13 +84,95 @@ jobs: uses: ./.github/actions/run-gate with: gate: build + # The canary records the frontmost app before anything can take focus. It stays ahead of the + # group because one of its members starts the simulator boot, which can raise the Simulator + # app. A host that cannot hand focus to Finder is not a usable automation host, so the iOS + # phase below requires its success: it no longer fails the phase by running inside it. + - name: Establish host focus canary + id: host-focus-canary + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + run: | + set -euo pipefail + osascript -e 'tell application "Finder" to activate' + sleep 1 + FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')" + test "$FRONTMOST" = "com.apple.finder" + mkdir -p .tmp + echo "$FRONTMOST" > .tmp/ios-host-focus-before + shell: bash + # The fixture wait policy decides how long the restore polls for a concurrently produced + # artifact, and a background step's outputs reach only the steps after its group, so this + # cheap decision stays sequential ahead of the group. + - name: Set fixture producer wait policy + id: fixture-producer + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + run: | + set -euo pipefail + WAIT_SECONDS=1800 + if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then + echo "Fork pull request cannot publish the shared fixture artifact; a cache miss will build inline." + WAIT_SECONDS=0 + fi + echo "wait-seconds=$WAIT_SECONDS" >> "$GITHUB_OUTPUT" + shell: bash + # None of these needs a booted simulator, and the iOS phase cannot start XCTest, preflight, + # or the fixture replays until all three finish. The phase ran them in sequence, so the + # runner paid the boot, the native build, and the fixture restore one after another while the + # simulator sat idle. They share no outputs and write disjoint paths (the simulator, the + # runner DerivedData, and .tmp/fixture-app through examples/test-app/node_modules), so they + # run as one parallel group and the phase pays the slowest instead of all three. The boot + # keeps its deferred readiness so it still overlaps the build it now starts beside. + - parallel: + - name: Boot iOS test simulator + id: ios-simulator + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + uses: ./.github/actions/boot-ios-test-simulator + with: + runtime-version: ${{ env.IOS_RUNTIME_VERSION }} + preferred-device-name: iPhone 17 Pro + wait-for-boot: 'false' + + - name: Restore and build iOS XCTest runner + id: ios-runner-build + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + uses: ./.github/actions/setup-apple-runner-build + with: + derived-path: ${{ env.IOS_DERIVED_PATH }} + cache-key-prefix: ios-runner-prebuilt + cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64 + gate: swift-runner-ios + xcuitest-platform: ios + xcuitest-destination: generic/platform=iOS Simulator + # The build variant is part of the cache key, so these must stay exactly as the phase + # passed them while the build ran inside it. + env: + AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' + AGENT_DEVICE_XCUITEST_ARCHS: arm64 + + - name: Restore fixture app + id: fixture-app + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + uses: ./.github/actions/setup-fixture-app + with: + install: 'false' + wait-for-artifact-seconds: ${{ steps.fixture-producer.outputs.wait-seconds }} + + # A background step publishes its outputs only after the group's wait, which is why the + # phase reads them as inputs rather than as its own steps. It also takes each member's + # outcome: a member that failed would otherwise leave the phase skipped, and the phase + # guard would report an unexplained skip instead of the member that caused it. - name: iOS smoke tests id: ios uses: ./.github/actions/smoke-ios-run continue-on-error: true timeout-minutes: 80 + with: + simulator-udid: ${{ steps.ios-simulator.outputs.simulator-udid }} + boot-timeout-seconds: ${{ steps.ios-simulator.outputs.boot-timeout-seconds }} + fixture-app-path: ${{ steps.fixture-app.outputs.app-path }} + fixture-app-id: ${{ steps.fixture-app.outputs.app-id }} + fixture-source: ${{ steps.fixture-app.outputs.source }} env: - IOS_RUNTIME_VERSION: '26.2' IOS_UDID: ${{ vars.IOS_UDID }} AGENT_DEVICE_STATE_DIR: ${{ env.IOS_STATE_DIR }} AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ env.IOS_DERIVED_PATH }} @@ -97,7 +180,7 @@ jobs: AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' AGENT_DEVICE_XCUITEST_ARCHS: arm64 AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload} - if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.host-focus-canary.outcome == 'success' && steps.ios-simulator.outcome == 'success' && steps.ios-runner-build.outcome == 'success' && steps.fixture-app.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} - name: Clean the iOS host id: ios-cleanup if: ${{ always() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} @@ -120,7 +203,9 @@ jobs: XCTEST_LANE: host if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.macos-impact.outputs.run != 'false' }} - name: Diagnose the Apple host after a phase failure - if: ${{ always() && (steps.ios.outcome == 'failure' || steps.macos.outcome == 'failure' || steps.ios-cleanup.outcome == 'failure') }} + # The canary, boot, runner build, and fixture restore can now fail outside the phase, where + # the phase's own outcome reads as skipped rather than failed. + if: ${{ always() && (steps.ios.outcome == 'failure' || steps.macos.outcome == 'failure' || steps.ios-cleanup.outcome == 'failure' || steps.host-focus-canary.outcome == 'failure' || steps.ios-simulator.outcome == 'failure' || steps.ios-runner-build.outcome == 'failure' || steps.fixture-app.outcome == 'failure') }} continue-on-error: true shell: bash run: | @@ -145,10 +230,14 @@ jobs: MACOS_OUTCOME: ${{ steps.macos.outcome }} IOS_CLEANUP_OUTCOME: ${{ steps.ios-cleanup.outcome }} MACOS_CLEANUP_OUTCOME: ${{ steps.macos-cleanup.outcome }} + IOS_SETUP_OUTCOMES: 'canary=${{ steps.host-focus-canary.outcome }} boot=${{ steps.ios-simulator.outcome }} runner=${{ steps.ios-runner-build.outcome }} fixture=${{ steps.fixture-app.outcome }}' shell: bash run: | set -euo pipefail printf "### Apple smoke phases\n\n| Phase | Outcome |\n| --- | --- |\n| iOS | %s |\n| macOS | %s |\n" "$IOS_OUTCOME" "$MACOS_OUTCOME" >> "$GITHUB_STEP_SUMMARY" + # A step hoisted out of the phase shows up below only as a skipped phase. Name them so the + # skip has a cause. + printf 'iOS setup outside the phase: %s\n' "$IOS_SETUP_OUTCOMES" >> "$GITHUB_STEP_SUMMARY" if [ "$IOS_SELECTED" = false ]; then test "$IOS_OUTCOME" = skipped test "$IOS_CLEANUP_OUTCOME" = skipped diff --git a/scripts/__tests__/apple-ci-impact.test.ts b/scripts/__tests__/apple-ci-impact.test.ts index 49e0e1aa7c..e2c2c4f080 100644 --- a/scripts/__tests__/apple-ci-impact.test.ts +++ b/scripts/__tests__/apple-ci-impact.test.ts @@ -16,6 +16,7 @@ import { selectChecks } from '../check-affected/model.ts'; const repoRoot = path.resolve(import.meta.dirname, '../..'); type AppleStep = { + name?: string; id?: string; uses?: string; if?: string; @@ -23,6 +24,7 @@ type AppleStep = { env?: Record; with?: Record; 'continue-on-error'?: boolean; + parallel?: AppleStep[]; }; function appleWorkflow() { @@ -323,6 +325,9 @@ test.each([ MACOS_OUTCOME: '${{ steps.macos.outcome }}', IOS_CLEANUP_OUTCOME: '${{ steps.ios-cleanup.outcome }}', MACOS_CLEANUP_OUTCOME: '${{ steps.macos-cleanup.outcome }}', + // A hoisted step that fails leaves the phase skipped, so the summary names them. + IOS_SETUP_OUTCOMES: + 'canary=${{ steps.host-focus-canary.outcome }} boot=${{ steps.ios-simulator.outcome }} runner=${{ steps.ios-runner-build.outcome }} fixture=${{ steps.fixture-app.outcome }}', }); const result = spawnSync('/bin/bash', ['-c', guard.run!], { env: { @@ -333,6 +338,7 @@ test.each([ MACOS_OUTCOME: 'success', IOS_CLEANUP_OUTCOME: 'success', MACOS_CLEANUP_OUTCOME: 'success', + IOS_SETUP_OUTCOMES: 'canary=success boot=success runner=success fixture=success', ...overrides, GITHUB_STEP_SUMMARY: path.join(mkdtempForTestSync('apple-smoke-status-'), 'summary.md'), }, @@ -666,28 +672,79 @@ test('a shallow PR merge still yields a known change set for all Apple selectors } }); -test('iOS boot overlaps the native build and readiness remains mandatory before XCTest', () => { +// None of these needs a booted simulator, so they overlap as one parallel group and the phase +// pays the slowest instead of all three. The readiness barrier stays inside the phase: overlapping +// the boot must never let a step reach a simulator that is still starting. +test('device-free iOS setup overlaps as one group and readiness stays mandatory before XCTest', () => { + const group = appleJob().steps.find((step) => step.parallel !== undefined)!; + const members = group.parallel!; + expect(members.map((step) => step.uses)).toEqual([ + './.github/actions/boot-ios-test-simulator', + './.github/actions/setup-apple-runner-build', + './.github/actions/setup-fixture-app', + ]); + expect( + members.find((step) => step.uses === './.github/actions/boot-ios-test-simulator')!.with?.[ + 'wait-for-boot' + ], + ).toBe('false'); + const document = parse( fs.readFileSync(path.join(repoRoot, '.github/actions/smoke-ios-run/action.yml'), 'utf8'), ) as { runs: { steps: AppleStep[] } }; const steps = document.runs.steps; - const boot = steps.findIndex((step) => step.id === 'ios-simulator'); - const build = steps.findIndex( - (step) => step.uses === './.github/actions/setup-apple-runner-build', + expect(steps.some((step) => step.uses === './.github/actions/boot-ios-test-simulator')).toBe( + false, ); + expect(steps.some((step) => step.uses === './.github/actions/setup-fixture-app')).toBe(false); const wait = steps.findIndex((step) => step.run?.includes('scripts/wait-for-ios-simulator.sh')); const tests = steps.findIndex((step) => step.id === 'ios-xctest'); - expect(boot).toBeGreaterThanOrEqual(0); - expect(build).toBeGreaterThan(boot); - expect(wait).toBeGreaterThan(build); + expect(wait).toBeGreaterThanOrEqual(0); expect(tests).toBeGreaterThan(wait); - expect(steps[boot]!.with?.['wait-for-boot']).toBe('false'); - expect(steps[wait]!.run).toContain('steps.ios-simulator.outputs.simulator-udid'); - expect(steps[wait]!.run).toContain('steps.ios-simulator.outputs.boot-timeout-seconds'); + expect(steps[wait]!.run).toContain('inputs.simulator-udid'); + expect(steps[wait]!.run).toContain('inputs.boot-timeout-seconds'); expect(steps[wait]!.if).toBeUndefined(); expect(steps[wait]!['continue-on-error']).not.toBe(true); }); +// The group's outputs cross into the phase as inputs, and a setup step that failed would leave the +// phase running on empty paths: `!cancelled()` does not see a sibling's failure. The canary moved +// out of the phase too, so a host that cannot hand focus to Finder must stop the phase here. +test('the iOS phase requires every hoisted setup step to have succeeded', () => { + const ios = appleJob().steps.find((step) => step.id === 'ios')!; + expect(ios.with).toEqual( + expect.objectContaining({ + 'simulator-udid': '${{ steps.ios-simulator.outputs.simulator-udid }}', + 'boot-timeout-seconds': '${{ steps.ios-simulator.outputs.boot-timeout-seconds }}', + 'fixture-app-path': '${{ steps.fixture-app.outputs.app-path }}', + 'fixture-app-id': '${{ steps.fixture-app.outputs.app-id }}', + 'fixture-source': '${{ steps.fixture-app.outputs.source }}', + }), + ); + for (const member of ['host-focus-canary', 'ios-simulator', 'ios-runner-build', 'fixture-app']) { + expect(ios.if).toContain(`steps.${member}.outcome == 'success'`); + } +}); + +// Each hoisted step can now fail where the phase reads as skipped, so the host diagnostic still has +// to fire for all of them. +test('a failed iOS setup step still captures the host diagnostic', () => { + const diagnose = appleJob().steps.find( + (step) => step.name === 'Diagnose the Apple host after a phase failure', + )!; + for (const member of [ + 'ios', + 'macos', + 'ios-cleanup', + 'host-focus-canary', + 'ios-simulator', + 'ios-runner-build', + 'fixture-app', + ]) { + expect(diagnose.if).toContain(`steps.${member}.outcome == 'failure'`); + } +}); + function simulatorBootFixture(mode: 'ready' | 'failed' | 'stalled') { const directory = mkdtempForTestSync('ios-boot-wait-'); const calls = path.join(directory, 'calls.ndjson');