Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 37 additions & 55 deletions .github/actions/smoke-ios-run/action.yml
Original file line number Diff line number Diff line change
@@ -1,58 +1,46 @@
name: Run iOS smoke tests
description: Run iOS smoke tests

# The caller boots the simulator, builds the runner, and restores the fixture app as one parallel
# group before it calls this action, because none of those three needs a booted simulator. Their
# outputs arrive as inputs: a composite cannot read the caller's step context.
inputs:
simulator-udid:
description: 'UDID of the iOS simulator the caller booted'
required: true
boot-timeout-seconds:
description: 'Boot wait budget the caller-owned readiness check must honor'
required: true
fixture-app-path:
description: 'Path to the restored fixture app bundle'
required: true
fixture-app-id:
description: 'Bundle identifier of the restored fixture app'
required: true
fixture-source:
description: 'Where the fixture binary came from: artifact or build'
required: true

runs:
using: composite
steps:
- name: Set fixture producer wait policy
id: fixture-producer
run: |
set -euo pipefail
WAIT_SECONDS=1800
if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
echo "Fork pull request cannot publish the shared fixture artifact; a cache miss will build inline."
WAIT_SECONDS=0
fi
echo "wait-seconds=$WAIT_SECONDS" >> "$GITHUB_OUTPUT"
shell: bash
- name: Select iOS runner XCTests
id: xctest-impact
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
node --experimental-strip-types scripts/apple-ci-impact.ts xctest
shell: bash
- name: Establish host focus canary
id: host-focus-canary
run: |
set -euo pipefail
osascript -e 'tell application "Finder" to activate'
sleep 1
FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')"
test "$FRONTMOST" = "com.apple.finder"
mkdir -p .tmp
echo "$FRONTMOST" > .tmp/ios-host-focus-before
shell: bash
- name: Boot iOS test simulator
id: ios-simulator
uses: ./.github/actions/boot-ios-test-simulator
with:
runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
preferred-device-name: iPhone 17 Pro
wait-for-boot: 'false'
- name: Restore and build iOS XCTest runner
uses: ./.github/actions/setup-apple-runner-build
with:
derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
cache-key-prefix: ios-runner-prebuilt
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64
gate: swift-runner-ios
xcuitest-platform: ios
xcuitest-destination: generic/platform=iOS Simulator
# The caller recorded the host focus canary before its boot started, so a simulator that takes
# focus during boot is still measured against the pre-boot frontmost app.
# The caller booted this simulator and built this runner alongside the fixture restore, so
# both are already in flight when this action starts. Readiness stays mandatory here: no step
# below may reach the device before it boots.
- name: Wait for iOS test simulator
run: |
bash scripts/wait-for-ios-simulator.sh \
"${{ steps.ios-simulator.outputs.simulator-udid }}" \
"${{ steps.ios-simulator.outputs.boot-timeout-seconds }}"
"${{ inputs.simulator-udid }}" \
"${{ inputs.boot-timeout-seconds }}"
shell: bash
- name: Run iOS runner platform checks
id: ios-xctest
Expand All @@ -70,7 +58,7 @@ runs:
mkdir -p "$GITHUB_WORKSPACE/test/artifacts"
xcodebuild test-without-building \
-xctestrun "$XCTESTRUN_PATH" \
-destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \
-destination "platform=iOS Simulator,id=${{ inputs.simulator-udid }}" \
-resultBundlePath "$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions.xcresult" \
"${IOS_PR_TEST_ARGS[@]}" 2>&1 | tee /tmp/agent-device-runner-regressions.log
xcrun xcresulttool get test-results summary \
Expand All @@ -92,34 +80,28 @@ runs:
const c = await ensureDaemon({ stateDir: process.env.AGENT_DEVICE_STATE_DIR });
console.log(JSON.stringify({ phase: "daemon_cold_startup", started: c.started, pid: c.daemon.pid, durationMs: Math.round(performance.now() - t0) }));
'
node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json
node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ inputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json
shell: bash
- name: Run iOS Settings replay smoke test
run: |
node bin/agent-device.mjs test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml
node bin/agent-device.mjs test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ inputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml
shell: bash
- name: Fetch current fixture app
id: fixture-app
uses: ./.github/actions/setup-fixture-app
with:
install: 'false'
wait-for-artifact-seconds: ${{ steps.fixture-producer.outputs.wait-seconds }}
- name: Report fixture cache source
run: |
echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY"
echo "Fixture app source: ${{ inputs.fixture-source }}" >> "$GITHUB_STEP_SUMMARY"
shell: bash
- name: Run gesture pan-duration smoke replay
run: |
xcrun simctl install "${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.fixture-app.outputs.app-path }}"
node bin/agent-device.mjs test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml
xcrun simctl install "${{ inputs.simulator-udid }}" "${{ inputs.fixture-app-path }}"
node bin/agent-device.mjs test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ inputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml
shell: bash
- name: Run fixture-backed iOS simulator E2E smoke
env:
AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }}
AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }}
AGENT_DEVICE_FIXTURE_APP_ID: ${{ inputs.fixture-app-id }}
AGENT_DEVICE_FIXTURE_APP_PATH: ${{ inputs.fixture-app-path }}
AGENT_DEVICE_IOS_E2E: '1'
AGENT_DEVICE_IOS_E2E_TIER: smoke
AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }}
AGENT_DEVICE_IOS_UDID: ${{ inputs.simulator-udid }}
run: |
xcrun simctl spawn "$AGENT_DEVICE_IOS_UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false
node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts
Expand Down
95 changes: 92 additions & 3 deletions .github/workflows/apple.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ jobs:
timeout-minutes: 90
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
IOS_RUNTIME_VERSION: '26.2'
IOS_STATE_DIR: ${{ github.workspace }}/.tmp/ios-agent-device-state
MACOS_STATE_DIR: ${{ github.workspace }}/.tmp/macos-agent-device-state
IOS_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
Expand Down Expand Up @@ -83,21 +84,103 @@ jobs:
uses: ./.github/actions/run-gate
with:
gate: build
# The canary records the frontmost app before anything can take focus. It stays ahead of the
# group because one of its members starts the simulator boot, which can raise the Simulator
# app. A host that cannot hand focus to Finder is not a usable automation host, so the iOS
# phase below requires its success: it no longer fails the phase by running inside it.
- name: Establish host focus canary
id: host-focus-canary
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }}
run: |
set -euo pipefail
osascript -e 'tell application "Finder" to activate'
sleep 1
FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')"
test "$FRONTMOST" = "com.apple.finder"
mkdir -p .tmp
echo "$FRONTMOST" > .tmp/ios-host-focus-before
shell: bash
Comment thread
thymikee marked this conversation as resolved.
Resolved
# The fixture wait policy decides how long the restore polls for a concurrently produced
# artifact, and a background step's outputs reach only the steps after its group, so this
# cheap decision stays sequential ahead of the group.
- name: Set fixture producer wait policy
id: fixture-producer
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }}
run: |
set -euo pipefail
WAIT_SECONDS=1800
if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
echo "Fork pull request cannot publish the shared fixture artifact; a cache miss will build inline."
WAIT_SECONDS=0
fi
echo "wait-seconds=$WAIT_SECONDS" >> "$GITHUB_OUTPUT"
shell: bash
# None of these needs a booted simulator, and the iOS phase cannot start XCTest, preflight,
# or the fixture replays until all three finish. The phase ran them in sequence, so the
# runner paid the boot, the native build, and the fixture restore one after another while the
# simulator sat idle. They share no outputs and write disjoint paths (the simulator, the
# runner DerivedData, and .tmp/fixture-app through examples/test-app/node_modules), so they
# run as one parallel group and the phase pays the slowest instead of all three. The boot
# keeps its deferred readiness so it still overlaps the build it now starts beside.
- parallel:
- name: Boot iOS test simulator
id: ios-simulator
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }}
uses: ./.github/actions/boot-ios-test-simulator
with:
runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
preferred-device-name: iPhone 17 Pro
wait-for-boot: 'false'

- name: Restore and build iOS XCTest runner
id: ios-runner-build
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }}
uses: ./.github/actions/setup-apple-runner-build
with:
derived-path: ${{ env.IOS_DERIVED_PATH }}
cache-key-prefix: ios-runner-prebuilt
cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-arm64
gate: swift-runner-ios
xcuitest-platform: ios
xcuitest-destination: generic/platform=iOS Simulator
# The build variant is part of the cache key, so these must stay exactly as the phase
# passed them while the build ran inside it.
env:
AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1'
AGENT_DEVICE_XCUITEST_ARCHS: arm64

- name: Restore fixture app
id: fixture-app
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }}
uses: ./.github/actions/setup-fixture-app
with:
install: 'false'
wait-for-artifact-seconds: ${{ steps.fixture-producer.outputs.wait-seconds }}

# A background step publishes its outputs only after the group's wait, which is why the
# phase reads them as inputs rather than as its own steps. It also takes each member's
# outcome: a member that failed would otherwise leave the phase skipped, and the phase
# guard would report an unexplained skip instead of the member that caused it.
- name: iOS smoke tests
id: ios
uses: ./.github/actions/smoke-ios-run
continue-on-error: true
timeout-minutes: 80
with:
simulator-udid: ${{ steps.ios-simulator.outputs.simulator-udid }}
boot-timeout-seconds: ${{ steps.ios-simulator.outputs.boot-timeout-seconds }}
fixture-app-path: ${{ steps.fixture-app.outputs.app-path }}
fixture-app-id: ${{ steps.fixture-app.outputs.app-id }}
fixture-source: ${{ steps.fixture-app.outputs.source }}
env:
IOS_RUNTIME_VERSION: '26.2'
IOS_UDID: ${{ vars.IOS_UDID }}
AGENT_DEVICE_STATE_DIR: ${{ env.IOS_STATE_DIR }}
AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ env.IOS_DERIVED_PATH }}
AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000'
AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1'
AGENT_DEVICE_XCUITEST_ARCHS: arm64
AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload}
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }}
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.host-focus-canary.outcome == 'success' && steps.ios-simulator.outcome == 'success' && steps.ios-runner-build.outcome == 'success' && steps.fixture-app.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }}
- name: Clean the iOS host
id: ios-cleanup
if: ${{ always() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }}
Expand All @@ -120,7 +203,9 @@ jobs:
XCTEST_LANE: host
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.macos-impact.outputs.run != 'false' }}
- name: Diagnose the Apple host after a phase failure
if: ${{ always() && (steps.ios.outcome == 'failure' || steps.macos.outcome == 'failure' || steps.ios-cleanup.outcome == 'failure') }}
# The canary, boot, runner build, and fixture restore can now fail outside the phase, where
# the phase's own outcome reads as skipped rather than failed.
if: ${{ always() && (steps.ios.outcome == 'failure' || steps.macos.outcome == 'failure' || steps.ios-cleanup.outcome == 'failure' || steps.host-focus-canary.outcome == 'failure' || steps.ios-simulator.outcome == 'failure' || steps.ios-runner-build.outcome == 'failure' || steps.fixture-app.outcome == 'failure') }}
continue-on-error: true
shell: bash
run: |
Expand All @@ -145,10 +230,14 @@ jobs:
MACOS_OUTCOME: ${{ steps.macos.outcome }}
IOS_CLEANUP_OUTCOME: ${{ steps.ios-cleanup.outcome }}
MACOS_CLEANUP_OUTCOME: ${{ steps.macos-cleanup.outcome }}
IOS_SETUP_OUTCOMES: 'canary=${{ steps.host-focus-canary.outcome }} boot=${{ steps.ios-simulator.outcome }} runner=${{ steps.ios-runner-build.outcome }} fixture=${{ steps.fixture-app.outcome }}'
shell: bash
run: |
set -euo pipefail
printf "### Apple smoke phases\n\n| Phase | Outcome |\n| --- | --- |\n| iOS | %s |\n| macOS | %s |\n" "$IOS_OUTCOME" "$MACOS_OUTCOME" >> "$GITHUB_STEP_SUMMARY"
# A step hoisted out of the phase shows up below only as a skipped phase. Name them so the
# skip has a cause.
printf 'iOS setup outside the phase: %s\n' "$IOS_SETUP_OUTCOMES" >> "$GITHUB_STEP_SUMMARY"
if [ "$IOS_SELECTED" = false ]; then
test "$IOS_OUTCOME" = skipped
test "$IOS_CLEANUP_OUTCOME" = skipped
Expand Down
Loading
Loading