diff --git a/.github/actions/setup-android-replay-host/action.yml b/.github/actions/setup-android-replay-host/action.yml index a50f321aa..27ef93920 100644 --- a/.github/actions/setup-android-replay-host/action.yml +++ b/.github/actions/setup-android-replay-host/action.yml @@ -1,10 +1,19 @@ name: 'Setup Android Replay Host' description: 'Prepare the Android CI host for replay tests and expose the agent-device home path' +inputs: + package-helpers: + description: 'Restore or package the npm-bundled Android snapshot and IME helpers' + required: false + default: 'false' + outputs: agent-home-dir: description: 'Resolved agent-device home directory' value: ${{ steps.agent-home.outputs.dir }} + helpers-cache-hit: + description: 'Whether packaged Android helpers were restored from cache' + value: ${{ steps.android-helpers-cache.outputs.cache-hit }} runs: using: 'composite' @@ -20,3 +29,64 @@ runs: sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm shell: bash + + - name: Resolve Android helper source hash + if: inputs.package-helpers == 'true' + id: android-helper-source + run: | + VERSION="$(node -p 'require("./package.json").version')" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "value=${{ hashFiles('android/snapshot-helper/AndroidManifest.xml', 'android/snapshot-helper/debug.keystore', 'android/snapshot-helper/src/**', 'android/ime-helper/AndroidManifest.xml', 'android/ime-helper/res/**', 'android/ime-helper/src/**', 'scripts/build-android-helper.sh', 'scripts/package-android-helper.sh', '.github/actions/setup-android-replay-host/action.yml') }}" >> "$GITHUB_OUTPUT" + shell: bash + + - name: Restore packaged Android helpers + if: inputs.package-helpers == 'true' + id: android-helpers-cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.3 + with: + path: | + android/snapshot-helper/dist + android/ime-helper/dist + key: android-helpers-${{ runner.os }}-${{ runner.arch }}-api36-v${{ steps.android-helper-source.outputs.version }}-${{ steps.android-helper-source.outputs.value }} + + - name: Install Android helper SDK packages + if: inputs.package-helpers == 'true' && steps.android-helpers-cache.outputs.cache-hit != 'true' + run: | + SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-/usr/local/lib/android/sdk}}" + SDKMANAGER="$SDK_ROOT/cmdline-tools/latest/bin/sdkmanager" + if [ ! -x "$SDKMANAGER" ]; then + SDKMANAGER="$SDK_ROOT/cmdline-tools/bin/sdkmanager" + fi + if [ ! -x "$SDKMANAGER" ]; then + echo "sdkmanager not found under $SDK_ROOT" >&2 + exit 1 + fi + yes | "$SDKMANAGER" --licenses >/dev/null || true + "$SDKMANAGER" "platforms;android-36" "build-tools;36.0.0" + shell: bash + + - name: Package npm-bundled Android helpers + if: inputs.package-helpers == 'true' && steps.android-helpers-cache.outputs.cache-hit != 'true' + run: | + VERSION="${{ steps.android-helper-source.outputs.version }}" + rm -rf android/snapshot-helper/dist android/ime-helper/dist + sh ./scripts/package-android-helper.sh snapshot "$VERSION" "v$VERSION" android/snapshot-helper/dist + sh ./scripts/package-android-helper.sh ime "$VERSION" android/ime-helper/dist + shell: bash + + - name: Verify packaged Android helpers + if: inputs.package-helpers == 'true' + run: | + VERSION="$(node -p 'require("./package.json").version')" + test -f "android/snapshot-helper/dist/agent-device-android-snapshot-helper-$VERSION.manifest.json" + test -f "android/ime-helper/dist/agent-device-android-ime-helper-$VERSION.manifest.json" + shell: bash + + - name: Save packaged Android helpers + if: inputs.package-helpers == 'true' && steps.android-helpers-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.3 + with: + path: | + android/snapshot-helper/dist + android/ime-helper/dist + key: android-helpers-${{ runner.os }}-${{ runner.arch }}-api36-v${{ steps.android-helper-source.outputs.version }}-${{ steps.android-helper-source.outputs.value }} diff --git a/.github/actions/setup-apple-replay/action.yml b/.github/actions/setup-apple-replay/action.yml deleted file mode 100644 index edea8a4fb..000000000 --- a/.github/actions/setup-apple-replay/action.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: 'Setup Apple Replay' -description: 'Resolve shared Apple replay cache/build state and expose the agent-device home path' - -inputs: - derived-path: - description: 'Derived data path used by the replay runner' - required: true - cache-key-prefix: - description: 'Prefix for the cache key' - required: true - cache-key-suffix: - description: 'Optional suffix for the cache key' - required: false - default: '' - build-command: - description: 'Command used to refresh replay artifacts before use' - required: true - xcuitest-platform: - description: 'Optional AGENT_DEVICE_XCUITEST_PLATFORM value' - required: false - default: '' - xcuitest-destination: - description: 'Optional AGENT_DEVICE_XCUITEST_DESTINATION value' - required: false - default: '' - build-before-use: - description: 'Whether this setup action should refresh replay artifacts after restoring the cache' - required: false - default: 'true' - -outputs: - agent-home-dir: - description: 'Resolved agent-device home directory' - value: ${{ steps.agent-home.outputs.dir }} - cache-hit: - description: 'Whether a replay cache was restored before any build refresh' - value: ${{ steps.restore-prebuilt.outputs.cache-hit }} - -runs: - using: 'composite' - steps: - - name: Resolve Xcode cache key - id: xcode - run: | - set -euo pipefail - XCODE_VERSION="$(xcodebuild -version | tr '\n' ' ' | sed -E 's/[[:space:]]+/ /g; s/[[:space:]]$//')" - XCODE_KEY="$(echo "$XCODE_VERSION" | tr ' ' '-' | tr -cd '[:alnum:]._-')" - echo "key=$XCODE_KEY" >> "$GITHUB_OUTPUT" - shell: bash - - - name: Resolve prebuild source hash - id: source-hash - run: | - set -euo pipefail - echo "value=${{ hashFiles('apple/runner/**', 'scripts/build-xcuitest-apple.sh', 'scripts/patch-xcuitest-runner-icon.ts', 'scripts/write-xcuitest-cache-metadata.mjs', 'src/platforms/ios/apple-runner-platform.ts', 'src/platforms/ios/runner-icon.ts', 'src/platforms/ios/runner-xctestrun.ts', 'src/platforms/ios/runner-xctestrun-products.ts', '.github/actions/setup-apple-replay/action.yml', 'package.json', 'pnpm-lock.yaml') }}" >> "$GITHUB_OUTPUT" - shell: bash - - - name: Cache replay prebuilt - id: restore-prebuilt - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.3 - with: - path: ${{ inputs.derived-path }} - key: ${{ inputs.cache-key-prefix }}-${{ steps.xcode.outputs.key }}${{ inputs.cache-key-suffix }}-${{ steps.source-hash.outputs.value }} - - - name: Resolve agent-device home - id: agent-home - run: echo "dir=${AGENT_DEVICE_STATE_DIR:-$HOME/.agent-device}" >> "$GITHUB_OUTPUT" - shell: bash - - - name: Build replay artifacts - if: inputs.build-before-use == 'true' - run: ${{ inputs.build-command }} - shell: bash - env: - AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ inputs.derived-path }} - AGENT_DEVICE_XCUITEST_PLATFORM: ${{ inputs.xcuitest-platform }} - AGENT_DEVICE_XCUITEST_DESTINATION: ${{ inputs.xcuitest-destination }} diff --git a/.github/actions/setup-apple-runner-build/action.yml b/.github/actions/setup-apple-runner-build/action.yml new file mode 100644 index 000000000..93047916c --- /dev/null +++ b/.github/actions/setup-apple-runner-build/action.yml @@ -0,0 +1,87 @@ +name: 'Set Up Apple Runner Build' +description: 'Restore an exact Apple runner build or build and cache it on a miss' + +inputs: + derived-path: + description: 'DerivedData path used by the Apple runner' + required: true + cache-key-prefix: + description: 'Prefix for the cache key' + required: true + cache-key-suffix: + description: 'Optional suffix for the cache key' + required: false + default: '' + build-command: + description: 'Command used to build Apple runner artifacts' + required: true + xcuitest-platform: + description: 'Optional AGENT_DEVICE_XCUITEST_PLATFORM value' + required: false + default: '' + xcuitest-destination: + description: 'Optional AGENT_DEVICE_XCUITEST_DESTINATION value' + required: false + default: '' +outputs: + cache-hit: + description: 'Whether an exact Apple runner build cache was restored' + value: ${{ steps.restore-runner-build.outputs.cache-hit }} + +runs: + using: 'composite' + steps: + - name: Resolve Xcode cache identity + id: xcode + run: | + set -euo pipefail + XCODE_VERSION="$(xcodebuild -version | tr '\n' ' ' | sed -E 's/[[:space:]]+/ /g; s/[[:space:]]$//')" + XCODE_KEY="$(echo "$XCODE_VERSION" | tr ' ' '-' | tr -cd '[:alnum:]._-')" + echo "key=$XCODE_KEY" >> "$GITHUB_OUTPUT" + shell: bash + + - name: Resolve Apple runner source hash + id: source-hash + run: | + set -euo pipefail + echo "value=${{ hashFiles('apple/runner/**', 'scripts/build-xcuitest-apple.sh', 'scripts/patch-xcuitest-runner-icon.ts', 'scripts/write-xcuitest-cache-metadata.mjs', 'src/platforms/apple/core/apple-runner-platform.ts', 'src/platforms/apple/core/runner/runner-cache-metadata.ts', 'src/platforms/apple/core/runner/runner-icon.ts', 'src/platforms/apple/core/runner/runner-xctestrun.ts', 'src/platforms/apple/core/runner/runner-xctestrun-products.ts', '.github/actions/setup-apple-runner-build/action.yml', 'package.json', 'pnpm-lock.yaml') }}" >> "$GITHUB_OUTPUT" + shell: bash + + - name: Resolve Apple runner build variant + id: build-variant + run: | + set -euo pipefail + VARIANT="$( + printf '%s\n' \ + '${{ inputs.build-command }}' \ + '${{ inputs.xcuitest-platform }}' \ + '${{ inputs.xcuitest-destination }}' \ + "${AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS:-0}" \ + | shasum -a 256 \ + | cut -c1-16 + )" + echo "key=$VARIANT" >> "$GITHUB_OUTPUT" + shell: bash + + - name: Restore Apple runner build cache + id: restore-runner-build + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.3 + with: + path: ${{ inputs.derived-path }} + key: ${{ inputs.cache-key-prefix }}-${{ steps.xcode.outputs.key }}${{ inputs.cache-key-suffix }}-${{ steps.build-variant.outputs.key }}-${{ steps.source-hash.outputs.value }} + + - name: Build Apple runner artifacts on cache miss + if: steps.restore-runner-build.outputs.cache-hit != 'true' + run: ${{ inputs.build-command }} + shell: bash + env: + AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ inputs.derived-path }} + AGENT_DEVICE_XCUITEST_PLATFORM: ${{ inputs.xcuitest-platform }} + AGENT_DEVICE_XCUITEST_DESTINATION: ${{ inputs.xcuitest-destination }} + + - name: Save Apple runner build cache + if: steps.restore-runner-build.outputs.cache-hit != 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.3 + with: + path: ${{ inputs.derived-path }} + key: ${{ inputs.cache-key-prefix }}-${{ steps.xcode.outputs.key }}${{ inputs.cache-key-suffix }}-${{ steps.build-variant.outputs.key }}-${{ steps.source-hash.outputs.value }} diff --git a/.github/actions/setup-fixture-app/action.yml b/.github/actions/setup-fixture-app/action.yml index 890e9fe63..6495842e5 100644 --- a/.github/actions/setup-fixture-app/action.yml +++ b/.github/actions/setup-fixture-app/action.yml @@ -11,7 +11,10 @@ description: 'Install a ready Release build of examples/test-app on the booted i # Release build embedded may be stale — @expo/repack-app swaps in a bundle built # from the current source. If no artifact exists yet (the producer has not run # for this fingerprint), it falls back to building inline, so a caller is never -# left without an app. +# left without an app. Callers that run alongside the producer can opt into a +# bounded wait first, avoiding two simultaneous native builds on a cold key. +# Artifact hits are accepted only from this repository's designated producer +# workflow at the current head or from a default-branch push. # # The caller's job needs `permissions: actions: read` for the artifact lookup. # Relevant to #320 (move replay coverage off system apps onto a stable fixture). @@ -25,7 +28,10 @@ inputs: description: 'Install the app onto the booted simulator' required: false default: 'true' - + wait-for-artifact-seconds: + description: 'How long to poll for a concurrently produced artifact before building inline' + required: false + default: '0' outputs: app-path: description: 'Path to the ready .app bundle' @@ -40,11 +46,10 @@ outputs: runs: using: 'composite' steps: - # Installed even on a hit, because the fingerprint is derived from the - # dependency graph and cannot be computed without it. - - name: Install test app dependencies - shell: bash - run: pnpm test-app:install + # The fingerprint/repack commands need the dependency graph. This shared + # action avoids recreating it on every warm native-artifact consumer. + - name: Setup test app dependencies + uses: ./.github/actions/setup-test-app-dependencies - name: Fetch the cached Release binary id: fetch @@ -53,19 +58,80 @@ runs: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - HASH="$(pnpm --dir examples/test-app exec fingerprint fingerprint:generate | jq -r .hash)" - NAME="fingerprint.${HASH}.ios" + NAME="$(sh "$GITHUB_ACTION_PATH/resolve-artifact-name.sh" ios)" DEST="${{ github.workspace }}/.tmp/fixture-app" + REPOSITORY="${{ github.repository }}" + EXPECTED_HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}" + TRUSTED_ARTIFACT="$GITHUB_ACTION_PATH/trusted-artifact.mjs" rm -rf "$DEST"; mkdir -p "$DEST" + WAIT_SECONDS="${{ inputs.wait-for-artifact-seconds }}" + case "$WAIT_SECONDS" in + ''|*[!0-9]*) + echo "::error::wait-for-artifact-seconds must be a non-negative integer." + exit 1 + ;; + esac # A lookup failure (API outage, auth, transient 5xx) must not fail the - # caller: it degrades to an inline build exactly as a cache miss does. The - # assignment sits in an `if` so `set -e` cannot exit on it, and an empty - # ART_ID then takes the same path as "no artifact found". - if ! ART_ID="$(gh api "repos/${{ github.repository }}/actions/artifacts?name=${NAME}&per_page=1" \ - --jq '[.artifacts[] | select(.expired == false)][0].id // empty' 2>/dev/null)"; then + # caller or trigger a pointless wait: it takes the inline-build path. + query_artifact() { + node "$TRUSTED_ARTIFACT" find "$REPOSITORY" "$NAME" "$EXPECTED_HEAD_SHA" 2>/dev/null + } + query_producer_state() { + node "$TRUSTED_ARTIFACT" producer-state "$REPOSITORY" "$EXPECTED_HEAD_SHA" 2>/dev/null + } + if ! ART_ID="$(query_artifact)"; then echo "::warning::Could not query the build cache for $NAME; building inline." ART_ID="" + WAIT_SECONDS=0 + fi + if [ -z "$ART_ID" ] && [ "$WAIT_SECONDS" -gt 0 ]; then + DEADLINE=$((SECONDS + WAIT_SECONDS)) + MISSING_PRODUCER_POLLS=0 + echo "$NAME is not available; waiting up to ${WAIT_SECONDS}s for its producer." + while [ -z "$ART_ID" ] && [ "$SECONDS" -lt "$DEADLINE" ]; do + if ! PRODUCER_STATE="$(query_producer_state)"; then + echo "::warning::Could not inspect the fixture producer; building inline." + break + fi + case "$PRODUCER_STATE" in + queued) + echo "Fixture producer is queued; building inline instead of waiting for a macOS runner." + break + ;; + failed) + echo "Fixture producer failed; building inline." + break + ;; + success) + echo "Fixture producer completed without $NAME; building inline." + break + ;; + absent) + MISSING_PRODUCER_POLLS=$((MISSING_PRODUCER_POLLS + 1)) + if [ "$MISSING_PRODUCER_POLLS" -ge 3 ]; then + echo "No fixture producer appeared for $EXPECTED_HEAD_SHA; building inline." + break + fi + ;; + in_progress) + MISSING_PRODUCER_POLLS=0 + ;; + *) + echo "::warning::Fixture producer returned unknown state '$PRODUCER_STATE'; building inline." + break + ;; + esac + sleep 30 + if ! ART_ID="$(query_artifact)"; then + echo "::warning::Build cache polling failed for $NAME; building inline." + ART_ID="" + break + fi + if [ -n "$ART_ID" ]; then + break + fi + done fi SOURCE=build @@ -84,7 +150,7 @@ runs: fi rm -rf "$STAGE" else - echo "$NAME not in the cache yet; building inline." + echo "$NAME not in the cache after the configured wait; building inline." fi echo "source=$SOURCE" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/setup-fixture-app/resolve-artifact-name.sh b/.github/actions/setup-fixture-app/resolve-artifact-name.sh new file mode 100644 index 000000000..b06533ecd --- /dev/null +++ b/.github/actions/setup-fixture-app/resolve-artifact-name.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu + +if [ "$#" -ne 1 ]; then + echo "usage: resolve-artifact-name.sh " >&2 + exit 2 +fi + +PLATFORM="$1" +case "$PLATFORM" in + ios|android) ;; + *) + echo "usage: resolve-artifact-name.sh " >&2 + exit 2 + ;; +esac + +FINGERPRINT_JSON="$(pnpm --dir examples/test-app exec fingerprint fingerprint:generate --platform "$PLATFORM")" +if ! HASH="$( + printf '%s\n' "$FINGERPRINT_JSON" | + jq -ser ' + select(length == 1) + | .[0].hash + | select(type == "string" and . != "null" and test("\\A[A-Za-z0-9_-]+\\z")) + ' +)"; then + echo "fingerprint:generate returned an invalid $PLATFORM hash" >&2 + exit 1 +fi +printf 'fingerprint.%s.%s\n' "$HASH" "$PLATFORM" diff --git a/.github/actions/setup-fixture-app/trusted-artifact.mjs b/.github/actions/setup-fixture-app/trusted-artifact.mjs new file mode 100644 index 000000000..5221799c7 --- /dev/null +++ b/.github/actions/setup-fixture-app/trusted-artifact.mjs @@ -0,0 +1,140 @@ +#!/usr/bin/env node +import process from 'node:process'; +import { pathToFileURL } from 'node:url'; + +const TRUSTED_WORKFLOW_PATH = '.github/workflows/test-app-build-cache.yml'; + +export async function findTrustedArtifact({ artifacts, expectedHeadSha, repository, loadRun }) { + for (const artifact of artifacts) { + if (!artifactBelongsToRepository(artifact, repository.id)) continue; + const run = await loadRun(artifact.workflow_run.id); + if (isTrustedProducerRun({ expectedHeadSha, repository, run })) return artifact.id; + } + return undefined; +} + +export function classifyProducerState(runs, { expectedHeadSha, repository }) { + const run = runs.find((candidate) => + isExpectedProducerRun(candidate, expectedHeadSha, repository.id), + ); + if (!run) return 'absent'; + return classifyRunState(run); +} + +function artifactBelongsToRepository(artifact, repositoryId) { + if (artifact.expired) return false; + return workflowRunBelongsToRepository(artifact.workflow_run, repositoryId); +} + +function workflowRunBelongsToRepository(run, repositoryId) { + if (!run) return false; + return run.repository_id === repositoryId && run.head_repository_id === repositoryId; +} + +function isExpectedProducerRun(run, expectedHeadSha, repositoryId) { + if (run.head_sha !== expectedHeadSha || run.path !== TRUSTED_WORKFLOW_PATH) return false; + return runBelongsToRepository(run, repositoryId); +} + +function classifyRunState(run) { + if (new Set(['pending', 'queued', 'waiting']).has(run.status)) return 'queued'; + if (run.status !== 'completed') return 'in_progress'; + return run.conclusion === 'success' ? 'success' : 'failed'; +} + +function isTrustedProducerRun({ expectedHeadSha, repository, run }) { + if (run.path !== TRUSTED_WORKFLOW_PATH) return false; + if (!runBelongsToRepository(run, repository.id)) return false; + if (run.head_sha === expectedHeadSha) return true; + return isDefaultBranchPush(run, repository.default_branch); +} + +function runBelongsToRepository(run, repositoryId) { + return run.repository?.id === repositoryId && run.head_repository?.id === repositoryId; +} + +function isDefaultBranchPush(run, defaultBranch) { + return run.event === 'push' && run.head_branch === defaultBranch; +} + +async function requestJson(path) { + const response = await fetch(new URL(path, resolveApiBaseUrl()), { + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${resolveToken()}`, + 'X-GitHub-Api-Version': '2022-11-28', + }, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) throw new Error(`GitHub API ${response.status} for ${path}`); + return await response.json(); +} + +function resolveApiBaseUrl() { + return `${process.env.GITHUB_API_URL || 'https://api.github.com'}/`; +} + +function resolveToken() { + return process.env.GH_TOKEN || process.env.GITHUB_TOKEN || ''; +} + +async function loadRepository(repositoryName) { + const repository = await requestJson(`repos/${repositoryName}`); + return { + default_branch: repository.default_branch, + id: repository.id, + }; +} + +async function runCli(argv) { + const [command, repositoryName, ...commandArgs] = argv; + requireArguments([command, repositoryName], 'command and repository'); + const repository = await loadRepository(repositoryName); + if (command === 'find') return await runFindCommand(repositoryName, repository, commandArgs); + if (command === 'producer-state') { + return await runProducerStateCommand(repositoryName, repository, commandArgs); + } + throw new Error(`unknown command: ${command}`); +} + +async function runFindCommand(repositoryName, repository, commandArgs) { + const [artifactName, expectedHeadSha] = commandArgs; + requireArguments([artifactName, expectedHeadSha], 'artifact name and expected head SHA'); + const result = await requestJson( + `repos/${repositoryName}/actions/artifacts?name=${encodeURIComponent(artifactName)}&per_page=100`, + ); + const artifactId = await findTrustedArtifact({ + artifacts: result.artifacts || [], + expectedHeadSha, + repository, + loadRun: (runId) => requestJson(`repos/${repositoryName}/actions/runs/${runId}`), + }); + process.stdout.write(artifactId === undefined ? '' : String(artifactId)); +} + +async function runProducerStateCommand(repositoryName, repository, commandArgs) { + const [expectedHeadSha] = commandArgs; + requireArguments([expectedHeadSha], 'expected head SHA'); + const result = await requestJson( + `repos/${repositoryName}/actions/workflows/test-app-build-cache.yml/runs?head_sha=${encodeURIComponent(expectedHeadSha)}&per_page=20`, + ); + process.stdout.write( + classifyProducerState(result.workflow_runs || [], { expectedHeadSha, repository }), + ); +} + +function requireArguments(values, description) { + if (values.every(Boolean)) return; + throw new Error(`trusted-artifact requires ${description}`); +} + +function formatError(error) { + return error instanceof Error ? error.message : String(error); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runCli(process.argv.slice(2)).catch((error) => { + process.stderr.write(`${formatError(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/.github/actions/setup-node-pnpm/action.yml b/.github/actions/setup-node-pnpm/action.yml index ac66c7d07..0d2a472b5 100644 --- a/.github/actions/setup-node-pnpm/action.yml +++ b/.github/actions/setup-node-pnpm/action.yml @@ -10,6 +10,10 @@ inputs: description: 'Whether to install dependencies with pnpm install --frozen-lockfile' required: false default: 'true' + cache-dependency-path: + description: 'Lockfile path(s) used to key the pnpm store cache' + required: false + default: 'pnpm-lock.yaml' runs: using: 'composite' @@ -45,6 +49,7 @@ runs: # Jobs with install-deps: false never create the store, and on a cache # miss (any lockfile-changing PR) the post-job cache save fails the job. cache: ${{ inputs.install-deps == 'true' && 'pnpm' || '' }} + cache-dependency-path: ${{ inputs.cache-dependency-path }} # Corepack resolves the same `packageManager` field locally, so a mismatch # here means CI is running a pnpm nobody develops against. diff --git a/.github/actions/setup-test-app-dependencies/action.yml b/.github/actions/setup-test-app-dependencies/action.yml new file mode 100644 index 000000000..993e51729 --- /dev/null +++ b/.github/actions/setup-test-app-dependencies/action.yml @@ -0,0 +1,39 @@ +name: 'Setup Test App Dependencies' +description: 'Restore the test app dependency graph, installing it on a cache miss' + +outputs: + cache-hit: + description: 'Whether the test app dependency graph was restored from cache' + value: ${{ steps.restore.outputs.cache-hit }} + +runs: + using: 'composite' + steps: + # Expo's dependency graph contains native binaries, so cache it by operating + # system, architecture, Node major, pnpm version, and the exact manifest. + - name: Resolve test app dependency cache key + id: key + shell: bash + run: | + set -euo pipefail + echo "node_major=$(node -p 'process.versions.node.split(".")[0]')" >> "$GITHUB_OUTPUT" + echo "pnpm_version=$(pnpm --version)" >> "$GITHUB_OUTPUT" + + - name: Restore test app dependencies + id: restore + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.3 + with: + path: examples/test-app/node_modules + key: test-app-dependencies-v2-${{ runner.os }}-${{ runner.arch }}-node${{ steps.key.outputs.node_major }}-pnpm${{ steps.key.outputs.pnpm_version }}-${{ hashFiles('examples/test-app/package.json', 'examples/test-app/pnpm-lock.yaml', 'examples/test-app/pnpm-workspace.yaml', '.github/actions/setup-test-app-dependencies/action.yml') }} + + - name: Install test app dependencies + if: steps.restore.outputs.cache-hit != 'true' + shell: bash + run: pnpm test-app:install + + - name: Save test app dependencies + if: steps.restore.outputs.cache-hit != 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.3 + with: + path: examples/test-app/node_modules + key: test-app-dependencies-v2-${{ runner.os }}-${{ runner.arch }}-node${{ steps.key.outputs.node_major }}-pnpm${{ steps.key.outputs.pnpm_version }}-${{ hashFiles('examples/test-app/package.json', 'examples/test-app/pnpm-lock.yaml', 'examples/test-app/pnpm-workspace.yaml', '.github/actions/setup-test-app-dependencies/action.yml') }} diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 588e9b1a6..35b353bca 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -36,25 +36,8 @@ jobs: - name: Setup Android replay host id: android-replay-host uses: ./.github/actions/setup-android-replay-host - - - name: Install Android SDK packages - run: | - SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-/usr/local/lib/android/sdk}}" - SDKMANAGER="$SDK_ROOT/cmdline-tools/latest/bin/sdkmanager" - if [ ! -x "$SDKMANAGER" ]; then - SDKMANAGER="$SDK_ROOT/cmdline-tools/bin/sdkmanager" - fi - if [ ! -x "$SDKMANAGER" ]; then - echo "sdkmanager not found under $SDK_ROOT" >&2 - exit 1 - fi - yes | "$SDKMANAGER" --licenses >/dev/null - "$SDKMANAGER" "platforms;android-36" "build-tools;36.0.0" - - - name: Package npm-bundled Android helpers - run: | - pnpm package:android-snapshot-helper:npm - pnpm package:android-ime-helper:npm + with: + package-helpers: 'true' - name: Run Android smoke checks uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78cc3fe2e..f21753d24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,13 +48,14 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm - - name: Compile Swift runner unit-test surface - uses: ./.github/actions/setup-apple-replay + - name: Restore and compile Swift runner unit-test surface + uses: ./.github/actions/setup-apple-runner-build with: derived-path: ${{ github.workspace }}/.tmp/swift-runner-unit-derived cache-key-prefix: swift-runner-unit build-command: AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS=1 pnpm build:xcuitest:macos xcuitest-platform: macos + xcuitest-destination: platform=macOS,arch=arm64 no-test-di-seams: name: No test-only DI seams diff --git a/.github/workflows/conformance-differential.yml b/.github/workflows/conformance-differential.yml index fcd103d7b..10e78e0e6 100644 --- a/.github/workflows/conformance-differential.yml +++ b/.github/workflows/conformance-differential.yml @@ -54,14 +54,15 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm - - name: Setup Apple replay - uses: ./.github/actions/setup-apple-replay + - name: Restore and build iOS XCTest runner + uses: ./.github/actions/setup-apple-runner-build with: derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} cache-key-prefix: ios-runner-prebuilt cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }} build-command: sh ./scripts/build-xcuitest-apple.sh xcuitest-platform: ios + xcuitest-destination: generic/platform=iOS Simulator - name: Boot simulator uses: ./.github/actions/boot-ios-test-simulator diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index ed4079734..f3958fb15 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -16,6 +16,7 @@ on: permissions: contents: read + actions: read concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} @@ -32,16 +33,42 @@ jobs: AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000' AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' + AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Set fixture producer wait policy + id: fixture-producer + run: | + set -euo pipefail + WAIT_SECONDS=1800 + if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then + echo "Fork pull request cannot publish the shared fixture artifact; a cache miss will build inline." + WAIT_SECONDS=0 + fi + echo "wait-seconds=$WAIT_SECONDS" >> "$GITHUB_OUTPUT" + - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + with: + cache-dependency-path: | + pnpm-lock.yaml + examples/test-app/pnpm-lock.yaml - - name: Setup Apple replay - id: apple-replay - uses: ./.github/actions/setup-apple-replay + - name: Establish host focus canary + id: host-focus-canary + run: | + set -euo pipefail + osascript -e 'tell application "Finder" to activate' + sleep 1 + FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')" + test "$FRONTMOST" = "com.apple.finder" + mkdir -p .tmp + echo "$FRONTMOST" > .tmp/ios-host-focus-before + + - name: Restore and build iOS XCTest runner + uses: ./.github/actions/setup-apple-runner-build with: derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} cache-key-prefix: ios-runner-prebuilt @@ -57,7 +84,7 @@ jobs: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro - - name: Run iOS runner gesture and lifecycle regressions + - name: Run targeted iOS runner XCTest regressions run: | XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" test -n "$XCTESTRUN_PATH" @@ -68,22 +95,53 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testMissingBundleCommandInvalidatesCompleteCachedTargetState \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCachedTargetInvalidationClearsProcessBoundState \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertAcceptTreatsOpenAsAffirmative \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSystemModalProbeSliceSharesAndClampsToPlanDeadline \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds - - name: Prepare iOS runner + - name: Preflight iOS runner through public CLI run: | pnpm clean:daemon node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json pnpm clean:daemon - - name: Run iOS simulator smoke replay + - name: Run iOS Settings replay smoke test run: | node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml + - name: Fetch current fixture app + id: fixture-app + uses: ./.github/actions/setup-fixture-app + with: + install: 'false' + wait-for-artifact-seconds: ${{ steps.fixture-producer.outputs.wait-seconds }} + + - name: Report fixture cache source + run: | + echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" + + - name: Run fixture-backed iOS simulator E2E smoke + env: + AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} + AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} + AGENT_DEVICE_IOS_E2E: '1' + AGENT_DEVICE_IOS_E2E_TIER: smoke + AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + run: | + pnpm build + node --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts + + - name: Assert simulator automation preserved host focus + if: ${{ always() }} + run: | + set -euo pipefail + test "$(cat .tmp/ios-host-focus-before)" = "com.apple.finder" + FRONTMOST="$(osascript -e 'tell application "System Events" to get bundle identifier of first application process whose frontmost is true')" + test "$FRONTMOST" = "com.apple.finder" + - name: Run iOS physical device smoke replay if: env.IOS_UDID != '' env: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 888f24de0..fafdcd55c 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -36,9 +36,8 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm - - name: Setup Apple replay - id: apple-replay - uses: ./.github/actions/setup-apple-replay + - name: Restore and build macOS XCTest runner + uses: ./.github/actions/setup-apple-runner-build with: derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} cache-key-prefix: macos-runner-prebuilt diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index 643236463..3ccf4213d 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -3,7 +3,7 @@ name: Perf Nightly # End-to-end command perf benchmark (scripts/perf). Scheduled + manual only — perf timing on # shared CI runners is noisy, so treat this as a trend/regression signal, not absolute numbers. # Reuses the same build artifacts as the device suites: the cached iOS XCUITest runner -# (setup-apple-replay, ios-runner-prebuilt cache) and the Android replay host, and runs the CLI +# (setup-apple-runner-build, ios-runner-prebuilt cache) and the Android replay host, and runs the CLI # from source via --experimental-strip-types (no dist build), matching the replay workflows. on: @@ -44,9 +44,8 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm - - name: Setup Apple replay - id: apple-replay - uses: ./.github/actions/setup-apple-replay + - name: Restore and build iOS XCTest runner + uses: ./.github/actions/setup-apple-runner-build with: derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} cache-key-prefix: ios-runner-prebuilt @@ -61,7 +60,7 @@ jobs: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro - - name: Prepare iOS runner + - name: Preflight iOS runner through public CLI run: | pnpm clean:daemon node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" @@ -97,11 +96,8 @@ jobs: - name: Setup Android replay host id: android-replay-host uses: ./.github/actions/setup-android-replay-host - - - name: Package npm-bundled Android helpers - run: | - pnpm package:android-snapshot-helper:npm - pnpm package:android-ime-helper:npm + with: + package-helpers: 'true' - name: Run Android command perf benchmark uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0 diff --git a/.github/workflows/replays-nightly.yml b/.github/workflows/replays-nightly.yml index 1776deb4d..f517043b1 100644 --- a/.github/workflows/replays-nightly.yml +++ b/.github/workflows/replays-nightly.yml @@ -15,6 +15,7 @@ on: permissions: contents: read + actions: read concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} @@ -102,6 +103,8 @@ jobs: - name: Setup Android replay host id: android-replay-host uses: ./.github/actions/setup-android-replay-host + with: + package-helpers: 'true' - name: Run Android replay suite uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0 @@ -129,16 +132,20 @@ jobs: AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000' + AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + with: + cache-dependency-path: | + pnpm-lock.yaml + examples/test-app/pnpm-lock.yaml - - name: Setup Apple replay - id: apple-replay - uses: ./.github/actions/setup-apple-replay + - name: Restore and build iOS XCTest runner + uses: ./.github/actions/setup-apple-runner-build with: derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} cache-key-prefix: ios-runner-prebuilt @@ -148,19 +155,41 @@ jobs: xcuitest-destination: generic/platform=iOS Simulator - name: Boot iOS test simulator + id: ios-simulator uses: ./.github/actions/boot-ios-test-simulator with: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro - - name: Prepare iOS runner + - name: Preflight iOS runner through public CLI run: | pnpm clean:daemon - node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json + node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json pnpm clean:daemon - name: Run iOS simulator replay suite - run: node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator --retries 2 --artifacts-dir test/artifacts/replays-ios-simulator --report-junit test/artifacts/replays-ios-simulator.junit.xml + run: node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --artifacts-dir test/artifacts/replays-ios-simulator --report-junit test/artifacts/replays-ios-simulator.junit.xml + + - name: Fetch current fixture app + id: fixture-app + uses: ./.github/actions/setup-fixture-app + with: + install: 'false' + + - name: Report fixture cache source + run: | + echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" + + - name: Run full fixture-backed iOS simulator E2E + env: + AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} + AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} + AGENT_DEVICE_IOS_E2E: '1' + AGENT_DEVICE_IOS_E2E_TIER: full + AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + run: | + pnpm build + node --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts - name: Run iOS physical device replay suite if: env.IOS_UDID != '' @@ -190,9 +219,8 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm - - name: Setup Apple replay - id: apple-replay - uses: ./.github/actions/setup-apple-replay + - name: Restore and build macOS XCTest runner + uses: ./.github/actions/setup-apple-runner-build with: derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} cache-key-prefix: macos-runner-prebuilt diff --git a/.github/workflows/test-app-build-cache.yml b/.github/workflows/test-app-build-cache.yml index 76425ce1d..06397f00f 100644 --- a/.github/workflows/test-app-build-cache.yml +++ b/.github/workflows/test-app-build-cache.yml @@ -10,19 +10,15 @@ name: Test App Build Cache # binary. Local development does not use this; it caches on disk via the # expo-build-disk-cache provider in app.config.js. # -# Only builds when the fingerprint has no artifact yet. Runs on every push to -# main, and on PRs that touch the cache so a change here is exercised before it -# merges (workflow_dispatch cannot reach a workflow that is not yet on main). +# Only builds when the fingerprint has no trusted artifact yet. Runs on every +# push to main and every PR so a same-head consumer can wait for a scheduled +# producer (workflow_dispatch cannot reach a workflow that is not yet on main). on: push: branches: - main pull_request: - paths: - - 'examples/test-app/**' - - '.github/workflows/test-app-build-cache.yml' - - '.github/actions/setup-fixture-app/**' workflow_dispatch: permissions: @@ -30,66 +26,114 @@ permissions: # Look up whether this fingerprint's artifact already exists across runs. actions: read -concurrency: - # One producer at a time so a fingerprint is never built twice over; queue - # rather than cancel, or a long build could be killed by the next merge and the - # cache would never populate. Queued runs are cheap: they find the artifact and - # skip the build. - group: ci-${{ github.workflow }} - cancel-in-progress: false - jobs: - release: - name: ${{ matrix.name }} - runs-on: ${{ matrix.runs-on }} - timeout-minutes: 60 - strategy: - # Independent caches; one platform failing must not withhold the other's. - fail-fast: false - matrix: - include: - - name: iOS Release - platform: ios - runs-on: macos-26 - - name: Android Release - platform: android - runs-on: ubuntu-latest + fingerprint: + name: Resolve native fingerprint + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + matrix: ${{ steps.fingerprint.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + with: + cache-dependency-path: examples/test-app/pnpm-lock.yaml - - name: Install test app dependencies - run: pnpm test-app:install + - name: Setup test app dependencies + uses: ./.github/actions/setup-test-app-dependencies - - name: Resolve fingerprint and whether it is already cached - id: fp + - name: Resolve native fingerprint + id: fingerprint env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail - HASH="$(pnpm --dir examples/test-app exec fingerprint fingerprint:generate | jq -r .hash)" - NAME="fingerprint.${HASH}.${{ matrix.platform }}" - # A non-expired artifact under this name means the native inputs are - # unchanged; there is nothing to rebuild. - COUNT="$(gh api "repos/${{ github.repository }}/actions/artifacts?name=${NAME}&per_page=1" \ - --jq '[.artifacts[] | select(.expired == false)] | length')" - echo "name=$NAME" >> "$GITHUB_OUTPUT" - if [ "$COUNT" -gt 0 ]; then - echo "cached=true" >> "$GITHUB_OUTPUT" - echo "$NAME already cached; nothing to build." - else - echo "cached=false" >> "$GITHUB_OUTPUT" - echo "$NAME not cached; building." + ARTIFACT_NAME_RESOLVER=".github/actions/setup-fixture-app/resolve-artifact-name.sh" + IOS_NAME="$(sh "$ARTIFACT_NAME_RESOLVER" ios)" + ANDROID_NAME="$(sh "$ARTIFACT_NAME_RESOLVER" android)" + EXPECTED_HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}" + TRUSTED_ARTIFACT=".github/actions/setup-fixture-app/trusted-artifact.mjs" + IOS_ARTIFACT="$(node "$TRUSTED_ARTIFACT" find "${{ github.repository }}" "$IOS_NAME" "$EXPECTED_HEAD_SHA")" + ANDROID_ARTIFACT="$(node "$TRUSTED_ARTIFACT" find "${{ github.repository }}" "$ANDROID_NAME" "$EXPECTED_HEAD_SHA")" + IOS_BUILD=true + if [ -n "$IOS_ARTIFACT" ] || + { [ "${{ github.event_name }}" = "pull_request" ] && + [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; }; then + IOS_BUILD=false fi + ANDROID_BUILD=true + if [ -n "$ANDROID_ARTIFACT" ]; then + ANDROID_BUILD=false + fi + MATRIX="$(jq -cn \ + --arg iosName "$IOS_NAME" \ + --argjson iosBuild "$IOS_BUILD" \ + --arg androidName "$ANDROID_NAME" \ + --argjson androidBuild "$ANDROID_BUILD" \ + '{ + include: [ + { + name: "iOS Release", + platform: "ios", + runsOn: (if $iosBuild then "macos-26" else "ubuntu-latest" end), + artifactName: $iosName, + build: $iosBuild + }, + { + name: "Android Release", + platform: "android", + runsOn: "ubuntu-latest", + artifactName: $androidName, + build: $androidBuild + } + ] + }')" + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + + release: + name: ${{ matrix.name }} + needs: fingerprint + runs-on: ${{ matrix.runsOn }} + timeout-minutes: 60 + concurrency: + group: test-app-${{ matrix.artifactName }}-${{ github.event.pull_request.number || github.ref_name }} + cancel-in-progress: false + strategy: + # Independent caches; one platform failing must not withhold the other's. + fail-fast: false + matrix: ${{ fromJSON(needs.fingerprint.outputs.matrix) }} + steps: + - name: Checkout + if: matrix.build + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup toolchain + if: matrix.build + uses: ./.github/actions/setup-node-pnpm + with: + cache-dependency-path: | + pnpm-lock.yaml + examples/test-app/pnpm-lock.yaml + + - name: Setup test app dependencies + if: matrix.build + uses: ./.github/actions/setup-test-app-dependencies # `--device generic` builds for the simulator without booting one or # installing. Release embeds the bundle (no Metro) and links a universal # x86_64+arm64 slice, so the artifact runs on any developer's simulator. + # Fork PRs cannot safely publish this binary; their iOS workflow builds the + # same Release app inline on a cache miss, so compiling it here would double + # the macOS cost without adding signal. - name: Build the iOS Release app - if: matrix.platform == 'ios' && steps.fp.outputs.cached == 'false' + if: | + matrix.platform == 'ios' && + matrix.build && + (github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository) run: | set -euo pipefail pnpm --dir examples/test-app exec expo run:ios \ @@ -99,14 +143,14 @@ jobs: --output "${{ github.workspace }}/.tmp/test-app-build" - name: Setup Android host - if: matrix.platform == 'android' && steps.fp.outputs.cached == 'false' + if: matrix.platform == 'android' && matrix.build uses: ./.github/actions/setup-android-replay-host - # Deliberately no `set -o pipefail`, matching android.yml: `yes` is killed - # by SIGPIPE once sdkmanager stops reading, and pipefail would report that - # 141 as the step's own failure. + # Deliberately no `set -o pipefail`: `yes` is killed by SIGPIPE once + # sdkmanager stops reading, and pipefail would report that 141 as the + # step's own failure. - name: Install Android SDK packages - if: matrix.platform == 'android' && steps.fp.outputs.cached == 'false' + if: matrix.platform == 'android' && matrix.build run: | SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-/usr/local/lib/android/sdk}}" SDKMANAGER="$SDK_ROOT/cmdline-tools/latest/bin/sdkmanager" @@ -125,7 +169,7 @@ jobs: # build already spans every ABI (the CLI only narrows debug builds to the # device ABI), so no --all-arch and no emulator-architecture coupling. - name: Build the Android Release apk - if: matrix.platform == 'android' && steps.fp.outputs.cached == 'false' + if: matrix.platform == 'android' && matrix.build uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0 with: api-level: 36 @@ -144,7 +188,11 @@ jobs: - name: Stage the binary for upload id: stage - if: steps.fp.outputs.cached == 'false' + if: | + matrix.build && + (matrix.platform == 'android' || + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository) run: | set -euo pipefail SRC="${{ github.workspace }}/.tmp/test-app-build" @@ -163,20 +211,21 @@ jobs: # leave an installed .app unable to launch. mkdir -p .tmp/test-app-artifact tar -czf .tmp/test-app-artifact/binary.tar.gz -C "$SRC" "$(basename "$BINARY")" - echo "publishing ${{ steps.fp.outputs.name }} ($(du -h .tmp/test-app-artifact/binary.tar.gz | cut -f1))" + echo "publishing ${{ matrix.artifactName }} ($(du -h .tmp/test-app-artifact/binary.tar.gz | cut -f1))" # Fork PRs run with a read-only token but can still upload artifacts, and a # consumer serves whatever it finds by name -- so a fork could publish a - # tampered binary that CI then executes. Only branches in this repo publish; - # a fork's run still builds, which is all the signal a fork PR needs. + # tampered binary that CI then executes. Only branches in this repo publish. + # Android still builds on forks; iOS gets equivalent build/install signal + # from the smoke workflow's inline cache-miss fallback. - name: Publish to the build cache if: | - steps.fp.outputs.cached == 'false' && + matrix.build && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: ${{ steps.fp.outputs.name }} + name: ${{ matrix.artifactName }} path: .tmp/test-app-artifact/binary.tar.gz if-no-files-found: error compression-level: 0 # already gzipped diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 5cdee9ae8..738c8427f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -225,10 +225,17 @@ extension RunnerTests { "yes", "continue", "done", + "open", "open settings" ].contains(normalized) || normalized.hasPrefix("confirm") } +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + func testAlertAcceptTreatsOpenAsAffirmative() { + XCTAssertTrue(isAcceptButton("Open")) + } +#endif + private func isDismissButton(_ label: String) -> Bool { [ "cancel", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 518e17694..45313773b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1894,21 +1894,7 @@ extension RunnerTests { pressHomeButton() return Response(ok: true, data: DataPayload(message: "home")) case .rotate: - guard let orientation = command.orientation?.trimmingCharacters(in: .whitespacesAndNewlines), - !orientation.isEmpty - else { - return Response(ok: false, error: ErrorPayload(message: "rotate requires orientation")) - } - if rotateDevice(to: orientation) { - return Response( - ok: true, - data: DataPayload(message: "rotate", orientation: orientation) - ) - } - return Response( - ok: false, - error: ErrorPayload(message: "unsupported rotate orientation: \(orientation)") - ) + return executeRotateCommand(command) case .appSwitcher: performAppSwitcherGesture(app: activeApp) return Response(ok: true, data: DataPayload(message: "appSwitcher")) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index da42b2133..c58d46e97 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -118,27 +118,6 @@ extension RunnerTests { #endif } - func rotateDevice(to orientationName: String) -> Bool { -#if os(macOS) || os(tvOS) || os(visionOS) - return false -#else - switch orientationName { - case "portrait": - XCUIDevice.shared.orientation = .portrait - case "portrait-upside-down": - XCUIDevice.shared.orientation = .portraitUpsideDown - case "landscape-left": - XCUIDevice.shared.orientation = .landscapeLeft - case "landscape-right": - XCUIDevice.shared.orientation = .landscapeRight - default: - return false - } - sleepFor(0.2) - return true -#endif - } - func findElement(app: XCUIApplication, text: String) -> XCUIElement? { let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text) let element = app.descendants(matching: .any).matching(predicate).firstMatch diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Orientation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Orientation.swift new file mode 100644 index 000000000..45ef96f5e --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Orientation.swift @@ -0,0 +1,92 @@ +import XCTest + +private enum OrientationObservationTiming { + static let pollInterval: TimeInterval = 0.05 + static let timeout: TimeInterval = 2 +} + +extension RunnerTests { + func executeRotateCommand(_ command: Command) -> Response { + guard let orientation = command.orientation?.trimmingCharacters(in: .whitespacesAndNewlines), + !orientation.isEmpty + else { + return Response(ok: false, error: ErrorPayload(message: "rotate requires orientation")) + } + let supportedOrientations = [ + "portrait", + "portrait-upside-down", + "landscape-left", + "landscape-right", + ] + guard supportedOrientations.contains(orientation) else { + return Response( + ok: false, + error: ErrorPayload(message: "unsupported rotate orientation: \(orientation)") + ) + } + guard let observedOrientation = rotateDevice(to: orientation) else { + return Response( + ok: false, + error: ErrorPayload(message: "unable to observe device orientation after rotate") + ) + } + guard observedOrientation == orientation else { + return Response( + ok: false, + error: ErrorPayload( + message: "device orientation is \(observedOrientation), expected \(orientation)" + ) + ) + } + return Response( + ok: true, + data: DataPayload(message: "rotate", orientation: observedOrientation) + ) + } + + private func rotateDevice(to orientationName: String) -> String? { +#if os(macOS) || os(tvOS) || os(visionOS) + return nil +#else + switch orientationName { + case "portrait": + XCUIDevice.shared.orientation = .portrait + case "portrait-upside-down": + XCUIDevice.shared.orientation = .portraitUpsideDown + case "landscape-left": + XCUIDevice.shared.orientation = .landscapeLeft + case "landscape-right": + XCUIDevice.shared.orientation = .landscapeRight + default: + return nil + } + let deadline = Date().addingTimeInterval(OrientationObservationTiming.timeout) + repeat { + if observedDeviceOrientation() == orientationName { + return orientationName + } + sleepFor(OrientationObservationTiming.pollInterval) + } while Date() < deadline + return observedDeviceOrientation() +#endif + } + + private func observedDeviceOrientation() -> String? { +#if os(macOS) || os(tvOS) || os(visionOS) + return nil +#else + switch XCUIDevice.shared.orientation { + case .portrait: + return "portrait" + case .portraitUpsideDown: + return "portrait-upside-down" + case .landscapeLeft: + return "landscape-left" + case .landscapeRight: + return "landscape-right" + default: + return nil + } +#endif + } +} diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 6e9648f78..0ccd648eb 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -324,6 +324,66 @@ envelope is written once, after **all** lane tests settle, and reports `fail` if replay self-check, or forced-contention guardrail) failed — a later-failing guardrail can never be published as a passing envelope. Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`. +## Live iOS simulator coverage + +The iOS lane combines three evidence layers instead of treating a catalog mention as E2E proof: + +- pull requests run a short JSON-asserting fixture smoke against the real built CLI, daemon, XCTest + runner, and simulator; +- the scheduled/manual nightly workflow adds device lifecycle, system UI, recording/trace, and + fixture replay scenarios without putting those slower operations on the pull-request merge gate; +- command-contract, workflow-live, and capability-denial rows explicitly own functionality that + requires remote sources, unavailable host permissions, or CI setup outside the app session. + +`test/integration/ios-simulator-e2e/coverage-manifest.ts` is the executable ownership source. A new +public command fails the always-running Node contract until it has one primary owner and an +observable assertion. Live scenario claims are credited only after the scenario runs every claimed +command and records command-specific app/device/artifact evidence. Replay and test run inside the +same full harness, so its coverage report cannot turn green before their semantic fixture canaries +and JUnit output pass. + +Command ownership guarantees at least one semantic path for every public command; it does not imply +that every optional collector or backend mode runs nightly. The complementary +`behavior-coverage.ts` matrix guards the cross-command mobile patterns from #320: cold deep-link +navigation, keyboard lifecycle, background resume, modal presentation, permission denial/reset/ +acceptance, interrupted Home/app-switcher recovery, long-list rediscovery, and host-focus +preservation. Existing focused command contracts remain the evidence for additional expensive or +host-permission-dependent modes. + +CI retrieves the Release fixture through `.github/actions/setup-fixture-app` with `install: false`; +the smoke then exercises the public `install` command. The artifact is keyed by the Expo native +fingerprint and repacked with current JavaScript, so screen and replay changes reuse the native +binary and do not need Metro. Both iOS workflows need `permissions.actions: read`; without it the +action deliberately falls back to an expensive inline native build. The pull-request consumer +polls a cold fingerprint while the producer workflow builds it, preventing two concurrent native +builds; hits proceed immediately. The pull-request lane also pins Finder as the frontmost host app +and, when the hosted runner can establish that canary, proves simulator automation does not steal +macOS focus. + +Run the static contract and documented live skip locally: + +```bash +node --test test/integration/smoke-ios-simulator-coverage.test.ts +``` + +Run a live tier after booting a simulator and obtaining a current Release `.app`: + +```bash +pnpm build +pnpm clean:daemon +AGENT_DEVICE_IOS_E2E=1 \ +AGENT_DEVICE_IOS_E2E_TIER=smoke \ +AGENT_DEVICE_IOS_UDID= \ +AGENT_DEVICE_FIXTURE_APP_PATH= \ +AGENT_DEVICE_FIXTURE_APP_ID=com.callstack.agentdevicelab \ +AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE='agent-device-test-app:///automation?event={event}&payload={payload}' \ +node --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts +``` + +Use `AGENT_DEVICE_IOS_E2E_TIER=full` for the nightly subset. Step history, coverage reports, +screenshots, recordings, traces, and failure context are written below +`test/artifacts/ios-simulator/` and uploaded by the existing shared artifact action. The six +Settings replays remain additive OS-chrome coverage and are not modified by this suite. ## Speed rules (experiment-backed, 2026-07-04) Measured on the full unit suite (340 files, 3,210 tests, 48s wall at ~7x parallelism): @@ -341,6 +401,7 @@ Measured on the full unit suite (340 files, 3,210 tests, 48s wall at ~7x paralle layer and assert the right `timeoutMs` constant is passed. Exec-layer timeout semantics are proven once, in exec's own tests. 3. *Fake clocks* where the code accepts an injected clock. + Never add a test-only DI seam for this — the CI gate forbids it; patterns 1–2 are production improvements and test restructurings respectively. - **The slow-test ratchet** (`scripts/vitest-slow-test-reporter.ts`) enforces this: unit budget diff --git a/examples/test-app/README.md b/examples/test-app/README.md index b1d762343..413816944 100644 --- a/examples/test-app/README.md +++ b/examples/test-app/README.md @@ -13,10 +13,11 @@ It is intentionally small, but each surface is dense with durable accessibility ## Screens - `Home`: visible-text checks, dismissible banner, modal open/close, async loading, status badge, switch state -- `Catalog`: search debounce, filter chips, long-list scroll, favorite toggles, cart updates, drill-in navigation +- `Catalog`: search debounce, filter chips, direction-aware scroll canary, favorite toggles, cart updates, drill-in navigation - `Product detail`: back navigation, quantity stepper, multiline notes, save action - `Checkout form`: required-field validation, fill vs type, checkbox state, choice groups, keyboard dismiss, success summary - `Settings`: switch rows, accordion content, loading and error states, retry flow, destructive-confirm modal +- `Automation lab`: long-press, alert-result, app-event, app-state, appearance, orientation, permission-recovery, and log canaries - `WebView accessibility`: a deterministic semantic fixture plus live websites with varied HTML for native accessibility snapshot verification Navigation uses Expo Router native bottom tabs, so the tab bar itself is also part of the test surface. @@ -84,6 +85,14 @@ named `fingerprint..`. Jobs that drive the app install it throug its JS with `@expo/repack-app` (~seconds) — so a JS-only change reuses the same native binary. A consuming job needs `permissions: actions: read`. +The `/automation` route is intentionally JavaScript-only and can be opened from +**Settings → Open automation lab** or with the +`agent-device-test-app:///automation` scheme. Its stable `automation-*` ids expose durable +outcomes for long press, native alert actions, app-event name/payload, app state, appearance, +window orientation, and microphone permission recovery. CI repacks JavaScript-only changes into the +cached Release app without starting Metro; native configuration changes intentionally produce one +new fingerprinted build that all simulator consumers share. + ### iOS simulator From the repo root, install dependencies and run the development build on the diff --git a/examples/test-app/app.config.js b/examples/test-app/app.config.js index 386ca960e..a4ec6d714 100644 --- a/examples/test-app/app.config.js +++ b/examples/test-app/app.config.js @@ -18,7 +18,17 @@ module.exports = { // rebuilding. CI does not rely on this — it builds Release and shares the // result through GitHub artifacts (see .github/workflows/test-app-build-cache.yml). buildCacheProvider: { plugin: 'expo-build-disk-cache' }, - plugins: ['expo-router'], + plugins: [ + 'expo-router', + [ + 'expo-audio', + { + microphonePermission: + 'Allow Agent Device Tester to exercise microphone permission recovery.', + recordAudioAndroid: false, + }, + ], + ], ios: { supportsTablet: true, bundleIdentifier: 'com.callstack.agentdevicelab', diff --git a/examples/test-app/app/(tabs)/settings.tsx b/examples/test-app/app/(tabs)/settings.tsx index 6047017e7..9f86565c5 100644 --- a/examples/test-app/app/(tabs)/settings.tsx +++ b/examples/test-app/app/(tabs)/settings.tsx @@ -16,6 +16,7 @@ export default function SettingsRoute() { diagnosticsState={state.diagnosticsState} notificationsEnabled={state.notificationsEnabled} onOpenAccessorySetup={() => router.push('/accessory-setup')} + onOpenAutomationLab={() => router.push('/automation')} onOpenInertSurface={() => router.push('/inert')} onOpenWebViewLab={() => router.push('/webview')} onConfirmReset={state.resetLabState} diff --git a/examples/test-app/app/_layout.tsx b/examples/test-app/app/_layout.tsx index cfd43e091..8ea767a7b 100644 --- a/examples/test-app/app/_layout.tsx +++ b/examples/test-app/app/_layout.tsx @@ -24,6 +24,7 @@ function RootLayoutContent() { > + diff --git a/examples/test-app/app/automation.tsx b/examples/test-app/app/automation.tsx new file mode 100644 index 000000000..9e11bf99f --- /dev/null +++ b/examples/test-app/app/automation.tsx @@ -0,0 +1,27 @@ +import { useLocalSearchParams, useRouter } from 'expo-router'; + +import { AppFrame } from '../src/components'; +import { AutomationLabScreen } from '../src/screens/AutomationLabScreen'; + +export default function AutomationRoute() { + const router = useRouter(); + const params = useLocalSearchParams<{ + event?: string | string[]; + payload?: string | string[]; + }>(); + + return ( + + router.replace('/(tabs)/catalog')} + /> + + ); +} + +function firstParam(value: string | string[] | undefined): string { + if (Array.isArray(value)) return value[0] ?? 'none'; + return value ?? 'none'; +} diff --git a/examples/test-app/replays/gesture-lab.ad b/examples/test-app/replays/gesture-lab.ad index ac3946e33..d366dc65c 100644 --- a/examples/test-app/replays/gesture-lab.ad +++ b/examples/test-app/replays/gesture-lab.ad @@ -14,10 +14,10 @@ wait "two-pointer pan activations 0" 5000 gesture pan 110 443 48 0 500 --pointer-count 2 wait "two-pointer pan activations 1" 5000 -gesture fling left 280 443 120 +gesture fling left 210 443 90 wait "fling 1" 5000 -gesture fling right 280 443 120 +gesture fling right 210 443 90 wait "fling 2" 5000 open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" @@ -25,7 +25,7 @@ wait "Gesture lab" 30000 wait "gesture canary ready" 5000 wait "pan changed no, pinch changed no, rotate changed no" 30000 -gesture pinch 1.5 280 443 +gesture pinch 1.5 210 443 wait "pan changed no, pinch changed yes, rotate changed no" 5000 open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" @@ -33,7 +33,7 @@ wait "Gesture lab" 30000 wait "gesture canary ready" 5000 wait "pan changed no, pinch changed no, rotate changed no" 30000 -gesture rotate 35 280 443 +gesture rotate 35 210 443 wait "pan changed no, pinch changed no, rotate changed yes" 5000 open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" @@ -41,7 +41,7 @@ wait "Gesture lab" 30000 wait "gesture canary ready" 5000 wait "pan changed no, pinch changed no, rotate changed no" 30000 -gesture transform 280 443 24 -24 1.35 30 300 +gesture transform 210 443 24 -24 1.35 30 300 wait "pan changed yes, pinch changed yes, rotate changed yes" 5000 close diff --git a/examples/test-app/src/screens/AutomationLabScreen.tsx b/examples/test-app/src/screens/AutomationLabScreen.tsx new file mode 100644 index 000000000..e95022926 --- /dev/null +++ b/examples/test-app/src/screens/AutomationLabScreen.tsx @@ -0,0 +1,268 @@ +import { useEffect, useRef, useState } from 'react'; +import { + Alert, + AppState, + Modal, + Pressable, + ScrollView, + StyleSheet, + Text, + useColorScheme, + useWindowDimensions, + View, +} from 'react-native'; +import { getRecordingPermissionsAsync, requestRecordingPermissionsAsync } from 'expo-audio'; + +import { ActionButton, ScreenTitle, SectionCard } from '../components'; +import { useAppColors, type AppColors } from '../theme'; + +export function AutomationLabScreen(props: { + eventName: string; + eventPayload: string; + onContinueToCatalog: () => void; +}) { + const colors = useAppColors(); + const styles = createStyles(colors); + const colorScheme = useColorScheme() ?? 'light'; + const dimensions = useWindowDimensions(); + const [appState, setAppState] = useState(AppState.currentState); + const [lastNonActiveState, setLastNonActiveState] = useState('none'); + const [alertResult, setAlertResult] = useState('none'); + const [lastInput, setLastInput] = useState('none'); + const [longPressCount, setLongPressCount] = useState(0); + const [microphonePermission, setMicrophonePermission] = useState('checking'); + const [sheetVisible, setSheetVisible] = useState(false); + const permissionReadGeneration = useRef(0); + const windowMode = dimensions.width > dimensions.height ? 'landscape' : 'portrait'; + + useEffect(() => { + let mounted = true; + const refreshMicrophonePermission = () => { + const generation = ++permissionReadGeneration.current; + setMicrophonePermission('checking'); + void getRecordingPermissionsAsync() + .then((permission) => { + if (mounted && generation === permissionReadGeneration.current) { + setMicrophonePermission(permission.status); + } + }) + .catch(() => { + if (mounted && generation === permissionReadGeneration.current) { + setMicrophonePermission('error'); + } + }); + }; + refreshMicrophonePermission(); + const subscription = AppState.addEventListener('change', (nextState) => { + setAppState(nextState); + if (nextState !== 'active') setLastNonActiveState(nextState); + if (nextState === 'active') refreshMicrophonePermission(); + }); + return () => { + mounted = false; + permissionReadGeneration.current += 1; + subscription.remove(); + }; + }, []); + + function showAutomationAlert() { + Alert.alert('Automation confirmation', 'Choose either result to update the visible canary.', [ + { + style: 'cancel', + text: 'Cancel', + onPress: () => setAlertResult('cancelled'), + }, + { + text: 'OK', + onPress: () => setAlertResult('accepted'), + }, + ]); + } + + async function requestMicrophonePermission() { + const generation = ++permissionReadGeneration.current; + setMicrophonePermission('checking'); + try { + const permission = await requestRecordingPermissionsAsync(); + if (generation === permissionReadGeneration.current) { + setMicrophonePermission(permission.status); + } + } catch { + if (generation === permissionReadGeneration.current) { + setMicrophonePermission('error'); + } + } + } + + return ( + + + + + + + + + + + + + + setSheetVisible(true)} + testID="automation-open-sheet" + /> + + + + setLastInput('press')} + testID="automation-press" + /> + { + setLastInput('longpress'); + setLongPressCount((count) => count + 1); + }} + onPress={() => setLastInput('tap')} + style={({ pressed }) => [styles.longPressTarget, pressed ? styles.pressed : null]} + testID="automation-longpress" + > + Hold this control + + + Last input: {lastInput} + + + Long presses: {longPressCount} + + + + + + + Alert result: {alertResult} + + + + + void requestMicrophonePermission()} + testID="automation-request-microphone" + /> + + + + setSheetVisible(false)} + presentationStyle="pageSheet" + visible={sheetVisible} + > + + + Automation sheet + + A deterministic modal presentation for open/close flows. + setSheetVisible(false)} + testID="automation-close-sheet" + /> + + + + ); +} + +function StateRow(props: { label: string; testID: string; value: string }) { + const colors = useAppColors(); + const styles = createStyles(colors); + return ( + + {props.label} + + {props.value} + + + ); +} + +function createStyles(colors: AppColors) { + return StyleSheet.create({ + content: { + paddingBottom: 28, + }, + label: { + color: colors.text, + fontSize: 15, + fontWeight: '600', + }, + longPressLabel: { + color: colors.text, + fontSize: 15, + fontWeight: '700', + }, + longPressTarget: { + alignItems: 'center', + borderColor: colors.lineStrong, + borderRadius: 4, + borderWidth: StyleSheet.hairlineWidth, + paddingHorizontal: 16, + paddingVertical: 16, + }, + pressed: { + opacity: 0.8, + }, + stateRow: { + alignItems: 'center', + flexDirection: 'row', + justifyContent: 'space-between', + }, + sheet: { + backgroundColor: colors.surface, + flex: 1, + gap: 20, + padding: 24, + paddingTop: 48, + }, + sheetTitle: { + color: colors.text, + fontSize: 28, + fontWeight: '700', + }, + value: { + color: colors.textSoft, + fontSize: 14, + fontWeight: '600', + }, + }); +} diff --git a/examples/test-app/src/screens/CatalogScreen.tsx b/examples/test-app/src/screens/CatalogScreen.tsx index ccc9dee50..a3bf322ea 100644 --- a/examples/test-app/src/screens/CatalogScreen.tsx +++ b/examples/test-app/src/screens/CatalogScreen.tsx @@ -1,4 +1,13 @@ -import { Pressable, ScrollView, StyleSheet, Text, View } from 'react-native'; +import { useRef, useState } from 'react'; +import { + type NativeScrollEvent, + type NativeSyntheticEvent, + Pressable, + ScrollView, + StyleSheet, + Text, + View, +} from 'react-native'; import { PRODUCT_CATEGORIES, type LabProduct, type ProductCategory } from '../data'; import { @@ -27,15 +36,42 @@ export interface CatalogScreenProps { export function CatalogScreen(props: CatalogScreenProps) { const colors = useAppColors(); const styles = createStyles(colors); + const lastScrollOffset = useRef(0); + const [scrollState, setScrollState] = useState<'top' | 'bottom' | 'down' | 'up'>('top'); + + function updateScrollState(event: NativeSyntheticEvent) { + const { contentOffset, contentSize, layoutMeasurement } = event.nativeEvent; + const offset = contentOffset.y; + const previous = lastScrollOffset.current; + lastScrollOffset.current = offset; + if (offset <= 4) { + setScrollState('top'); + } else if (offset + layoutMeasurement.height >= contentSize.height - 4) { + setScrollState('bottom'); + } else if (offset > previous + 4) { + setScrollState('down'); + } else if (offset < previous - 4) { + setScrollState('up'); + } + } return ( - + + + Catalog scroll: {scrollState} + props.onChange('email', value)} + onSubmitEditing={() => Keyboard.dismiss()} placeholder="ada@example.com" + returnKeyType="done" testID="field-email" value={props.form.email} /> diff --git a/examples/test-app/src/screens/SettingsScreen.tsx b/examples/test-app/src/screens/SettingsScreen.tsx index 87055c81c..cc52a7c15 100644 --- a/examples/test-app/src/screens/SettingsScreen.tsx +++ b/examples/test-app/src/screens/SettingsScreen.tsx @@ -18,6 +18,7 @@ export interface SettingsScreenProps { notificationsEnabled: boolean; reducedMotionEnabled: boolean; onOpenAccessorySetup: () => void; + onOpenAutomationLab: () => void; onOpenInertSurface: () => void; onOpenWebViewLab: () => void; onLoadDiagnostics: () => void; @@ -62,6 +63,17 @@ export function SettingsScreen(props: SettingsScreenProps) { testID="settings-title" /> + + + + ; back(mode?: BackMode): Promise; home(): Promise; - setOrientation(orientation: DeviceRotation): Promise; + setOrientation(orientation: DeviceRotation): Promise<{ orientation?: DeviceRotation } | void>; performGesture?(plan: GesturePlan): Promise | void>; appSwitcher(): Promise; tvRemote(button: TvRemoteButton, durationMs?: number): Promise; diff --git a/src/core/__tests__/dispatch-orientation.test.ts b/src/core/__tests__/dispatch-orientation.test.ts index c5ed04a09..6dd6ba2c0 100644 --- a/src/core/__tests__/dispatch-orientation.test.ts +++ b/src/core/__tests__/dispatch-orientation.test.ts @@ -17,7 +17,6 @@ const mockRunAppleRunnerCommand = vi.mocked(runAppleRunnerCommand); beforeEach(() => { vi.resetAllMocks(); - mockRunAppleRunnerCommand.mockResolvedValue({ message: 'rotate', orientation: 'landscape-left' }); }); test('dispatch orientation normalizes value aliases before Android execution', async () => { @@ -34,6 +33,10 @@ test('dispatch orientation normalizes value aliases before Android execution', a }); test('dispatch orientation sends normalized orientation to the iOS runner', async () => { + mockRunAppleRunnerCommand.mockResolvedValue({ + message: 'rotate', + orientation: 'landscape-right', + }); const result = await dispatchCommand(IOS_DEVICE, 'orientation', ['right'], undefined, { appBundleId: 'com.example.app', }); @@ -48,3 +51,15 @@ test('dispatch orientation sends normalized orientation to the iOS runner', asyn appBundleId: 'com.example.app', }); }); + +test('dispatch orientation rejects a mismatched iOS runner readback', async () => { + mockRunAppleRunnerCommand.mockResolvedValue({ + message: 'rotate', + orientation: 'portrait', + }); + + await assert.rejects( + dispatchCommand(IOS_DEVICE, 'orientation', ['left']), + /observed portrait after requesting landscape-left/, + ); +}); diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 67dbfdac5..85afbdada 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -168,8 +168,9 @@ const DISPATCH_HANDLERS: Record = { return { action: 'home', ...successText('Home') }; }, orientation: async ({ interactor, positionals }) => { - const orientation = parseDeviceRotation(positionals[0]); - await interactor.setOrientation(orientation); + const requestedOrientation = parseDeviceRotation(positionals[0]); + const result = await interactor.setOrientation(requestedOrientation); + const orientation = result?.orientation ?? requestedOrientation; return { action: 'orientation', orientation, ...successText(`Rotated to ${orientation}`) }; }, 'app-switcher': async ({ interactor }) => { diff --git a/src/daemon/__tests__/request-execution-scope.test.ts b/src/daemon/__tests__/request-execution-scope.test.ts index 0d4c7b362..c040a4b38 100644 --- a/src/daemon/__tests__/request-execution-scope.test.ts +++ b/src/daemon/__tests__/request-execution-scope.test.ts @@ -2,7 +2,11 @@ import { afterAll, test, expect } from 'vitest'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { flushDiagnosticsToSessionFile, withDiagnosticsScope } from '../../utils/diagnostics.ts'; +import { + emitDiagnostic, + flushDiagnosticsToSessionFile, + withDiagnosticsScope, +} from '../../utils/diagnostics.ts'; import { makeAndroidSession, makeIosSession, @@ -570,6 +574,34 @@ test('prepareLockedRequestScope passes the session runner log path into handler } }); +test('prepareLockedRequestScope streams ordinary diagnostics into the active trace', async () => { + const sessionStore = makeSessionStore('agent-device-request-scope-'); + const tracePath = path.join(TEST_ROOT, 'active-session.trace'); + sessionStore.set( + 'default', + makeIosSession('default', { + trace: { outPath: tracePath, startedAt: Date.now() }, + }), + ); + const scope = await createRequestExecutionScope({ + req: makeRequest({ command: 'snapshot' }), + sessionStore, + leaseRegistry: new LeaseRegistry(), + }); + + await withDiagnosticsScope({ command: 'snapshot', logPath: LOG_PATH }, async () => { + const result = prepareLockedRequestScope({ + scope, + sessionStore, + trackDownloadableArtifact: () => 'artifact-id', + }); + expect(result.type).toBe('scope'); + emitDiagnostic({ phase: 'trace_regression_canary' }); + }); + + expect(fs.readFileSync(tracePath, 'utf8')).toContain('"phase":"trace_regression_canary"'); +}); + test('runLocked rejects a canceled request before executing work', async () => { const requestId = 'request-scope-canceled-before-lock'; const scope = await createRequestExecutionScope({ diff --git a/src/daemon/handlers/__tests__/install-source.test.ts b/src/daemon/handlers/__tests__/install-source.test.ts index 69c55edc4..e499450f8 100644 --- a/src/daemon/handlers/__tests__/install-source.test.ts +++ b/src/daemon/handlers/__tests__/install-source.test.ts @@ -31,17 +31,40 @@ vi.mock('../../../platforms/android/app-lifecycle.ts', () => ({ inferAndroidAppName: vi.fn(() => 'App'), })); -function makeRequest(meta?: DaemonRequest['meta']): DaemonRequest { +vi.mock('../../../platforms/apple/core/apps.ts', () => ({ + installIosInstallablePath: vi.fn(async () => {}), +})); + +function makeRequest( + meta?: DaemonRequest['meta'], + platform: 'android' | 'ios' = 'android', +): DaemonRequest { return { token: 't', session: 'default', command: 'install_source', positionals: [], - flags: { platform: 'android' }, + flags: { platform }, meta, }; } +function makeIosSession(name: string): SessionState { + return { + name, + createdAt: Date.now(), + actions: [], + device: { + platform: 'apple', + appleOs: 'ios', + id: 'sim-1', + name: 'iPhone', + kind: 'simulator', + booted: true, + }, + }; +} + function makeSessionStore(): SessionStore { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-install-source-session-')); return new SessionStore(path.join(root, 'sessions')); @@ -123,3 +146,56 @@ test('install_from_source returns an error when Android package identity cannot expect(response.error.message).toMatch(/identity could not be resolved/i); } }); + +test('install_from_source prepares and installs a local iOS simulator app source with typed identity', async () => { + const { resolveTargetDevice } = await import('../../../core/dispatch.ts'); + const { installIosInstallablePath } = await import('../../../platforms/apple/core/apps.ts'); + const sourceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-ios-source-')); + const appPath = path.join(sourceRoot, 'AgentDeviceTester.app'); + fs.mkdirSync(appPath); + fs.writeFileSync( + path.join(appPath, 'Info.plist'), + ` + + + CFBundleIdentifier + com.callstack.agentdevicelab + CFBundleDisplayName + Agent Device Tester + + +`, + ); + + const sessionStore = makeSessionStore(); + const session = makeIosSession('ios-source'); + vi.mocked(resolveTargetDevice).mockResolvedValueOnce(session.device); + try { + const response = await handleInstallFromSourceCommand({ + req: makeRequest( + { + installSource: { + kind: 'path', + path: appPath, + }, + }, + 'ios', + ), + sessionName: session.name, + sessionStore, + }); + + expect(response).toEqual({ + ok: true, + data: { + appName: 'Agent Device Tester', + bundleId: 'com.callstack.agentdevicelab', + launchTarget: 'com.callstack.agentdevicelab', + message: 'Installed: Agent Device Tester', + }, + }); + expect(installIosInstallablePath).toHaveBeenCalledWith(session.device, appPath); + } finally { + fs.rmSync(sourceRoot, { recursive: true, force: true }); + } +}); diff --git a/src/daemon/handlers/__tests__/record-trace-ios-simulator-recording.test.ts b/src/daemon/handlers/__tests__/record-trace-ios-simulator-recording.test.ts new file mode 100644 index 000000000..3184a9820 --- /dev/null +++ b/src/daemon/handlers/__tests__/record-trace-ios-simulator-recording.test.ts @@ -0,0 +1,277 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { afterEach, test, vi } from 'vitest'; + +import { IOS_SIMULATOR } from '../../../__tests__/test-utils/device-fixtures.ts'; +import type { RecordTraceDeps } from '../record-trace-types.ts'; +import { startIosSimulatorRecording } from '../record-trace-ios-simulator-recording.ts'; + +const temporaryRoots: string[] = []; + +afterEach(() => { + vi.useRealTimers(); + for (const root of temporaryRoots.splice(0)) { + fs.rmSync(root, { force: true, recursive: true }); + } +}); + +test('startIosSimulatorRecording waits for the zero-byte simctl destination before reporting ready', async () => { + vi.useFakeTimers(); + const testStartedAt = Date.now(); + const root = makeTemporaryRoot('agent-device-record-ready-'); + const outPath = path.join(root, 'recording.mp4'); + const kill = vi.fn((_signal?: NodeJS.Signals) => true); + const wait = new Promise(() => {}); + setTimeout(() => fs.writeFileSync(outPath, ''), 600); + + const resultPromise = startIosSimulatorRecording({ + req: { + token: 'test-token', + session: 'record-ready', + command: 'record', + positionals: ['start', outPath], + flags: {}, + }, + activeSession: { + name: 'record-ready', + device: IOS_SIMULATOR, + createdAt: Date.now(), + actions: [], + appBundleId: 'com.example.app', + }, + device: IOS_SIMULATOR, + deps: makeDeps({ + startIosSimulatorRecording: () => ({ + child: { kill, pid: 1234 }, + wait, + }), + }), + recordingBase: { + outPath, + startedAt: 0, + showTouches: false, + gestureEvents: [], + }, + resolvedOut: outPath, + }); + + await vi.advanceTimersByTimeAsync(800); + const result = await resultPromise; + assert.ok(!('ok' in result), JSON.stringify(result)); + assert.equal(result.platform, 'ios'); + assert.equal(result.startedAt, testStartedAt + 750); + assert.equal(kill.mock.calls.length, 0); +}); + +test('startIosSimulatorRecording reports an early recorder exit instead of a false start', async () => { + const root = makeTemporaryRoot('agent-device-record-exit-'); + const outPath = path.join(root, 'recording.mp4'); + const result = await startIosSimulatorRecording({ + req: { + token: 'test-token', + session: 'record-exit', + command: 'record', + positionals: ['start', outPath], + flags: {}, + }, + activeSession: { + name: 'record-exit', + device: IOS_SIMULATOR, + createdAt: Date.now(), + actions: [], + appBundleId: 'com.example.app', + }, + device: IOS_SIMULATOR, + deps: makeDeps({ + startIosSimulatorRecording: () => ({ + child: { kill: vi.fn(() => true), pid: 1234 }, + wait: Promise.resolve({ stdout: '', stderr: 'capture unavailable', exitCode: 1 }), + }), + }), + recordingBase: { + outPath, + startedAt: 0, + showTouches: false, + gestureEvents: [], + }, + resolvedOut: outPath, + }); + + assert.equal('ok' in result && result.ok, false); + assert.match(JSON.stringify(result), /failed to start recording: capture unavailable/); +}); + +test('startIosSimulatorRecording escalates cleanup when its wait monitor rejects', async () => { + const root = makeTemporaryRoot('agent-device-record-wait-failed-'); + const outPath = path.join(root, 'recording.mp4'); + const kill = vi.fn((_signal?: NodeJS.Signals) => true); + const result = await startIosSimulatorRecording({ + req: { + token: 'test-token', + session: 'record-wait-failed', + command: 'record', + positionals: ['start', outPath], + flags: {}, + }, + activeSession: { + name: 'record-wait-failed', + device: IOS_SIMULATOR, + createdAt: Date.now(), + actions: [], + appBundleId: 'com.example.app', + }, + device: IOS_SIMULATOR, + deps: makeDeps({ + startIosSimulatorRecording: () => ({ + child: { kill, pid: 1234 }, + wait: Promise.reject(new Error('recorder wait monitor failed')), + }), + }), + recordingBase: { + outPath, + startedAt: 0, + showTouches: false, + gestureEvents: [], + }, + resolvedOut: outPath, + }); + + assert.equal('ok' in result && result.ok, false); + assert.match(JSON.stringify(result), /recorder wait monitor failed/); + assert.deepEqual(kill.mock.calls, [['SIGINT'], ['SIGTERM'], ['SIGKILL']]); +}); + +test.each([ + { exitCode: 0, exitDelayMs: 0 }, + { exitCode: 1, exitDelayMs: 0 }, + { exitCode: 1, exitDelayMs: 25 }, +])( + 'startIosSimulatorRecording rejects recorder exit code $exitCode after $exitDelayMs ms when its destination exists', + async ({ exitCode, exitDelayMs }) => { + if (exitDelayMs > 0) vi.useFakeTimers(); + const root = makeTemporaryRoot('agent-device-record-file-exit-'); + const outPath = path.join(root, 'recording.mp4'); + fs.writeFileSync(outPath, ''); + const wait = + exitDelayMs === 0 + ? Promise.resolve({ stdout: '', stderr: 'recorder exited', exitCode }) + : new Promise<{ stdout: string; stderr: string; exitCode: number }>((resolve) => { + setTimeout( + () => resolve({ stdout: '', stderr: 'recorder exited', exitCode }), + exitDelayMs, + ); + }); + + const resultPromise = startIosSimulatorRecording({ + req: { + token: 'test-token', + session: 'record-file-exit', + command: 'record', + positionals: ['start', outPath], + flags: {}, + }, + activeSession: { + name: 'record-file-exit', + device: IOS_SIMULATOR, + createdAt: Date.now(), + actions: [], + appBundleId: 'com.example.app', + }, + device: IOS_SIMULATOR, + deps: makeDeps({ + startIosSimulatorRecording: () => ({ + child: { kill: vi.fn(() => true), pid: 1234 }, + wait, + }), + }), + recordingBase: { + outPath, + startedAt: 0, + showTouches: false, + gestureEvents: [], + }, + resolvedOut: outPath, + }); + if (exitDelayMs > 0) await vi.advanceTimersByTimeAsync(50); + const result = await resultPromise; + + assert.equal('ok' in result && result.ok, false); + assert.match(JSON.stringify(result), /failed to start recording/); + assert.equal(fs.existsSync(outPath), false); + }, +); + +test('startIosSimulatorRecording times out and cleans up a recorder that never becomes ready', async () => { + vi.useFakeTimers(); + const root = makeTemporaryRoot('agent-device-record-timeout-'); + const outPath = path.join(root, 'recording.mp4'); + const kill = vi.fn((_signal?: NodeJS.Signals) => true); + const resultPromise = startIosSimulatorRecording({ + req: { + token: 'test-token', + session: 'record-timeout', + command: 'record', + positionals: ['start', outPath], + flags: {}, + }, + activeSession: { + name: 'record-timeout', + device: IOS_SIMULATOR, + createdAt: Date.now(), + actions: [], + appBundleId: 'com.example.app', + }, + device: IOS_SIMULATOR, + deps: makeDeps({ + startIosSimulatorRecording: () => ({ + child: { kill, pid: undefined }, + wait: new Promise(() => {}), + }), + }), + recordingBase: { + outPath, + startedAt: 0, + showTouches: false, + gestureEvents: [], + }, + resolvedOut: outPath, + }); + + // Let the readiness loop schedule its first fake-timer poll before advancing + // the clock. Unlike the success case above, this test has no independent + // timer to yield that initial microtask turn. + await Promise.resolve(); + await vi.runAllTimersAsync(); + const result = await resultPromise; + + assert.equal('ok' in result && result.ok, false); + assert.match(JSON.stringify(result), /did not create its output within 15000ms/); + assert.deepEqual( + kill.mock.calls.map(([signal]) => signal), + ['SIGINT', 'SIGTERM', 'SIGKILL'], + ); + assert.equal(fs.existsSync(outPath), false); +}); + +function makeDeps(overrides: Pick): RecordTraceDeps { + return { + runCmd: async () => ({ stdout: '', stderr: '', exitCode: 0 }), + startIosSimulatorRecording: overrides.startIosSimulatorRecording, + runAppleRunnerCommand: async () => ({}), + waitForRecordingTail: async () => {}, + waitForStableFile: async () => {}, + isPlayableVideo: async () => true, + trimRecordingStart: async () => {}, + resizeRecording: async () => {}, + overlayRecordingTouches: async () => {}, + }; +} + +function makeTemporaryRoot(prefix: string): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryRoots.push(root); + return root; +} diff --git a/src/daemon/handlers/__tests__/record-trace.test.ts b/src/daemon/handlers/__tests__/record-trace.test.ts index 395769f24..e76c55f5e 100644 --- a/src/daemon/handlers/__tests__/record-trace.test.ts +++ b/src/daemon/handlers/__tests__/record-trace.test.ts @@ -96,6 +96,7 @@ const mockWaitForStableFile = vi.mocked(waitForStableFile); const mockIsPlayableVideo = vi.mocked(isPlayableVideo); const overlaySupportWarning = getRecordingOverlaySupportWarning(); +const mockedIosRecordingOutputs: string[] = []; function makeSessionStore(): SessionStore { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-record-trace-')); @@ -198,6 +199,35 @@ function makeIosSimulatorRecordingSession( return session; } +function mockIosSimulatorRecordingStart( + options: { pid?: number; onStart?: () => void } = {}, +): void { + mockRunCmdBackground.mockImplementation((_cmd, args) => { + options.onStart?.(); + const outPath = args.at(-1); + if (!outPath) throw new Error('simctl recordVideo output path is required'); + const resolvedOutPath = path.resolve(outPath); + fs.writeFileSync(resolvedOutPath, ''); + mockedIosRecordingOutputs.push(resolvedOutPath); + let resolveWait: + | ((value: { stdout: string; stderr: string; exitCode: number }) => void) + | undefined; + const wait = new Promise<{ stdout: string; stderr: string; exitCode: number }>((resolve) => { + resolveWait = resolve; + }); + return { + child: { + kill: () => { + resolveWait?.({ stdout: '', stderr: '', exitCode: 0 }); + return true; + }, + pid: options.pid, + } as any, + wait, + }; + }); +} + function isAndroidScreenrecordStartCommand(command: string): boolean { return /^-s emulator-5554 shell screenrecord (?:--size 756x1344 )?--bit-rate (?:8000000|20000000) \/(?:sdcard|data\/local\/tmp)\/agent-device-recording-\d+\.mp4 >\/dev\/null 2>&1 & echo \$!$/.test( command, @@ -289,6 +319,9 @@ beforeEach(() => { afterEach(() => { vi.useRealTimers(); + for (const outPath of mockedIosRecordingOutputs.splice(0)) { + fs.rmSync(outPath, { force: true }); + } }); test('record stop derives telemetry artifact local path from client outPath', async () => { @@ -900,10 +933,7 @@ test('record stop resizes iOS simulator recording when max-size is explicit', as const sessionName = 'ios-sim-quality'; sessionStore.set(sessionName, makeOpenedIosSimulatorSession(sessionName)); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {} } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart(); await runRecordCommand({ sessionStore, @@ -932,10 +962,7 @@ test('record stop forwards the requested quality to the resize step', async () = const sessionName = 'ios-sim-export-quality'; sessionStore.set(sessionName, makeOpenedIosSimulatorSession(sessionName)); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {} } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart(); await runRecordCommand({ sessionStore, @@ -1246,10 +1273,7 @@ test('record start with explicit missing session can opt into device scope', asy kind: 'simulator', booted: true, }); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {}, pid: 5153 } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart({ pid: 5153 }); const response = await runRecordCommand({ sessionStore, @@ -1300,10 +1324,7 @@ test('record start on iOS simulator supports explicit device-scope capture witho kind: 'simulator', booted: true, }); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {}, pid: 5152 } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart({ pid: 5152 }); const response = await runRecordCommand({ sessionStore, @@ -1325,10 +1346,7 @@ test('record start stores iOS simulator recorder pid for scoped cleanup', async const sessionStore = makeSessionStore(); const sessionName = 'ios-sim-recorder-pid'; sessionStore.set(sessionName, makeOpenedIosSimulatorSession(sessionName)); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {}, pid: 5151 } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart({ pid: 5151 }); const response = await runRecordCommand({ sessionStore, @@ -1405,7 +1423,7 @@ test('record stop prefers session-owned iOS recorder processes before path fallb ['-P', '1111'], ]); expect(processKill.mock.calls.map((call) => call[0])).toEqual([ - 1111, 2222, 1111, 2222, 1111, 2222, + 1111, 2222, 1111, 2222, 1111, 2222, 1111, ]); expect(processKill.mock.calls.map((call) => call[1])).toEqual([ 'SIGINT', @@ -1414,6 +1432,7 @@ test('record stop prefers session-owned iOS recorder processes before path fallb 'SIGTERM', 'SIGKILL', 'SIGKILL', + 0, ]); } finally { processKill.mockRestore(); @@ -1475,10 +1494,7 @@ test('record stop keeps iOS simulator video when overlay export fails', async () const sessionName = 'ios-sim-overlay-warning'; sessionStore.set(sessionName, makeOpenedIosSimulatorSession(sessionName)); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {} } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart(); mockOverlayRecordingTouches.mockImplementation(async () => { throw new Error('swift export failed'); }); @@ -1512,10 +1528,7 @@ test('record stop skips touch overlay export when no gestures were recorded', as const sessionName = 'ios-sim-no-gestures'; sessionStore.set(sessionName, makeOpenedIosSimulatorSession(sessionName)); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {} } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart(); await runRecordCommand({ sessionStore, @@ -1539,10 +1552,7 @@ test('record stop keeps iOS simulator video when resize export fails', async () const sessionName = 'ios-sim-resize-fail'; sessionStore.set(sessionName, makeOpenedIosSimulatorSession(sessionName)); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {} } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart(); mockResizeRecording.mockImplementation(async () => { throw new Error('resize failed'); @@ -1579,13 +1589,7 @@ test('record start does not fail when iOS simulator runner warm-up fails', async sessionStore.set(sessionName, session); let started = false; - mockRunCmdBackground.mockImplementation(() => { - started = true; - return { - child: { kill: () => {} } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - }; - }); + mockIosSimulatorRecordingStart({ onStart: () => (started = true) }); const runnerCalls: RunnerCall[] = []; mockRunAppleRunnerCommand.mockImplementation(async (_device, command) => { runnerCalls.push({ command: command.command }); @@ -1625,10 +1629,7 @@ test('record start anchors gesture clock from simulator warm-up and skips standa session.appBundleId = 'com.apple.Preferences'; sessionStore.set(sessionName, session); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {} } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart(); const runnerCalls: RunnerCall[] = []; mockRunAppleRunnerCommand.mockImplementation(async (_device, command) => { runnerCalls.push({ command: command.command }); @@ -1670,10 +1671,7 @@ test('record start falls back to standalone uptime when warm response lacks curr session.appBundleId = 'com.apple.Preferences'; sessionStore.set(sessionName, session); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {} } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart(); const runnerCalls: RunnerCall[] = []; mockRunAppleRunnerCommand.mockImplementation(async (_device, command) => { runnerCalls.push({ command: command.command }); @@ -1712,10 +1710,7 @@ test('record start rejects non-finite or non-positive warm anchors', async () => session.appBundleId = 'com.apple.Preferences'; sessionStore.set(sessionName, session); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {} } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart(); const runnerCalls: RunnerCall[] = []; mockRunAppleRunnerCommand.mockImplementation(async (_device, command) => { runnerCalls.push({ command: command.command }); @@ -1754,10 +1749,7 @@ test('record start degrades to wall-clock when warm anchor missing and uptime fa session.appBundleId = 'com.apple.Preferences'; sessionStore.set(sessionName, session); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {} } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart(); mockRunAppleRunnerCommand.mockImplementation(async (_device, command) => { if (command.command === 'uptime') { throw new Error('uptime unavailable'); @@ -1793,10 +1785,7 @@ test('record start skips iOS simulator runner warm-up when touch overlays are hi session.appBundleId = 'com.apple.Preferences'; sessionStore.set(sessionName, session); - mockRunCmdBackground.mockImplementation(() => ({ - child: { kill: () => {} } as any, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - })); + mockIosSimulatorRecordingStart(); const response = await runRecordCommand({ sessionStore, diff --git a/src/daemon/handlers/record-trace-ios-simulator-recording.ts b/src/daemon/handlers/record-trace-ios-simulator-recording.ts index 6e13f05c3..2b0ac7395 100644 --- a/src/daemon/handlers/record-trace-ios-simulator-recording.ts +++ b/src/daemon/handlers/record-trace-ios-simulator-recording.ts @@ -21,12 +21,21 @@ import type { RecordTraceDeps, RecordingBase } from './record-trace-types.ts'; import { errorResponse } from './response.ts'; const LOCAL_RECORDING_READY_POLL_MS = 250; -const LOCAL_RECORDING_READY_SETTLE_POLLS = 2; +const LOCAL_RECORDING_LIVENESS_GRACE_MS = 50; +// CoreSimulator may delay creating the zero-byte recordVideo destination while +// a just-booted simulator finishes service startup. This is still much shorter +// than recording itself, but avoids reporting a false start under CI load. +const LOCAL_RECORDING_READY_TIMEOUT_MS = 15_000; const IOS_SIMULATOR_VIDEO_READY_POLL_MS = 150; const IOS_SIMULATOR_VIDEO_READY_ATTEMPTS = 12; type ActiveRecording = NonNullable; type IosSimulatorRecording = Extract; +type LocalRecordingReadiness = + | { kind: 'ready'; readyAt: number } + | { kind: 'exited'; result: Awaited } + | { kind: 'failed'; error: unknown } + | { kind: 'timeout' }; export async function startIosSimulatorRecording(params: { req: DaemonRequest; @@ -47,7 +56,26 @@ export async function startIosSimulatorRecording(params: { ? await warmIosSimulatorRunner({ req, activeSession, device, logPath, deps }) : undefined; const { child, wait } = deps.startIosSimulatorRecording({ device, outPath: resolvedOut }); - const readyAt = await waitForLocalRecordingSettleWindow(resolvedOut); + const readiness = await waitForLocalRecordingReadiness(resolvedOut, wait); + if (readiness.kind !== 'ready') { + if (readiness.kind === 'timeout' || readiness.kind === 'failed') { + await stopIosSimulatorRecordingProcess({ + deps, + recording: { + platform: 'ios', + child, + wait, + ...recordingBase, + outPath: resolvedOut, + recorderPid: child.pid, + startedAt: Date.now(), + }, + }); + } + removeInvalidRecordingOutput(resolvedOut); + return errorResponse('COMMAND_FAILED', formatRecordingStartFailure(readiness)); + } + const readyAt = readiness.readyAt; let gestureClockOriginAtMs: number | undefined; let gestureClockOriginUptimeMs: number | undefined; if (warmAnchor) { @@ -187,28 +215,61 @@ export async function stopIosSimulatorRecording(params: { return null; } -async function waitForLocalRecordingSettleWindow(outPath: string): Promise { - // simctl recordVideo can take a beat to open its output even though recording has already - // started. This is a short settle window, not a strict readiness guarantee. We prefer a - // close recorder anchor over blocking start indefinitely waiting for non-zero bytes. - for (let attempt = 0; attempt < LOCAL_RECORDING_READY_SETTLE_POLLS; attempt += 1) { +async function waitForLocalRecordingReadiness( + outPath: string, + wait: IosSimulatorRecording['wait'], +): Promise { + let settledProcessExit: LocalRecordingReadiness | undefined; + const processExit: Promise = wait.then( + (result) => (settledProcessExit = { kind: 'exited', result }), + (error: unknown) => (settledProcessExit = { kind: 'failed', error }), + ); + // Give an already-settled recorder wait precedence over a destination the process touched + // immediately before exiting. + await Promise.resolve(); + const attempts = Math.ceil(LOCAL_RECORDING_READY_TIMEOUT_MS / LOCAL_RECORDING_READY_POLL_MS); + for (let attempt = 0; attempt <= attempts; attempt += 1) { + if (settledProcessExit) return settledProcessExit; try { - const stat = fs.statSync(outPath); - if (stat.size > 0) { - return Date.now(); - } + fs.statSync(outPath); + const readyAt = Date.now(); + // `simctl recordVideo` creates a zero-byte destination when capture is ready and writes + // the finalized MP4 only after SIGINT. Existence, not size, is the readiness signal, but + // keep a short liveness window for an immediate post-create process exit to win. + const exit = await Promise.race([ + processExit, + sleep(LOCAL_RECORDING_LIVENESS_GRACE_MS).then(() => undefined), + ]); + if (exit) return exit; + return { kind: 'ready', readyAt }; } catch { // Wait for the recorder to create the output file. } - if (attempt + 1 >= LOCAL_RECORDING_READY_SETTLE_POLLS) { - return Date.now(); - } - - await sleep(LOCAL_RECORDING_READY_POLL_MS); + if (attempt === attempts) return { kind: 'timeout' }; + const exit = await Promise.race([ + processExit, + sleep(LOCAL_RECORDING_READY_POLL_MS).then(() => undefined), + ]); + if (exit) return exit; } - return Date.now(); + return { kind: 'timeout' }; +} + +function formatRecordingStartFailure( + readiness: Exclude, +): string { + if (readiness.kind === 'timeout') { + return `failed to start recording: simctl recordVideo did not create its output within ${LOCAL_RECORDING_READY_TIMEOUT_MS}ms`; + } + if (readiness.kind === 'failed') { + return `failed to start recording: ${formatRecordTraceError(readiness.error)}`; + } + return `failed to start recording: ${formatRecordTraceExecFailure( + readiness.result, + 'simctl recordVideo', + )}`; } function buildIosSimulatorRecordingStopFailure( diff --git a/src/daemon/handlers/record-trace-ios-simulator.ts b/src/daemon/handlers/record-trace-ios-simulator.ts index 4fff00866..c7df5cc21 100644 --- a/src/daemon/handlers/record-trace-ios-simulator.ts +++ b/src/daemon/handlers/record-trace-ios-simulator.ts @@ -55,17 +55,40 @@ export async function stopIosSimulatorRecordingProcess(params: { recording.child.kill('SIGKILL'); await signalIosSimulatorRecorderCleanup(deps, recording, 'SIGKILL'); - return await waitForRecordingProcessExit( + result = await waitForRecordingProcessExit( recording.wait, IOS_SIMULATOR_RECORDING_FORCE_STOP_TIMEOUT_MS, ); + if (result) return result; + if (recording.recorderPid !== undefined && !isProcessAlive(recording.recorderPid)) { + return { exitCode: 0, stderr: '', stdout: '' }; + } + return null; } async function waitForRecordingProcessExit( wait: Promise, timeoutMs: number, ): Promise { - return await Promise.race([wait, sleep(timeoutMs).then(() => null)]); + // A rejected monitor means we lost the ability to confirm process exit; it does not mean the + // recorder exited. Treat it like an unconfirmed timeout so the caller continues through the + // PID-backed SIGINT/SIGTERM/SIGKILL cleanup sequence. + return await Promise.race([ + wait.then( + (result) => result, + () => null, + ), + sleep(timeoutMs).then(() => null), + ]); +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } } async function signalIosSimulatorRecorderCleanup( diff --git a/src/daemon/request-execution-scope.ts b/src/daemon/request-execution-scope.ts index 1c902b9fc..2c4fe9b17 100644 --- a/src/daemon/request-execution-scope.ts +++ b/src/daemon/request-execution-scope.ts @@ -241,6 +241,7 @@ export function prepareLockedRequestScope(params: { }); const lockedReq = binding.req; existingSession = binding.existingSession; + updateDiagnosticsScope({ traceLogPath: existingSession?.trace?.outPath }); const finalize = (response: DaemonResponse): DaemonResponse => { const finalized = finalizeDaemonResponse(lockedReq, response, trackDownloadableArtifact); if (shouldRecordEventForRequest(lockedReq)) { diff --git a/src/daemon/server/daemon-runtime-recording-teardown.test.ts b/src/daemon/server/daemon-runtime-recording-teardown.test.ts index d407b0ce1..c1e457004 100644 --- a/src/daemon/server/daemon-runtime-recording-teardown.test.ts +++ b/src/daemon/server/daemon-runtime-recording-teardown.test.ts @@ -98,7 +98,12 @@ test('daemon shutdown lets a slow recorder run its full stop escalation instead // The recorder was escalated all the way to SIGKILL before shutdown moved on. expect(kill?.mock.calls.map((call) => call[0])).toEqual(['SIGINT', 'SIGTERM', 'SIGKILL']); - expect(processKill.mock.calls.map((call) => call[1])).toEqual(['SIGINT', 'SIGTERM', 'SIGKILL']); + expect(processKill.mock.calls.map((call) => call[1])).toEqual([ + 'SIGINT', + 'SIGTERM', + 'SIGKILL', + 0, + ]); // The extended budget covered the escalation: teardown completed (surfacing // the recorder-stop failure) rather than being abandoned by the timeout. expect(stderrChunks.join('')).toMatch(/Daemon session teardown error .*recording/); diff --git a/src/platforms/apple/__tests__/interactor-runner-provider.test.ts b/src/platforms/apple/__tests__/interactor-runner-provider.test.ts index d4db40251..819941d0b 100644 --- a/src/platforms/apple/__tests__/interactor-runner-provider.test.ts +++ b/src/platforms/apple/__tests__/interactor-runner-provider.test.ts @@ -144,6 +144,8 @@ function runnerResultFor(command: RunnerCommand): Record { }; case 'gestureViewport': return { x: 0, y: 0, x2: 390, y2: 844 }; + case 'rotate': + return { orientation: command.orientation }; default: return {}; } diff --git a/src/platforms/apple/core/config.ts b/src/platforms/apple/core/config.ts index 5bb49975c..47e87c171 100644 --- a/src/platforms/apple/core/config.ts +++ b/src/platforms/apple/core/config.ts @@ -14,6 +14,10 @@ export const IOS_SIMULATOR_TERMINATE_TIMEOUT_MS = 15_000; export const IOS_SIMULATOR_SCREENSHOT_TIMEOUT_MS = 20_000; +// CoreSimulator can briefly stall while it services the scale lookup immediately +// after a keyboard transition. Keep this bounded below the full capture budget. +export const IOS_SIMULATOR_SCREENSHOT_SCALE_TIMEOUT_MS = 15_000; + export const IOS_RUNNER_SCREENSHOT_COPY_TIMEOUT_MS = 20_000; export const IOS_SIMULATOR_SCREENSHOT_RETRY_MAX_ATTEMPTS = 5; diff --git a/src/platforms/apple/core/screenshot.ts b/src/platforms/apple/core/screenshot.ts index 516dc6735..aac88709d 100644 --- a/src/platforms/apple/core/screenshot.ts +++ b/src/platforms/apple/core/screenshot.ts @@ -14,6 +14,7 @@ import { IOS_SIMULATOR_SCREENSHOT_RETRY_BASE_DELAY_MS, IOS_SIMULATOR_SCREENSHOT_RETRY_MAX_ATTEMPTS, IOS_SIMULATOR_SCREENSHOT_RETRY_MAX_DELAY_MS, + IOS_SIMULATOR_SCREENSHOT_SCALE_TIMEOUT_MS, IOS_SIMULATOR_SCREENSHOT_TIMEOUT_MS, } from './config.ts'; import { runAppleRunnerCommand, IOS_RUNNER_CONTAINER_BUNDLE_IDS } from './runner/runner-client.ts'; @@ -446,7 +447,7 @@ async function readIosSimulatorMainScreenScale(device: DeviceInfo): Promise { - await runAppleRunnerCommand( + const result = await runAppleRunnerCommand( device, // `rotate` is the runner-protocol command name (its own namespace); the // CLI-facing command/method is `orientation`. { command: 'rotate', orientation, appBundleId: runnerContext.appBundleId }, runnerOpts, ); + const observed = readRunnerOrientation(result); + if (observed !== orientation) { + throw new AppError( + 'COMMAND_FAILED', + `iOS runner observed ${observed} after requesting ${orientation}`, + { requestedOrientation: orientation, observedOrientation: observed }, + ); + } + return { orientation: observed }; }, appSwitcher: async () => { await runAppleRunnerCommand( @@ -168,6 +178,16 @@ export function createAppleInteractor( return withInjectedAppleRunnerTransport(device, runnerContext, interactor, runnerProvider); } +function readRunnerOrientation(result: Record): DeviceRotation { + const orientation = result.orientation; + if (typeof orientation === 'string' && DEVICE_ROTATIONS.includes(orientation as DeviceRotation)) { + return orientation as DeviceRotation; + } + throw new AppError('COMMAND_FAILED', 'iOS runner returned an invalid orientation result', { + orientation, + }); +} + /** * Partitions the interactor for an injected provider transport. Its unique * jobs are the local-tooling rejection and in-process scoping for interactors diff --git a/src/utils/diagnostics.ts b/src/utils/diagnostics.ts index 131790355..dc0839d17 100644 --- a/src/utils/diagnostics.ts +++ b/src/utils/diagnostics.ts @@ -133,17 +133,17 @@ export function emitDiagnostic(event: { }; scope.events.push(payload); scope.phaseCounts.set(event.phase, (scope.phaseCounts.get(event.phase) ?? 0) + 1); - if (!scope.debug) return; + if (!scope.debug && !scope.traceLogPath) return; const fileLine = `${JSON.stringify(payload)}\n`; try { - if (scope.logPath) { + if (scope.debug && scope.logPath) { appendDiagnosticLine(scope.logPath, fileLine); scope.liveWrittenEventCount = scope.events.length; } if (scope.traceLogPath) { appendDiagnosticLine(scope.traceLogPath, fileLine); } - if (!scope.logPath && !scope.traceLogPath) { + if (scope.debug && !scope.logPath && !scope.traceLogPath) { process.stderr.write(`[agent-device][diag] ${fileLine}`); } } catch { diff --git a/src/utils/scroll-edge-state.test.ts b/src/utils/scroll-edge-state.test.ts new file mode 100644 index 000000000..dc264a7a3 --- /dev/null +++ b/src/utils/scroll-edge-state.test.ts @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; + +import { captureScrollEdgeState, runScrollEdgePasses } from './scroll-edge-state.ts'; +import type { SnapshotNode } from '../kernel/snapshot.ts'; + +test('duplicate scroll-container labels do not scope edge verification to a child', async () => { + const scopes: Array = []; + const nodes: SnapshotNode[] = [ + { + index: 1, + ref: 'e1', + type: 'ScrollView', + label: 'Automation lab', + hiddenContentAbove: true, + rect: { x: 18, y: 178, width: 366, height: 662 }, + }, + { + index: 2, + ref: 'e2', + parentIndex: 1, + type: 'StaticText', + label: 'Automation lab', + rect: { x: 18, y: -344, width: 311, height: 36 }, + }, + ]; + + const state = await captureScrollEdgeState({ + edge: 'top', + captureNodes: async (scope) => { + scopes.push(scope); + return nodes; + }, + }); + + assert.equal(state.canScroll, true); + assert.equal(state.scope, undefined); + assert.deepEqual(scopes, [undefined]); +}); + +for (const collision of ['Automation lab details', 'AUTOMATION LAB']) { + test(`native-style scope collision does not select ${JSON.stringify(collision)}`, async () => { + const nodes: SnapshotNode[] = [ + { + index: 1, + ref: 'e1', + type: 'ScrollView', + label: 'Automation lab', + hiddenContentAbove: true, + rect: { x: 18, y: 178, width: 366, height: 662 }, + }, + { + index: 2, + ref: 'e2', + parentIndex: 1, + type: 'StaticText', + label: collision, + rect: { x: 18, y: -344, width: 311, height: 36 }, + }, + ]; + + const state = await captureScrollEdgeState({ + edge: 'top', + captureNodes: async () => nodes, + }); + + assert.equal(state.scope, undefined); + }); +} + +test('value collision does not scope edge verification to an ambiguous subtree', async () => { + const nodes: SnapshotNode[] = [ + { + index: 1, + ref: 'e1', + type: 'ScrollView', + label: 'Automation lab', + hiddenContentAbove: true, + rect: { x: 18, y: 178, width: 366, height: 662 }, + }, + { + index: 2, + ref: 'e2', + type: 'Other', + value: 'Automation lab ready', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + ]; + + const state = await captureScrollEdgeState({ + edge: 'top', + captureNodes: async () => nodes, + }); + + assert.equal(state.scope, undefined); +}); + +test('unique container scope is retained across edge pass captures', async () => { + const scopes: Array = []; + const snapshots = [scrollSnapshot(true), scrollSnapshot(true), scrollSnapshot(false)]; + let captureIndex = 0; + + const result = await runScrollEdgePasses({ + edge: 'bottom', + captureState: async (scope) => + await captureScrollEdgeState({ + edge: 'bottom', + scope, + captureNodes: async (capturedScope) => { + scopes.push(capturedScope); + return snapshots[Math.min(captureIndex++, snapshots.length - 1)] ?? []; + }, + }), + scroll: async () => ({ scrolled: true }), + }); + + assert.equal(result.passes, 1); + assert.deepEqual(scopes, [undefined, 'Messages', 'Messages']); +}); + +function scrollSnapshot(hiddenContentBelow: boolean): SnapshotNode[] { + return [ + { + index: 1, + ref: 'e1', + type: 'ScrollView', + label: 'Messages', + hiddenContentBelow: hiddenContentBelow ? true : undefined, + rect: { x: 0, y: 100, width: 400, height: 600 }, + }, + { + index: 2, + ref: 'e2', + parentIndex: 1, + type: 'Button', + label: hiddenContentBelow ? 'Middle message' : 'Latest message', + rect: { x: 0, y: 640, width: 400, height: 56 }, + }, + ]; +} diff --git a/src/utils/scroll-edge-state.ts b/src/utils/scroll-edge-state.ts index eed7a492d..a1416e412 100644 --- a/src/utils/scroll-edge-state.ts +++ b/src/utils/scroll-edge-state.ts @@ -61,7 +61,7 @@ function analyzeScrollEdgeState( canScroll, emptySnapshot: false, signature, - scope: buildScrollContainerScope(container), + scope: buildScrollContainerScope(container, nodes), }; } @@ -256,10 +256,28 @@ function hasHiddenContentAtEdge( return node.hiddenContentAbove === true || hint?.hiddenContentAbove === true; } -function buildScrollContainerScope(node: SnapshotNode): string | undefined { - return [node.identifier, node.label, node.value] +function buildScrollContainerScope( + node: SnapshotNode, + nodes: readonly SnapshotNode[], +): string | undefined { + return [node.identifier, node.label] .map((value) => (typeof value === 'string' ? value.trim() : '')) - .find(isUsefulScope); + .find((value) => isUsefulScope(value) && isUniqueScopeValue(value, node, nodes)); +} + +function isUniqueScopeValue( + value: string, + target: SnapshotNode, + nodes: readonly SnapshotNode[], +): boolean { + const normalized = value.toLowerCase(); + const matches = nodes.filter((node) => + [node.identifier, node.label, node.value].some( + (candidate) => + typeof candidate === 'string' && candidate.trim().toLowerCase().includes(normalized), + ), + ); + return matches.length === 1 && matches[0]?.index === target.index; } function isUsefulScope(value: string): boolean { diff --git a/test/ci/trusted-fixture-artifact.test.mjs b/test/ci/trusted-fixture-artifact.test.mjs new file mode 100644 index 000000000..dd957df6c --- /dev/null +++ b/test/ci/trusted-fixture-artifact.test.mjs @@ -0,0 +1,281 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { parse } from 'yaml'; +import { + classifyProducerState, + findTrustedArtifact, +} from '../../.github/actions/setup-fixture-app/trusted-artifact.mjs'; + +const repository = { default_branch: 'main', id: 42 }; +const trustedRun = { + event: 'pull_request', + head_branch: 'feature', + head_repository: { id: 42 }, + head_sha: 'current-head', + path: '.github/workflows/test-app-build-cache.yml', + repository: { id: 42 }, + status: 'in_progress', +}; + +test('producer, consumer, upload, and concurrency use the canonical artifact name', () => { + const action = parse(fs.readFileSync('.github/actions/setup-fixture-app/action.yml', 'utf8')); + const workflow = parse(fs.readFileSync('.github/workflows/test-app-build-cache.yml', 'utf8')); + const fetchStep = action.runs.steps.find((step) => step.id === 'fetch'); + const fingerprintStep = workflow.jobs.fingerprint.steps.find((step) => step.id === 'fingerprint'); + const uploadStep = workflow.jobs.release.steps.find((step) => + step.uses?.startsWith('actions/upload-artifact@'), + ); + + assert.match( + fetchStep.run, + /NAME="\$\(sh "\$GITHUB_ACTION_PATH\/resolve-artifact-name\.sh" ios\)"/, + ); + assert.match(fetchStep.run, /find "\$REPOSITORY" "\$NAME" "\$EXPECTED_HEAD_SHA"/); + assert.match( + fingerprintStep.run, + /ARTIFACT_NAME_RESOLVER="\.github\/actions\/setup-fixture-app\/resolve-artifact-name\.sh"/, + ); + assert.match(fingerprintStep.run, /IOS_NAME="\$\(sh "\$ARTIFACT_NAME_RESOLVER" ios\)"/); + assert.match(fingerprintStep.run, /ANDROID_NAME="\$\(sh "\$ARTIFACT_NAME_RESOLVER" android\)"/); + assert.equal(uploadStep.with.name, '${{ matrix.artifactName }}'); + assert.equal( + workflow.jobs.release.concurrency.group, + 'test-app-${{ matrix.artifactName }}-${{ github.event.pull_request.number || github.ref_name }}', + ); +}); + +test('producer maps each platform to its resolved lookup and matrix artifact name', (t) => { + const workflow = parse(fs.readFileSync('.github/workflows/test-app-build-cache.yml', 'utf8')); + const fingerprintStep = workflow.jobs.fingerprint.steps.find((step) => step.id === 'fingerprint'); + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'fixture-producer-name-')); + t.after(() => fs.rmSync(tempRoot, { force: true, recursive: true })); + const actionDir = path.join(tempRoot, '.github/actions/setup-fixture-app'); + const binDir = path.join(tempRoot, 'bin'); + const outputPath = path.join(tempRoot, 'output'); + const resolverLog = path.join(tempRoot, 'resolver-calls'); + const nodeLog = path.join(tempRoot, 'node-calls'); + fs.mkdirSync(actionDir, { recursive: true }); + fs.mkdirSync(binDir); + fs.writeFileSync( + path.join(actionDir, 'resolve-artifact-name.sh'), + [ + '#!/bin/sh', + 'printf "%s\\n" "$1" >> "$TEST_RESOLVER_LOG"', + 'printf "fingerprint.%s-hash.%s\\n" "$1" "$1"', + '', + ].join('\n'), + ); + fs.writeFileSync( + path.join(binDir, 'node'), + ['#!/bin/sh', 'printf "%s\\n" "$*" >> "$TEST_NODE_LOG"', ''].join('\n'), + ); + fs.chmodSync(path.join(binDir, 'node'), 0o755); + const run = fingerprintStep.run + .replaceAll('${{ github.event.pull_request.head.sha || github.sha }}', 'current-head') + .replaceAll('${{ github.repository }}', 'octo/repo') + .replaceAll('${{ github.event_name }}', 'pull_request') + .replaceAll('${{ github.event.pull_request.head.repo.full_name }}', 'octo/repo'); + const result = spawnSync('bash', ['-c', run], { + cwd: tempRoot, + encoding: 'utf8', + env: { + ...process.env, + GITHUB_OUTPUT: outputPath, + PATH: `${binDir}:${process.env.PATH}`, + TEST_NODE_LOG: nodeLog, + TEST_RESOLVER_LOG: resolverLog, + }, + }); + assert.equal(result.status, 0, result.stderr); + const matrixLine = fs.readFileSync(outputPath, 'utf8').trim(); + assert.match(matrixLine, /^matrix=/); + const matrix = JSON.parse(matrixLine.slice('matrix='.length)); + assert.deepEqual( + matrix.include.map(({ platform, artifactName }) => ({ platform, artifactName })), + [ + { platform: 'ios', artifactName: 'fingerprint.ios-hash.ios' }, + { platform: 'android', artifactName: 'fingerprint.android-hash.android' }, + ], + ); + assert.deepEqual(fs.readFileSync(resolverLog, 'utf8').trim().split('\n'), ['ios', 'android']); + assert.deepEqual(fs.readFileSync(nodeLog, 'utf8').trim().split('\n'), [ + '.github/actions/setup-fixture-app/trusted-artifact.mjs find octo/repo fingerprint.ios-hash.ios current-head', + '.github/actions/setup-fixture-app/trusted-artifact.mjs find octo/repo fingerprint.android-hash.android current-head', + ]); +}); + +test('artifact name resolver scopes both platforms and rejects invalid output', (t) => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'fixture-artifact-name-')); + t.after(() => fs.rmSync(tempRoot, { force: true, recursive: true })); + const callLog = path.join(tempRoot, 'calls'); + const pnpmStub = path.join(tempRoot, 'pnpm'); + fs.writeFileSync( + pnpmStub, + [ + '#!/bin/sh', + 'printf "%s\\n" "$*" >> "$TEST_CALL_LOG"', + 'if [ "$TEST_PNPM_EXIT" -ne 0 ]; then exit "$TEST_PNPM_EXIT"; fi', + 'if [ -n "$TEST_RAW_OUTPUT" ]; then', + ' printf "%s\\n" "$TEST_RAW_OUTPUT"', + 'else', + ' printf \'{"hash":"%s"}\\n\' "$TEST_HASH"', + 'fi', + '', + ].join('\n'), + ); + fs.chmodSync(pnpmStub, 0o755); + const resolver = '.github/actions/setup-fixture-app/resolve-artifact-name.sh'; + const runResolver = (platform, hash, pnpmExit = 0, rawOutput = '') => + spawnSync('sh', platform === undefined ? [resolver] : [resolver, platform], { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + PATH: `${tempRoot}:${process.env.PATH}`, + TEST_CALL_LOG: callLog, + TEST_HASH: hash, + TEST_PNPM_EXIT: String(pnpmExit), + TEST_RAW_OUTPUT: rawOutput, + }, + }); + + const ios = runResolver('ios', 'ios-hash'); + assert.equal(ios.status, 0, ios.stderr); + assert.equal(ios.stdout, 'fingerprint.ios-hash.ios\n'); + + const android = runResolver('android', 'android-hash'); + assert.equal(android.status, 0, android.stderr); + assert.equal(android.stdout, 'fingerprint.android-hash.android\n'); + + assert.deepEqual(fs.readFileSync(callLog, 'utf8').trim().split('\n'), [ + '--dir examples/test-app exec fingerprint fingerprint:generate --platform ios', + '--dir examples/test-app exec fingerprint fingerprint:generate --platform android', + ]); + assert.equal(runResolver(undefined, 'unused').status, 2); + assert.equal( + spawnSync('sh', [resolver, 'ios', 'extra'], { + cwd: process.cwd(), + encoding: 'utf8', + env: process.env, + }).status, + 2, + ); + assert.equal(runResolver('windows', 'unused').status, 2); + assert.equal(runResolver('ios', 'null').status, 1); + assert.equal(runResolver('ios', 'bad/hash').status, 1); + assert.equal(runResolver('ios', 'unused', 0, '{"hash":"bad\\nhash"}').status, 1); + assert.equal(runResolver('ios', 'unused', 0, '{"hash":"first"}\n{"hash":"second"}').status, 1); + assert.equal(runResolver('ios', 'unused', 17).status, 17); +}); + +test('same-name artifact from an untrusted repository cannot suppress a trusted candidate', async () => { + const artifacts = [ + { + expired: false, + id: 100, + workflow_run: { head_repository_id: 7, id: 10, repository_id: 42 }, + }, + { + expired: false, + id: 200, + workflow_run: { head_repository_id: 42, id: 20, repository_id: 42 }, + }, + ]; + const loaded = []; + const artifactId = await findTrustedArtifact({ + artifacts, + expectedHeadSha: 'current-head', + repository, + loadRun: async (runId) => { + loaded.push(runId); + return trustedRun; + }, + }); + assert.equal(artifactId, 200); + assert.deepEqual(loaded, [20]); +}); + +test('same-name artifact from another workflow falls back instead of being executed', async () => { + const artifactId = await findTrustedArtifact({ + artifacts: [ + { + expired: false, + id: 100, + workflow_run: { head_repository_id: 42, id: 10, repository_id: 42 }, + }, + ], + expectedHeadSha: 'current-head', + repository, + loadRun: async () => ({ ...trustedRun, path: '.github/workflows/untrusted.yml' }), + }); + assert.equal(artifactId, undefined); +}); + +test('same-repository artifact from an unrelated feature head is rejected', async () => { + const artifactId = await findTrustedArtifact({ + artifacts: [ + { + expired: false, + id: 100, + workflow_run: { head_repository_id: 42, id: 10, repository_id: 42 }, + }, + ], + expectedHeadSha: 'current-head', + repository, + loadRun: async () => ({ ...trustedRun, head_sha: 'another-feature-head' }), + }); + assert.equal(artifactId, undefined); +}); + +test('default-branch producer artifacts remain reusable across native-equivalent heads', async () => { + const artifactId = await findTrustedArtifact({ + artifacts: [ + { + expired: false, + id: 300, + workflow_run: { head_repository_id: 42, id: 30, repository_id: 42 }, + }, + ], + expectedHeadSha: 'current-head', + repository, + loadRun: async () => ({ + ...trustedRun, + event: 'push', + head_branch: 'main', + head_sha: 'older-main-head', + }), + }); + assert.equal(artifactId, 300); +}); + +test('producer state is derived only from the trusted exact-head workflow run', () => { + assert.equal( + classifyProducerState([], { expectedHeadSha: 'current-head', repository }), + 'absent', + ); + assert.equal( + classifyProducerState([{ ...trustedRun, status: 'queued' }], { + expectedHeadSha: 'current-head', + repository, + }), + 'queued', + ); + assert.equal( + classifyProducerState([{ ...trustedRun, conclusion: 'failure', status: 'completed' }], { + expectedHeadSha: 'current-head', + repository, + }), + 'failed', + ); + assert.equal( + classifyProducerState([{ ...trustedRun, conclusion: 'success', status: 'completed' }], { + expectedHeadSha: 'current-head', + repository, + }), + 'success', + ); +}); diff --git a/test/integration/cli-json.ts b/test/integration/cli-json.ts index 65d40fff6..a1ca8b15b 100644 --- a/test/integration/cli-json.ts +++ b/test/integration/cli-json.ts @@ -28,11 +28,12 @@ export function runSourceCliJsonSync( export async function runBuiltCliJson( args: string[], env: NodeJS.ProcessEnv, + options?: { timeoutMs?: number }, ): Promise { const result = await runCmd(process.execPath, ['bin/agent-device.mjs', ...args], { allowFailure: true, env, - timeoutMs: CLI_TIMEOUT_MS, + timeoutMs: options?.timeoutMs ?? CLI_TIMEOUT_MS, }); return cliJsonResult(result); } diff --git a/test/integration/ios-simulator-e2e/behavior-coverage.ts b/test/integration/ios-simulator-e2e/behavior-coverage.ts new file mode 100644 index 000000000..687237389 --- /dev/null +++ b/test/integration/ios-simulator-e2e/behavior-coverage.ts @@ -0,0 +1,75 @@ +export type IosSimulatorBehaviorId = + | 'background-foreground-resume' + | 'cold-start-deep-link-navigation' + | 'host-focus-preservation' + | 'interrupted-system-ui-flow' + | 'long-list-scroll-recovery' + | 'modal-open-close' + | 'permission-state-recovery' + | 'text-entry-keyboard-lifecycle'; + +type BehaviorCoverageEntry = + | { + assertion: string; + level: 'live'; + owner: string; + } + | { + assertion: string; + level: 'workflow-live'; + owner: { path: string; test: string }; + }; + +/** + * Cross-command mobile usage patterns requested by #320. Command ownership + * remains exhaustive in coverage-manifest.ts; this table prevents that + * command-level view from hiding missing end-to-end journeys. + */ +export const IOS_SIMULATOR_BEHAVIOR_COVERAGE = { + 'cold-start-deep-link-navigation': { + assertion: 'a terminated fixture opens a deep route, renders payload, and navigates back', + level: 'live', + owner: 'smoke:automation-input', + }, + 'text-entry-keyboard-lifecycle': { + assertion: + 'fill opens the keyboard, before/after pixels prove dismissal, and type appends text', + level: 'live', + owner: 'smoke:form-input', + }, + 'background-foreground-resume': { + assertion: 'Home backgrounds the fixture and its persisted transition survives foregrounding', + level: 'live', + owner: 'full:lifecycle-system', + }, + 'modal-open-close': { + assertion: 'a native page-sheet modal opens structurally and closes through its control', + level: 'live', + owner: 'smoke:automation-input', + }, + 'permission-state-recovery': { + assertion: + 'microphone reset, grant, denial, and second reset produce exact app-observed states', + level: 'live', + owner: 'full:lifecycle-system', + }, + 'interrupted-system-ui-flow': { + assertion: 'Home and app switcher expose distinct system pixels before fixture restoration', + level: 'live', + owner: 'full:lifecycle-system', + }, + 'long-list-scroll-recovery': { + assertion: 'direction, bottom footer, reverse movement, and top rediscovery are all observed', + level: 'live', + owner: 'full:fixture-replays', + }, + 'host-focus-preservation': { + assertion: + 'when the hosted runner can establish a Finder canary, it remains frontmost across simulator automation', + level: 'workflow-live', + owner: { + path: '.github/workflows/ios.yml', + test: 'Assert simulator automation preserved host focus', + }, + }, +} as const satisfies Record; diff --git a/test/integration/ios-simulator-e2e/coverage-manifest.ts b/test/integration/ios-simulator-e2e/coverage-manifest.ts new file mode 100644 index 000000000..75c129135 --- /dev/null +++ b/test/integration/ios-simulator-e2e/coverage-manifest.ts @@ -0,0 +1,217 @@ +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; + +type PublicCommand = (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS]; + +type RepositoryEvidence = { + path: string; + test: string; +}; + +export type IosSimulatorCoverageEntry = + | { + assertion: string; + level: 'live'; + owner: string; + } + | { + assertion: string; + level: 'command-contract' | 'workflow-live' | 'capability-denial'; + owner: RepositoryEvidence; + } + | { + assertion: string; + level: 'known-gap'; + owner: string; + trackingIssue: string; + }; + +const C = PUBLIC_COMMANDS; +const live = (owner: string, assertion: string): IosSimulatorCoverageEntry => ({ + assertion, + level: 'live', + owner, +}); +const contract = (path: string, test: string, assertion: string): IosSimulatorCoverageEntry => ({ + assertion, + level: 'command-contract', + owner: { path, test }, +}); + +/** + * One primary owner for every public command on an iOS mobile simulator. + * + * "live" means a real simulator scenario does more than check exit status: it + * asserts app/device state, typed data, or an artifact. Command-contract rows + * are deliberately not presented as E2E coverage; they point to a named test + * for functionality whose host permissions or source transport makes it a poor + * fit for the shared hosted-simulator lane. This is catalog-complete command + * ownership, not a claim that every optional backend or subcommand runs + * nightly; cross-command mobile journeys are tracked separately in + * behavior-coverage.ts. + */ +export const IOS_SIMULATOR_E2E_COVERAGE = { + [C.alert]: live('smoke:automation-input', 'native alert get, dismiss, and accept update the app'), + [C.appSwitcher]: live( + 'full:lifecycle-system', + 'app switcher covers fixture controls and differs from Home before restoration', + ), + [C.apps]: live('smoke:inventory-install', 'installed fixture bundle appears in app inventory'), + [C.appState]: live( + 'full:lifecycle-system', + 'session-backed state names the fixture bundle and selected simulator', + ), + [C.artifacts]: contract( + 'src/daemon/__tests__/http-server-artifacts.test.ts', + 'daemon artifact inventory lists artifacts and downloads consume them', + 'daemon inventory exposes a typed non-empty artifact and its downloadable bytes', + ), + [C.audio]: contract( + 'src/daemon/handlers/__tests__/session-audio.test.ts', + 'audio probe starts host helper for iOS simulator audio', + 'host audio permission and probe lifecycle; ScreenCaptureKit is not available on hosted CI', + ), + [C.back]: live('smoke:automation-input', 'back navigation returns from the automation route'), + [C.batch]: live('full:observability-artifacts', 'nested get/is results are asserted'), + [C.boot]: live( + 'full:device-lifecycle', + 'shutdown simulator boots again and inventory confirms it', + ), + [C.capabilities]: live( + 'smoke:inventory-install', + 'typed capability response includes fixture-driving commands', + ), + [C.click]: live('smoke:automation-input', 'selector click opens the automation lab'), + [C.clipboard]: live('full:lifecycle-system', 'Unicode clipboard value round-trips exactly'), + [C.close]: live('smoke:capture-close', 'session inventory proves the app lease is removed'), + [C.devices]: live('smoke:inventory-install', 'selected simulator UDID appears in inventory'), + [C.diff]: live('smoke:form-input', 'snapshot diff observes a form state mutation'), + [C.doctor]: live('smoke:inventory-install', 'doctor discovers the installed fixture app'), + [C.events]: live('full:observability-artifacts', 'timeline contains commands from this session'), + [C.fill]: live('smoke:form-input', 'replacement text is read back from the fixture input'), + [C.find]: live('smoke:automation-input', 'find reports the automation heading'), + [C.focus]: live( + 'smoke:form-input', + 'snapshot-derived field coordinates focus the target before typed text is read back', + ), + [C.gesture]: live( + 'full:fixture-replays', + 'fixture gesture counters prove pan/fling/pinch/rotate', + ), + [C.get]: live('smoke:automation-input', 'get returns automation canary text/attributes'), + [C.home]: live( + 'full:lifecycle-system', + 'fixture AppState becomes non-active and system pixels replace foreground pixels', + ), + [C.install]: live('smoke:inventory-install', 'public CLI installs the cached fixture .app'), + [C.installFromSource]: contract( + 'src/daemon/handlers/__tests__/install-source.test.ts', + 'install_from_source prepares and installs a local iOS simulator app source with typed identity', + 'real local .app preparation, simulator install dispatch, and typed identity', + ), + [C.is]: live('smoke:automation-input', 'visible/editable predicates pass on fixture nodes'), + [C.keyboard]: live( + 'smoke:form-input', + 'real keyboard dismissal reports dismissed=true and visible=false', + ), + [C.logs]: live( + 'full:observability-artifacts', + 'iOS simulator stream starts, exposes its concrete app.log path, and stops', + ), + [C.longPress]: live( + 'smoke:automation-input', + 'an 800ms hold increments the durable long-press counter', + ), + [C.network]: contract( + 'src/daemon/handlers/__tests__/session-network.test.ts', + 'network dump recovers iOS simulator entries from simctl log show when the live stream is empty', + 'iOS simulator recovery parses HTTP status, duration, and URL from bounded logs', + ), + [C.open]: live('smoke:automation-input', 'fixture cold launch and deep route become visible'), + [C.orientation]: live( + 'full:lifecycle-system', + 'native runner reads back exact landscape-left and portrait device states', + ), + [C.perf]: live( + 'full:observability-artifacts', + 'startup duration, resident memory, and CPU usage are typed and numeric', + ), + [C.prepare]: { + assertion: 'cached XCTest runner is prepared once before Settings and fixture suites', + level: 'workflow-live', + owner: { + path: '.github/workflows/ios.yml', + test: 'Preflight iOS runner through public CLI', + }, + }, + [C.press]: live('smoke:automation-input', 'semantic press updates a durable input canary'), + [C.push]: contract( + 'src/platforms/apple/core/__tests__/apps.test.ts', + 'pushIosNotification uses simctl push with temporary payload file', + 'simctl push dispatch; fixture has no notification entitlement or UI oracle', + ), + [C.reactNative]: live( + 'full:observability-artifacts', + 'Release fixture returns typed detected=false and dismissed=false overlay state', + ), + [C.record]: live('full:observability-artifacts', 'visible mutation produces a playable MP4'), + [C.reinstall]: live( + 'full:device-lifecycle', + 'cached fixture is reinstalled with typed bundle identity and app path', + ), + [C.replay]: live('full:fixture-replays', 'a fixture .ad flow runs through public replay'), + [C.screenshot]: live('smoke:capture-close', 'captured file has a valid PNG signature'), + [C.scroll]: live( + 'full:lifecycle-system', + 'bottom-edge traversal executes a live scroll pass and reports the reached edge', + ), + [C.settings]: live( + 'full:lifecycle-system', + 'appearance changes are visible in useColorScheme and restored', + ), + [C.shutdown]: live( + 'full:device-lifecycle', + 'shutdown succeeds and inventory reports the selected simulator stopped', + ), + [C.snapshot]: live( + 'smoke:automation-input', + 'scoped interactive tree includes the stable fixture title', + ), + [C.swipe]: live( + 'full:fixture-replays', + 'fixture direction canary proves both compact-safe directional swipes move content', + ), + [C.test]: live('full:fixture-replays', 'fixture scripts run as a suite with JUnit artifacts'), + [C.trace]: live( + 'full:observability-artifacts', + 'typed start/stop lifecycle retains the requested path and captures non-empty diagnostics', + ), + [C.triggerAppEvent]: live( + 'full:lifecycle-system', + 'custom-scheme event name and JSON payload render in the fixture', + ), + [C.tvRemote]: { + assertion: 'iOS mobile simulator capability model rejects TV remote input', + level: 'capability-denial', + owner: { + path: 'test/integration/smoke-ios-simulator-coverage.test.ts', + test: 'capability classifications match executable simulator behavior', + }, + }, + [C.type]: live('smoke:form-input', 'typed suffix is read back from a focused fixture field'), + [C.viewport]: { + assertion: 'capability currently admits iOS while the Apple interactor has no viewport backend', + level: 'known-gap', + owner: 'full:known-gaps', + trackingIssue: '#1407', + }, + [C.wait]: live('smoke:automation-input', 'polling observes durable fixture state'), +} satisfies Record; + +export function liveCommandsForScenario(scenarioId: string): PublicCommand[] { + return Object.entries(IOS_SIMULATOR_E2E_COVERAGE) + .filter( + ([, entry]) => + (entry.level === 'live' || entry.level === 'known-gap') && entry.owner === scenarioId, + ) + .map(([command]) => command as PublicCommand); +} diff --git a/test/integration/ios-simulator-e2e/event-timeline.ts b/test/integration/ios-simulator-e2e/event-timeline.ts new file mode 100644 index 000000000..037551a59 --- /dev/null +++ b/test/integration/ios-simulator-e2e/event-timeline.ts @@ -0,0 +1,79 @@ +export type EventTimelinePage = { + events?: unknown; + nextCursor?: unknown; +}; + +type EventTimeline = { + commands: string[]; + pages: Array<{ + cursor: string; + eventCount: number; + nextCursor?: string; + }>; +}; + +const MAX_EVENT_TIMELINE_PAGES = 100; + +export async function collectPagedEventTimeline( + readPage: (cursor?: string) => Promise, +): Promise { + const commands: string[] = []; + const pages: EventTimeline['pages'] = []; + let cursor: string | undefined; + let cursorOffset = 0; + + while (pages.length < MAX_EVENT_TIMELINE_PAGES) { + const page = await readPage(cursor); + const pageCommands = readPageCommands(page.events, cursor); + const next = readNextCursor(page.nextCursor, cursorOffset); + commands.push(...pageCommands); + pages.push({ + cursor: cursor ?? '0', + eventCount: pageCommands.length, + ...(next === undefined ? {} : { nextCursor: next.cursor }), + }); + if (next === undefined) return { commands, pages }; + + cursorOffset = next.offset; + cursor = next.cursor; + } + + throw new Error(`events pagination exceeded ${MAX_EVENT_TIMELINE_PAGES} pages`); +} + +function readPageCommands(events: unknown, cursor: string | undefined): string[] { + if (!Array.isArray(events)) { + throw new Error('events page must contain an events array'); + } + return events.map((event, index) => readEventCommand(event, index, cursor)); +} + +function readNextCursor( + value: unknown, + currentOffset: number, +): { cursor: string; offset: number } | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'string' || !/^(?:0|[1-9]\d*)$/.test(value)) { + throw new Error('events nextCursor must be a canonical non-negative integer string'); + } + const offset = Number(value); + if (!Number.isSafeInteger(offset)) { + throw new Error('events nextCursor must be a safe non-negative integer string'); + } + if (offset <= currentOffset) { + throw new Error(`events pagination did not advance beyond cursor ${currentOffset}`); + } + return { cursor: value, offset }; +} + +function readEventCommand(event: unknown, index: number, cursor: string | undefined): string { + if ( + typeof event !== 'object' || + event === null || + Array.isArray(event) || + typeof (event as { command?: unknown }).command !== 'string' + ) { + throw new Error(`events entry ${index} at cursor ${cursor ?? '0'} must name a command`); + } + return (event as { command: string }).command; +} diff --git a/test/integration/ios-simulator-e2e/live-assertions.ts b/test/integration/ios-simulator-e2e/live-assertions.ts new file mode 100644 index 000000000..73bca634b --- /dev/null +++ b/test/integration/ios-simulator-e2e/live-assertions.ts @@ -0,0 +1,115 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; + +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import { isPlayableVideo } from '../../../src/utils/video.ts'; +import { assertPngFile } from '../provider-scenarios/assertions.ts'; +import type { CliJsonResult } from '../cli-json.ts'; +import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; + +export function assertJsonContains(result: CliJsonResult, expected: string, message: string): void { + const serialized = JSON.stringify(result.json?.data ?? result.json); + assert.ok(serialized.includes(expected), `${message}\nreceived: ${serialized}`); +} + +export async function assertWaitText(context: LiveContext, expected: string): Promise { + const result = await runStep(context, `wait for ${expected}`, [ + 'wait', + 'text', + expected, + '10000', + ]); + assertJsonContains(result, expected, `wait should observe ${expected}`); + verifyCommand(context, PUBLIC_COMMANDS.wait, `wait observes durable text: ${expected}`); +} + +export async function assertElementText( + context: LiveContext, + selector: string, + expected: string, +): Promise { + const result = await runStep(context, `read ${selector}`, ['get', 'text', selector]); + assert.equal( + result.json?.data?.text, + expected, + `${selector} should expose ${expected}: ${JSON.stringify(result.json)}`, + ); +} + +export async function assertElementTextAfterScrolling( + context: LiveContext, + selector: string, + expected: string, +): Promise { + for (let attempt = 1; attempt <= 4; attempt += 1) { + const visible = await runStep( + context, + `wait for ${selector} after scroll (attempt ${attempt})`, + ['wait', selector, '1000'], + { allowFailure: attempt < 4 }, + ); + if (visible.status === 0) { + await assertElementText(context, selector, expected); + return; + } + await runStep(context, `scroll toward ${selector}`, ['scroll', 'down', '0.75']); + } + assert.fail(`${selector} did not become visible after scrolling`); +} + +export function assertNonEmptyFile(filePath: string, name: string): void { + assert.ok(fs.statSync(filePath).size > 0, `${name} artifact is empty: ${filePath}`); +} + +export async function assertMp4File(filePath: string): Promise { + assertNonEmptyFile(filePath, 'recording'); + assert.equal( + await isPlayableVideo(filePath), + true, + `recording is not a finalized playable video: ${filePath}`, + ); +} + +export async function capturePng( + context: LiveContext, + step: string, + outputPath: string, +): Promise { + await runStep(context, step, ['screenshot', outputPath, '--max-size', '900']); + assertPngFile(outputPath); +} + +export function assertFilesDiffer(first: string, second: string, message: string): void { + assert.notDeepEqual(fs.readFileSync(first), fs.readFileSync(second), message); +} + +function requireNode( + result: CliJsonResult, + identifier: string, +): { label?: unknown; rect?: { height: number; width: number; x: number; y: number } } { + const nodes = Array.isArray(result.json?.data?.nodes) ? result.json.data.nodes : []; + const node = nodes.find( + (candidate: { identifier?: unknown }) => candidate.identifier === identifier, + ); + assert.ok(node, `snapshot missing ${identifier}: ${JSON.stringify(result.json)}`); + return node; +} + +export function requireNodeRect( + result: CliJsonResult, + identifier: string, +): { height: number; width: number; x: number; y: number } { + const rect = requireNode(result, identifier).rect; + assert.ok(rect, `snapshot node ${identifier} has no rect: ${JSON.stringify(result.json)}`); + for (const value of [rect.x, rect.y, rect.width, rect.height]) { + assert.ok(Number.isFinite(value), `snapshot node ${identifier} has invalid rect`); + } + return rect; +} + +export function requireDevice(result: CliJsonResult, udid: string): { booted?: unknown } { + const devices = Array.isArray(result.json?.data?.devices) ? result.json.data.devices : []; + const device = devices.find((candidate: { id?: unknown }) => candidate.id === udid); + assert.ok(device, `device inventory missing ${udid}: ${JSON.stringify(result.json)}`); + return device; +} diff --git a/test/integration/ios-simulator-e2e/live-automation-scenario.ts b/test/integration/ios-simulator-e2e/live-automation-scenario.ts new file mode 100644 index 000000000..6fb56fcd2 --- /dev/null +++ b/test/integration/ios-simulator-e2e/live-automation-scenario.ts @@ -0,0 +1,174 @@ +import assert from 'node:assert/strict'; + +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import { assertElementText, assertJsonContains, assertWaitText } from './live-assertions.ts'; +import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; + +const C = PUBLIC_COMMANDS; +const FIXTURE_HOME_TITLE = 'Agent Device Tester'; +const AUTOMATION_DEEP_LINK = + 'agent-device-test-app:///automation?event=cold.start&payload=%7B%22source%22%3A%22deep-link%22%7D'; + +async function observeFixtureHome(context: LiveContext) { + await assertWaitText(context, FIXTURE_HOME_TITLE); + const snapshot = await runStep(context, 'capture fixture home', [ + 'snapshot', + '-i', + '-s', + FIXTURE_HOME_TITLE, + ]); + const nodes = Array.isArray(snapshot.json?.data?.nodes) ? snapshot.json.data.nodes : []; + assert.ok( + nodes.some((node: { label?: unknown }) => node.label === FIXTURE_HOME_TITLE), + `home snapshot nodes should expose ${FIXTURE_HOME_TITLE}: ${JSON.stringify(snapshot.json)}`, + ); + return snapshot; +} + +export async function assertAutomationInput(context: LiveContext): Promise { + const opened = await runStep(context, 'cold launch fixture', [ + 'open', + context.appId, + '--relaunch', + ]); + assertJsonContains(opened, context.appId, 'open response should retain fixture identity'); + if (context.tier === 'full') { + await runStep(context, 'normalize simulator orientation', ['orientation', 'portrait']); + await runStep(context, 'normalize simulator appearance', ['settings', 'appearance', 'light']); + } + + await observeFixtureHome(context); + verifyCommand(context, C.snapshot, 'interactive fixture tree exposes its stable title'); + verifyCommand(context, C.open, 'cold launch exposes the fixture UI through snapshot and wait'); + + await openAutomationDeepLink(context, 'cold launch fixture through a deep link'); + await acceptDeepLinkConfirmationIfPresent(context); + await assertWaitText(context, 'Automation lab'); + await assertElementText(context, 'id="automation-event-name"', 'cold.start'); + await assertElementText(context, 'id="automation-event-payload"', '{"source":"deep-link"}'); + await runStep(context, 'navigate onward from cold deep link', [ + 'click', + 'id="automation-continue-catalog"', + ]); + await runStep(context, 'wait for exact catalog destination', [ + 'wait', + 'id="catalog-title"', + '10000', + ]); + verifyBehavior( + context, + 'cold-start-deep-link-navigation', + 'cold deep route rendered decoded payload and continued into the fixture catalog', + ); + + await runStep(context, 'wait for settings tab target', ['wait', 'label="Settings"', '10000']); + await runStep(context, 'open settings tab', ['click', 'label="Settings"']); + await assertWaitText(context, 'Settings'); + await runStep(context, 'open automation route', ['click', 'id="open-automation-lab"']); + await assertWaitText(context, 'Automation lab'); + verifyCommand(context, C.click, 'selector click opens the automation route'); + + await runStep(context, 'open fixture sheet', ['click', 'id="automation-open-sheet"']); + await assertWaitText(context, 'Automation sheet'); + await runStep(context, 'close fixture sheet', ['click', 'id="automation-close-sheet"']); + await assertWaitText(context, 'Automation lab'); + await runStep(context, 'restore automation route top after sheet', ['scroll', 'top']); + verifyBehavior( + context, + 'modal-open-close', + 'page-sheet modal exposed structural content and returned to the automation route', + ); + + const heading = await runStep(context, 'read automation heading', [ + 'get', + 'text', + 'text="Automation lab"', + ]); + assertJsonContains(heading, 'Automation lab', 'get text should return automation heading'); + verifyCommand(context, C.get, 'get returns exact automation heading text'); + + const visible = await runStep(context, 'assert automation heading visible', [ + 'is', + 'visible', + 'id="automation-title"', + ]); + assert.equal(visible.json?.data?.pass, true, JSON.stringify(visible.json)); + verifyCommand(context, C.is, 'visible predicate passes for the automation heading'); + + const found = await runStep(context, 'find automation heading', [ + 'find', + 'text', + 'Automation lab', + 'exists', + ]); + assert.equal(found.json?.data?.found, true, JSON.stringify(found.json)); + verifyCommand(context, C.find, 'find reports the automation heading'); + + await runStep(context, 'reveal automation input canaries', ['scroll', 'down', '--pixels', '120']); + for (const identifier of ['automation-press', 'automation-longpress']) { + const inputVisible = await runStep(context, `assert ${identifier} is visible`, [ + 'is', + 'visible', + `id="${identifier}"`, + ]); + assert.equal(inputVisible.json?.data?.pass, true, JSON.stringify(inputVisible.json)); + } + + await runStep(context, 'press semantic canary', ['press', 'id="automation-press"']); + await assertWaitText(context, 'Last input: press'); + verifyCommand(context, C.press, 'semantic press changes the durable fixture canary'); + + await runStep(context, 'long press semantic canary', [ + 'longpress', + 'id="automation-longpress"', + '800', + ]); + await assertWaitText(context, 'Long presses: 1'); + verifyCommand(context, C.longPress, '800ms hold increments the long-press counter'); + + await runStep(context, 'scroll native alert canary into view', ['scroll', 'down', '1']); + await runStep(context, 'open native alert', ['click', 'id="automation-open-alert"']); + const alert = await runStep(context, 'wait for native alert', ['alert', 'wait', '5000']); + assertJsonContains(alert, 'Automation confirmation', 'alert wait should return fixture alert'); + await runStep(context, 'inspect native alert', ['alert', 'get']); + await runStep(context, 'dismiss native alert', ['alert', 'dismiss']); + await assertWaitText(context, 'Alert result: cancelled'); + + await runStep(context, 'reopen native alert', ['click', 'id="automation-open-alert"']); + await runStep(context, 'accept native alert', ['alert', 'accept']); + await assertWaitText(context, 'Alert result: accepted'); + verifyCommand(context, C.alert, 'alert wait/get/dismiss/accept produce both fixture outcomes'); + + await runStep(context, 'return from automation route', ['back']); + await assertWaitText(context, 'Settings'); + verifyCommand(context, C.back, 'back returns from automation route to Settings'); +} + +async function acceptDeepLinkConfirmationIfPresent(context: LiveContext): Promise { + const destination = await runStep( + context, + 'wait for deep-link destination before inspecting system UI', + ['wait', 'text', 'Automation lab', '2500'], + { allowFailure: true }, + ); + if (destination.status === 0) return; + + const alert = await runStep(context, 'inspect delayed deep-link system alert', ['alert', 'get']); + const alertInfo = alert.json?.data; + assert.match(String(alertInfo?.message), /^Open in\b/, JSON.stringify(alert.json)); + assert.ok( + Array.isArray(alertInfo?.items) && alertInfo.items.includes('Open'), + JSON.stringify(alert.json), + ); + await runStep(context, 'accept deep-link confirmation', ['alert', 'accept']); +} + +async function openAutomationDeepLink(context: LiveContext, step: string): Promise { + await runStep(context, step, [ + 'open', + context.appId, + '--relaunch', + '--launch-url', + AUTOMATION_DEEP_LINK, + ]); +} diff --git a/test/integration/ios-simulator-e2e/live-coverage-report.ts b/test/integration/ios-simulator-e2e/live-coverage-report.ts new file mode 100644 index 000000000..a252d7917 --- /dev/null +++ b/test/integration/ios-simulator-e2e/live-coverage-report.ts @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { + IOS_SIMULATOR_BEHAVIOR_COVERAGE, + type IosSimulatorBehaviorId, +} from './behavior-coverage.ts'; +import { liveCommandsForScenario } from './coverage-manifest.ts'; +import type { LiveContext, Tier } from './live-harness.ts'; +import { IOS_SIMULATOR_LIVE_SCENARIOS, type IosSimulatorScenario } from './scenarios.ts'; + +export function assertCoverageComplete(context: LiveContext): void { + const missing = computeMissing(context); + assert.deepEqual( + missing, + { missingBehaviors: [], missingCommands: [], missingScenarios: [] }, + 'iOS simulator E2E coverage is incomplete', + ); +} + +export function writeCoverageReport(context: LiveContext): void { + const missing = computeMissing(context); + fs.writeFileSync( + path.join(context.artifactDir, 'coverage-report.json'), + JSON.stringify( + { + behaviorEvidence: context.behaviorEvidence, + commandEvidence: context.commandEvidence, + completedScenarios: context.completedScenarios, + ...missing, + tier: context.tier, + }, + null, + 2, + ), + ); +} + +function computeMissing(context: LiveContext) { + const requiredScenarios = scenariosForTier(context.tier); + return { + missingBehaviors: requiredScenarios + .flatMap((scenario) => liveBehaviorsForScenario(scenario.id)) + .filter((behavior) => (context.behaviorEvidence[behavior]?.length ?? 0) === 0), + missingCommands: requiredScenarios + .flatMap((scenario) => liveCommandsForScenario(scenario.id)) + .filter((command) => (context.commandEvidence[command]?.length ?? 0) === 0), + missingScenarios: requiredScenarios + .map((scenario) => scenario.id) + .filter((id) => !context.completedScenarios.includes(id)), + }; +} + +function scenariosForTier(tier: Tier): readonly IosSimulatorScenario[] { + return IOS_SIMULATOR_LIVE_SCENARIOS.filter( + (scenario) => scenario.tier === 'smoke' || tier === 'full', + ); +} + +export function liveBehaviorsForScenario(scenarioId: string): IosSimulatorBehaviorId[] { + return Object.entries(IOS_SIMULATOR_BEHAVIOR_COVERAGE) + .filter(([, entry]) => entry.level === 'live' && entry.owner === scenarioId) + .map(([behavior]) => behavior as IosSimulatorBehaviorId); +} diff --git a/test/integration/ios-simulator-e2e/live-device-lifecycle.ts b/test/integration/ios-simulator-e2e/live-device-lifecycle.ts new file mode 100644 index 000000000..55048af6e --- /dev/null +++ b/test/integration/ios-simulator-e2e/live-device-lifecycle.ts @@ -0,0 +1,76 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; + +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import { requireDevice } from './live-assertions.ts'; +import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; + +const C = PUBLIC_COMMANDS; + +export async function assertDeviceLifecycle(context: LiveContext): Promise { + const reinstall = await runStep(context, 'reinstall cached fixture', [ + 'reinstall', + context.appId, + context.appPath, + ]); + assert.equal(reinstall.json?.data?.bundleId, context.appId, JSON.stringify(reinstall.json)); + assert.equal( + path.resolve(String(reinstall.json?.data?.appPath)), + path.resolve(context.appPath), + JSON.stringify(reinstall.json), + ); + verifyCommand(context, C.reinstall, 'typed reinstall result retains fixture identity and path'); + + let primaryError: unknown; + try { + const shutdown = await runStep(context, 'shutdown selected simulator', ['shutdown']); + assert.equal(shutdown.json?.data?.id, context.udid, JSON.stringify(shutdown.json)); + assert.equal(shutdown.json?.data?.shutdown?.success, true, JSON.stringify(shutdown.json)); + const stoppedInventory = await runStep(context, 'verify simulator shutdown', ['devices']); + assert.equal(requireDevice(stoppedInventory, context.udid).booted, false); + verifyCommand(context, C.shutdown, 'inventory reports the selected simulator as shut down'); + + const boot = await runStep(context, 'boot selected simulator', ['boot'], { + timeoutMs: 300_000, + }); + assert.equal(boot.json?.data?.id, context.udid, JSON.stringify(boot.json)); + assert.equal(boot.json?.data?.booted, true, JSON.stringify(boot.json)); + const bootedInventory = await runStep(context, 'verify simulator boot', ['devices']); + assert.equal(requireDevice(bootedInventory, context.udid).booted, true); + verifyCommand(context, C.boot, 'typed result and inventory report the simulator booted again'); + } catch (error) { + primaryError = error; + } + + if (primaryError === undefined) return; + + let recoveryError: unknown; + try { + const inventory = await runStep(context, 'cleanup: inspect simulator boot state', ['devices']); + if (!requireDevice(inventory, context.udid).booted) { + await runStep(context, 'cleanup: recover selected simulator', ['boot'], { + timeoutMs: 300_000, + }); + } + } catch (error) { + recoveryError = error; + } + if (recoveryError !== undefined) { + throw new AggregateError( + [primaryError, recoveryError], + 'device lifecycle failed and simulator recovery also failed', + ); + } + throw primaryError; +} + +export async function assertKnownGaps(context: LiveContext): Promise { + const viewport = await runStep( + context, + 'pin unsupported Apple viewport backend', + ['viewport', '390', '844'], + { expectFailure: true }, + ); + assert.equal(viewport.json?.error?.code, 'UNSUPPORTED_OPERATION', JSON.stringify(viewport.json)); + verifyCommand(context, C.viewport, 'live Apple dispatch returns typed UNSUPPORTED_OPERATION'); +} diff --git a/test/integration/ios-simulator-e2e/live-full-scenarios.ts b/test/integration/ios-simulator-e2e/live-full-scenarios.ts new file mode 100644 index 000000000..d66f6a622 --- /dev/null +++ b/test/integration/ios-simulator-e2e/live-full-scenarios.ts @@ -0,0 +1,324 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import { + assertElementText, + assertElementTextAfterScrolling, + assertFilesDiffer, + assertJsonContains, + assertMp4File, + assertNonEmptyFile, + assertWaitText, + capturePng, +} from './live-assertions.ts'; +import { collectPagedEventTimeline, type EventTimelinePage } from './event-timeline.ts'; +import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; + +const C = PUBLIC_COMMANDS; + +export async function assertLifecycleAndSystem(context: LiveContext): Promise { + const appState = await runStep(context, 'read fixture app state', ['appstate']); + assert.equal(appState.json?.data?.appBundleId, context.appId, JSON.stringify(appState.json)); + assert.equal(appState.json?.data?.source, 'session', JSON.stringify(appState.json)); + assert.equal(appState.json?.data?.device_udid, context.udid, JSON.stringify(appState.json)); + verifyCommand(context, C.appState, 'typed appstate retains active session and fixture identity'); + + const clipboardValue = 'Zażółć gęślą 🧪'; + await runStep(context, 'write Unicode clipboard', ['clipboard', 'write', clipboardValue]); + const clipboard = await runStep(context, 'read Unicode clipboard', ['clipboard', 'read']); + assertJsonContains(clipboard, clipboardValue, 'clipboard should round-trip Unicode text'); + verifyCommand(context, C.clipboard, 'Unicode clipboard value round-trips exactly'); + + await runStep(context, 'open automation route through app event', [ + 'trigger-app-event', + 'fixture.ready', + '{"source":"ios-e2e","count":7}', + ]); + await runStep(context, 'return to top-level event canaries', ['scroll', 'top']); + await assertElementText(context, 'id="automation-event-name"', 'fixture.ready'); + await assertElementText( + context, + 'id="automation-event-payload"', + '{"source":"ios-e2e","count":7}', + ); + verifyCommand(context, C.triggerAppEvent, 'deep event name and JSON payload render exactly'); + + const bottomEdge = await runStep(context, 'scroll automation route to the bottom edge', [ + 'scroll', + 'bottom', + '0.75', + ]); + assert.equal(bottomEdge.json?.data?.edge, 'bottom', JSON.stringify(bottomEdge.json)); + assert.ok( + Number(bottomEdge.json?.data?.passes) > 0, + `bottom edge traversal must execute a live scroll: ${JSON.stringify(bottomEdge.json)}`, + ); + verifyCommand( + context, + C.scroll, + 'bottom-edge traversal executes at least one live scroll pass and reports the reached edge', + ); + + await setMicrophonePermissionAndRestart(context, 'initial', 'reset', 'undetermined'); + await setMicrophonePermissionAndRestart(context, 'grant', 'grant', 'granted'); + await setMicrophonePermissionAndRestart(context, 'denial', 'deny', 'denied'); + await setMicrophonePermissionAndRestart(context, 'recovery', 'reset', 'undetermined'); + await runStep(context, 'reset microphone permission after scenario', [ + 'settings', + 'permission', + 'reset', + 'microphone', + ]); + await runStep(context, 'clear fixture state after permission scenario', [ + 'settings', + 'clear-app-state', + context.appId, + ]); + await runStep(context, 'open fixture after permission cleanup', ['open', context.appId]); + await runStep(context, 'restore automation route after permission cleanup', [ + 'trigger-app-event', + 'fixture.permission.cleanup', + '{"source":"permission-cleanup"}', + ]); + await assertWaitText(context, 'Automation lab'); + verifyBehavior( + context, + 'permission-state-recovery', + 'reset, grant, denial, and a second reset were all observed through the fixture permission API', + ); + await runStep(context, 'set dark appearance', ['settings', 'appearance', 'dark']); + await assertElementText(context, 'id="automation-appearance"', 'dark'); + await runStep(context, 'restore light appearance', ['settings', 'appearance', 'light']); + await assertElementText(context, 'id="automation-appearance"', 'light'); + verifyCommand( + context, + C.settings, + 'permission reset/deny/grant and appearance dark/light all produce durable evidence', + ); + + // XCUIDevice models physical device orientation, which is intentionally not + // coupled to an app's interface orientation. Assert the exact typed states + // accepted by the native runner instead of overclaiming a window resize. + const landscape = await runStep(context, 'rotate landscape', ['orientation', 'landscape-left']); + assert.equal(landscape.json?.data?.orientation, 'landscape-left', JSON.stringify(landscape.json)); + const portrait = await runStep(context, 'restore portrait', ['orientation', 'portrait']); + assert.equal(portrait.json?.data?.orientation, 'portrait', JSON.stringify(portrait.json)); + verifyCommand( + context, + C.orientation, + 'native runner reads back exact landscape-left and portrait device states', + ); + + const foregroundPath = path.join(context.artifactDir, 'system-foreground.png'); + await capturePng(context, 'capture foreground system baseline', foregroundPath); + await runStep(context, 'background fixture to home', ['home']); + const homePath = path.join(context.artifactDir, 'system-home.png'); + await capturePng(context, 'capture Home screen', homePath); + assertFilesDiffer(foregroundPath, homePath, 'Home should replace the foreground fixture'); + + await runStep(context, 'restore fixture after Home', ['open', context.appId]); + await assertWaitText(context, 'Automation lab'); + const lastNonActive = await runStep(context, 'read persisted Home transition', [ + 'get', + 'text', + 'id="automation-last-nonactive"', + ]); + const lastNonActiveState = lastNonActive.json?.data?.text; + assert.ok( + lastNonActiveState === 'inactive' || lastNonActiveState === 'background', + `Home should produce a non-active app transition: ${JSON.stringify(lastNonActive.json)}`, + ); + const restoredForegroundPath = path.join(context.artifactDir, 'system-restored-foreground.png'); + await capturePng(context, 'capture restored foreground fixture', restoredForegroundPath); + verifyCommand(context, C.home, 'Home pixels replace the fixture and its transition is persisted'); + verifyBehavior( + context, + 'background-foreground-resume', + 'Home transition persisted while the fixture was backgrounded and survived foreground restore', + ); + + await runStep(context, 'open app switcher', ['app-switcher']); + const switcherSurface = await runStep(context, 'inspect covered fixture in app switcher', [ + 'snapshot', + '-i', + ]); + const switcherNodes = Array.isArray(switcherSurface.json?.data?.nodes) + ? switcherSurface.json.data.nodes + : []; + const coveredFixtureControl = switcherNodes.find( + (node: { identifier?: unknown }) => node.identifier === 'automation-press-canary', + ); + assert.ok(coveredFixtureControl, JSON.stringify(switcherSurface.json)); + assert.equal(coveredFixtureControl.hittable, false, JSON.stringify(coveredFixtureControl)); + assert.equal( + switcherNodes.some( + (node: { hittable?: unknown; type?: unknown }) => + node.type === 'Button' && node.hittable === true, + ), + false, + 'app switcher should cover every fixture button', + ); + const switcherPath = path.join(context.artifactDir, 'system-app-switcher.png'); + await capturePng(context, 'capture app switcher', switcherPath); + assertFilesDiffer(homePath, switcherPath, 'app switcher should differ from Home'); + assertFilesDiffer( + restoredForegroundPath, + switcherPath, + 'app switcher should differ from the foreground fixture', + ); + verifyCommand( + context, + C.appSwitcher, + 'app switcher covers fixture controls and differs from Home and foreground pixels', + ); + verifyBehavior( + context, + 'interrupted-system-ui-flow', + 'Home and app switcher produced distinct system surfaces before fixture restoration', + ); + + await runStep(context, 'restore fixture after system UI', ['open', context.appId]); + await assertWaitText(context, 'Automation lab'); +} + +async function setMicrophonePermissionAndRestart( + context: LiveContext, + phase: 'initial' | 'denial' | 'recovery' | 'grant', + action: 'deny' | 'grant' | 'reset', + expected: 'denied' | 'granted' | 'undetermined', +): Promise { + // Terminate the app before changing TCC so its live permission requester + // cannot retain the previous state. Clearing data preserves the installed + // cached binary and exercises settings permission without depending on a + // transient system prompt racing XCTest reconnection on hosted simulators. + await runStep(context, `clear fixture state before ${phase} permission ${action}`, [ + 'settings', + 'clear-app-state', + context.appId, + ]); + await runStep(context, `${action} microphone permission for ${phase} state`, [ + 'settings', + 'permission', + action, + 'microphone', + ]); + await runStep(context, `open fixture after ${phase} permission ${action}`, [ + 'open', + context.appId, + ]); + await runStep(context, `restore automation route after ${phase} permission ${action}`, [ + 'trigger-app-event', + `fixture.permission.${phase}`, + `{"source":"permission-${phase}"}`, + ]); + await assertWaitText(context, 'Automation lab'); + await assertElementTextAfterScrolling(context, 'id="automation-microphone-permission"', expected); +} + +export async function assertObservabilityAndArtifacts(context: LiveContext): Promise { + const perf = await runStep(context, 'read fixture performance metrics', ['perf', 'metrics']); + const metrics = perf.json?.data?.metrics; + assert.equal(metrics?.startup?.available, true, JSON.stringify(perf.json)); + assert.ok(Number(metrics?.startup?.lastDurationMs) > 0, JSON.stringify(perf.json)); + assert.equal(metrics?.memory?.available, true, JSON.stringify(perf.json)); + assert.ok(Number(metrics?.memory?.residentMemoryKb) > 0, JSON.stringify(perf.json)); + assert.equal(metrics?.cpu?.available, true, JSON.stringify(perf.json)); + assert.ok(Number.isFinite(Number(metrics?.cpu?.usagePercent)), JSON.stringify(perf.json)); + verifyCommand(context, C.perf, 'startup, memory, and CPU process metrics are typed and numeric'); + + const logsStart = await runStep(context, 'start fixture log stream', ['logs', 'start']); + assert.equal(logsStart.json?.data?.started, true, JSON.stringify(logsStart.json)); + assert.ok(fs.existsSync(String(logsStart.json?.data?.path)), JSON.stringify(logsStart.json)); + const logsPath = await runStep(context, 'inspect fixture log stream', ['logs', 'path']); + assert.equal(logsPath.json?.data?.active, true, JSON.stringify(logsPath.json)); + assert.equal(logsPath.json?.data?.backend, 'ios-simulator', JSON.stringify(logsPath.json)); + assert.ok(fs.existsSync(String(logsPath.json?.data?.path)), JSON.stringify(logsPath.json)); + const logsStop = await runStep(context, 'stop fixture log stream', ['logs', 'stop']); + assert.equal(logsStop.json?.data?.stopped, true, JSON.stringify(logsStop.json)); + verifyCommand(context, C.logs, 'iOS simulator log stream starts, exposes app.log, and stops'); + + const tracePath = path.join(context.artifactDir, 'fixture.adtrace'); + const traceStart = await runStep(context, 'start interaction trace', [ + 'trace', + 'start', + tracePath, + ]); + assertJsonContains(traceStart, 'started', 'trace start should return typed started state'); + assertJsonContains(traceStart, tracePath, 'trace start should retain the requested output path'); + await runStep(context, 'trace a visible mutation', ['press', 'id="automation-press"']); + const traceStop = await runStep(context, 'stop interaction trace', ['trace', 'stop', tracePath]); + assertJsonContains(traceStop, 'stopped', 'trace stop should return typed stopped state'); + assertJsonContains(traceStop, tracePath, 'trace stop should retain the requested output path'); + assertNonEmptyFile(tracePath, 'trace'); + verifyCommand(context, C.trace, 'traced press produces a non-empty trace at the requested path'); + + const recordingPath = path.join(context.artifactDir, 'fixture.mp4'); + await runStep(context, 'start screen recording', [ + 'record', + 'start', + recordingPath, + '--hide-touches', + ]); + await runStep(context, 'record a visible mutation', ['press', 'id="automation-press"']); + await runStep(context, 'stop screen recording', ['record', 'stop']); + await assertMp4File(recordingPath); + verifyCommand(context, C.record, 'visible mutation produces a non-empty playable MP4'); + + const batch = await runStep(context, 'run semantic read batch', [ + 'batch', + '--steps', + JSON.stringify([ + { + command: 'get', + input: { + format: 'text', + target: { kind: 'selector', selector: 'id="automation-press"' }, + }, + }, + { + command: 'is', + input: { predicate: 'visible', selector: 'id="automation-press"' }, + }, + ]), + ]); + assertJsonContains(batch, 'Press canary', 'batch should contain nested get result'); + assertJsonContains(batch, '"pass":true', 'batch should contain passing nested is result'); + verifyCommand(context, C.batch, 'batch returns semantic nested get and is results'); + + const eventTimeline = await collectPagedEventTimeline(async (cursor) => { + const events = await runStep( + context, + cursor === undefined + ? 'read session event timeline' + : `read session event timeline from cursor ${cursor}`, + cursor === undefined ? ['events', '100'] : ['events', '100', cursor], + ); + return (events.json?.data ?? {}) as EventTimelinePage; + }); + assert.ok( + eventTimeline.pages.length > 1, + `live events coverage must traverse multiple pages: ${JSON.stringify(eventTimeline.pages)}`, + ); + + for (const command of [C.open, C.press, C.snapshot]) { + assert.ok( + eventTimeline.commands.includes(command), + `events missing ${command}: ${JSON.stringify(eventTimeline)}`, + ); + } + verifyCommand( + context, + C.events, + 'paged typed event entries name open, press, and snapshot across the full timeline', + ); + + const reactNative = await runStep(context, 'inspect Release overlay state', [ + 'react-native', + 'dismiss-overlay', + ]); + assert.equal(reactNative.json?.data?.detected, false, JSON.stringify(reactNative.json)); + assert.equal(reactNative.json?.data?.dismissed, false, JSON.stringify(reactNative.json)); + verifyCommand(context, C.reactNative, 'Release fixture returns a typed no-overlay verdict'); +} diff --git a/test/integration/ios-simulator-e2e/live-harness.ts b/test/integration/ios-simulator-e2e/live-harness.ts new file mode 100644 index 000000000..69d763cda --- /dev/null +++ b/test/integration/ios-simulator-e2e/live-harness.ts @@ -0,0 +1,277 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { type CliJsonResult, formatResultDebug, runBuiltCliJson } from '../cli-json.ts'; +import { type IosSimulatorBehaviorId } from './behavior-coverage.ts'; +import { liveCommandsForScenario } from './coverage-manifest.ts'; +import { liveBehaviorsForScenario, writeCoverageReport } from './live-coverage-report.ts'; + +export { assertCoverageComplete, writeCoverageReport } from './live-coverage-report.ts'; + +export type Tier = 'smoke' | 'full'; + +type StepRecord = { + accepted: boolean; + command: string; + commandName?: string; + durationMs: number; + errorCode?: string; + errorMessage?: string; + scenario: string; + status: number; + step: string; +}; + +export type LiveContext = { + appId: string; + appPath: string; + artifactDir: string; + behaviorEvidence: Partial>; + commandEvidence: Record; + completedScenarios: string[]; + currentScenario: string; + env: NodeJS.ProcessEnv; + session: string; + sessionOpen: boolean; + stepHistory: StepRecord[]; + tier: Tier; + udid: string; +}; + +export function createContext(): LiveContext { + const tier = requiredEnv('AGENT_DEVICE_IOS_E2E_TIER'); + assert.ok(tier === 'smoke' || tier === 'full', `unsupported iOS E2E tier: ${tier}`); + if (tier === 'full') requiredEnv('AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE'); + const runId = `${Date.now()}-${process.pid}`; + const artifactDir = path.resolve('test/artifacts/ios-simulator', tier, runId); + fs.mkdirSync(artifactDir, { recursive: true }); + + return { + appId: requiredEnv('AGENT_DEVICE_FIXTURE_APP_ID'), + appPath: requiredEnv('AGENT_DEVICE_FIXTURE_APP_PATH'), + artifactDir, + behaviorEvidence: {}, + commandEvidence: {}, + completedScenarios: [], + currentScenario: 'bootstrap', + env: process.env, + session: `ios-e2e-${tier}-${process.pid.toString(36)}`, + sessionOpen: false, + stepHistory: [], + tier, + udid: requiredEnv('AGENT_DEVICE_IOS_UDID'), + }; +} + +export async function runScenario( + context: LiveContext, + scenario: { + id: string; + run: (context: LiveContext) => Promise; + }, +): Promise { + context.currentScenario = scenario.id; + const commands = liveCommandsForScenario(scenario.id); + const evidenceCounts = new Map( + commands.map((command) => [command, context.commandEvidence[command]?.length ?? 0]), + ); + const behaviorCounts = new Map( + liveBehaviorsForScenario(scenario.id).map((behavior) => [ + behavior, + context.behaviorEvidence[behavior]?.length ?? 0, + ]), + ); + await scenario.run(context); + assertScenarioCommandsVerified(scenario.id, commands, context, evidenceCounts); + assertScenarioBehaviorsVerified(scenario.id, context, behaviorCounts); + context.completedScenarios.push(scenario.id); + writeCoverageReport(context); +} + +export async function runStep( + context: LiveContext, + step: string, + args: string[], + options: { + allowFailure?: boolean; + commonFlags?: boolean; + expectFailure?: boolean; + timeoutMs?: number; + } = {}, +): Promise { + const fullArgs = options.commonFlags === false ? withJson(args) : withCommonFlags(context, args); + const startedAt = Date.now(); + const result = await runBuiltCliJson(fullArgs, context.env, { + timeoutMs: options.timeoutMs, + }); + context.stepHistory.push({ + accepted: result.status === 0 || (options.expectFailure === true && result.status !== 0), + command: `agent-device ${fullArgs.join(' ')}`, + commandName: args[0], + durationMs: Date.now() - startedAt, + errorCode: stringValue(result.json?.error?.code), + errorMessage: stringValue(result.json?.error?.message), + scenario: context.currentScenario, + status: result.status, + step, + }); + writeStepHistory(context); + const failedAsExpected = options.expectFailure === true && result.status !== 0; + if (result.status !== 0 && !failedAsExpected && options.allowFailure !== true) { + const message = [ + formatResultDebug(step, fullArgs, result), + `scenario: ${context.currentScenario}`, + `artifacts: ${context.artifactDir}`, + ].join('\n'); + fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message); + assert.fail(message); + } + if (options.expectFailure === true && result.status === 0) { + assert.fail(`${step} unexpectedly succeeded\ncommand: agent-device ${fullArgs.join(' ')}`); + } + if (result.status === 0 && args[0] === 'open') context.sessionOpen = true; + if (result.status === 0 && args[0] === 'close') context.sessionOpen = false; + return result; +} + +export function verifyCommand(context: LiveContext, command: string, evidence: string): void { + recordCommandEvidence(context, command, command, evidence); +} + +export function verifyNestedReplayCommand( + context: LiveContext, + command: 'gesture' | 'swipe', + executedVia: 'replay' | 'test', + evidence: string, +): void { + recordCommandEvidence(context, command, executedVia, evidence); +} + +function recordCommandEvidence( + context: LiveContext, + command: string, + executedCommand: string, + evidence: string, +): void { + assert.ok( + context.stepHistory.some( + (step) => + step.scenario === context.currentScenario && + step.commandName === executedCommand && + step.accepted, + ), + `${context.currentScenario} credited ${command} without a successful ${executedCommand} execution`, + ); + const existing = context.commandEvidence[command] ?? []; + context.commandEvidence[command] = [...existing, evidence]; +} + +export function verifyBehavior( + context: LiveContext, + behavior: IosSimulatorBehaviorId, + evidence: string, +): void { + const existing = context.behaviorEvidence[behavior] ?? []; + context.behaviorEvidence[behavior] = [...existing, evidence]; +} + +export async function cleanupSession(context: LiveContext): Promise { + const failures: unknown[] = []; + const cleanupSteps: Array<[string, string[]]> = []; + if (context.tier === 'full') { + cleanupSteps.push( + ['reset microphone permission', ['settings', 'permission', 'reset', 'microphone']], + ['restore light appearance', ['settings', 'appearance', 'light']], + ['restore portrait orientation', ['orientation', 'portrait']], + ); + } + cleanupSteps.push(['close fixture session', ['close']]); + for (const [step, args] of cleanupSteps) { + for (let attempt = 1; attempt <= 3; attempt += 1) { + try { + const result = await runStep(context, `cleanup: ${step} (attempt ${attempt})`, args, { + allowFailure: attempt < 3, + }); + if (result.status === 0) break; + await new Promise((resolve) => setTimeout(resolve, 500)); + } catch (error) { + failures.push(error); + break; + } + } + } + if (failures.length === 0) return; + const errorPath = path.join(context.artifactDir, 'cleanup-error.txt'); + fs.writeFileSync(errorPath, failures.map(String).join('\n\n')); + throw new AggregateError(failures, `iOS E2E cleanup failed; details: ${errorPath}`); +} + +export async function sessionExists(context: LiveContext): Promise { + const inventory = await runStep(context, 'inspect final session ownership', ['session', 'list'], { + commonFlags: false, + }); + const sessions = Array.isArray(inventory.json?.data?.sessions) + ? inventory.json.data.sessions + : []; + return sessions.some((session: { name?: unknown }) => session.name === context.session); +} + +function assertScenarioCommandsVerified( + scenarioId: string, + commands: readonly string[], + context: LiveContext, + evidenceCounts: ReadonlyMap, +): void { + for (const command of commands) { + const before = evidenceCounts.get(command) ?? 0; + const after = context.commandEvidence[command]?.length ?? 0; + assert.ok(after > before, `${scenarioId} produced no command-specific evidence for ${command}`); + } +} + +function assertScenarioBehaviorsVerified( + scenarioId: string, + context: LiveContext, + evidenceCounts: ReadonlyMap, +): void { + for (const behavior of liveBehaviorsForScenario(scenarioId)) { + const before = evidenceCounts.get(behavior) ?? 0; + const after = context.behaviorEvidence[behavior]?.length ?? 0; + assert.ok(after > before, `${scenarioId} produced no behavior evidence for ${behavior}`); + } +} + +function withCommonFlags(context: LiveContext, args: string[]): string[] { + return [ + ...args, + '--platform', + 'ios', + '--udid', + context.udid, + '--session', + context.session, + ...(args.includes('--json') ? [] : ['--json']), + ]; +} + +function withJson(args: string[]): string[] { + return args.includes('--json') ? args : [...args, '--json']; +} + +function writeStepHistory(context: LiveContext): void { + fs.writeFileSync( + path.join(context.artifactDir, 'step-history.json'), + JSON.stringify(context.stepHistory, null, 2), + ); +} + +function requiredEnv(name: string): string { + const value = process.env[name]?.trim(); + assert.ok(value, `${name} is required when AGENT_DEVICE_IOS_E2E=1`); + return value; +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} diff --git a/test/integration/ios-simulator-e2e/live-replay-scenarios.ts b/test/integration/ios-simulator-e2e/live-replay-scenarios.ts new file mode 100644 index 000000000..a54b71aaa --- /dev/null +++ b/test/integration/ios-simulator-e2e/live-replay-scenarios.ts @@ -0,0 +1,101 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; + +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import { assertNonEmptyFile } from './live-assertions.ts'; +import { + type LiveContext, + runStep, + verifyBehavior, + verifyCommand, + verifyNestedReplayCommand, +} from './live-harness.ts'; +import { + assertReplayCommands, + readReplayCommands, + replayAttemptTimeoutMs, + replaySuiteHostTimeoutMs, +} from './replay-evidence.ts'; + +const C = PUBLIC_COMMANDS; + +export async function assertFixtureReplays(context: LiveContext): Promise { + const navigationReplay = path.resolve( + 'test/integration/replays/ios/fixture/01-navigation-scroll.ad', + ); + const navigationActions = readReplayCommands(navigationReplay); + const navigationDaemonTimeoutMs = replayAttemptTimeoutMs(navigationReplay) + 30_000; + const navigation = await runStep( + context, + 'run navigation replay through public command', + ['replay', navigationReplay, '--timeout', String(navigationDaemonTimeoutMs)], + { timeoutMs: navigationDaemonTimeoutMs + 30_000 }, + ); + assert.equal( + navigation.json?.data?.replayed, + navigationActions.length, + JSON.stringify(navigation.json), + ); + assertReplayCommands(navigationReplay, navigationActions, [C.swipe]); + verifyCommand(context, C.replay, 'public replay completes the navigation fixture'); + verifyNestedReplayCommand( + context, + C.swipe, + C.replay, + 'direction canary proves both compact-safe swipes moved content', + ); + verifyBehavior( + context, + 'long-list-scroll-recovery', + 'replay proved down, bottom footer, up, and top rediscovery in one fixture journey', + ); + + const junitPath = path.join(context.artifactDir, 'fixture-replays.junit.xml'); + const suiteArtifacts = path.join(context.artifactDir, 'fixture-replays'); + const checkoutReplay = path.resolve( + 'test/integration/replays/ios/fixture/02-checkout-release.ad', + ); + const gestureReplay = path.resolve('examples/test-app/replays/gesture-lab.ad'); + const suiteRetries = 2; + const suite = await runStep( + context, + 'run fixture suite through public test command', + [ + 'test', + checkoutReplay, + gestureReplay, + '--retries', + String(suiteRetries), + '--artifacts-dir', + suiteArtifacts, + '--report-junit', + junitPath, + ], + { + timeoutMs: replaySuiteHostTimeoutMs([checkoutReplay, gestureReplay], suiteRetries), + }, + ); + assert.equal(suite.json?.data?.failed, 0, JSON.stringify(suite.json)); + assert.equal(suite.json?.data?.passed, 2, JSON.stringify(suite.json)); + const suiteTests = Array.isArray(suite.json?.data?.tests) ? suite.json.data.tests : []; + for (const [replayPath, expectedCommands] of [ + [checkoutReplay, [C.swipe]], + [gestureReplay, [C.gesture]], + ] as const) { + const commands = readReplayCommands(replayPath); + const result = suiteTests.find( + (entry: { file?: unknown }) => path.resolve(String(entry.file)) === replayPath, + ); + assert.equal(result?.status, 'passed', JSON.stringify(suite.json)); + assert.equal(result?.replayed, commands.length, JSON.stringify(result)); + assertReplayCommands(replayPath, commands, expectedCommands); + } + assertNonEmptyFile(junitPath, 'fixture JUnit'); + verifyNestedReplayCommand( + context, + C.gesture, + C.test, + 'gesture fixture counters prove pan/fling/pinch/rotate/transform', + ); + verifyCommand(context, C.test, 'two fixture scripts pass as a suite and emit non-empty JUnit'); +} diff --git a/test/integration/ios-simulator-e2e/live-runner.ts b/test/integration/ios-simulator-e2e/live-runner.ts new file mode 100644 index 000000000..324850446 --- /dev/null +++ b/test/integration/ios-simulator-e2e/live-runner.ts @@ -0,0 +1,231 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; + +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import { assertPngFile } from '../provider-scenarios/assertions.ts'; +import { + assertFilesDiffer, + assertJsonContains, + assertWaitText, + capturePng, + requireNodeRect, +} from './live-assertions.ts'; +import { assertAutomationInput } from './live-automation-scenario.ts'; +import { assertDeviceLifecycle, assertKnownGaps } from './live-device-lifecycle.ts'; +import { + assertLifecycleAndSystem, + assertObservabilityAndArtifacts, +} from './live-full-scenarios.ts'; +import { assertFixtureReplays } from './live-replay-scenarios.ts'; +import { + assertCoverageComplete, + cleanupSession, + createContext, + type LiveContext, + runScenario, + runStep, + sessionExists, + verifyBehavior, + verifyCommand, + writeCoverageReport, +} from './live-harness.ts'; +import { bindIosSimulatorScenarios } from './scenarios.ts'; + +const C = PUBLIC_COMMANDS; +const LIVE_SCENARIOS = bindIosSimulatorScenarios({ + automationInput: assertAutomationInput, + captureClose: async (context) => { + await assertCapture(context); + await assertClose(context); + }, + deviceLifecycle: assertDeviceLifecycle, + fixtureReplays: assertFixtureReplays, + formInput: assertFormInput, + inventoryInstall: assertInventoryAndInstall, + knownGaps: async (context) => { + await assertKnownGaps(context); + await assertClose(context); + }, + lifecycleSystem: assertLifecycleAndSystem, + observabilityArtifacts: assertObservabilityAndArtifacts, +}); + +export async function runIosSimulatorE2E(): Promise { + const context = createContext(); + let primaryError: unknown; + try { + await executeLiveScenarios(context); + } catch (error) { + primaryError = error; + } + const cleanupError = await finalizeLiveRun(context); + throwLiveRunErrors(primaryError, cleanupError); +} + +async function executeLiveScenarios(context: LiveContext): Promise { + for (const scenario of LIVE_SCENARIOS.filter((candidate) => candidate.tier === 'smoke')) { + await runScenario(context, scenario); + } + if (context.tier === 'full') { + await runStep(context, 'reopen fixture for full tier', ['open', context.appId, '--relaunch']); + for (const scenario of LIVE_SCENARIOS.filter((candidate) => candidate.tier === 'full')) { + await runScenario(context, scenario); + } + } + assertCoverageComplete(context); +} + +async function finalizeLiveRun(context: LiveContext): Promise { + let cleanupError: unknown; + try { + context.sessionOpen = context.sessionOpen || (await sessionExists(context)); + } catch (error) { + cleanupError = error; + } + if (context.sessionOpen) { + try { + await cleanupSession(context); + } catch (error) { + cleanupError = combineErrors(cleanupError, error, 'session inspection and cleanup failed'); + } + } + try { + writeCoverageReport(context); + } catch (error) { + cleanupError = combineErrors(cleanupError, error, 'cleanup and coverage reporting failed'); + } + return cleanupError; +} + +function throwLiveRunErrors(primaryError: unknown, cleanupError: unknown): void { + if (primaryError !== undefined && cleanupError !== undefined) { + throw new AggregateError( + [primaryError, cleanupError], + 'iOS simulator E2E failed and cleanup also failed', + ); + } + if (primaryError !== undefined) throw primaryError; + if (cleanupError !== undefined) throw cleanupError; +} + +function combineErrors(existing: unknown, next: unknown, message: string): unknown { + return existing === undefined ? next : new AggregateError([existing, next], message); +} + +async function assertInventoryAndInstall(context: LiveContext): Promise { + const devices = await runStep(context, 'list iOS devices', ['devices']); + assertJsonContains(devices, context.udid, 'device inventory should include selected UDID'); + verifyCommand(context, C.devices, 'selected simulator UDID appears in the typed inventory'); + + const capabilities = await runStep(context, 'read simulator capabilities', ['capabilities']); + for (const command of ['click', 'fill', 'gesture', 'snapshot']) { + assertJsonContains(capabilities, command, `capabilities should include ${command}`); + } + verifyCommand(context, C.capabilities, 'typed capability response includes fixture commands'); + + await runStep(context, 'install cached fixture through public CLI', ['install', context.appPath]); + const apps = await runStep(context, 'list installed user apps', ['apps']); + assertJsonContains(apps, context.appId, 'app inventory should include fixture bundle id'); + verifyCommand(context, C.install, 'fixture installed through the public CLI and appears in apps'); + verifyCommand(context, C.apps, 'installed fixture bundle appears in app inventory'); + + const doctor = await runStep(context, 'doctor fixture discovery', [ + 'doctor', + '--app', + context.appId, + ]); + assertJsonContains(doctor, context.appId, 'doctor should discover fixture app'); + verifyCommand(context, C.doctor, 'doctor discovers the installed fixture bundle'); +} + +async function assertFormInput(context: LiveContext): Promise { + await runStep(context, 'open form tab', ['click', 'label="Form"']); + await assertWaitText(context, 'Checkout form'); + + await runStep(context, 'establish diff baseline', ['snapshot', '-i']); + await runStep(context, 'fill full name', ['fill', 'id="field-name"', 'Ada Lovelace']); + const diff = await runStep(context, 'observe filled-name diff', ['diff', 'snapshot', '-i']); + const additions = Number(diff.json?.data?.summary?.additions ?? 0); + const removals = Number(diff.json?.data?.summary?.removals ?? 0); + assert.ok( + additions + removals > 0, + `expected non-empty snapshot diff: ${JSON.stringify(diff.json)}`, + ); + verifyCommand(context, C.diff, 'snapshot diff reports a non-empty form mutation'); + const name = await runStep(context, 'read filled full name', ['get', 'attrs', 'id="field-name"']); + assertJsonContains(name, 'Ada Lovelace', 'filled name should be observable'); + verifyCommand(context, C.fill, 'replacement name text is read back from the fixture field'); + + const editable = await runStep(context, 'assert email is editable', [ + 'is', + 'editable', + 'id="field-email"', + ]); + assert.equal(editable.json?.data?.pass, true, JSON.stringify(editable.json)); + + await runStep(context, 'seed email field', ['fill', 'id="field-email"', 'ada@example']); + const keyboardVisiblePath = path.join(context.artifactDir, 'keyboard-visible.png'); + await capturePng(context, 'capture visible input keyboard', keyboardVisiblePath); + const keyboard = await runStep(context, 'dismiss input keyboard', ['keyboard', 'dismiss']); + assert.equal(keyboard.json?.data?.dismissed, true, JSON.stringify(keyboard.json)); + assert.equal(keyboard.json?.data?.visible, false, JSON.stringify(keyboard.json)); + const keyboardHiddenPath = path.join(context.artifactDir, 'keyboard-hidden.png'); + await capturePng(context, 'capture dismissed input keyboard', keyboardHiddenPath); + assertFilesDiffer( + keyboardVisiblePath, + keyboardHiddenPath, + 'keyboard dismissal should change simulator pixels', + ); + verifyCommand( + context, + C.keyboard, + 'dismiss reports dismissed=true and visible=false while before/after pixels differ', + ); + + const formSnapshot = await runStep(context, 'locate email coordinates', ['snapshot', '-i']); + const emailRect = requireNodeRect(formSnapshot, 'field-email'); + await runStep(context, 'focus email by snapshot-derived coordinates', [ + 'focus', + String(emailRect.x + emailRect.width / 2), + String(emailRect.y + emailRect.height / 2), + ]); + await runStep(context, 'append email suffix from coordinate focus', ['type', '.test']); + const email = await runStep(context, 'read typed email', ['get', 'attrs', 'id="field-email"']); + assertJsonContains(email, 'ada@example.test', 'typed email suffix should be observable'); + verifyBehavior( + context, + 'text-entry-keyboard-lifecycle', + 'fill showed the keyboard, dismissal changed pixels, and coordinate focus enabled typed text', + ); + verifyCommand(context, C.focus, 'snapshot-derived coordinate focus directs subsequent typing'); + verifyCommand(context, C.type, 'typed suffix is read back from the coordinate-focused field'); +} + +async function assertCapture(context: LiveContext): Promise { + const screenshotPath = path.join(context.artifactDir, 'fixture-smoke.png'); + const screenshot = await runStep(context, 'capture fixture screenshot', [ + 'screenshot', + screenshotPath, + '--max-size', + '900', + ]); + assertJsonContains(screenshot, screenshotPath, 'screenshot response should return artifact path'); + assertPngFile(screenshotPath); + verifyCommand(context, C.screenshot, 'captured fixture file has a valid PNG signature'); +} + +async function assertClose(context: LiveContext): Promise { + await runStep(context, 'close fixture session', ['close']); + const inventory = await runStep(context, 'verify fixture session released', ['session', 'list'], { + commonFlags: false, + }); + const sessions = Array.isArray(inventory.json?.data?.sessions) + ? inventory.json.data.sessions + : []; + assert.equal( + sessions.some((session: { name?: unknown }) => session.name === context.session), + false, + JSON.stringify(inventory.json), + ); + verifyCommand(context, C.close, 'session inventory proves the fixture lease was removed'); +} diff --git a/test/integration/ios-simulator-e2e/replay-evidence.ts b/test/integration/ios-simulator-e2e/replay-evidence.ts new file mode 100644 index 000000000..ccffecf78 --- /dev/null +++ b/test/integration/ios-simulator-e2e/replay-evidence.ts @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; + +import { parseReplayScriptDetailed, readReplayScriptMetadata } from '../../../src/replay/script.ts'; + +const DEFAULT_REPLAY_TIMEOUT_MS = 90_000; +const HOST_TIMEOUT_MARGIN_MS = 60_000; + +export function readReplayCommands(replayPath: string): string[] { + return parseReplayScriptDetailed(fs.readFileSync(replayPath, 'utf8')).actions.map( + (action) => action.command, + ); +} + +export function assertReplayCommands( + replayPath: string, + commands: readonly string[], + expectedCommands: readonly string[], +): void { + for (const command of expectedCommands) { + assert.ok(commands.includes(command), `${replayPath} did not execute ${command}`); + } +} + +export function replayAttemptTimeoutMs(replayPath: string): number { + const script = fs.readFileSync(replayPath, 'utf8'); + return readReplayScriptMetadata(script).timeoutMs ?? DEFAULT_REPLAY_TIMEOUT_MS; +} + +export function replaySuiteHostTimeoutMs(replayPaths: readonly string[], retries: number): number { + const maximumAttemptTime = replayPaths.reduce( + (total, replayPath) => total + replayAttemptTimeoutMs(replayPath), + 0, + ); + return maximumAttemptTime * (retries + 1) + HOST_TIMEOUT_MARGIN_MS; +} diff --git a/test/integration/ios-simulator-e2e/scenarios.ts b/test/integration/ios-simulator-e2e/scenarios.ts new file mode 100644 index 000000000..e2bceb944 --- /dev/null +++ b/test/integration/ios-simulator-e2e/scenarios.ts @@ -0,0 +1,47 @@ +export type IosSimulatorScenario = { + id: string; + tier: 'smoke' | 'full'; +}; + +type ScenarioRunnerKey = + | 'automationInput' + | 'captureClose' + | 'deviceLifecycle' + | 'fixtureReplays' + | 'formInput' + | 'inventoryInstall' + | 'knownGaps' + | 'lifecycleSystem' + | 'observabilityArtifacts'; + +type ScenarioDefinition = IosSimulatorScenario & { + runner: ScenarioRunnerKey; +}; + +const SCENARIO_DEFINITIONS: readonly ScenarioDefinition[] = [ + { id: 'smoke:inventory-install', runner: 'inventoryInstall', tier: 'smoke' }, + { id: 'smoke:automation-input', runner: 'automationInput', tier: 'smoke' }, + { id: 'smoke:form-input', runner: 'formInput', tier: 'smoke' }, + { id: 'smoke:capture-close', runner: 'captureClose', tier: 'smoke' }, + { id: 'full:lifecycle-system', runner: 'lifecycleSystem', tier: 'full' }, + { + id: 'full:observability-artifacts', + runner: 'observabilityArtifacts', + tier: 'full', + }, + { id: 'full:known-gaps', runner: 'knownGaps', tier: 'full' }, + { id: 'full:fixture-replays', runner: 'fixtureReplays', tier: 'full' }, + { id: 'full:device-lifecycle', runner: 'deviceLifecycle', tier: 'full' }, +] as const; + +export const IOS_SIMULATOR_LIVE_SCENARIOS: readonly IosSimulatorScenario[] = SCENARIO_DEFINITIONS; + +export function bindIosSimulatorScenarios( + runners: Record Promise>, +): readonly (IosSimulatorScenario & { run: (context: Context) => Promise })[] { + return SCENARIO_DEFINITIONS.map(({ id, runner, tier }) => ({ + id, + run: runners[runner], + tier, + })); +} diff --git a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts index 0e720e4d3..ecb277707 100644 --- a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts +++ b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts @@ -8,8 +8,8 @@ import type { RecordingProvider } from '../../../src/daemon/recording-provider.t import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; import { PROVIDER_SCENARIO_IOS_SIMULATOR, PROVIDER_SCENARIO_MACOS } from './fixtures.ts'; import { + createProviderIosSimulatorRecordingProcess, createProviderScenarioHarness, - likelyPlayableMp4Container, type ProviderScenarioHarness, type ProviderScenarioRpcResult, } from './harness.ts'; @@ -333,13 +333,8 @@ function permissiveTool(world: World) { function permissiveRecording(): RecordingProvider { return { - startIosSimulatorRecording: ({ outPath }) => { - fs.writeFileSync(outPath, likelyPlayableMp4Container()); - return { - child: { kill: () => true }, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - }; - }, + startIosSimulatorRecording: ({ outPath }) => + createProviderIosSimulatorRecordingProcess(outPath), }; } diff --git a/test/integration/provider-scenarios/harness.ts b/test/integration/provider-scenarios/harness.ts index 9d93eae36..aa49c6b58 100644 --- a/test/integration/provider-scenarios/harness.ts +++ b/test/integration/provider-scenarios/harness.ts @@ -9,10 +9,12 @@ import { createRequestHandler, type RequestRouterDeps, } from '../../../src/daemon/request-router.ts'; +import type { RecordingProcess } from '../../../src/daemon/recording-provider.ts'; import { trackDownloadableArtifact } from '../../../src/daemon/artifact-tracking.ts'; import { LeaseRegistry } from '../../../src/daemon/lease-registry.ts'; import { SessionStore } from '../../../src/daemon/session-store.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from '../../../src/daemon/types.ts'; +import type { ExecResult } from '../../../src/utils/exec.ts'; const PROVIDER_SCENARIO_TOKEN = 'provider-scenario-token'; const PROVIDER_SCENARIO_TEMP_REMOVE_OPTIONS = { @@ -140,6 +142,28 @@ export function likelyPlayableMp4Container(): Buffer { return Buffer.concat([atom('ftyp', Buffer.from('isom0000isom')), atom('moov')]); } +export function createProviderIosSimulatorRecordingProcess( + outPath: string, + onSignal?: (signal: NodeJS.Signals | number | undefined) => void, +): RecordingProcess { + fs.writeFileSync(outPath, Buffer.alloc(0)); + let resolveWait: ((result: ExecResult) => void) | undefined; + const wait = new Promise((resolve) => { + resolveWait = resolve; + }); + return { + child: { + kill: (signal) => { + onSignal?.(signal); + fs.writeFileSync(outPath, likelyPlayableMp4Container()); + resolveWait?.({ stdout: '', stderr: '', exitCode: 0 }); + return true; + }, + }, + wait, + }; +} + function atom(type: string, payload = Buffer.alloc(0)): Buffer { const header = Buffer.alloc(8); header.writeUInt32BE(8 + payload.length, 0); diff --git a/test/integration/provider-scenarios/ios-record-trace.test.ts b/test/integration/provider-scenarios/ios-record-trace.test.ts index 15ac586a8..7a8e843a6 100644 --- a/test/integration/provider-scenarios/ios-record-trace.test.ts +++ b/test/integration/provider-scenarios/ios-record-trace.test.ts @@ -10,6 +10,7 @@ import { } from './assertions.ts'; import { PROVIDER_SCENARIO_IOS_DEVICE, PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { + createProviderIosSimulatorRecordingProcess, createProviderScenarioHarness, likelyPlayableMp4Container, restoreEnv, @@ -226,17 +227,10 @@ test('Provider-backed integration iOS simulator recording flow uses semantic rec startIosSimulatorRecording: ({ device, outPath }) => { assert.equal(device.id, PROVIDER_SCENARIO_IOS_SIMULATOR.id); recordingStarts.push(outPath); - fs.writeFileSync(outPath, likelyPlayableMp4Container()); - return { - child: { - kill: (signal) => { - assert.equal(signal, 'SIGINT'); - stopped = true; - return true; - }, - }, - wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), - }; + return createProviderIosSimulatorRecordingProcess(outPath, (signal) => { + assert.equal(signal, 'SIGINT'); + stopped = true; + }); }, }; const daemon = await createProviderScenarioHarness({ diff --git a/test/integration/replays/android/01-settings.ad b/test/integration/replays/android/01-settings.ad index 47b9ff049..68728f69d 100644 --- a/test/integration/replays/android/01-settings.ad +++ b/test/integration/replays/android/01-settings.ad @@ -10,6 +10,7 @@ wait 1000 snapshot -i is exists "label=Apps || label=\"Apps & notifications\" || label=\"Network & internet\" || label=\"Network and internet\" || label=\"Connected devices\" || label=Display || label=Battery || label=Notifications || label=Security || label=Privacy" click "label=Apps || label=\"Apps & notifications\" || label=\"Network & internet\" || label=\"Network and internet\" || label=\"Connected devices\" || label=Display || label=Battery || label=Notifications || label=Security || label=Privacy" +wait "label=\"Navigate up\"" 5000 snapshot -i appstate back diff --git a/test/integration/replays/ios/fixture/01-navigation-scroll.ad b/test/integration/replays/ios/fixture/01-navigation-scroll.ad new file mode 100644 index 000000000..86668d48f --- /dev/null +++ b/test/integration/replays/ios/fixture/01-navigation-scroll.ad @@ -0,0 +1,21 @@ +# Stable fixture navigation and long-list recovery. +context platform=ios kind=simulator timeout=120000 + +env APP_TARGET="Agent Device Tester" + +open "${APP_TARGET}" --relaunch +wait "Agent Device Tester" 30000 +click "label=\"Catalog\"" +wait "Search, filter, scroll, favorite, and drill into detail without extra dependencies." 5000 +wait "Catalog scroll: top" 5000 +swipe 160 480 160 200 +wait "Catalog scroll: down" 5000 +scroll bottom +wait "Catalog scroll: bottom" 5000 +wait "Seasonal footer target" 5000 +swipe 160 200 160 480 +wait "Catalog scroll: up" 5000 +scroll top +wait "Catalog scroll: top" 5000 +wait "Search, filter, scroll, favorite, and drill into detail without extra dependencies." 5000 +close diff --git a/test/integration/replays/ios/fixture/02-checkout-release.ad b/test/integration/replays/ios/fixture/02-checkout-release.ad new file mode 100644 index 000000000..796c97567 --- /dev/null +++ b/test/integration/replays/ios/fixture/02-checkout-release.ad @@ -0,0 +1,24 @@ +# Release fixture checkout: no development overlay is present. +context platform=ios kind=simulator timeout=120000 + +env APP_TARGET="Agent Device Tester" + +open "${APP_TARGET}" --relaunch +wait "label=\"Form\"" 30000 +click "label=\"Form\"" +wait "Checkout form" 5000 +fill id="field-name" "Ada Lovelace" +fill id="field-email" "ada@example.com" +keyboard dismiss +wait "Checkout form" 5000 +swipe 190 480 190 180 +is visible id="shipping-pickup" +click id="shipping-pickup" +click id="payment-cash" +wait "Delivery choices" 5000 +is visible id="checkbox-agree" +click id="checkbox-agree" +is visible id="submit-order" +click id="submit-order" +wait "Order summary" 5000 +close diff --git a/test/integration/smoke-ios-simulator-coverage.test.ts b/test/integration/smoke-ios-simulator-coverage.test.ts new file mode 100644 index 000000000..ec83310c4 --- /dev/null +++ b/test/integration/smoke-ios-simulator-coverage.test.ts @@ -0,0 +1,260 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; + +import { PUBLIC_COMMANDS } from '../../src/command-catalog.ts'; +import { + gesturePayloadFromPositionals, + normalizePublicGesture, + normalizePublicSwipeMotion, + swipePayloadFromPositionals, +} from '../../src/contracts/gesture-normalization.ts'; +import { buildGesturePlan } from '../../src/contracts/gesture-plan.ts'; +import { isCommandSupportedOnDevice } from '../../src/core/capabilities.ts'; +import { parseReplayScriptDetailed } from '../../src/replay/script.ts'; +import { IOS_SIMULATOR_BEHAVIOR_COVERAGE } from './ios-simulator-e2e/behavior-coverage.ts'; +import { + IOS_SIMULATOR_E2E_COVERAGE, + liveCommandsForScenario, +} from './ios-simulator-e2e/coverage-manifest.ts'; +import { collectPagedEventTimeline } from './ios-simulator-e2e/event-timeline.ts'; +import { IOS_SIMULATOR_LIVE_SCENARIOS } from './ios-simulator-e2e/scenarios.ts'; + +const IOS_SIMULATOR = { + appleOs: 'ios' as const, + id: 'ci-ios-simulator', + kind: 'simulator' as const, + name: 'CI iPhone', + platform: 'apple' as const, + target: 'mobile' as const, +}; + +test('iOS simulator coverage exhaustively classifies the public catalog', () => { + const publicCommands = Object.values(PUBLIC_COMMANDS).sort(); + assert.deepEqual(Object.keys(IOS_SIMULATOR_E2E_COVERAGE).sort(), publicCommands); + + for (const command of publicCommands) { + const entry = IOS_SIMULATOR_E2E_COVERAGE[command]; + assert.ok(entry.assertion.trim().length > 0, `${command} needs an observable assertion`); + if (typeof entry.owner === 'string') { + assert.ok(entry.owner.trim().length > 0, `${command} needs a concrete scenario owner`); + } else { + assert.ok(entry.owner.path.trim().length > 0, `${command} needs an evidence path`); + assert.ok(entry.owner.test.trim().length > 0, `${command} needs named evidence`); + } + if (entry.level === 'known-gap') { + assert.match(entry.trackingIssue, /^#\d+$/, `${command} gap needs a tracking issue`); + } + } +}); + +test('live command claims are owned by executable scenarios', () => { + const scenariosById = new Map( + IOS_SIMULATOR_LIVE_SCENARIOS.map((scenario) => [scenario.id, scenario]), + ); + + for (const [command, entry] of Object.entries(IOS_SIMULATOR_E2E_COVERAGE)) { + if (entry.level !== 'live' && entry.level !== 'known-gap') continue; + const scenario = scenariosById.get(entry.owner); + assert.ok(scenario, `${command} references missing scenario ${entry.owner}`); + assert.ok( + liveCommandsForScenario(scenario.id).some((candidate) => candidate === command), + `${entry.owner} does not execute ${command}`, + ); + } + + const scenarioIds = IOS_SIMULATOR_LIVE_SCENARIOS.map((scenario) => scenario.id); + assert.equal(new Set(scenarioIds).size, scenarioIds.length, 'scenario ids must be unique'); + const claimedCommands = IOS_SIMULATOR_LIVE_SCENARIOS.flatMap((scenario) => + liveCommandsForScenario(scenario.id), + ); + assert.equal( + new Set(claimedCommands).size, + claimedCommands.length, + 'runtime command claims must have one primary scenario', + ); +}); + +test('mobile behavior patterns are owned by live scenarios or executable workflows', () => { + const scenarioIds = new Set(IOS_SIMULATOR_LIVE_SCENARIOS.map((scenario) => scenario.id)); + for (const [behavior, entry] of Object.entries(IOS_SIMULATOR_BEHAVIOR_COVERAGE)) { + assert.ok(entry.assertion.trim().length > 0, `${behavior} needs an observable assertion`); + if (entry.level === 'live') { + assert.ok(scenarioIds.has(entry.owner), `${behavior} references missing ${entry.owner}`); + } else { + const ownerPath = path.resolve(entry.owner.path); + assert.ok(fs.existsSync(ownerPath), `${behavior} owner does not exist`); + assert.ok( + fs.readFileSync(ownerPath, 'utf8').includes(entry.owner.test), + `${behavior} owner does not contain ${entry.owner.test}`, + ); + } + } +}); + +test('non-live owners name concrete executable repository evidence', () => { + for (const [command, entry] of Object.entries(IOS_SIMULATOR_E2E_COVERAGE)) { + if (entry.level === 'live' || entry.level === 'known-gap') continue; + const ownerPath = path.resolve(entry.owner.path); + assert.ok(fs.existsSync(ownerPath), `${command} owner does not exist: ${entry.owner.path}`); + assert.ok( + fs.readFileSync(ownerPath, 'utf8').includes(entry.owner.test), + `${command} owner does not contain named evidence: ${entry.owner.test}`, + ); + } +}); + +test('capability classifications match executable simulator behavior', () => { + for (const [command, entry] of Object.entries(IOS_SIMULATOR_E2E_COVERAGE)) { + const supported = isCommandSupportedOnDevice(command, IOS_SIMULATOR); + if (command === PUBLIC_COMMANDS.audio) { + assert.equal( + supported, + process.platform === 'darwin', + 'simulator audio admission follows host ScreenCaptureKit availability', + ); + continue; + } + if (entry.level === 'capability-denial') { + assert.equal(supported, false, `${command} denial must match capability admission`); + } else { + assert.equal(supported, true, `${command} evidence requires simulator capability admission`); + } + } + + assert.equal(isCommandSupportedOnDevice(PUBLIC_COMMANDS.tvRemote, IOS_SIMULATOR), false); + assert.equal(IOS_SIMULATOR_E2E_COVERAGE[PUBLIC_COMMANDS.tvRemote].level, 'capability-denial'); + + assert.equal(isCommandSupportedOnDevice(PUBLIC_COMMANDS.viewport, IOS_SIMULATOR), true); + assert.equal(IOS_SIMULATOR_E2E_COVERAGE[PUBLIC_COMMANDS.viewport].level, 'known-gap'); + const viewportScenario = IOS_SIMULATOR_LIVE_SCENARIOS.find( + (scenario) => scenario.id === IOS_SIMULATOR_E2E_COVERAGE[PUBLIC_COMMANDS.viewport].owner, + ); + assert.ok( + viewportScenario && + liveCommandsForScenario(viewportScenario.id).includes(PUBLIC_COMMANDS.viewport), + ); +}); + +type ReplayAction = ReturnType['actions'][number]; +type Viewport = { height: number; width: number; x: number; y: number }; + +function assertReplayActionFitsViewport(action: ReplayAction, viewport: Viewport): void { + const positionals = action.positionals ?? []; + if (action.command === 'swipe') { + const payload = swipePayloadFromPositionals(positionals); + buildGesturePlan(normalizePublicSwipeMotion(payload).gesture, viewport, 'ios'); + } + if (action.command === 'gesture') { + const pointerCount = + typeof action.flags.pointerCount === 'number' ? action.flags.pointerCount : undefined; + const payload = gesturePayloadFromPositionals(positionals, pointerCount); + buildGesturePlan(normalizePublicGesture(payload).gesture, viewport, 'ios'); + } +} + +function assertReplayFitsViewports(replayPath: string, viewports: readonly Viewport[]): void { + const actions = parseReplayScriptDetailed(fs.readFileSync(replayPath, 'utf8')).actions; + for (const action of actions) { + for (const viewport of viewports) assertReplayActionFitsViewport(action, viewport); + } +} + +test('fixture replay gestures fit the smallest supported iPhone viewport', () => { + const compactViewports = [ + { x: 0, y: 0, width: 320, height: 568 }, + { x: 0, y: 0, width: 375, height: 667 }, + ]; + const replayPaths = [ + 'test/integration/replays/ios/fixture/01-navigation-scroll.ad', + 'test/integration/replays/ios/fixture/02-checkout-release.ad', + 'examples/test-app/replays/gesture-lab.ad', + ]; + for (const replayPath of replayPaths) assertReplayFitsViewports(replayPath, compactViewports); +}); + +test('fixture navigation uses edge-aware traversal without losing direct swipe evidence', () => { + const actions = parseReplayScriptDetailed( + fs.readFileSync('test/integration/replays/ios/fixture/01-navigation-scroll.ad', 'utf8'), + ).actions; + assert.equal( + actions.filter((action) => action.command === 'swipe').length, + 2, + 'one directional swipe per edge proves replay swipe execution', + ); + assert.deepEqual( + actions + .filter((action) => action.command === 'scroll') + .map((action) => action.positionals?.[0]), + ['bottom', 'top'], + 'edge traversal must terminate from observed scroll state rather than viewport-tuned swipes', + ); +}); + +test('event timeline coverage follows cursors beyond the first page', async () => { + const requestedCursors: Array = []; + const timeline = await collectPagedEventTimeline(async (cursor) => { + requestedCursors.push(cursor); + if (cursor === undefined) { + return { + events: [{ command: PUBLIC_COMMANDS.open }, { command: PUBLIC_COMMANDS.snapshot }], + nextCursor: '100', + }; + } + return { events: [{ command: PUBLIC_COMMANDS.press }] }; + }); + + assert.deepEqual(requestedCursors, [undefined, '100']); + assert.deepEqual(timeline.commands, [ + PUBLIC_COMMANDS.open, + PUBLIC_COMMANDS.snapshot, + PUBLIC_COMMANDS.press, + ]); + assert.deepEqual(timeline.pages, [ + { cursor: '0', eventCount: 2, nextCursor: '100' }, + { cursor: '100', eventCount: 1 }, + ]); +}); + +test('event timeline coverage rejects malformed and non-advancing pages', async () => { + await assert.rejects( + collectPagedEventTimeline(async () => ({})), + /events page must contain an events array/, + ); + await assert.rejects( + collectPagedEventTimeline(async () => ({ events: [{ command: 42 }] })), + /must name a command/, + ); + await assert.rejects( + collectPagedEventTimeline(async () => ({ events: [], nextCursor: { offset: 100 } })), + /nextCursor must be a canonical non-negative integer string/, + ); + await assert.rejects( + collectPagedEventTimeline(async () => ({ events: [], nextCursor: '1e2' })), + /nextCursor must be a canonical non-negative integer string/, + ); + await assert.rejects( + collectPagedEventTimeline(async () => ({ events: [], nextCursor: '100' })), + /did not advance beyond cursor 100/, + ); + let page = 0; + await assert.rejects( + collectPagedEventTimeline(async () => ({ + events: [], + nextCursor: page++ === 0 ? '100' : '50', + })), + /did not advance beyond cursor 100/, + ); +}); + +test('event timeline coverage fails fast on runaway pagination', async () => { + let nextCursor = 0; + await assert.rejects( + collectPagedEventTimeline(async () => ({ + events: [], + nextCursor: String(++nextCursor), + })), + /events pagination exceeded 100 pages/, + ); +}); diff --git a/test/integration/smoke-ios-simulator.test.ts b/test/integration/smoke-ios-simulator.test.ts new file mode 100644 index 000000000..7f29985df --- /dev/null +++ b/test/integration/smoke-ios-simulator.test.ts @@ -0,0 +1,15 @@ +import test from 'node:test'; + +import { runIosSimulatorE2E } from './ios-simulator-e2e/live-runner.ts'; + +const enabled = process.env.AGENT_DEVICE_IOS_E2E === '1'; + +test( + 'live iOS simulator fixture E2E', + { + skip: enabled + ? false + : 'Set AGENT_DEVICE_IOS_E2E=1 with fixture path/id and simulator UDID to run.', + }, + runIosSimulatorE2E, +); diff --git a/test/scripts/setup-fixture-app-fallback-smoke.sh b/test/scripts/setup-fixture-app-fallback-smoke.sh index fc3cf84d9..8d9785bd9 100755 --- a/test/scripts/setup-fixture-app-fallback-smoke.sh +++ b/test/scripts/setup-fixture-app-fallback-smoke.sh @@ -1,12 +1,12 @@ #!/bin/sh -# Regression test for setup-fixture-app: a build-cache lookup failure must -# degrade to an inline build, never fail the caller. +# Regression tests for setup-fixture-app's trusted artifact selection and +# cache-outage fallback. # -# The fetch step runs under `set -euo pipefail`, so a bare `ART_ID="$(gh api …)"` -# would exit the whole composite on any API outage — turning a cache-service -# blip into a conformance failure, the opposite of what the action promises. This -# extracts that step's actual shell from action.yml (so it cannot drift from a -# copy), runs it with a `gh` that fails the lookup, and asserts it still reaches +# The fetch step runs under `set -euo pipefail`, so an unhandled API error would +# exit the whole composite — turning a cache-service blip into a conformance +# failure, the opposite of what the action promises. This extracts that step's +# actual shell from action.yml (so it cannot drift from a copy), points its real +# API client at an unavailable endpoint, and asserts it still reaches # `source=build` and exits 0. set -eu @@ -16,37 +16,53 @@ WORK="$(mktemp -d)" export WORK trap 'rm -rf "$WORK"' EXIT +pnpm test:fixture-cache + # Extract the exact `run:` body of the fetch step, then substitute the GitHub # expressions the composite would have expanded. -node -e ' - const fs = require("fs"); - const yaml = require("yaml"); - const doc = yaml.parse(fs.readFileSync(process.argv[1], "utf8")); - const step = doc.runs.steps.find((s) => s.id === "fetch"); - if (!step) { console.error("no fetch step in action.yml"); process.exit(2); } - let body = step.run - .replaceAll("${{ github.repository }}", "octo/repo") - .replaceAll("${{ github.workspace }}", process.env.WORK); - fs.writeFileSync(process.env.WORK + "/fetch.sh", body); -' "$ACTION" +extract_fetch() { + node -e ' + const fs = require("fs"); + const yaml = require("yaml"); + const doc = yaml.parse(fs.readFileSync(process.argv[1], "utf8")); + const step = doc.runs.steps.find((s) => s.id === "fetch"); + if (!step) { console.error("no fetch step in action.yml"); process.exit(2); } + let body = step.run + .replaceAll("${{ github.repository }}", "octo/repo") + .replaceAll("${{ github.event.pull_request.head.sha || github.sha }}", "current-head") + .replaceAll("${{ github.workspace }}", process.env.WORK) + .replaceAll("${{ inputs.wait-for-artifact-seconds }}", process.env.TEST_WAIT_SECONDS); + fs.writeFileSync(process.env.WORK + "/fetch.sh", body); + ' "$ACTION" +} + +export TEST_WAIT_SECONDS=0 +extract_fetch -# Stubs on PATH: gh fails every call (simulated outage); pnpm yields a fixed -# fingerprint so the step has a name to look up without installing the app. +# pnpm yields a fixed fingerprint so the step has a name to look up without +# installing the app. The real selector uses the deliberately unavailable API. mkdir -p "$WORK/bin" -cat > "$WORK/bin/gh" <<'STUB' -#!/bin/sh -echo "gh: simulated API outage" >&2 -exit 1 -STUB +REAL_NODE="$(command -v node)" +export REAL_NODE cat > "$WORK/bin/pnpm" <<'STUB' #!/bin/sh +printf '%s\n' "$*" >> "$WORK/fingerprint-calls" echo '{"hash":"deadbeefcafe"}' STUB -chmod +x "$WORK/bin/gh" "$WORK/bin/pnpm" +cat > "$WORK/bin/node" <<'STUB' +#!/bin/sh +printf '%s\n' "$*" >> "$WORK/node-calls" +exec "$REAL_NODE" "$@" +STUB +chmod +x "$WORK/bin/pnpm" "$WORK/bin/node" GITHUB_OUTPUT="$WORK/out" : > "$GITHUB_OUTPUT" export GITHUB_OUTPUT +GITHUB_ACTION_PATH="$ROOT/.github/actions/setup-fixture-app" +GITHUB_API_URL="http://127.0.0.1:1" +GH_TOKEN="test-token" +export GITHUB_ACTION_PATH GITHUB_API_URL GH_TOKEN set +e PATH="$WORK/bin:$PATH" bash "$WORK/fetch.sh" > "$WORK/log" 2>&1 @@ -70,8 +86,70 @@ if ! grep -q "building inline" "$WORK/log"; then echo "FAIL: expected a warning that it is building inline." >&2 FAIL=1 fi +if ! grep -qx -- '--dir examples/test-app exec fingerprint fingerprint:generate --platform ios' "$WORK/fingerprint-calls"; then + echo "FAIL: fixture consumer did not request the iOS-scoped fingerprint." >&2 + FAIL=1 +fi +if ! grep -q -- ' find octo/repo fingerprint.deadbeefcafe.ios current-head$' "$WORK/node-calls"; then + echo "FAIL: fixture consumer did not query the exact platform-scoped artifact name." >&2 + FAIL=1 +fi if [ "$FAIL" -eq 0 ]; then echo "PASS: build-cache lookup failure degrades to an inline build." fi + +# Exercise the composite's real wait loop without sleeping. The selector's +# provenance and state classification are tested above; these stubs constrain +# only its output so this test can prove each terminal state reaches fallback. +cat > "$WORK/bin/node" <<'STUB' +#!/bin/sh +case "${2:-}" in + find) + exit 0 + ;; + producer-state) + printf '%s' "$TEST_PRODUCER_STATE" + exit 0 + ;; +esac +exit 2 +STUB +cat > "$WORK/bin/sleep" <<'STUB' +#!/bin/sh +exit 0 +STUB +chmod +x "$WORK/bin/node" "$WORK/bin/sleep" + +run_terminal_producer_case() { + TEST_PRODUCER_STATE="$1" + EXPECTED_LOG="$2" + export TEST_PRODUCER_STATE EXPECTED_LOG + TEST_WAIT_SECONDS=1800 + export TEST_WAIT_SECONDS + extract_fetch + : > "$GITHUB_OUTPUT" + if ! PATH="$WORK/bin:$PATH" bash "$WORK/fetch.sh" > "$WORK/log-$TEST_PRODUCER_STATE" 2>&1; then + echo "FAIL: $TEST_PRODUCER_STATE producer state did not fall back cleanly." >&2 + FAIL=1 + return + fi + if ! grep -q '^source=build$' "$GITHUB_OUTPUT"; then + echo "FAIL: $TEST_PRODUCER_STATE producer state did not select inline build." >&2 + FAIL=1 + fi + if ! grep -q "$EXPECTED_LOG" "$WORK/log-$TEST_PRODUCER_STATE"; then + echo "FAIL: $TEST_PRODUCER_STATE producer state did not explain its bounded exit." >&2 + FAIL=1 + fi +} + +run_terminal_producer_case queued 'producer is queued' +run_terminal_producer_case failed 'producer failed' +run_terminal_producer_case absent 'No fixture producer appeared' + +if [ "$FAIL" -eq 0 ]; then + echo "PASS: queued, failed, and absent producers exit the wait path early." +fi + exit "$FAIL"