From 593416f2158f51bd5b94f1dca7445496e1b5f5c8 Mon Sep 17 00:00:00 2001 From: Dor Blayzer <59066376+Dor-bl@users.noreply.github.com> Date: Sat, 21 Nov 2026 21:29:09 +0000 Subject: [PATCH] ci: reduce iOS functional test flakiness iOS functional test jobs fail often on hosted macOS runners: the first Appium session on a freshly booted simulator times out, the simulator boot sometimes never finishes, and setup-uv can time out fetching its version manifest. - Set up Python, uv and dependencies before booting the simulator, and pin the uv version in every job. - Open Simulator.app after the boot, and cap the boot step at 20 min. - Warm up the simulator (install WDA, launch Safari once) and WDA (one throwaway Appium session) before the tests; the WDA warm-up never fails the job. - Retry session creation once in the iOS and Flutter fixtures when WDA fails to start. The XCUITest driver ignores wdaStartupRetries with usePreinstalledWDA, and deletes the failed session before raising. - Raise wdaLaunchTimeout to 240 s for the Safari tests. - Use read-only workflow token permissions; quote python-version values. --- .github/workflows/functional-test.yml | 109 +++++++++++++----- .../flutter_integration/conftest.py | 3 +- test/functional/ios/helper/options.py | 2 + test/functional/ios/helper/warm_up.py | 55 +++++++++ test/functional/ios/safari_tests.py | 4 +- test/functional/test_helper.py | 21 ++++ 6 files changed, 164 insertions(+), 30 deletions(-) create mode 100644 test/functional/ios/helper/warm_up.py diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 2658a9d6..528c9519 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -6,6 +6,9 @@ on: pull_request: branches: [ master ] +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -15,6 +18,7 @@ env: APPIUM_TEST_SERVER_HOST: '127.0.0.1' PYTHONUNBUFFERED: 1 UV_LOCKED: 'true' + UV_VERSION: '0.11.32' jobs: ios_test: @@ -49,6 +53,21 @@ jobs: with: xcode-version: ${{ env.XCODE_VERSION }} + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.14' + + - name: Set up uv + uses: astral-sh/setup-uv@v10.4.0 + with: + enable-cache: true + python-version: '3.14' + version: ${{ env.UV_VERSION }} + + - name: Install Python dependencies + run: uv sync + - name: Install Appium and drivers run: | npm install -g appium @@ -62,11 +81,8 @@ jobs: host: ${{ env.APPIUM_TEST_SERVER_HOST }} server_args: '--relaxed-security' - - name: Start iOS Simulator UI - run: | - defaults write com.apple.iphonesimulator PasteboardAutomaticSync -bool false - open -Fn "$(xcode-select --print-path)/Applications/Simulator.app" - name: Prepare iOS simulator + timeout-minutes: 20 uses: futureware-tech/simulator-action@v6 with: model: ${{ env.IPHONE_MODEL }} @@ -77,16 +93,27 @@ jobs: - name: Finalize iOS simulator boot run: xcrun --sdk iphonesimulator --show-sdk-version - - name: Set up Python - uses: actions/setup-python@v7 - with: - python-version: 3.14 + - name: Start iOS Simulator UI + run: | + defaults write com.apple.iphonesimulator PasteboardAutomaticSync -bool false + open -Fn "$(xcode-select --print-path)/Applications/Simulator.app" - - name: Set up uv - uses: astral-sh/setup-uv@v10.4.0 - with: - enable-cache: true - python-version: '3.14' + - name: Warm up iOS simulator + # The first app launch on a fresh simulator is often slow enough to make + # the first Appium session time out, so do it once before the tests. + run: | + xcrun simctl install booted "$PREBUILT_WDA_PATH" + xcrun simctl launch booted com.apple.mobilesafari + sleep 15 + xcrun simctl terminate booted com.apple.mobilesafari || true + + - name: Warm up WebDriverAgent + # Start and quit one Appium session so WDA's first launch happens here, not in a test. + timeout-minutes: 12 + continue-on-error: true + run: uv run python -m test.functional.ios.helper.warm_up + env: + LOCAL_PREBUILT_WDA: ${{ env.PREBUILT_WDA_PATH }} - name: Run Tests run: | @@ -187,13 +214,14 @@ jobs: - name: Set up Python 3.14 uses: actions/setup-python@v7 with: - python-version: 3.14 + python-version: '3.14' - name: Set up uv uses: astral-sh/setup-uv@v10.4.0 with: enable-cache: true python-version: '3.14' + version: ${{ env.UV_VERSION }} - name: Run tests uses: reactivecircus/android-emulator-runner@v2 @@ -269,7 +297,17 @@ jobs: - name: Set up Python 3.14 uses: actions/setup-python@v7 with: - python-version: 3.14 + python-version: '3.14' + + - name: Set up uv + uses: astral-sh/setup-uv@v10.4.0 + with: + enable-cache: true + python-version: '3.14' + version: ${{ env.UV_VERSION }} + + - name: Install Python dependencies + run: uv sync - name: Install Node.js uses: actions/setup-node@v7 @@ -305,12 +343,6 @@ jobs: server_args: '--relaxed-security' log_file: appium-${{ matrix.e2e-tests }}.log - - name: Set up uv - uses: astral-sh/setup-uv@v10.4.0 - with: - enable-cache: true - python-version: '3.14' - - name: Run Android tests if: matrix.e2e-tests == 'flutter-android' uses: reactivecircus/android-emulator-runner@v2 @@ -322,14 +354,10 @@ jobs: disable-spellchecker: true disable-animations: true - - name: Start iOS Simulator UI - if: matrix.e2e-tests == 'flutter-ios' - run: | - defaults write com.apple.iphonesimulator PasteboardAutomaticSync -bool false - open -Fn "$(xcode-select --print-path)/Applications/Simulator.app" - - - uses: futureware-tech/simulator-action@v6 + - name: Prepare iOS simulator + uses: futureware-tech/simulator-action@v6 if: matrix.e2e-tests == 'flutter-ios' + timeout-minutes: 20 with: # https://github.com/actions/runner-images/blob/main/images/macos/macos-14-arm64-Readme.md model: ${{ env.IPHONE_MODEL }} @@ -341,6 +369,31 @@ jobs: if: matrix.e2e-tests == 'flutter-ios' run: xcrun --sdk iphonesimulator --show-sdk-version + - name: Start iOS Simulator UI + if: matrix.e2e-tests == 'flutter-ios' + run: | + defaults write com.apple.iphonesimulator PasteboardAutomaticSync -bool false + open -Fn "$(xcode-select --print-path)/Applications/Simulator.app" + + - name: Warm up iOS simulator + if: matrix.e2e-tests == 'flutter-ios' + # The first app launch on a fresh simulator is often slow enough to make + # the first Appium session time out, so do it once before the tests. + run: | + xcrun simctl install booted "$PREBUILT_WDA_PATH" + xcrun simctl launch booted com.apple.mobilesafari + sleep 15 + xcrun simctl terminate booted com.apple.mobilesafari || true + + - name: Warm up WebDriverAgent + if: matrix.e2e-tests == 'flutter-ios' + # Start and quit one Appium session so WDA's first launch happens here, not in a test. + timeout-minutes: 12 + continue-on-error: true + run: uv run python -m test.functional.ios.helper.warm_up + env: + LOCAL_PREBUILT_WDA: ${{ env.PREBUILT_WDA_PATH }} + - name: Run IOS tests if: matrix.e2e-tests == 'flutter-ios' run: | diff --git a/test/functional/flutter_integration/conftest.py b/test/functional/flutter_integration/conftest.py index 011338bb..bfc4be24 100644 --- a/test/functional/flutter_integration/conftest.py +++ b/test/functional/flutter_integration/conftest.py @@ -20,6 +20,7 @@ from appium import webdriver from appium.webdriver.client_config import AppiumClientConfig from appium.webdriver.extensions.flutter_integration.flutter_commands import FlutterCommand +from test.functional.test_helper import create_session_with_wda_retry from test.helpers.constants import SERVER_URL_BASE from .helper.options import make_options @@ -36,7 +37,7 @@ def driver() -> Generator['WebDriver', None, None]: client_config = AppiumClientConfig(remote_server_addr=SERVER_URL_BASE) client_config.timeout = 600 - driver = webdriver.Remote(options=options, client_config=client_config) + driver = create_session_with_wda_retry(lambda: webdriver.Remote(options=options, client_config=client_config)) yield driver diff --git a/test/functional/ios/helper/options.py b/test/functional/ios/helper/options.py index c2d2fc3f..0dcd33fd 100644 --- a/test/functional/ios/helper/options.py +++ b/test/functional/ios/helper/options.py @@ -28,6 +28,8 @@ def make_options(app: str | None = None) -> XCUITestOptions: options.allow_touch_id_enroll = True options.wda_local_port = get_wda_port() options.simple_is_visible_check = True + # CI simulators are often slow to launch WDA right after boot. + options.wda_launch_timeout = 240000 if app is not None: options.app = app diff --git a/test/functional/ios/helper/warm_up.py b/test/functional/ios/helper/warm_up.py new file mode 100644 index 00000000..9e8d9e29 --- /dev/null +++ b/test/functional/ios/helper/warm_up.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Start and quit one Appium session so WebDriverAgent's first, slowest launch happens before the tests. + +Used by CI only. Failures are reported but don't fail the step: the tests retry on their own. +It makes a single attempt, so that it normally finishes within the step's 12-minute timeout. +""" + +import sys + +from appium import webdriver +from appium.webdriver.client_config import AppiumClientConfig +from test.helpers.constants import SERVER_URL_BASE + +from .options import make_options + + +def main() -> int: + client_config = AppiumClientConfig(remote_server_addr=SERVER_URL_BASE) + # Match the tests' timeout: giving up earlier would leave the server still creating this + # session, so the first test's session request would queue behind it. + client_config.timeout = 600 + try: + driver = webdriver.Remote(options=make_options(), client_config=client_config) + except Exception as e: # noqa: BLE001 + print(f'Warm-up session failed: {e}', file=sys.stderr) + return 0 + # Give teardown its own short budget, so a session that started near the 600 s limit + # normally still finishes within the step's 12-minute timeout, even if urllib3 retries + # the request three times (600 s + 4 x 15 s). This is a margin, not a hard deadline: + # the timeout applies per socket read, so a server that keeps trickling data can exceed it. + client_config.timeout = 15 + try: + driver.quit() + except Exception as e: # noqa: BLE001 + print(f'Warm-up session started but failed to quit: {e}', file=sys.stderr) + return 0 + print('Warm-up session started and quit successfully') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/functional/ios/safari_tests.py b/test/functional/ios/safari_tests.py index 21199587..db1ba721 100644 --- a/test/functional/ios/safari_tests.py +++ b/test/functional/ios/safari_tests.py @@ -20,6 +20,7 @@ from appium import webdriver from appium.webdriver.client_config import AppiumClientConfig +from test.functional.test_helper import create_session_with_wda_retry from test.helpers.constants import SERVER_URL_BASE from .helper.options import make_options @@ -28,6 +29,7 @@ from appium.webdriver.webdriver import WebDriver +# One session per test: if a session fails to start, the next test still gets a fresh attempt. @pytest.fixture def driver() -> Generator['WebDriver', None, None]: """Create and configure Safari driver for testing.""" @@ -39,7 +41,7 @@ def driver() -> Generator['WebDriver', None, None]: client_config = AppiumClientConfig(remote_server_addr=SERVER_URL_BASE) client_config.timeout = 600 - driver = webdriver.Remote(options=options, client_config=client_config) + driver = create_session_with_wda_retry(lambda: webdriver.Remote(options=options, client_config=client_config)) # Fresh iOS 17.4 simulator may not show up the webview context with "safari" # after a fresh simlator instance creation. diff --git a/test/functional/test_helper.py b/test/functional/test_helper.py index 550c26e4..737908f8 100644 --- a/test/functional/test_helper.py +++ b/test/functional/test_helper.py @@ -6,6 +6,7 @@ from time import sleep from typing import TYPE_CHECKING, Any +from selenium.common.exceptions import WebDriverException from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait @@ -141,3 +142,23 @@ def get_wda_port() -> int: """ worker_info = get_worker_info() return 8100 + (worker_info.worker_number or 0) + + +# Errors the XCUITest driver raises after it has already cleaned up a failed WDA startup, +# so creating a new session afterwards is safe. +_WDA_STARTUP_ERRORS = ('Unable to launch WebDriverAgent', 'Unable to start WebDriverAgent session') + + +def create_session_with_wda_retry(create: Callable[[], 'WebDriver'], interval_sec: float = 10) -> 'WebDriver': + """Create a session, retrying once when WebDriverAgent fails to start. + + The XCUITest driver does not retry WDA startup itself when `usePreinstalledWDA` is set, + and the first launch on a freshly booted CI simulator often fails. + """ + try: + return create() + except WebDriverException as e: + if not any(msg in str(e) for msg in _WDA_STARTUP_ERRORS): + raise + sleep(interval_sec) + return create()