From f3754869180c93b77104d14ea75e294826784e81 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Thu, 10 Sep 2026 20:12:40 +0300 Subject: [PATCH 01/24] [Enhancement]Build infra improvements --- .github/actions/cache-webrtc/action.yml | 69 ++ .github/actions/deps-android/action.yml | 54 ++ .github/actions/deps-apple/action.yml | 53 ++ .github/actions/prepare-android/action.yml | 256 ------ .github/actions/prepare-apple/action.yml | 250 ------ .github/actions/prepare-common/action.yml | 38 +- .github/actions/restore-tree/action.yml | 34 + .github/actions/setup-webrtc/action.yml | 93 +++ .github/workflows/_make.yml | 851 ++++++++++++++++++++ .github/workflows/manual-platform-tests.yml | 552 +------------ .github/workflows/publish.yml | 610 +------------- .github/workflows/release.yml | 78 ++ .github/workflows/test.yml | 47 ++ .gitignore | 2 + stream_build/AGENTS.md | 90 +++ stream_build/Makefile | 354 ++++++++ stream_build/gn/android.args | 1 + stream_build/gn/apple.args | 7 + stream_build/gn/common.args | 10 + stream_build/gn/ios-test.args | 10 + stream_build/gn/macos-test.args | 6 + stream_build/gn/slices.tsv | 14 + stream_build/gn/windows-test.args | 6 + stream_build/gn/windows.args | 1 + stream_build/scripts/check.sh | 148 ++++ stream_build/scripts/combine-apple.sh | 139 ++++ stream_build/scripts/common.sh | 210 +++++ stream_build/scripts/deps.sh | 204 +++++ stream_build/scripts/gn-gen.sh | 85 ++ stream_build/scripts/macos-gtest-filter.txt | 1 + stream_build/scripts/package-android.sh | 95 +++ stream_build/scripts/package-apple.sh | 203 +++++ stream_build/scripts/package-windows.sh | 59 ++ stream_build/scripts/rename-android.sh | 34 + stream_build/scripts/rename-apple.sh | 95 +++ stream_build/scripts/run-ios-tests.sh | 116 +++ 36 files changed, 3277 insertions(+), 1598 deletions(-) create mode 100644 .github/actions/cache-webrtc/action.yml create mode 100644 .github/actions/deps-android/action.yml create mode 100644 .github/actions/deps-apple/action.yml delete mode 100644 .github/actions/prepare-android/action.yml delete mode 100644 .github/actions/prepare-apple/action.yml create mode 100644 .github/actions/restore-tree/action.yml create mode 100644 .github/actions/setup-webrtc/action.yml create mode 100644 .github/workflows/_make.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml create mode 100644 stream_build/AGENTS.md create mode 100644 stream_build/Makefile create mode 100644 stream_build/gn/android.args create mode 100644 stream_build/gn/apple.args create mode 100644 stream_build/gn/common.args create mode 100644 stream_build/gn/ios-test.args create mode 100644 stream_build/gn/macos-test.args create mode 100644 stream_build/gn/slices.tsv create mode 100644 stream_build/gn/windows-test.args create mode 100644 stream_build/gn/windows.args create mode 100755 stream_build/scripts/check.sh create mode 100755 stream_build/scripts/combine-apple.sh create mode 100755 stream_build/scripts/common.sh create mode 100755 stream_build/scripts/deps.sh create mode 100755 stream_build/scripts/gn-gen.sh create mode 100644 stream_build/scripts/macos-gtest-filter.txt create mode 100755 stream_build/scripts/package-android.sh create mode 100755 stream_build/scripts/package-apple.sh create mode 100755 stream_build/scripts/package-windows.sh create mode 100755 stream_build/scripts/rename-android.sh create mode 100755 stream_build/scripts/rename-apple.sh create mode 100755 stream_build/scripts/run-ios-tests.sh diff --git a/.github/actions/cache-webrtc/action.yml b/.github/actions/cache-webrtc/action.yml new file mode 100644 index 0000000000..5ee050df48 --- /dev/null +++ b/.github/actions/cache-webrtc/action.yml @@ -0,0 +1,69 @@ +name: Cache WebRTC deps and out +description: OS-keyed cache of gclient trees plus DEPS_ROOT/out. Never share across OS. + +inputs: + operation: + required: true + description: restore or save + target_os_cache_key: + required: true + description: Cache-safe target_os suffix (ios-mac, android-unix, win). + config: + required: true + description: release, debug, or deps-only label included in the key. + ignore_cache: + required: false + default: "false" + description: Skip restore (save still runs). + +runs: + using: composite + steps: + - id: key + name: Compute cache key + shell: bash + run: | + set -euo pipefail + deps_hash="${{ hashFiles('DEPS') }}" + prefix="webrtc-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ inputs.config }}-${deps_hash}" + echo "prefix=${prefix}" >> "${GITHUB_OUTPUT}" + echo "key=${prefix}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}" >> "${GITHUB_OUTPUT}" + echo "Cache prefix: ${prefix}" + + - name: Restore deps and out cache + if: ${{ inputs.operation == 'restore' && inputs.ignore_cache != 'true' }} + uses: actions/cache/restore@v4 + with: + path: | + .gclient_deps + third_party + build + buildtools + testing + ios + tools + key: ${{ steps.key.outputs.key }} + restore-keys: | + ${{ steps.key.outputs.prefix }}- + + - name: Re-link DEPS_ROOT src after cache restore + if: ${{ inputs.operation == 'restore' }} + shell: bash + run: | + set -euo pipefail + mkdir -p "${GITHUB_WORKSPACE}/.gclient_deps" + ln -sfn .. "${GITHUB_WORKSPACE}/.gclient_deps/src" + + - name: Save deps and out cache + if: ${{ inputs.operation == 'save' }} + uses: actions/cache/save@v4 + with: + path: | + .gclient_deps + third_party + build + buildtools + testing + ios + tools + key: ${{ steps.key.outputs.key }} diff --git a/.github/actions/deps-android/action.yml b/.github/actions/deps-android/action.yml new file mode 100644 index 0000000000..d6d45d6ab2 --- /dev/null +++ b/.github/actions/deps-android/action.yml @@ -0,0 +1,54 @@ +name: Deps Android +description: Linux gclient sync for Android. Caches deps trees and out/. + +inputs: + webrtc_ref: + required: true + description: Branch, tag, or SHA for GetStream/webrtc. + target_os: + required: true + description: Comma-separated gclient target_os (android,unix). + target_os_cache_key: + required: true + description: Cache-safe target_os key suffix. + config: + required: true + description: Cache key config (release or debug). + ignore_cache: + required: true + description: Skip restoring caches and force a fresh sync. + +runs: + using: composite + steps: + - name: Setup WebRTC checkout + uses: ./.github/actions/setup-webrtc + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + install_android_packages: "true" + + - name: Restore Linux deps and out + uses: ./.github/actions/cache-webrtc + with: + operation: restore + target_os_cache_key: ${{ inputs.target_os_cache_key }} + config: ${{ inputs.config }} + ignore_cache: ${{ inputs.ignore_cache }} + + - name: gclient sync (Android) + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + TARGET_OS: ${{ inputs.target_os }} + JOBS: "2" + WEBRTC_REPO: https://github.com/GetStream/webrtc.git + run: make deps + + - name: Save Linux deps and out + uses: ./.github/actions/cache-webrtc + with: + operation: save + target_os_cache_key: ${{ inputs.target_os_cache_key }} + config: ${{ inputs.config }} + ignore_cache: ${{ inputs.ignore_cache }} diff --git a/.github/actions/deps-apple/action.yml b/.github/actions/deps-apple/action.yml new file mode 100644 index 0000000000..f505ae8977 --- /dev/null +++ b/.github/actions/deps-apple/action.yml @@ -0,0 +1,53 @@ +name: Deps Apple +description: Darwin gclient sync for iOS/macOS. Caches deps trees and out/. + +inputs: + webrtc_ref: + required: true + description: Branch, tag, or SHA for GetStream/webrtc. + target_os: + required: true + description: Comma-separated gclient target_os (ios and/or mac). + target_os_cache_key: + required: true + description: Cache-safe target_os key suffix. + config: + required: true + description: Cache key config (release or debug). + ignore_cache: + required: true + description: Skip restoring caches and force a fresh sync. + +runs: + using: composite + steps: + - name: Setup WebRTC checkout + uses: ./.github/actions/setup-webrtc + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + + - name: Restore Darwin deps and out + uses: ./.github/actions/cache-webrtc + with: + operation: restore + target_os_cache_key: ${{ inputs.target_os_cache_key }} + config: ${{ inputs.config }} + ignore_cache: ${{ inputs.ignore_cache }} + + - name: gclient sync (Apple) + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + TARGET_OS: ${{ inputs.target_os }} + JOBS: "2" + WEBRTC_REPO: https://github.com/GetStream/webrtc.git + run: make deps + + - name: Save Darwin deps and out + uses: ./.github/actions/cache-webrtc + with: + operation: save + target_os_cache_key: ${{ inputs.target_os_cache_key }} + config: ${{ inputs.config }} + ignore_cache: ${{ inputs.ignore_cache }} diff --git a/.github/actions/prepare-android/action.yml b/.github/actions/prepare-android/action.yml deleted file mode 100644 index 7a7f932b2c..0000000000 --- a/.github/actions/prepare-android/action.yml +++ /dev/null @@ -1,256 +0,0 @@ -name: Prepare Android dependencies -description: Prepare a Linux-host WebRTC checkout for Android validation jobs. - -inputs: - webrtc_ref: - required: true - description: Branch, tag, or SHA for GetStream/webrtc. - target_os: - required: true - description: Comma-separated gclient target_os values. - target_os_cache_key: - required: true - description: Cache-safe target_os key suffix. - ignore_cache: - required: true - description: Whether to skip cache restore. - release_pipeline_token: - required: true - description: Token used to check out the release pipeline. - -runs: - using: composite - steps: - - name: Preserve local action metadata - shell: bash - run: | - set -euo pipefail - mkdir -p "${RUNNER_TEMP}/local-actions" - cp -R "${GITHUB_WORKSPACE}/.github/actions/prepare-android" \ - "${RUNNER_TEMP}/local-actions/prepare-android" - - - name: Check out this repository (WebRTC tree) - uses: actions/checkout@v4 - with: - ref: ${{ inputs.webrtc_ref }} - - - name: Restore local action metadata - shell: bash - run: | - set -euo pipefail - mkdir -p "${GITHUB_WORKSPACE}/.github/actions" - rm -rf "${GITHUB_WORKSPACE}/.github/actions/prepare-android" - cp -R "${RUNNER_TEMP}/local-actions/prepare-android" \ - "${GITHUB_WORKSPACE}/.github/actions/prepare-android" - - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ inputs.release_pipeline_token }} - - - name: Install system dependencies - shell: bash - run: | - sudo apt-get update - sudo apt-get install -y python3-pip openjdk-8-jdk lsb-release software-properties-common - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Git authentication for HTTPS fetches - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - git config --global credential.helper store - git credential approve < "${GITHUB_WORKSPACE}/.github-git-askpass" <<'EOF' - #!/usr/bin/env bash - case "$1" in - *Username*) echo "x-access-token" ;; - *Password*) echo "${GITHUB_TOKEN}" ;; - *) echo "" ;; - esac - EOF - chmod +x "${GITHUB_WORKSPACE}/.github-git-askpass" - - - name: Restore gclient source Git cache - if: ${{ inputs.ignore_cache != 'true' }} - id: gclient-cache - uses: actions/cache/restore@v4 - with: - path: | - ${{ github.workspace }}/.output/src/.git - ${{ github.workspace }}/.gclient-git-cache - key: webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: | - webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}- - webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}- - webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}- - - - name: Materialize cached WebRTC source checkout - if: ${{ inputs.ignore_cache != 'true' }} - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass - GIT_TERMINAL_PROMPT: "0" - GIT_CONFIG_COUNT: "2" - GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_0: "git@github.com:" - GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_1: "ssh://git@github.com/" - run: | - set -euo pipefail - src="${GITHUB_WORKSPACE}/.output/src" - if [[ ! -d "${src}/.git" ]]; then - echo "No cached Git checkout found at ${src}/.git; gclient sync will clone it." - exit 0 - fi - - alternates="${src}/.git/objects/info/alternates" - if [[ -f "${alternates}" ]]; then - while IFS= read -r alternate; do - [[ -z "${alternate}" ]] && continue - if [[ ! -d "${alternate}" ]]; then - echo "Discarding cached Git checkout with missing alternate object store: ${alternate}" - rm -rf "${src}/.git" - exit 0 - fi - done < "${alternates}" - fi - - if ! git -C "${src}" remote set-url origin https://github.com/GetStream/webrtc.git || - ! git -C "${src}" fetch --prune origin "${{ inputs.webrtc_ref }}" || - ! git -C "${src}" checkout --force FETCH_HEAD || - ! git -C "${src}" reset --hard FETCH_HEAD || - ! git -C "${src}" clean -ffdx; then - echo "Discarding unusable cached Git checkout; gclient sync will recreate it." - rm -rf "${src}/.git" - fi - - - name: Install depot_tools - shell: bash - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: gclient sync (deps lane) - if: ${{ inputs.ignore_cache != 'true' }} - shell: bash - working-directory: release-pipeline - env: - GITHUB_TOKEN: ${{ github.token }} - GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass - GIT_TERMINAL_PROMPT: "0" - GIT_CONFIG_COUNT: "2" - GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_0: "git@github.com:" - GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_1: "ssh://git@github.com/" - GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache - run: >- - bundle exec fastlane deps sync - "root:${{ github.workspace }}" - "target_os:${{ inputs.target_os }}" - number_of_jobs:2 - run_hooks:false - - - name: gclient sync without cache (deps lane) - if: ${{ inputs.ignore_cache == 'true' }} - shell: bash - working-directory: release-pipeline - env: - GITHUB_TOKEN: ${{ github.token }} - GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass - GIT_TERMINAL_PROMPT: "0" - GIT_CONFIG_COUNT: "2" - GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_0: "git@github.com:" - GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_1: "ssh://git@github.com/" - run: >- - bundle exec fastlane deps sync - "root:${{ github.workspace }}" - "target_os:${{ inputs.target_os }}" - number_of_jobs:2 - run_hooks:false - - - name: Restore gclient hook download cache - if: ${{ inputs.ignore_cache != 'true' }} - id: gclient-hook-cache - uses: actions/cache/restore@v4 - with: - path: | - ${{ github.workspace }}/.output/src/resources - ${{ github.workspace }}/.output/src/tools/clang/dsymutil/bin - key: webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: | - webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}- - webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}- - - - name: gclient runhooks (deps lane) - shell: bash - working-directory: release-pipeline - run: >- - bundle exec fastlane deps runhooks - "root:${{ github.workspace }}" - - - name: Save gclient hook download cache - if: ${{ steps.gclient-hook-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 - with: - path: | - ${{ github.workspace }}/.output/src/resources - ${{ github.workspace }}/.output/src/tools/clang/dsymutil/bin - key: webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ github.run_id }}-${{ github.run_attempt }} - - - name: Save gclient source Git cache - if: ${{ steps.gclient-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 - with: - path: | - ${{ github.workspace }}/.output/src/.git - ${{ github.workspace }}/.gclient-git-cache - key: webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}-${{ github.run_id }}-${{ github.run_attempt }} - - - name: Archive prepared WebRTC tree - shell: bash - run: | - set -euo pipefail - archive="${RUNNER_TEMP}/webrtc-src-prepared-android.tar.gz" - tar -czf "${archive}" \ - -C "${GITHUB_WORKSPACE}" \ - --exclude='.output/src/.git' \ - --exclude='.output/src/.git/**' \ - .output - mv "${archive}" "${GITHUB_WORKSPACE}/webrtc-src-prepared-android.tar.gz" - - - name: Upload dependency artifact - uses: actions/upload-artifact@v4 - with: - name: webrtc-src-prepared-android - path: webrtc-src-prepared-android.tar.gz - retention-days: 2 - compression-level: 0 diff --git a/.github/actions/prepare-apple/action.yml b/.github/actions/prepare-apple/action.yml deleted file mode 100644 index e23a2cf231..0000000000 --- a/.github/actions/prepare-apple/action.yml +++ /dev/null @@ -1,250 +0,0 @@ -name: Prepare Apple dependencies -description: Prepare a macOS-host WebRTC checkout for iOS and macOS validation jobs. - -inputs: - webrtc_ref: - required: true - description: Branch, tag, or SHA for GetStream/webrtc. - target_os: - required: true - description: Comma-separated gclient target_os values. - target_os_cache_key: - required: true - description: Cache-safe target_os key suffix. - ignore_cache: - required: true - description: Whether to skip cache restore. - release_pipeline_token: - required: true - description: Token used to check out the release pipeline. - -runs: - using: composite - steps: - - name: Preserve local action metadata - shell: bash - run: | - set -euo pipefail - mkdir -p "${RUNNER_TEMP}/local-actions" - cp -R "${GITHUB_WORKSPACE}/.github/actions/prepare-apple" \ - "${RUNNER_TEMP}/local-actions/prepare-apple" - - - name: Check out this repository (WebRTC tree) - uses: actions/checkout@v4 - with: - ref: ${{ inputs.webrtc_ref }} - - - name: Restore local action metadata - shell: bash - run: | - set -euo pipefail - mkdir -p "${GITHUB_WORKSPACE}/.github/actions" - rm -rf "${GITHUB_WORKSPACE}/.github/actions/prepare-apple" - cp -R "${RUNNER_TEMP}/local-actions/prepare-apple" \ - "${GITHUB_WORKSPACE}/.github/actions/prepare-apple" - - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ inputs.release_pipeline_token }} - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Git authentication for HTTPS fetches - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - git config --global credential.helper store - git credential approve < "${GITHUB_WORKSPACE}/.github-git-askpass" <<'EOF' - #!/usr/bin/env bash - case "$1" in - *Username*) echo "x-access-token" ;; - *Password*) echo "${GITHUB_TOKEN}" ;; - *) echo "" ;; - esac - EOF - chmod +x "${GITHUB_WORKSPACE}/.github-git-askpass" - - - name: Restore gclient source Git cache - if: ${{ inputs.ignore_cache != 'true' }} - id: gclient-cache - uses: actions/cache/restore@v4 - with: - path: | - ${{ github.workspace }}/.output/src/.git - ${{ github.workspace }}/.gclient-git-cache - key: webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: | - webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}- - webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}- - webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}- - - - name: Materialize cached WebRTC source checkout - if: ${{ inputs.ignore_cache != 'true' }} - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass - GIT_TERMINAL_PROMPT: "0" - GIT_CONFIG_COUNT: "2" - GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_0: "git@github.com:" - GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_1: "ssh://git@github.com/" - run: | - set -euo pipefail - src="${GITHUB_WORKSPACE}/.output/src" - if [[ ! -d "${src}/.git" ]]; then - echo "No cached Git checkout found at ${src}/.git; gclient sync will clone it." - exit 0 - fi - - alternates="${src}/.git/objects/info/alternates" - if [[ -f "${alternates}" ]]; then - while IFS= read -r alternate; do - [[ -z "${alternate}" ]] && continue - if [[ ! -d "${alternate}" ]]; then - echo "Discarding cached Git checkout with missing alternate object store: ${alternate}" - rm -rf "${src}/.git" - exit 0 - fi - done < "${alternates}" - fi - - if ! git -C "${src}" remote set-url origin https://github.com/GetStream/webrtc.git || - ! git -C "${src}" fetch --prune origin "${{ inputs.webrtc_ref }}" || - ! git -C "${src}" checkout --force FETCH_HEAD || - ! git -C "${src}" reset --hard FETCH_HEAD || - ! git -C "${src}" clean -ffdx; then - echo "Discarding unusable cached Git checkout; gclient sync will recreate it." - rm -rf "${src}/.git" - fi - - - name: Install depot_tools - shell: bash - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: gclient sync (deps lane) - if: ${{ inputs.ignore_cache != 'true' }} - shell: bash - working-directory: release-pipeline - env: - GITHUB_TOKEN: ${{ github.token }} - GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass - GIT_TERMINAL_PROMPT: "0" - GIT_CONFIG_COUNT: "2" - GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_0: "git@github.com:" - GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_1: "ssh://git@github.com/" - GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache - run: >- - bundle exec fastlane deps sync - "root:${{ github.workspace }}" - "target_os:${{ inputs.target_os }}" - number_of_jobs:2 - run_hooks:false - - - name: gclient sync without cache (deps lane) - if: ${{ inputs.ignore_cache == 'true' }} - shell: bash - working-directory: release-pipeline - env: - GITHUB_TOKEN: ${{ github.token }} - GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass - GIT_TERMINAL_PROMPT: "0" - GIT_CONFIG_COUNT: "2" - GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_0: "git@github.com:" - GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf - GIT_CONFIG_VALUE_1: "ssh://git@github.com/" - run: >- - bundle exec fastlane deps sync - "root:${{ github.workspace }}" - "target_os:${{ inputs.target_os }}" - number_of_jobs:2 - run_hooks:false - - - name: Restore gclient hook download cache - if: ${{ inputs.ignore_cache != 'true' }} - id: gclient-hook-cache - uses: actions/cache/restore@v4 - with: - path: | - ${{ github.workspace }}/.output/src/resources - ${{ github.workspace }}/.output/src/tools/clang/dsymutil/bin - key: webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: | - webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}- - webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}- - - - name: gclient runhooks (deps lane) - shell: bash - working-directory: release-pipeline - run: >- - bundle exec fastlane deps runhooks - "root:${{ github.workspace }}" - - - name: Save gclient hook download cache - if: ${{ steps.gclient-hook-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 - with: - path: | - ${{ github.workspace }}/.output/src/resources - ${{ github.workspace }}/.output/src/tools/clang/dsymutil/bin - key: webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ github.run_id }}-${{ github.run_attempt }} - - - name: Save gclient source Git cache - if: ${{ steps.gclient-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v4 - with: - path: | - ${{ github.workspace }}/.output/src/.git - ${{ github.workspace }}/.gclient-git-cache - key: webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}-${{ github.run_id }}-${{ github.run_attempt }} - - - name: Archive prepared WebRTC tree - shell: bash - run: | - set -euo pipefail - archive="${RUNNER_TEMP}/webrtc-src-prepared-apple.tar.gz" - COPYFILE_DISABLE=1 tar -czf "${archive}" \ - -C "${GITHUB_WORKSPACE}" \ - --exclude='.output/src/.git' \ - --exclude='.output/src/.git/**' \ - .output - mv "${archive}" "${GITHUB_WORKSPACE}/webrtc-src-prepared-apple.tar.gz" - - - name: Upload dependency artifact - uses: actions/upload-artifact@v4 - with: - name: webrtc-src-prepared-apple - path: webrtc-src-prepared-apple.tar.gz - retention-days: 2 - compression-level: 0 diff --git a/.github/actions/prepare-common/action.yml b/.github/actions/prepare-common/action.yml index a441753034..b8b4400714 100644 --- a/.github/actions/prepare-common/action.yml +++ b/.github/actions/prepare-common/action.yml @@ -1,16 +1,20 @@ -name: Plan platform validation -description: Resolve selected platform flags into host-specific prepare plans. +name: Plan platform deps +description: Map selected platforms to gclient target_os and cache key suffixes. inputs: platform_ios: required: true - description: Whether iOS validation is enabled. + description: Whether iOS is selected. platform_macos: required: true - description: Whether macOS validation is enabled. + description: Whether macOS is selected. platform_android: required: true - description: Whether Android validation is enabled. + description: Whether Android is selected. + platform_windows: + required: false + default: "false" + description: Whether Windows is selected. outputs: run_ios: @@ -19,16 +23,24 @@ outputs: value: ${{ steps.flags.outputs.run_macos }} run_android: value: ${{ steps.flags.outputs.run_android }} + run_windows: + value: ${{ steps.flags.outputs.run_windows }} run_apple: value: ${{ steps.flags.outputs.run_apple }} apple_target_os: value: ${{ steps.flags.outputs.apple_target_os }} apple_target_os_cache_key: value: ${{ steps.flags.outputs.apple_target_os_cache_key }} + apple_target_os_label: + value: ${{ steps.flags.outputs.apple_target_os_label }} android_target_os: value: ${{ steps.flags.outputs.android_target_os }} android_target_os_cache_key: value: ${{ steps.flags.outputs.android_target_os_cache_key }} + windows_target_os: + value: ${{ steps.flags.outputs.windows_target_os }} + windows_target_os_cache_key: + value: ${{ steps.flags.outputs.windows_target_os_cache_key }} runs: using: composite @@ -41,10 +53,7 @@ runs: ios='${{ inputs.platform_ios }}' macos='${{ inputs.platform_macos }}' android='${{ inputs.platform_android }}' - if [[ "${ios}" != "true" && "${macos}" != "true" && "${android}" != "true" ]]; then - echo "Select at least one platform (enable one or more checkboxes above)." - exit 1 - fi + windows='${{ inputs.platform_windows }}' apple_os_list=() if [[ "${ios}" == "true" ]]; then apple_os_list+=("ios"); fi @@ -52,10 +61,12 @@ runs: run_apple=false apple_target_os="" + apple_target_os_label="" if [[ ${#apple_os_list[@]} -gt 0 ]]; then run_apple=true IFS=',' apple_target_os="${apple_os_list[*]}" + apple_target_os_label="${apple_target_os//,/, }" fi android_target_os="" @@ -63,11 +74,20 @@ runs: android_target_os="android,unix" fi + windows_target_os="" + if [[ "${windows}" == "true" ]]; then + windows_target_os="win" + fi + echo "run_ios=${ios}" >> "${GITHUB_OUTPUT}" echo "run_macos=${macos}" >> "${GITHUB_OUTPUT}" echo "run_android=${android}" >> "${GITHUB_OUTPUT}" + echo "run_windows=${windows}" >> "${GITHUB_OUTPUT}" echo "run_apple=${run_apple}" >> "${GITHUB_OUTPUT}" echo "apple_target_os=${apple_target_os}" >> "${GITHUB_OUTPUT}" echo "apple_target_os_cache_key=${apple_target_os//,/-}" >> "${GITHUB_OUTPUT}" + echo "apple_target_os_label=${apple_target_os_label}" >> "${GITHUB_OUTPUT}" echo "android_target_os=${android_target_os}" >> "${GITHUB_OUTPUT}" echo "android_target_os_cache_key=${android_target_os//,/-}" >> "${GITHUB_OUTPUT}" + echo "windows_target_os=${windows_target_os}" >> "${GITHUB_OUTPUT}" + echo "windows_target_os_cache_key=${windows_target_os}" >> "${GITHUB_OUTPUT}" diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml new file mode 100644 index 0000000000..2975b86abd --- /dev/null +++ b/.github/actions/restore-tree/action.yml @@ -0,0 +1,34 @@ +name: Restore WebRTC tree +description: Same-OS cache restore, then SKIP_DEPS=1 consumers can make. + +inputs: + webrtc_ref: + required: true + description: Branch, tag, or SHA for GetStream/webrtc. + target_os_cache_key: + required: true + description: Cache-safe target_os key suffix. + config: + required: true + description: Cache key config (release or debug). + install_android_packages: + required: false + default: "false" + description: Install apt packages needed for Android builds. + +runs: + using: composite + steps: + - name: Setup WebRTC checkout + uses: ./.github/actions/setup-webrtc + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + install_android_packages: ${{ inputs.install_android_packages }} + + - name: Restore same-OS deps and out + uses: ./.github/actions/cache-webrtc + with: + operation: restore + target_os_cache_key: ${{ inputs.target_os_cache_key }} + config: ${{ inputs.config }} + ignore_cache: "false" diff --git a/.github/actions/setup-webrtc/action.yml b/.github/actions/setup-webrtc/action.yml new file mode 100644 index 0000000000..47c9e067cf --- /dev/null +++ b/.github/actions/setup-webrtc/action.yml @@ -0,0 +1,93 @@ +name: Setup WebRTC checkout +description: Check out webrtc_ref, keep this workflow's stream_build, install depot_tools. + +inputs: + webrtc_ref: + required: true + description: Branch, tag, or SHA for GetStream/webrtc. + install_android_packages: + required: false + default: "false" + description: Install apt packages needed for Android builds. + +runs: + using: composite + steps: + - name: Preserve pipeline from the workflow revision + shell: bash + run: | + set -euo pipefail + test -d "${GITHUB_WORKSPACE}/stream_build" || { + echo "::error::stream_build is missing from the workflow checkout." + exit 1 + } + mkdir -p "${RUNNER_TEMP}/pipeline" + rm -rf "${RUNNER_TEMP}/pipeline/stream_build" "${RUNNER_TEMP}/pipeline/actions" + cp -R "${GITHUB_WORKSPACE}/stream_build" "${RUNNER_TEMP}/pipeline/stream_build" + cp -R "${GITHUB_WORKSPACE}/.github/actions" "${RUNNER_TEMP}/pipeline/actions" + + - name: Check out GetStream/webrtc + uses: actions/checkout@v4 + with: + ref: ${{ inputs.webrtc_ref }} + + - name: Restore pipeline from the workflow revision + shell: bash + run: | + set -euo pipefail + rm -rf "${GITHUB_WORKSPACE}/stream_build" "${GITHUB_WORKSPACE}/.github/actions" + mkdir -p "${GITHUB_WORKSPACE}/.github" + cp -R "${RUNNER_TEMP}/pipeline/stream_build" "${GITHUB_WORKSPACE}/stream_build" + cp -R "${RUNNER_TEMP}/pipeline/actions" "${GITHUB_WORKSPACE}/.github/actions" + test -f "${GITHUB_WORKSPACE}/DEPS" + test -f "${GITHUB_WORKSPACE}/stream_build/Makefile" + + - name: Install Android host packages + if: ${{ inputs.install_android_packages == 'true' }} + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y python3-pip openjdk-8-jdk lsb-release software-properties-common + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Git authentication for HTTPS fetches + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + git config --global credential.helper store + git credential approve <> "${GITHUB_PATH}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + echo "DEPS_ROOT=${GITHUB_WORKSPACE}/.gclient_deps" >> "${GITHUB_ENV}" + echo "WEBRTC_SRC=${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}" + echo "WEBRTC_REPO=https://github.com/GetStream/webrtc.git" >> "${GITHUB_ENV}" + + - name: Ensure DEPS_ROOT src symlink + shell: bash + run: | + set -euo pipefail + mkdir -p "${GITHUB_WORKSPACE}/.gclient_deps" + ln -sfn .. "${GITHUB_WORKSPACE}/.gclient_deps/src" + test -f "${GITHUB_WORKSPACE}/.gclient_deps/src/DEPS" diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml new file mode 100644 index 0000000000..90a36bfb33 --- /dev/null +++ b/.github/workflows/_make.yml @@ -0,0 +1,851 @@ +# Called by Build / Test / Package / Release dispatch workflows. +# Not listed in the Actions dispatch UI (workflow_call only). +# Callers pass mode: build | test | package | release. + +name: WebRTC make + +on: + workflow_call: + inputs: + mode: + description: build, test, package, or release + required: true + type: string + webrtc_ref: + required: true + type: string + ios: + required: true + type: boolean + macos: + required: true + type: boolean + android: + required: true + type: boolean + windows: + required: true + type: boolean + config: + required: true + type: string + ignore_cache: + required: true + type: boolean + android_arch: + required: false + type: string + default: "" + version: + required: false + type: string + default: "" + prerelease: + required: false + type: boolean + default: false + release_notes: + required: false + type: string + default: "" + +jobs: + validate_inputs: + name: Validate inputs + runs-on: ubuntu-latest + steps: + - name: Ensure at least one platform is selected + env: + BUILD_IOS: ${{ inputs.ios }} + BUILD_MACOS: ${{ inputs.macos }} + BUILD_ANDROID: ${{ inputs.android }} + BUILD_WINDOWS: ${{ inputs.windows }} + MODE: ${{ inputs.mode }} + run: | + set -euo pipefail + case "${MODE}" in + build|test|package|release) ;; + *) + echo "mode must be build, test, package, or release (got '${MODE}')." + exit 1 + ;; + esac + if [[ "${BUILD_IOS}" != "true" && "${BUILD_MACOS}" != "true" && + "${BUILD_ANDROID}" != "true" && "${BUILD_WINDOWS}" != "true" ]]; then + echo "Select at least one platform." + exit 1 + fi + if [[ "${MODE}" == "test" && "${BUILD_ANDROID}" == "true" ]]; then + echo "make test android is not wired. Disable Android on Test." + exit 1 + fi + if [[ "${MODE}" == "release" && -z "${{ inputs.version }}" ]]; then + echo "Release requires a version." + exit 1 + fi + + - name: Validate Android build options + if: ${{ inputs.android }} + env: + ANDROID_ARCH: ${{ inputs.android_arch }} + run: | + set -euo pipefail + if [[ -n "${ANDROID_ARCH}" && "${ANDROID_ARCH}" =~ [[:space:]] ]]; then + echo "::error::android_arch accepts a single ABI, for example arm64-v8a." + exit 1 + fi + + plan: + name: Plan + needs: validate_inputs + runs-on: ubuntu-latest + outputs: + apple_target_os: ${{ steps.plan.outputs.apple_target_os }} + apple_target_os_cache_key: ${{ steps.plan.outputs.apple_target_os_cache_key }} + apple_target_os_label: ${{ steps.plan.outputs.apple_target_os_label }} + android_target_os: ${{ steps.plan.outputs.android_target_os }} + android_target_os_cache_key: ${{ steps.plan.outputs.android_target_os_cache_key }} + steps: + - uses: actions/checkout@v4 + - id: plan + name: Plan target_os + uses: ./.github/actions/prepare-common + with: + platform_ios: ${{ inputs.ios }} + platform_macos: ${{ inputs.macos }} + platform_android: ${{ inputs.android }} + platform_windows: ${{ inputs.windows }} + + deps_apple: + name: Deps macOS (${{ needs.plan.outputs.apple_target_os_label }}) + needs: plan + if: ${{ inputs.ios || inputs.macos }} + runs-on: macos-26 + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - name: Deps Apple + uses: ./.github/actions/deps-apple + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.plan.outputs.apple_target_os }} + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + ignore_cache: ${{ inputs.ignore_cache }} + + deps_android: + name: Deps Android + needs: plan + if: ${{ inputs.android }} + runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - name: Deps Android + uses: ./.github/actions/deps-android + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.plan.outputs.android_target_os }} + target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} + config: ${{ inputs.config }} + ignore_cache: ${{ inputs.ignore_cache }} + + build_ios: + name: Build iOS (${{ inputs.config }}) + needs: [plan, deps_apple] + if: ${{ inputs.mode == 'build' && inputs.ios }} + runs-on: macos-26 + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/restore-tree + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + - name: Build iOS + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make build ios + - uses: ./.github/actions/cache-webrtc + continue-on-error: true + with: + operation: save + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + + build_macos: + name: Build macOS (${{ inputs.config }}) + needs: [plan, deps_apple] + if: ${{ inputs.mode == 'build' && inputs.macos }} + runs-on: macos-26 + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/restore-tree + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + - name: Build macOS + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make build macos + - uses: ./.github/actions/cache-webrtc + continue-on-error: true + with: + operation: save + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + + build_android: + name: Build Android (${{ inputs.config }}) + needs: [plan, deps_android] + if: ${{ inputs.mode == 'build' && inputs.android }} + runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/restore-tree + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} + config: ${{ inputs.config }} + install_android_packages: "true" + - name: Build Android + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + ANDROID_ARCH: ${{ inputs.android_arch }} + run: | + set -euo pipefail + extra=() + if [[ -n "${ANDROID_ARCH}" ]]; then + extra+=(ARCHS="${ANDROID_ARCH}") + fi + make build android "${extra[@]}" + - uses: ./.github/actions/cache-webrtc + continue-on-error: true + with: + operation: save + target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} + config: ${{ inputs.config }} + + test_ios: + name: Test iOS (build + debug test) + needs: [plan, deps_apple] + if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.ios }} + runs-on: macos-26 + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/restore-tree + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + - name: Build iOS + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make build ios + - name: Test iOS + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + SKIP_DEPS: "1" + run: make test ios + - uses: ./.github/actions/cache-webrtc + continue-on-error: true + with: + operation: save + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + + test_macos: + name: Test macOS (build + debug test) + needs: [plan, deps_apple] + if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.macos }} + runs-on: macos-26 + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/restore-tree + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + - name: Build macOS + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make build macos + - name: Test macOS + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + SKIP_DEPS: "1" + run: make test macos + - uses: ./.github/actions/cache-webrtc + continue-on-error: true + with: + operation: save + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + + test_windows: + name: Test Windows (build + debug test) + needs: plan + if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.windows }} + runs-on: windows-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - name: Check Windows WebRTC host + shell: bash + run: | + set -euo pipefail + missing=() + command -v make >/dev/null || missing+=("GNU make") + command -v python3 >/dev/null || missing+=("python3") + if [[ ${#missing[@]} -gt 0 ]]; then + echo "::error::Windows WebRTC host is not ready on windows-latest (missing: ${missing[*]}). This job is wired for make test windows, but the runner image cannot run it yet." + exit 1 + fi + - uses: ./.github/actions/restore-tree + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os_cache_key: win + config: ${{ inputs.config }} + - name: Windows deps + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + TARGET_OS: win + JOBS: "2" + WEBRTC_REPO: https://github.com/GetStream/webrtc.git + run: make deps + - name: Build Windows + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make build windows + - name: Test Windows + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + SKIP_DEPS: "1" + run: make test windows + + tests_passed: + name: Tests passed + needs: [test_ios, test_macos, test_windows] + if: ${{ always() && !cancelled() && (inputs.mode == 'package' || inputs.mode == 'release') }} + runs-on: ubuntu-latest + steps: + - name: Require tests before package on Release + env: + MODE: ${{ inputs.mode }} + WANT_IOS: ${{ inputs.ios }} + WANT_MACOS: ${{ inputs.macos }} + WANT_WINDOWS: ${{ inputs.windows }} + IOS_RESULT: ${{ needs.test_ios.result }} + MACOS_RESULT: ${{ needs.test_macos.result }} + WINDOWS_RESULT: ${{ needs.test_windows.result }} + run: | + set -euo pipefail + if [[ "${MODE}" != "release" ]]; then + echo "Package mode: tests are not required." + exit 0 + fi + failed=0 + if [[ "${WANT_IOS}" == "true" && "${IOS_RESULT}" != "success" ]]; then + echo "::error::Release is blocked: iOS tests ${IOS_RESULT}." + failed=1 + fi + if [[ "${WANT_MACOS}" == "true" && "${MACOS_RESULT}" != "success" ]]; then + echo "::error::Release is blocked: macOS tests ${MACOS_RESULT}." + failed=1 + fi + if [[ "${WANT_WINDOWS}" == "true" && "${WINDOWS_RESULT}" != "success" ]]; then + echo "::error::Release is blocked: Windows tests ${WINDOWS_RESULT}." + failed=1 + fi + if [[ "${failed}" -ne 0 ]]; then + echo "If tests fail, do not package or publish." + exit 1 + fi + echo "Selected-platform tests passed (Android tests are unwired and skipped)." + + package_ios: + name: Package iOS (${{ inputs.config }}) + needs: [plan, deps_apple, tests_passed] + if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.ios && !cancelled() && needs.tests_passed.result == 'success' }} + runs-on: macos-26 + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/restore-tree + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + - name: Build iOS + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + PRODUCTS: ${{ github.workspace }}/products + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make build ios + - name: Package iOS + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + PRODUCTS: ${{ github.workspace }}/products + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make package ios + - uses: ./.github/actions/cache-webrtc + continue-on-error: true + with: + operation: save + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + - name: Upload products-ios + uses: actions/upload-artifact@v4 + with: + name: products-ios + path: products/ios + if-no-files-found: error + retention-days: 7 + + package_macos: + name: Package macOS (${{ inputs.config }}) + needs: [plan, deps_apple, tests_passed] + if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.macos && !cancelled() && needs.tests_passed.result == 'success' }} + runs-on: macos-26 + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/restore-tree + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + - name: Build macOS + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + PRODUCTS: ${{ github.workspace }}/products + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make build macos + - name: Package macOS + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + PRODUCTS: ${{ github.workspace }}/products + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make package macos + - uses: ./.github/actions/cache-webrtc + continue-on-error: true + with: + operation: save + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + config: ${{ inputs.config }} + - name: Upload products-macos + uses: actions/upload-artifact@v4 + with: + name: products-macos + path: products/macos + if-no-files-found: error + retention-days: 7 + + package_android: + name: Package Android (${{ inputs.config }}) + needs: [plan, deps_android, tests_passed] + if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.android && !cancelled() && needs.tests_passed.result == 'success' }} + runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/restore-tree + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} + config: ${{ inputs.config }} + install_android_packages: "true" + - name: Build Android + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + PRODUCTS: ${{ github.workspace }}/products + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + ANDROID_ARCH: ${{ inputs.android_arch }} + run: | + set -euo pipefail + extra=() + if [[ -n "${ANDROID_ARCH}" ]]; then + extra+=(ARCHS="${ANDROID_ARCH}") + fi + make build android "${extra[@]}" + - name: Package Android + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + PRODUCTS: ${{ github.workspace }}/products + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + ANDROID_ARCH: ${{ inputs.android_arch }} + run: | + set -euo pipefail + extra=() + if [[ -n "${ANDROID_ARCH}" ]]; then + extra+=(ARCHS="${ANDROID_ARCH}") + fi + make package android "${extra[@]}" + - uses: ./.github/actions/cache-webrtc + continue-on-error: true + with: + operation: save + target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} + config: ${{ inputs.config }} + - name: Upload products-android + uses: actions/upload-artifact@v4 + with: + name: products-android + path: products + if-no-files-found: error + retention-days: 7 + + build_windows: + name: Build Windows (${{ inputs.config }}) + needs: plan + if: ${{ inputs.mode == 'build' && inputs.windows }} + runs-on: windows-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - name: Check Windows WebRTC host + shell: bash + run: | + set -euo pipefail + missing=() + command -v make >/dev/null || missing+=("GNU make") + command -v python3 >/dev/null || missing+=("python3") + if [[ ${#missing[@]} -gt 0 ]]; then + echo "::error::Windows WebRTC host is not ready on windows-latest (missing: ${missing[*]}). This job is wired for make build windows, but the runner image cannot run it yet." + exit 1 + fi + - uses: ./.github/actions/restore-tree + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os_cache_key: win + config: ${{ inputs.config }} + - name: Windows deps + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + TARGET_OS: win + JOBS: "2" + WEBRTC_REPO: https://github.com/GetStream/webrtc.git + run: make deps + - name: Build Windows + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make build windows + + package_windows: + name: Package Windows (${{ inputs.config }}) + needs: [plan, tests_passed] + if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.windows && !cancelled() && needs.tests_passed.result == 'success' }} + runs-on: windows-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - name: Check Windows WebRTC host + shell: bash + run: | + set -euo pipefail + missing=() + command -v make >/dev/null || missing+=("GNU make") + command -v python3 >/dev/null || missing+=("python3") + if [[ ${#missing[@]} -gt 0 ]]; then + echo "::error::Windows WebRTC host is not ready on windows-latest (missing: ${missing[*]}). This job is wired for make package windows, but the runner image cannot run it yet." + exit 1 + fi + - uses: ./.github/actions/restore-tree + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os_cache_key: win + config: ${{ inputs.config }} + - name: Windows deps + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + TARGET_OS: win + JOBS: "2" + WEBRTC_REPO: https://github.com/GetStream/webrtc.git + run: make deps + - name: Build Windows + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + PRODUCTS: ${{ github.workspace }}/products + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make build windows + - name: Package Windows + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + PRODUCTS: ${{ github.workspace }}/products + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make package windows + - name: Upload products-windows + uses: actions/upload-artifact@v4 + with: + name: products-windows + path: products/windows + if-no-files-found: error + retention-days: 7 + + finalise_package: + name: Finalise package + needs: [tests_passed, package_ios, package_macos, package_android, package_windows] + if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && !cancelled() && needs.tests_passed.result == 'success' && !failure() }} + runs-on: ${{ (inputs.ios || inputs.macos) && 'macos-26' || 'ubuntu-latest' }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Download product artifacts + uses: actions/download-artifact@v4 + with: + pattern: products-* + path: combine-in + + - name: Combine Apple xcframeworks + if: ${{ inputs.ios || inputs.macos }} + working-directory: stream_build + env: + PRODUCTS: ${{ github.workspace }}/combine-in + SKIP_LICENSES: "1" + run: make combine + + - name: Attach LICENSE.md and zip Apple artifact + if: ${{ inputs.ios || inputs.macos }} + run: | + set -euo pipefail + dest="${GITHUB_WORKSPACE}/combine-in/WebRTC.xcframework" + test -d "${dest}" + if [[ ! -f "${dest}/LICENSE.md" ]]; then + license="$(find "${GITHUB_WORKSPACE}/combine-in" -path '*/WebRTC.xcframework/LICENSE.md' ! -path "${dest}/LICENSE.md" | head -n 1 || true)" + if [[ -n "${license}" ]]; then + cp "${license}" "${dest}/LICENSE.md" + fi + fi + ditto -c -k --sequesterRsrc --keepParent \ + "${dest}" \ + "${GITHUB_WORKSPACE}/WebRTC.xcframework.zip" + + - name: Pass through Android AAR + if: ${{ inputs.android }} + run: | + set -euo pipefail + src="$(find "${GITHUB_WORKSPACE}/combine-in" -name libwebrtc.aar -type f | head -n 1)" + test -n "${src}" + cp "${src}" "${GITHUB_WORKSPACE}/libwebrtc.aar" + mkdir -p "${GITHUB_WORKSPACE}/final-android" + cp "${src}" "${GITHUB_WORKSPACE}/final-android/libwebrtc.aar" + license="$(find "${GITHUB_WORKSPACE}/combine-in" -name LICENSE.md -type f | head -n 1 || true)" + if [[ -n "${license}" ]]; then + cp "${license}" "${GITHUB_WORKSPACE}/final-android/LICENSE.md" + fi + + - name: Pass through Windows products + if: ${{ inputs.windows }} + run: | + set -euo pipefail + mkdir -p "${GITHUB_WORKSPACE}/windows-libs" + cp -R "${GITHUB_WORKSPACE}/combine-in/products-windows/." "${GITHUB_WORKSPACE}/windows-libs/" + + - name: Rename copies for wrapper SDKs + if: ${{ inputs.mode == 'release' }} + working-directory: stream_build + env: + PRODUCTS: ${{ github.workspace }}/combine-in + run: | + set -euo pipefail + if [[ "${{ inputs.ios }}" == "true" || "${{ inputs.macos }}" == "true" ]]; then + make rename apple XCFRAMEWORK="${PRODUCTS}/WebRTC.xcframework" + ditto -c -k --sequesterRsrc --keepParent \ + "${PRODUCTS}/renamed/StreamWebRTC.xcframework" \ + "${GITHUB_WORKSPACE}/StreamWebRTC.xcframework.zip" + fi + if [[ "${{ inputs.android }}" == "true" ]]; then + make rename android AAR="${GITHUB_WORKSPACE}/libwebrtc.aar" + cp "${PRODUCTS}/renamed/libwebrtc.aar" \ + "${GITHUB_WORKSPACE}/libwebrtc-renamed.aar" + fi + + - name: Upload WebRTC.xcframework.zip + if: ${{ inputs.ios || inputs.macos }} + uses: actions/upload-artifact@v4 + with: + name: final-apple + path: WebRTC.xcframework.zip + if-no-files-found: error + retention-days: 7 + + - name: Upload libwebrtc.aar + if: ${{ inputs.android }} + uses: actions/upload-artifact@v4 + with: + name: final-android + path: final-android + if-no-files-found: error + retention-days: 7 + + - name: Upload Windows libs + if: ${{ inputs.windows }} + uses: actions/upload-artifact@v4 + with: + name: final-windows + path: windows-libs + if-no-files-found: error + retention-days: 7 + + - name: Upload StreamWebRTC.xcframework.zip + if: ${{ inputs.mode == 'release' && (inputs.ios || inputs.macos) }} + uses: actions/upload-artifact@v4 + with: + name: final-apple-renamed + path: StreamWebRTC.xcframework.zip + if-no-files-found: error + retention-days: 7 + + - name: Upload renamed Android AAR + if: ${{ inputs.mode == 'release' && inputs.android }} + uses: actions/upload-artifact@v4 + with: + name: final-android-renamed + path: libwebrtc-renamed.aar + if-no-files-found: error + retention-days: 7 + + github_release: + name: Release ${{ inputs.version }} + needs: finalise_package + if: ${{ inputs.mode == 'release' && !cancelled() && !failure() }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.webrtc_ref }} + + - name: Download finalised artifacts + uses: actions/download-artifact@v4 + with: + pattern: final-* + merge-multiple: true + path: release-assets + + - name: Create GitHub release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_REF: ${{ inputs.webrtc_ref }} + IS_PRERELEASE: ${{ inputs.prerelease }} + RELEASE_NOTES: ${{ inputs.release_notes }} + run: | + set -euo pipefail + shopt -s nullglob + assets=(release-assets/*) + if [[ ${#assets[@]} -eq 0 ]]; then + echo "::error::No finalised artifacts to attach." + exit 1 + fi + notes_file="$(mktemp)" + if [[ -n "${RELEASE_NOTES}" ]]; then + printf '%s\n' "${RELEASE_NOTES}" > "${notes_file}" + else + printf 'Automated WebRTC SDK release %s.\n' "${RELEASE_VERSION}" > "${notes_file}" + fi + release_args=( + "${RELEASE_VERSION}" + "${assets[@]}" + --repo "${{ github.repository }}" + --target "${RELEASE_REF}" + --title "${RELEASE_VERSION}" + --notes-file "${notes_file}" + ) + if [[ "${IS_PRERELEASE}" == "true" ]]; then + release_args+=(--prerelease) + else + release_args+=(--latest) + fi + gh release create "${release_args[@]}" + + trigger_downstream_releases: + name: Trigger downstream WebRTC releases + needs: github_release + if: ${{ inputs.mode == 'release' && !cancelled() && !failure() }} + runs-on: ubuntu-latest + steps: + - name: Trigger stream-video-swift-webrtc release + if: ${{ inputs.ios || inputs.macos }} + env: + GH_TOKEN: ${{ secrets.CROSS_REPO_TRIGGER_RELEASE_TOKEN }} + RELEASE_VERSION: ${{ inputs.version }} + IS_PRERELEASE: ${{ inputs.prerelease }} + run: | + set -euo pipefail + webrtc_release_url="https://github.com/${{ github.repository }}/releases/tag/${RELEASE_VERSION}" + gh workflow run publish-from-webrtc.yml \ + --repo GetStream/stream-video-swift-webrtc \ + --field "webrtc_release_url=${webrtc_release_url}" \ + --field "pre_release=${IS_PRERELEASE}" + + - name: Trigger stream-video-android-webrtc release + if: ${{ inputs.android }} + env: + GH_TOKEN: ${{ secrets.CROSS_REPO_TRIGGER_RELEASE_TOKEN }} + RELEASE_VERSION: ${{ inputs.version }} + IS_PRERELEASE: ${{ inputs.prerelease }} + run: | + set -euo pipefail + webrtc_release_url="https://github.com/${{ github.repository }}/releases/tag/${RELEASE_VERSION}" + gh workflow run publish-from-webrtc.yml \ + --repo GetStream/stream-video-android-webrtc \ + --field "webrtc_release_url=${webrtc_release_url}" \ + --field "pre_release=${IS_PRERELEASE}" diff --git a/.github/workflows/manual-platform-tests.yml b/.github/workflows/manual-platform-tests.yml index cbf8cae2de..74e7f96263 100644 --- a/.github/workflows/manual-platform-tests.yml +++ b/.github/workflows/manual-platform-tests.yml @@ -1,8 +1,11 @@ -# Manual WebRTC build. Shared planning runs first, host-specific prepare jobs -# produce platform-ready trees, then selected platform builds/tests fan out. +# Manual WebRTC build. Dispatchable from a PR because this path exists on the +# default branch. Implementation: .github/workflows/_make.yml name: Build +run-name: >- + Build config:${{ inputs.config }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true') && ', Android' || ' Android') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} + permissions: contents: read @@ -13,530 +16,49 @@ on: description: Branch, tag, or SHA for this repo (GetStream/webrtc) required: true default: main - apple: - description: Build and package Apple xcframework artifacts + ios: + description: Build iOS type: boolean default: true - android: - description: Build and package Android AAR artifact + macos: + description: Build macOS type: boolean default: true - debug: - description: Build WebRTC artifacts with GN is_debug=true and include Apple dSYMs when available - type: boolean - default: true - run_ios_tests: - description: Run iOS simulator tests when Apple is enabled + android: + description: Build Android type: boolean default: true - run_macos_tests: - description: Run macOS host tests when Apple is enabled + windows: + description: Build Windows (fails clearly if the runner is not a WebRTC host) type: boolean default: false + config: + description: GN configuration + type: choice + options: + - release + - debug + default: release + android_arch: + description: Optional Android ABI, for example arm64-v8a. Empty = default ABI set. + required: false + default: "" ignore_cache: description: Skip restoring dependency caches and force a fresh sync type: boolean default: false - android_arch: - description: Optional Android ABI to build, for example arm64-v8a. Leave empty to build the default ABI set. - required: false - default: "" jobs: - # Fail invalid manual input before any expensive WebRTC sync/build work starts. - validate_inputs: - name: Validate build inputs - runs-on: ubuntu-latest - steps: - - name: Ensure at least one platform is selected - env: - BUILD_APPLE: ${{ inputs.apple }} - BUILD_ANDROID: ${{ inputs.android }} - run: | - set -euo pipefail - if [[ "${BUILD_APPLE}" != "true" && "${BUILD_ANDROID}" != "true" ]]; then - echo "At least one of apple or android must be true." - exit 1 - fi - - - name: Validate Android build options - env: - ANDROID_ARCH: ${{ inputs.android_arch }} - run: | - set -euo pipefail - - if [[ -n "${ANDROID_ARCH}" && "${ANDROID_ARCH}" =~ [[:space:]] ]]; then - echo "::error::android_arch accepts a single ABI, for example arm64-v8a." - exit 1 - fi - - prepare_common: - name: Plan platform build - needs: validate_inputs - runs-on: ubuntu-latest - outputs: - apple_target_os: ${{ steps.plan.outputs.apple_target_os }} - apple_target_os_cache_key: ${{ steps.plan.outputs.apple_target_os_cache_key }} - android_target_os: ${{ steps.plan.outputs.android_target_os }} - android_target_os_cache_key: ${{ steps.plan.outputs.android_target_os_cache_key }} - steps: - - name: Check out workflow actions - uses: actions/checkout@v4 - - - id: plan - name: Plan platform build - uses: ./.github/actions/prepare-common - with: - # Apple artifacts include both iOS and macOS slices, so both platform flags follow apple. - platform_ios: ${{ inputs.apple }} - platform_macos: ${{ inputs.apple }} - platform_android: ${{ inputs.android }} - - prepare_apple: - name: Prepare Apple dependencies - needs: prepare_common - if: ${{ inputs.apple }} - runs-on: macos-26 - timeout-minutes: 360 - steps: - - name: Check out workflow actions - uses: actions/checkout@v4 - - - name: Prepare Apple dependencies - uses: ./.github/actions/prepare-apple - with: - webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.prepare_common.outputs.apple_target_os }} - target_os_cache_key: ${{ needs.prepare_common.outputs.apple_target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} - release_pipeline_token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - prepare_android: - name: Prepare Android dependencies - needs: prepare_common - if: ${{ inputs.android }} - runs-on: ubuntu-latest - timeout-minutes: 360 - steps: - - name: Check out workflow actions - uses: actions/checkout@v4 - - - name: Prepare Android dependencies - uses: ./.github/actions/prepare-android - with: - webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.prepare_common.outputs.android_target_os }} - target_os_cache_key: ${{ needs.prepare_common.outputs.android_target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} - release_pipeline_token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - ios_test: - name: Validate iOS simulator tests - needs: prepare_apple - if: ${{ inputs.apple && inputs.run_ios_tests }} - runs-on: macos-26 - timeout-minutes: 180 - steps: - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - - name: Download prepared WebRTC tree - uses: actions/download-artifact@v4 - with: - name: webrtc-src-prepared-apple - - - name: Extract WebRTC tree - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/webrtc-tree" - tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" - - - name: CI verification (fan-out does not run gclient sync) - run: | - echo "ios test uses only the restored tree; Fastlane lane :test does not call configure_google_client." - test -f "${{ github.workspace }}/webrtc-tree/.output/src/tools/mb/mb.py" - test -f "${{ github.workspace }}/webrtc-tree/.output/src/tools_webrtc/mb/mb_config.pyl" - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install depot_tools - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: Generate local iOS mb config - run: | - python3 - <<'PY' - import ast - import pathlib - import pprint - - base_path = pathlib.Path("${{ github.workspace }}/webrtc-tree/.output/src/tools_webrtc/mb/mb_config.pyl") - output_path = pathlib.Path("${{ github.workspace }}/release-pipeline/fastlane/mb_configs/local_ios.pyl") - config = ast.literal_eval(base_path.read_text(encoding="utf-8")) - - mixins = dict(config.get("mixins", {})) - mixins["stream_local_ios_tests"] = { - "gn_args": "use_siso=false use_remoteexec=false use_reclient=false", - } - config["mixins"] = mixins - - configs = {name: list(value) for name, value in config.get("configs", {}).items()} - configs["ios_debug_local_bot_arm64"] = configs["ios_debug_bot_arm64"] + ["stream_local_ios_tests"] - config["configs"] = configs - - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(f"{pprint.pformat(config, width=120)}\n", encoding="utf-8") - PY - - - name: Run iOS tests (no gclient sync in lane) - working-directory: release-pipeline - run: >- - bundle exec fastlane ios test - "root:${{ github.workspace }}/webrtc-tree/.output/src" - "builder_config:ios_debug_local_bot_arm64" - "config_file:${{ github.workspace }}/release-pipeline/fastlane/mb_configs/local_ios.pyl" - - macos_test: - name: Validate macOS host tests - needs: prepare_apple - if: ${{ inputs.apple && inputs.run_macos_tests }} - runs-on: macos-26 - timeout-minutes: 180 - steps: - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - - name: Download prepared WebRTC tree - uses: actions/download-artifact@v4 - with: - name: webrtc-src-prepared-apple - - - name: Extract WebRTC tree - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/webrtc-tree" - tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install depot_tools - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: Run macOS host tests (no gclient sync in lane) - working-directory: release-pipeline - run: >- - bundle exec fastlane macos test - "root:${{ github.workspace }}/webrtc-tree/.output/src" - - build_ios: - name: Build iOS SDK - needs: prepare_apple - if: ${{ inputs.apple }} - runs-on: macos-26 - timeout-minutes: 360 - steps: - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - - name: Download prepared Apple WebRTC tree - uses: actions/download-artifact@v4 - with: - name: webrtc-src-prepared-apple - - - name: Extract WebRTC tree - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/webrtc-tree" - tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install depot_tools - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: Build iOS SDK artifact - working-directory: release-pipeline - run: >- - bundle exec fastlane ios build - "root:${{ github.workspace }}/webrtc-tree/.output/src" - "build_root:${{ github.workspace }}/webrtc-tree/.output" - "products_root:${{ github.workspace }}/ios-products" - configure_google_client_skip:true - prepare_signing_skip:false - sign_product_skip:true - verify_signatures_skip:true - zip_product_skip:true - "build_product_arg_is_debug:${{ inputs.debug }}" - build_product_arg_treat_warnings_as_errors:false - build_product_arg_rtc_build_examples:false - skip_licenses:true - - - name: Package iOS XCFramework artifact - run: | - set -euo pipefail - ditto -c -k --sequesterRsrc --keepParent \ - "${{ github.workspace }}/ios-products/WebRTC.xcframework" \ - "${{ github.workspace }}/WebRTC-ios.xcframework.zip" - - - name: Upload iOS SDK artifact - uses: actions/upload-artifact@v4 - with: - name: apple-ios-xcframework - path: WebRTC-ios.xcframework.zip - retention-days: 2 - - build_macos: - name: Build macOS SDK - needs: prepare_apple - if: ${{ inputs.apple }} - runs-on: macos-26 - timeout-minutes: 360 - steps: - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - - name: Download prepared Apple WebRTC tree - uses: actions/download-artifact@v4 - with: - name: webrtc-src-prepared-apple - - - name: Extract WebRTC tree - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/webrtc-tree" - tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install depot_tools - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: Build macOS SDK artifact - working-directory: release-pipeline - run: >- - bundle exec fastlane macos build - "root:${{ github.workspace }}/webrtc-tree/.output/src" - "products_root:${{ github.workspace }}/macos-products" - "build_product_arg_is_debug:${{ inputs.debug }}" - build_product_arg_treat_warnings_as_errors:false - build_product_arg_rtc_build_examples:false - zip_product_skip:true - - - name: Package macOS XCFramework artifact - run: | - set -euo pipefail - ditto -c -k --sequesterRsrc --keepParent \ - "${{ github.workspace }}/macos-products/WebRTC.xcframework" \ - "${{ github.workspace }}/WebRTC-macos.xcframework.zip" - - - name: Upload macOS SDK artifact - uses: actions/upload-artifact@v4 - with: - name: apple-macos-xcframework - path: WebRTC-macos.xcframework.zip - retention-days: 2 - - package_apple: - name: Package Apple build artifact - needs: [build_ios, build_macos, ios_test, macos_test] - # Test jobs can be skipped by input. Continue when selected Apple build jobs succeeded and no selected test failed. - if: ${{ inputs.apple && !cancelled() && !failure() }} - runs-on: macos-26 - timeout-minutes: 60 - steps: - - name: Download iOS SDK artifact - uses: actions/download-artifact@v4 - with: - name: apple-ios-xcframework - path: apple-inputs/ios - - - name: Download macOS SDK artifact - uses: actions/download-artifact@v4 - with: - name: apple-macos-xcframework - path: apple-inputs/macos - - - name: Combine Apple XCFrameworks - env: - ENABLE_DEBUG: ${{ inputs.debug }} - run: | - set -euo pipefail - ditto -x -k apple-inputs/ios/WebRTC-ios.xcframework.zip apple-inputs/ios - ditto -x -k apple-inputs/macos/WebRTC-macos.xcframework.zip apple-inputs/macos - - command=(xcodebuild -create-xcframework) - while IFS= read -r framework; do - command+=(-framework "${framework}") - if [[ "${ENABLE_DEBUG}" == "true" ]]; then - mapfile -t dsyms < <(find "$(dirname "${framework}")" -name "$(basename "${framework}").dSYM" -type d | sort) - if [[ "${#dsyms[@]}" -eq 0 ]]; then - echo "::warning::Missing dSYM for ${framework}" - continue - fi - command+=(-debug-symbols "${dsyms[0]}") - fi - done < <(find apple-inputs -name '*.framework' -type d | sort) - - "${command[@]}" -output WebRTC.xcframework - ditto -c -k --sequesterRsrc --keepParent WebRTC.xcframework WebRTC.xcframework.zip - - - name: Upload Apple build artifact - uses: actions/upload-artifact@v4 - with: - name: release-apple - path: WebRTC.xcframework.zip - retention-days: 7 - - build_android: - name: Build Android AAR - needs: prepare_android - if: ${{ inputs.android }} - runs-on: ubuntu-latest - timeout-minutes: 360 - steps: - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - - name: Download prepared Android WebRTC tree - uses: actions/download-artifact@v4 - with: - name: webrtc-src-prepared-android - - - name: Extract WebRTC tree - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/webrtc-tree" - tar -xzf webrtc-src-prepared-android.tar.gz -C "${{ github.workspace }}/webrtc-tree" - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y python3-pip openjdk-8-jdk lsb-release software-properties-common - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install depot_tools - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: Build Android SDK artifact - working-directory: release-pipeline - env: - ANDROID_ARCH: ${{ inputs.android_arch }} - run: | - set -euo pipefail - - fastlane_args=( - android build - "root:${{ github.workspace }}/webrtc-tree/.output/src" - "products_root:${{ github.workspace }}/android-products" - "build_product_arg_is_debug:${{ inputs.debug }}" - build_product_arg_treat_warnings_as_errors:false - build_product_arg_rtc_build_examples:false - ) - - if [[ -n "${ANDROID_ARCH}" ]]; then - fastlane_args+=("archs:${ANDROID_ARCH}") - fi - - bundle exec fastlane "${fastlane_args[@]}" - - - name: Upload Android build artifact - uses: actions/upload-artifact@v4 - with: - name: release-android - path: android-products/libwebrtc.aar - retention-days: 7 + build: + uses: ./.github/workflows/_make.yml + secrets: inherit + with: + mode: build + webrtc_ref: ${{ inputs.webrtc_ref }} + ios: ${{ inputs.ios }} + macos: ${{ inputs.macos }} + android: ${{ inputs.android }} + windows: ${{ inputs.windows }} + config: ${{ inputs.config }} + ignore_cache: ${{ inputs.ignore_cache }} + android_arch: ${{ inputs.android_arch }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d74465e668..34b65860a7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,36 +1,46 @@ -name: Publish +# Package WebRTC artifacts. Same path as today's Publish so a PR can dispatch it. +# Does not create a GitHub release. + +name: Package + +run-name: >- + Package config:${{ inputs.config }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true') && ', Android' || ' Android') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} permissions: - contents: write + contents: read on: workflow_dispatch: inputs: - version: - description: Release version/tag to create - required: true - branch: - description: Branch, tag, or SHA from GetStream/webrtc to publish + webrtc_ref: + description: Branch, tag, or SHA for this repo (GetStream/webrtc) required: true default: main - alpha: - description: Mark the GitHub release as an alpha/pre-release - type: boolean - default: false - debug: - description: Build WebRTC artifacts with GN is_debug=true - type: boolean - default: false ios: - description: Build, validate, publish Apple artifacts and trigger stream-video-swift-webrtc + description: Package iOS xcframework + type: boolean + default: true + macos: + description: Package macOS xcframework type: boolean default: true android: - description: Build, validate, publish Android artifacts and trigger stream-video-android-webrtc + description: Package Android AAR type: boolean default: true - changelog: - description: Optional release notes markdown. Leave empty to use generated notes. + windows: + description: Package Windows libs (fails clearly if the runner is not a WebRTC host) + type: boolean + default: false + config: + description: GN configuration + type: choice + options: + - release + - debug + default: release + android_arch: + description: Optional Android ABI, for example arm64-v8a. Empty = default ABI set. required: false default: "" ignore_cache: @@ -39,552 +49,16 @@ on: default: false jobs: - # Keep this as a separate first job so invalid manual input fails before any expensive WebRTC sync/build work starts. - validate_inputs: - name: Validate release inputs - runs-on: ubuntu-latest - steps: - - name: Ensure at least one platform is selected - env: - BUILD_IOS: ${{ inputs.ios }} - BUILD_ANDROID: ${{ inputs.android }} - run: | - set -euo pipefail - if [[ "${BUILD_IOS}" != "true" && "${BUILD_ANDROID}" != "true" ]]; then - echo "At least one of ios or android must be true." - exit 1 - fi - - plan: - name: Plan release build - needs: validate_inputs - runs-on: ubuntu-latest - outputs: - apple_target_os: ${{ steps.plan.outputs.apple_target_os }} - apple_target_os_cache_key: ${{ steps.plan.outputs.apple_target_os_cache_key }} - android_target_os: ${{ steps.plan.outputs.android_target_os }} - android_target_os_cache_key: ${{ steps.plan.outputs.android_target_os_cache_key }} - steps: - - name: Check out workflow actions - uses: actions/checkout@v4 - - - id: plan - name: Plan release build - uses: ./.github/actions/prepare-common - with: - # Platform inputs control which prepare/build/test legs run. iOS releases need both iOS and macOS - # Apple artifacts, so platform_macos follows the iOS input. - platform_ios: ${{ inputs.ios }} - platform_macos: ${{ inputs.ios }} - platform_android: ${{ inputs.android }} - - prepare_apple: - name: Prepare Apple dependencies - needs: plan - if: ${{ inputs.ios }} - runs-on: macos-26 - timeout-minutes: 360 - steps: - - name: Check out workflow actions - uses: actions/checkout@v4 - - - name: Prepare Apple dependencies - uses: ./.github/actions/prepare-apple - with: - webrtc_ref: ${{ inputs.branch }} - target_os: ${{ needs.plan.outputs.apple_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} - release_pipeline_token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - prepare_android: - name: Prepare Android dependencies - needs: plan - if: ${{ inputs.android }} - runs-on: ubuntu-latest - timeout-minutes: 360 - steps: - - name: Check out workflow actions - uses: actions/checkout@v4 - - - name: Prepare Android dependencies - uses: ./.github/actions/prepare-android - with: - webrtc_ref: ${{ inputs.branch }} - target_os: ${{ needs.plan.outputs.android_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} - release_pipeline_token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - validate_ios: - name: Validate iOS simulator tests - needs: prepare_apple - if: ${{ inputs.ios }} - runs-on: macos-26 - timeout-minutes: 180 - steps: - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - - name: Download prepared Apple WebRTC tree - uses: actions/download-artifact@v4 - with: - name: webrtc-src-prepared-apple - - - name: Extract WebRTC tree - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/webrtc-tree" - tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install depot_tools - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: Generate local iOS mb config - run: | - python3 - <<'PY' - import ast - import pathlib - import pprint - - base_path = pathlib.Path("${{ github.workspace }}/webrtc-tree/.output/src/tools_webrtc/mb/mb_config.pyl") - output_path = pathlib.Path("${{ github.workspace }}/release-pipeline/fastlane/mb_configs/local_ios.pyl") - config = ast.literal_eval(base_path.read_text(encoding="utf-8")) - - mixins = dict(config.get("mixins", {})) - mixins["stream_local_ios_tests"] = { - "gn_args": "use_siso=false use_remoteexec=false use_reclient=false", - } - config["mixins"] = mixins - - configs = {name: list(value) for name, value in config.get("configs", {}).items()} - configs["ios_debug_local_bot_arm64"] = configs["ios_debug_bot_arm64"] + ["stream_local_ios_tests"] - config["configs"] = configs - - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(f"{pprint.pformat(config, width=120)}\n", encoding="utf-8") - PY - - - name: Run iOS tests - working-directory: release-pipeline - run: >- - bundle exec fastlane ios test - "root:${{ github.workspace }}/webrtc-tree/.output/src" - "builder_config:ios_debug_local_bot_arm64" - "config_file:${{ github.workspace }}/release-pipeline/fastlane/mb_configs/local_ios.pyl" - - validate_macos: - name: Validate macOS host tests - needs: prepare_apple - if: ${{ inputs.ios }} - runs-on: macos-26 - timeout-minutes: 180 - steps: - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - - name: Download prepared Apple WebRTC tree - uses: actions/download-artifact@v4 - with: - name: webrtc-src-prepared-apple - - - name: Extract WebRTC tree - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/webrtc-tree" - tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install depot_tools - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: Run macOS host tests - working-directory: release-pipeline - run: >- - bundle exec fastlane macos test - "root:${{ github.workspace }}/webrtc-tree/.output/src" - - build_ios: - name: Build iOS SDK - needs: prepare_apple - if: ${{ inputs.ios }} - runs-on: macos-26 - timeout-minutes: 360 - steps: - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - - name: Download prepared Apple WebRTC tree - uses: actions/download-artifact@v4 - with: - name: webrtc-src-prepared-apple - - - name: Extract WebRTC tree - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/webrtc-tree" - tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install depot_tools - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: Build iOS SDK artifact - working-directory: release-pipeline - run: >- - bundle exec fastlane ios build - "root:${{ github.workspace }}/webrtc-tree/.output/src" - "build_root:${{ github.workspace }}/webrtc-tree/.output" - "products_root:${{ github.workspace }}/ios-products" - configure_google_client_skip:true - prepare_signing_skip:false - sign_product_skip:true - verify_signatures_skip:true - zip_product_skip:true - "build_product_arg_is_debug:${{ inputs.debug }}" - skip_licenses:true - - - name: Package iOS XCFramework artifact - run: | - set -euo pipefail - ditto -c -k --sequesterRsrc --keepParent \ - "${{ github.workspace }}/ios-products/WebRTC.xcframework" \ - "${{ github.workspace }}/WebRTC-ios.xcframework.zip" - - - name: Upload iOS SDK artifact - uses: actions/upload-artifact@v4 - with: - name: apple-ios-xcframework - path: WebRTC-ios.xcframework.zip - retention-days: 2 - - build_macos: - name: Build macOS SDK - needs: prepare_apple - if: ${{ inputs.ios }} - runs-on: macos-26 - timeout-minutes: 360 - steps: - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - - name: Download prepared Apple WebRTC tree - uses: actions/download-artifact@v4 - with: - name: webrtc-src-prepared-apple - - - name: Extract WebRTC tree - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/webrtc-tree" - tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install depot_tools - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: Build macOS SDK artifact - working-directory: release-pipeline - run: >- - bundle exec fastlane macos build - "root:${{ github.workspace }}/webrtc-tree/.output/src" - "products_root:${{ github.workspace }}/macos-products" - "build_product_arg_is_debug:${{ inputs.debug }}" - zip_product_skip:true - - - name: Package macOS XCFramework artifact - run: | - set -euo pipefail - ditto -c -k --sequesterRsrc --keepParent \ - "${{ github.workspace }}/macos-products/WebRTC.xcframework" \ - "${{ github.workspace }}/WebRTC-macos.xcframework.zip" - - - name: Upload macOS SDK artifact - uses: actions/upload-artifact@v4 - with: - name: apple-macos-xcframework - path: WebRTC-macos.xcframework.zip - retention-days: 2 - - package_apple: - name: Package Apple release artifact - needs: [build_ios, build_macos, validate_ios, validate_macos] - if: ${{ inputs.ios }} - runs-on: macos-26 - timeout-minutes: 60 - steps: - - name: Download iOS SDK artifact - uses: actions/download-artifact@v4 - with: - name: apple-ios-xcframework - path: apple-inputs/ios - - - name: Download macOS SDK artifact - uses: actions/download-artifact@v4 - with: - name: apple-macos-xcframework - path: apple-inputs/macos - - - name: Combine Apple XCFrameworks - env: - ENABLE_DEBUG: ${{ inputs.debug }} - run: | - set -euo pipefail - ditto -x -k apple-inputs/ios/WebRTC-ios.xcframework.zip apple-inputs/ios - ditto -x -k apple-inputs/macos/WebRTC-macos.xcframework.zip apple-inputs/macos - - command=(xcodebuild -create-xcframework) - while IFS= read -r framework; do - command+=(-framework "${framework}") - if [[ "${ENABLE_DEBUG}" == "true" ]]; then - mapfile -t dsyms < <(find "$(dirname "${framework}")" -name "$(basename "${framework}").dSYM" -type d | sort) - if [[ "${#dsyms[@]}" -eq 0 ]]; then - echo "::warning::Missing dSYM for ${framework}" - continue - fi - command+=(-debug-symbols "${dsyms[0]}") - fi - done < <(find apple-inputs -name '*.framework' -type d | sort) - - "${command[@]}" -output WebRTC.xcframework - ditto -c -k --sequesterRsrc --keepParent WebRTC.xcframework WebRTC.xcframework.zip - - - name: Upload Apple release artifact - uses: actions/upload-artifact@v4 - with: - name: release-apple - path: WebRTC.xcframework.zip - retention-days: 2 - - build_android: - name: Build Android AAR - needs: prepare_android - if: ${{ inputs.android }} - runs-on: ubuntu-latest - timeout-minutes: 360 - steps: - - name: Check out stream-webrtc-release-pipeline - uses: actions/checkout@v4 - with: - repository: GetStream/stream-webrtc-release-pipeline - path: release-pipeline - token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} - - - name: Download prepared Android WebRTC tree - uses: actions/download-artifact@v4 - with: - name: webrtc-src-prepared-android - - - name: Extract WebRTC tree - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/webrtc-tree" - tar -xzf webrtc-src-prepared-android.tar.gz -C "${{ github.workspace }}/webrtc-tree" - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y python3-pip openjdk-8-jdk lsb-release software-properties-common - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - working-directory: release-pipeline - bundler-cache: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install depot_tools - run: | - if [[ ! -d "${HOME}/depot_tools" ]]; then - git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" - fi - echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" - echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - - - name: Build Android SDK artifact - working-directory: release-pipeline - run: >- - bundle exec fastlane android build - "root:${{ github.workspace }}/webrtc-tree/.output/src" - "products_root:${{ github.workspace }}/android-products" - "build_product_arg_is_debug:${{ inputs.debug }}" - - - name: Upload Android SDK artifact - uses: actions/upload-artifact@v4 - with: - name: release-android - path: android-products/libwebrtc.aar - retention-days: 2 - - publish: - name: Publish GitHub release - # GitHub Actions needs are static. Jobs for disabled platforms are skipped quickly, and this condition - # lets publish continue as long as no selected platform failed or was cancelled. - needs: [validate_ios, validate_macos, package_apple, build_android] - if: ${{ !cancelled() && !failure() }} - runs-on: ubuntu-latest - steps: - - name: Check out release branch - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - - - name: Download release artifacts - uses: actions/download-artifact@v4 - with: - pattern: release-* - merge-multiple: true - path: release-assets - - - name: Create GitHub release - env: - GH_TOKEN: ${{ github.token }} - RELEASE_VERSION: ${{ inputs.version }} - RELEASE_BRANCH: ${{ inputs.branch }} - IS_ALPHA: ${{ inputs.alpha }} - CHANGELOG: ${{ inputs.changelog }} - run: | - set -euo pipefail - notes_file="$(mktemp)" - if [[ -n "${CHANGELOG}" ]]; then - printf '%s\n' "${CHANGELOG}" > "${notes_file}" - else - printf 'Automated WebRTC SDK release %s.\n' "${RELEASE_VERSION}" > "${notes_file}" - fi - - release_args=( - "${RELEASE_VERSION}" - release-assets/* - --repo "${{ github.repository }}" - --target "${RELEASE_BRANCH}" - --title "${RELEASE_VERSION}" - --notes-file "${notes_file}" - ) - if [[ "${IS_ALPHA}" == "true" ]]; then - release_args+=(--prerelease) - else - release_args+=(--latest) - fi - gh release create "${release_args[@]}" - - trigger_downstream_releases: - name: Trigger downstream WebRTC releases - needs: publish - if: ${{ !cancelled() && !failure() }} - runs-on: ubuntu-latest - steps: - - name: Trigger stream-video-swift-webrtc release - # Only trigger the Swift wrapper when Apple artifacts were part of this WebRTC release. - if: ${{ inputs.ios }} - env: - GH_TOKEN: ${{ secrets.CROSS_REPO_TRIGGER_RELEASE_TOKEN }} - RELEASE_VERSION: ${{ inputs.version }} - IS_ALPHA: ${{ inputs.alpha }} - run: | - set -euo pipefail - webrtc_release_url="https://github.com/${{ github.repository }}/releases/tag/${RELEASE_VERSION}" - gh workflow run publish-from-webrtc.yml \ - --repo GetStream/stream-video-swift-webrtc \ - --field "webrtc_release_url=${webrtc_release_url}" \ - --field "pre_release=${IS_ALPHA}" - - - name: Trigger stream-video-android-webrtc release - # Only trigger the Android wrapper when Android artifacts were part of this WebRTC release. - if: ${{ inputs.android }} - env: - GH_TOKEN: ${{ secrets.CROSS_REPO_TRIGGER_RELEASE_TOKEN }} - RELEASE_VERSION: ${{ inputs.version }} - IS_ALPHA: ${{ inputs.alpha }} - run: | - set -euo pipefail - webrtc_release_url="https://github.com/${{ github.repository }}/releases/tag/${RELEASE_VERSION}" - gh workflow run publish-from-webrtc.yml \ - --repo GetStream/stream-video-android-webrtc \ - --field "webrtc_release_url=${webrtc_release_url}" \ - --field "pre_release=${IS_ALPHA}" + package: + uses: ./.github/workflows/_make.yml + secrets: inherit + with: + mode: package + webrtc_ref: ${{ inputs.webrtc_ref }} + ios: ${{ inputs.ios }} + macos: ${{ inputs.macos }} + android: ${{ inputs.android }} + windows: ${{ inputs.windows }} + config: ${{ inputs.config }} + ignore_cache: ${{ inputs.ignore_cache }} + android_arch: ${{ inputs.android_arch }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..27959538af --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,78 @@ +# GitHub release of packaged WebRTC artifacts, then downstream wrapper publishes. +# Not dispatchable until this file exists on the default branch. + +name: Release + +run-name: >- + Release config:${{ inputs.config }}${{ inputs.prerelease == 'true' && ' Pre-Release' || '' }} ${{ inputs.version }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true') && ', Android' || ' Android') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} + +permissions: + contents: write + +on: + workflow_dispatch: + inputs: + version: + description: Release version/tag to create + required: true + prerelease: + description: Mark the GitHub release as a pre-release + type: boolean + default: false + release_notes: + description: Optional release notes markdown. Empty uses a one-line fallback. + required: false + default: "" + webrtc_ref: + description: Branch, tag, or SHA for this repo (GetStream/webrtc) + required: true + default: main + ios: + description: Include iOS xcframework + type: boolean + default: true + macos: + description: Include macOS xcframework + type: boolean + default: true + android: + description: Include Android AAR + type: boolean + default: true + windows: + description: Include Windows libs (fails clearly if the runner is not a WebRTC host) + type: boolean + default: false + config: + description: GN configuration + type: choice + options: + - release + - debug + default: release + android_arch: + description: Optional Android ABI, for example arm64-v8a. Empty = default ABI set. + required: false + default: "" + ignore_cache: + description: Skip restoring dependency caches and force a fresh sync + type: boolean + default: false + +jobs: + release: + uses: ./.github/workflows/_make.yml + secrets: inherit + with: + mode: release + webrtc_ref: ${{ inputs.webrtc_ref }} + ios: ${{ inputs.ios }} + macos: ${{ inputs.macos }} + android: ${{ inputs.android }} + windows: ${{ inputs.windows }} + config: ${{ inputs.config }} + ignore_cache: ${{ inputs.ignore_cache }} + android_arch: ${{ inputs.android_arch }} + version: ${{ inputs.version }} + prerelease: ${{ inputs.prerelease }} + release_notes: ${{ inputs.release_notes }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..1b50e0dd79 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,47 @@ +# WebRTC tests. Not dispatchable until this file exists on the default branch. + +name: Test + +run-name: >- + Test config:debug${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true') && ', Windows' || ' Windows') || '' }} + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + webrtc_ref: + description: Branch, tag, or SHA for this repo (GetStream/webrtc) + required: true + default: main + ios: + description: Run iOS simulator tests + type: boolean + default: true + macos: + description: Run macOS host tests + type: boolean + default: true + windows: + description: Run Windows tests (fails clearly if the runner is not a WebRTC host) + type: boolean + default: false + ignore_cache: + description: Skip restoring dependency caches and force a fresh sync + type: boolean + default: false + +jobs: + test: + uses: ./.github/workflows/_make.yml + secrets: inherit + with: + mode: test + webrtc_ref: ${{ inputs.webrtc_ref }} + ios: ${{ inputs.ios }} + macos: ${{ inputs.macos }} + android: false + windows: ${{ inputs.windows }} + config: debug + ignore_cache: ${{ inputs.ignore_cache }} diff --git a/.gitignore b/.gitignore index 9b7a2aa225..08d318a6b1 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,5 @@ out_ios_libs .output .products out_macos_libs +.gclient_deps/ +.gclient-git-cache diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md new file mode 100644 index 0000000000..12c7107f91 --- /dev/null +++ b/stream_build/AGENTS.md @@ -0,0 +1,90 @@ +# WebRTC Makefile wrapper + +Public API: + +``` +cd stream_build +make build|test|package ios|android|macos|windows [VAR=value ...] +make combine +make rename apple|android +``` + +`build` is gn+ninja. `package` only copies/lipo/zips artifacts already in `OUT`, then writes `LICENSE.md` unless `SKIP_LICENSES=1`. +Do not reintroduce Fastlane or wrap `tools_webrtc/ios/build_ios_libs.py`. + +Apple package writes per-platform trees so they do not overwrite: + +``` +$(PRODUCTS)/ios/WebRTC.xcframework +$(PRODUCTS)/macos/WebRTC.xcframework +$(PRODUCTS)/WebRTC.xcframework # make combine +$(PRODUCTS)/renamed/StreamWebRTC.xcframework +$(PRODUCTS)/renamed/libwebrtc.aar +``` + +`make combine` globs `$(PRODUCTS)/*/WebRTC.xcframework` (ios, macos, and any +future sibling such as visionos/tvos). One match is copied to the stable +output path; two or more are merged with `xcodebuild -create-xcframework`. + +`make rename` copies the original artifact and rebrands the copy. The +GetStream/webrtc release keeps `WebRTC.xcframework` / `libwebrtc.aar`. +Renamed copies feed stream-video-swift-webrtc and stream-video-android-webrtc. + +## Layout + +- `Makefile` — verb + platform dispatch +- `gn/common.args` — Stream policy GN args +- `gn/slices.tsv` — slice → ninja target + GN overlay +- `scripts/deps.sh` — gclient parent at `DEPS_ROOT` with `src` → git root +- `scripts/gn-gen.sh` — args.gn + gn gen +- `scripts/package-apple.sh` — lipo + create-xcframework +- `scripts/combine-apple.sh` — discover platform xcframeworks and merge +- `scripts/rename-apple.sh` — copy WebRTC.xcframework → StreamWebRTC +- `scripts/rename-android.sh` — copy libwebrtc.aar into PRODUCTS/renamed/ +- `scripts/package-android.sh` — zip libwebrtc.aar +- `scripts/package-windows.sh` — copy Windows libs +- `scripts/run-ios-tests.sh` +- `scripts/check.sh` + +Default `DEPS_ROOT` is `/.gclient_deps` (gitignored). `src` there is a +symlink, not a nested clone; `make deps` does not reset that git checkout. +Ninja output is `DEPS_ROOT/out`. + +## Host gates + +- ios / macos / combine / rename apple: Darwin +- android / rename android: Linux for build/package; rename android is a file copy on any host +- windows: Windows +- deps / runhooks: any host with depot_tools + +## Overrides + +| Name | Role | +|------|------| +| `CONFIG` | `release` (default, `is_debug=false`) or `debug`. `make test` always uses debug. | +| `GN_ARGS` | extra `key=value` tokens, applied last | +| `DEPS_ROOT` | gclient parent (`.gclient` + `src` symlink + `out/`) | +| `WEBRTC_SRC` | git root containing `DEPS` (default: parent of `stream_build/`) | +| `OUT` / `PRODUCTS` | ninja dirs / packaged output (default under `DEPS_ROOT`) | +| `ARCHS` | android ABI or windows cpu (`arm64-v8a`, `x64`, …) | +| `JOBS` | ninja/gclient parallelism | +| `ZIP` | `1` to zip Apple/Windows products | +| `XCFRAMEWORK` | input for `make rename apple` (default `$(PRODUCTS)/WebRTC.xcframework`) | +| `AAR` | input for `make rename android` (default `$(PRODUCTS)/libwebrtc.aar`) | +| `RENAMED` | output dir for `make rename` (default `$(PRODUCTS)/renamed`) | +| `SKIP_DEPS` | `1` skips gclient sync only; build still runs. Default `0`. | +| `SKIP_LICENSES` | `1` skips `LICENSE.md` generation only; lipo/zip still run. Default `0`. | +| `SKIP_MACCATALYST` | `1` drops `catalyst-arm64` and `catalyst-x64` from ios build+package only. Default `0`. | + +```bash +make build ios +make build ios SKIP_DEPS=1 +make build ios DEPS_ROOT=/tmp/webrtc-deps +make build ios SKIP_MACCATALYST=1 +make build android ARCHS=arm64-v8a SKIP_DEPS=1 +make package ios SKIP_DEPS=1 SKIP_LICENSES=1 +make package macos SKIP_DEPS=1 SKIP_LICENSES=1 +make combine SKIP_LICENSES=1 +make rename apple +make rename android +``` diff --git a/stream_build/Makefile b/stream_build/Makefile new file mode 100644 index 0000000000..e007587734 --- /dev/null +++ b/stream_build/Makefile @@ -0,0 +1,354 @@ +# Public API: +# make build|test|package ios|android|macos|windows [VAR=value ...] +# make combine +# make rename apple|android +# GN overrides: CONFIG=debug GN_ARGS='key=value key2=value' +# +# From the git root: cd stream_build && make build ios +# SKIP_DEPS=1 skips gclient sync only; gn/ninja still run. +# SKIP_LICENSES=1 skips license generation only; lipo/zip still run. +# SKIP_MACCATALYST=1 drops catalyst-* from ios build+package only. +# package ios/macos write PRODUCTS/ios and PRODUCTS/macos. +# make combine merges whatever PRODUCTS/*/WebRTC.xcframework exist. +# make rename copies WebRTC.xcframework / libwebrtc.aar to PRODUCTS/renamed/. + +SHELL := /bin/bash +.SUFFIXES: + +PIPELINE := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +SCRIPTS := $(PIPELINE)scripts + +CONFIG ?= release +TARGET_OS ?= ios +JOBS ?= $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 8) +SKIP_DEPS ?= 0 +SKIP_LICENSES ?= 0 +SKIP_MACCATALYST ?= 0 +RUN_HOOKS ?= 1 +GN_ARGS ?= +EXTRA_GN_ARGS ?= +WEBRTC_REPO ?= git@github.com:GetStream/webrtc.git +WEBRTC_ROOT ?= +WEBRTC_SRC ?= $(abspath $(PIPELINE)/..) +ifneq ($(WEBRTC_ROOT),) +DEPS_ROOT ?= $(WEBRTC_ROOT) +endif +DEPS_ROOT ?= $(WEBRTC_SRC)/.gclient_deps +OUT ?= $(DEPS_ROOT)/out +PRODUCTS ?= $(DEPS_ROOT)/products +TARGET ?= +NINJA_TARGET ?= +ARCHS ?= +TEST_TARGETS ?= +ZIP ?= 0 +XCFRAMEWORK ?= $(PRODUCTS)/WebRTC.xcframework +AAR ?= $(PRODUCTS)/libwebrtc.aar +RENAMED ?= $(PRODUCTS)/renamed +SIMULATOR_PLATFORM ?= +SIMULATOR_VERSION ?= +EXTRA_ARGS ?= + +ALL_GN_ARGS := $(strip $(EXTRA_GN_ARGS) $(GN_ARGS)) + +CATALYST_SLICES := catalyst-arm64 catalyst-x64 +IOS_SLICES := ios-arm64-device ios-arm64-simulator ios-x64-simulator $(CATALYST_SLICES) +MACOS_SLICES := macos-arm64 macos-x64 +ANDROID_SLICES := android-armeabi-v7a android-arm64-v8a android-x86 android-x86_64 +WINDOWS_SLICES := windows-x64 windows-arm64 +IOS_TEST_TARGETS ?= sdk_unittests sdk_framework_unittests +MACOS_TEST_TARGETS ?= rtc_unittests rtc_pc_unittests rtc_stats_unittests +WINDOWS_TEST_TARGETS ?= rtc_unittests rtc_pc_unittests rtc_stats_unittests + +ios_slices = $(if $(filter 1,$(SKIP_MACCATALYST)),$(filter-out $(CATALYST_SLICES),$(IOS_SLICES)),$(IOS_SLICES)) +android_slices = $(if $(ARCHS),$(addprefix android-,$(ARCHS)),$(ANDROID_SLICES)) +windows_slices = $(if $(ARCHS),$(addprefix windows-,$(ARCHS)),$(WINDOWS_SLICES)) + +PLATFORMS := ios android macos windows +VERBS := build test package +PLATFORM := $(firstword $(filter $(PLATFORMS),$(MAKECMDGOALS))) + +export DEPS_ROOT WEBRTC_SRC WEBRTC_ROOT WEBRTC_REPO WEBRTC_REVISION WEBRTC_REF +export TARGET_OS JOBS RUN_HOOKS CONFIG OUT PRODUCTS ARCHS SKIP_LICENSES +export GN_ARGS EXTRA_GN_ARGS ALL_GN_ARGS + +.DEFAULT_GOAL := help + +.PHONY: help check deps runhooks gen ninja slice clean print-gn-args \ + announce require-src require-darwin require-linux require-windows maybe-deps \ + combine apple rename rename-apple rename-android \ + $(PLATFORMS) $(VERBS) \ + build-ios build-macos build-android build-windows \ + test-ios test-macos test-android test-windows \ + package-ios package-macos package-android package-windows \ + $(addprefix build-slice-,$(IOS_SLICES) $(MACOS_SLICES) $(ANDROID_SLICES) $(WINDOWS_SLICES)) + +ifneq ($(filter build,$(MAKECMDGOALS)),) + ifneq ($(filter package,$(MAKECMDGOALS)),) + package: build + endif +endif + +help: + @echo "make build|test|package ios|android|macos|windows [VAR=value ...]" + @echo "make combine" + @echo "make rename apple|android" + @echo + @echo " build gn gen + ninja" + @echo " test build and run tests" + @echo " package assemble artifacts (xcframework, aar, zip)" + @echo " combine merge PRODUCTS/*/WebRTC.xcframework into one xcframework" + @echo " rename copy+rebrand for wrapper SDKs (original untouched)" + @echo + @echo " CONFIG=release (default, is_debug=false) or CONFIG=debug" + @echo " tests always gn-gen with debug, ignoring CONFIG" + @echo " SKIP_DEPS=1 skips gclient sync only; build/test/package still run" + @echo " SKIP_LICENSES=1 skips license generation only; lipo/zip still run" + @echo " SKIP_MACCATALYST=1 drops catalyst-arm64 catalyst-x64 from ios" + @echo + @echo " make package ios" + @echo " make package macos" + @echo " make combine # merges whatever platform dirs exist under PRODUCTS" + @echo " make rename apple XCFRAMEWORK=path/to/WebRTC.xcframework" + @echo " make rename android AAR=path/to/libwebrtc.aar" + @echo " make build package ios compile then pack" + @echo " make build android ARCHS=arm64-v8a" + @echo " make build ios GN_ARGS='rtc_use_h264=false' CONFIG=debug" + @echo " make build ios DEPS_ROOT=/tmp/webrtc-deps" + @echo " make build ios SKIP_MACCATALYST=1" + @echo + @echo "Also: deps runhooks check clean print-gn-args TARGET=slice" + @echo "Vars: DEPS_ROOT WEBRTC_SRC OUT PRODUCTS CONFIG GN_ARGS JOBS ARCHS ZIP XCFRAMEWORK AAR RENAMED SKIP_DEPS SKIP_LICENSES SKIP_MACCATALYST" + +check: + @$(SCRIPTS)/check.sh + +deps: + @$(SCRIPTS)/deps.sh sync + +runhooks: + @$(SCRIPTS)/deps.sh runhooks + +print-gn-args: + @test -n "$(TARGET)" || { echo "TARGET is required (slice name)"; exit 1; } + @$(SCRIPTS)/gn-gen.sh --print --slice "$(TARGET)" --config "$(CONFIG)" --extra "$(ALL_GN_ARGS)" + +require-src: + @test -n "$(WEBRTC_SRC)" || { echo "WEBRTC_SRC is required"; exit 1; } + @test -f "$(WEBRTC_SRC)/DEPS" || { echo "No WebRTC checkout at $(WEBRTC_SRC)"; exit 1; } + +require-darwin: + @test "$$(uname -s)" = Darwin || { echo "Apple targets require macOS"; exit 1; } + +require-linux: + @test "$$(uname -s)" = Linux || { echo "Android builds require Linux"; exit 1; } + +require-windows: + @case "$$(uname -s)" in MINGW*|MSYS*|CYGWIN*) exit 0 ;; esac; \ + test "$${OS}" = Windows_NT || { echo "Windows targets require Windows"; exit 1; } + +maybe-deps: +ifneq ($(SKIP_DEPS),1) + $(MAKE) deps +endif + +$(PLATFORMS) apple: + @: + +announce: + @effective="$(CONFIG)"; \ + note=""; \ + if [[ "$(VERB)" == test ]]; then \ + effective=debug; \ + note=" (tests always debug)"; \ + fi; \ + echo "==> $(strip $(VERB) $(PLATFORM))"; \ + echo " config: $$effective$$note"; \ + echo " deps_root: $(DEPS_ROOT)"; \ + echo " src: $(WEBRTC_SRC)"; \ + echo " out: $(OUT)"; \ + products="$(PRODUCTS)"; \ + case "$(PLATFORM)" in \ + ios) products="$(PRODUCTS)/ios" ;; \ + macos) products="$(PRODUCTS)/macos" ;; \ + esac; \ + echo " products: $$products"; \ + echo " jobs: $(JOBS)"; \ + echo " skip_deps: $(SKIP_DEPS)"; \ + echo " skip_licenses: $(SKIP_LICENSES)"; \ + echo " skip_maccatalyst: $(SKIP_MACCATALYST)"; \ + if [[ -n "$(ALL_GN_ARGS)" ]]; then echo " gn_args: $(ALL_GN_ARGS)"; fi; \ + case "$(PLATFORM)" in \ + ios) echo " slices: $(ios_slices)" ;; \ + macos) echo " slices: $(MACOS_SLICES)" ;; \ + android) \ + if [[ "$(VERB)" != rename ]]; then echo " slices: $(android_slices)"; fi ;; \ + windows) echo " slices: $(windows_slices)" ;; \ + esac; \ + if [[ "$(VERB)" == test ]]; then \ + case "$(PLATFORM)" in \ + ios) echo " tests: $(if $(TEST_TARGETS),$(TEST_TARGETS),$(IOS_TEST_TARGETS))" ;; \ + macos) echo " tests: $(if $(TEST_TARGETS),$(TEST_TARGETS),$(MACOS_TEST_TARGETS))" ;; \ + windows) echo " tests: $(if $(TEST_TARGETS),$(TEST_TARGETS),$(WINDOWS_TEST_TARGETS))" ;; \ + esac; \ + fi; \ + if [[ "$(VERB)" == rename ]]; then \ + case "$(PLATFORM)" in \ + apple) \ + echo " input: $(XCFRAMEWORK)"; \ + echo " output: $(RENAMED)/StreamWebRTC.xcframework" ;; \ + android) \ + echo " input: $(AAR)"; \ + echo " output: $(RENAMED)/libwebrtc.aar" ;; \ + esac; \ + fi; \ + if [[ "$$(uname -s)" == Darwin ]]; then \ + echo " xcode: $$(xcodebuild -version 2>/dev/null | paste -sd ' ' -)"; \ + echo " developer: $${DEVELOPER_DIR:-$$(xcode-select -p 2>/dev/null)}"; \ + fi + +build test package: + @test -n "$(PLATFORM)" || { echo "usage: make $@ ios|android|macos|windows"; exit 1; } + @$(MAKE) --no-print-directory announce VERB=$@ PLATFORM="$(PLATFORM)" + @$(MAKE) --no-print-directory $@-$(PLATFORM) + +combine: require-darwin require-src + @$(MAKE) --no-print-directory announce VERB=combine PLATFORM= + @$(SCRIPTS)/combine-apple.sh \ + --src "$(WEBRTC_SRC)" \ + --out "$(OUT)" \ + --products "$(PRODUCTS)" \ + $(if $(filter 1,$(ZIP)),--zip,) + +rename: + @target="$(firstword $(filter apple android,$(MAKECMDGOALS)))"; \ + test -n "$$target" || { echo "usage: make rename apple|android"; exit 1; }; \ + $(MAKE) --no-print-directory announce VERB=rename PLATFORM="$$target"; \ + $(MAKE) --no-print-directory rename-$$target + +rename-apple: require-darwin + @$(SCRIPTS)/rename-apple.sh \ + --src "$(XCFRAMEWORK)" \ + --dest "$(RENAMED)" + +rename-android: + @$(SCRIPTS)/rename-android.sh \ + --src "$(AAR)" \ + --dest "$(RENAMED)" + +gen: require-src + @test -n "$(TARGET)" || { echo "TARGET is required (slice name)"; exit 1; } + @$(SCRIPTS)/gn-gen.sh \ + --src "$(WEBRTC_SRC)" \ + --out "$(OUT)/$(TARGET)" \ + --slice "$(TARGET)" \ + --config "$(CONFIG)" \ + --extra "$(ALL_GN_ARGS)" + +ninja: require-src + @test -n "$(TARGET)" || { echo "TARGET is required (slice name)"; exit 1; } + @ninja_bin="$$($(SCRIPTS)/common.sh ninja "$(WEBRTC_SRC)")"; \ + ninja_target="$(NINJA_TARGET)"; \ + if [[ -z "$$ninja_target" ]]; then \ + ninja_target="$$($(SCRIPTS)/gn-gen.sh --ninja-target "$(TARGET)")"; \ + fi; \ + echo "$$ninja_bin -C $(OUT)/$(TARGET) $$ninja_target"; \ + "$$ninja_bin" -C "$(OUT)/$(TARGET)" $$ninja_target -j"$(JOBS)" + +slice: gen ninja + +$(addprefix build-slice-,$(IOS_SLICES) $(MACOS_SLICES) $(ANDROID_SLICES) $(WINDOWS_SLICES)): build-slice-%: require-src + @$(MAKE) --no-print-directory slice TARGET=$* + +build-ios: require-darwin maybe-deps require-src $(addprefix build-slice-,$(ios_slices)) +build-macos: require-darwin maybe-deps require-src $(addprefix build-slice-,$(MACOS_SLICES)) +build-android: require-linux maybe-deps require-src $(addprefix build-slice-,$(android_slices)) +build-windows: require-windows maybe-deps require-src $(addprefix build-slice-,$(windows_slices)) + +package-ios: require-darwin require-src + @$(SCRIPTS)/package-apple.sh \ + --src "$(WEBRTC_SRC)" \ + --out "$(OUT)" \ + --products "$(PRODUCTS)/ios" \ + --slices "$(ios_slices)" \ + $(if $(filter 1,$(ZIP)),--zip,) + +package-macos: require-darwin require-src + @$(SCRIPTS)/package-apple.sh \ + --src "$(WEBRTC_SRC)" \ + --out "$(OUT)" \ + --products "$(PRODUCTS)/macos" \ + --slices "$(MACOS_SLICES)" \ + $(if $(filter 1,$(ZIP)),--zip,) + +package-android: require-linux require-src + @$(SCRIPTS)/package-android.sh \ + --src "$(WEBRTC_SRC)" \ + --out "$(OUT)" \ + --products "$(PRODUCTS)" \ + --slices "$(android_slices)" + +package-windows: require-windows require-src + @$(SCRIPTS)/package-windows.sh \ + --out "$(OUT)" \ + --products "$(PRODUCTS)" \ + --slices "$(windows_slices)" \ + $(if $(filter 1,$(ZIP)),--zip,) + +test-ios: require-darwin maybe-deps require-src + @$(SCRIPTS)/gn-gen.sh \ + --src "$(WEBRTC_SRC)" \ + --out "$(OUT)/ios_tests" \ + --config debug \ + --overlay ios-test \ + --extra "$(ALL_GN_ARGS)" + @ninja_bin="$$($(SCRIPTS)/common.sh ninja "$(WEBRTC_SRC)")"; \ + targets="$(if $(TEST_TARGETS),$(TEST_TARGETS),$(IOS_TEST_TARGETS))"; \ + echo "$$ninja_bin -C $(OUT)/ios_tests $$targets"; \ + "$$ninja_bin" -C "$(OUT)/ios_tests" $$targets -j"$(JOBS)" + @$(SCRIPTS)/run-ios-tests.sh \ + --build-dir "$(OUT)/ios_tests" \ + --targets "$(if $(TEST_TARGETS),$(TEST_TARGETS),$(IOS_TEST_TARGETS))" \ + $(if $(SIMULATOR_PLATFORM),--platform "$(SIMULATOR_PLATFORM)",) \ + $(if $(SIMULATOR_VERSION),--version "$(SIMULATOR_VERSION)",) \ + $(if $(EXTRA_ARGS),--extra "$(EXTRA_ARGS)",) + +test-macos: require-darwin maybe-deps require-src + @$(SCRIPTS)/gn-gen.sh \ + --src "$(WEBRTC_SRC)" \ + --out "$(OUT)/webrtc_tests" \ + --config debug \ + --overlay macos-test \ + --extra "$(ALL_GN_ARGS)" + @ninja_bin="$$($(SCRIPTS)/common.sh ninja "$(WEBRTC_SRC)")"; \ + targets="$(if $(TEST_TARGETS),$(TEST_TARGETS),$(MACOS_TEST_TARGETS))"; \ + echo "$$ninja_bin -C $(OUT)/webrtc_tests $$targets"; \ + "$$ninja_bin" -C "$(OUT)/webrtc_tests" $$targets -j"$(JOBS)"; \ + filter="$$(tr -d '\n' < "$(SCRIPTS)/macos-gtest-filter.txt")"; \ + for target in $$targets; do \ + echo "$(OUT)/webrtc_tests/$$target"; \ + "$(OUT)/webrtc_tests/$$target" --gtest_filter="-$$filter" $(EXTRA_ARGS); \ + done + +test-android: + @echo "error: make test android is not wired (no device runner)." >&2 + @echo "use: make build android && make package android" >&2 + @exit 1 + +test-windows: require-windows maybe-deps require-src + @$(SCRIPTS)/gn-gen.sh \ + --src "$(WEBRTC_SRC)" \ + --out "$(OUT)/windows_tests" \ + --config debug \ + --overlay windows-test \ + --extra "$(ALL_GN_ARGS)" + @ninja_bin="$$($(SCRIPTS)/common.sh ninja "$(WEBRTC_SRC)")"; \ + targets="$(if $(TEST_TARGETS),$(TEST_TARGETS),$(WINDOWS_TEST_TARGETS))"; \ + echo "$$ninja_bin -C $(OUT)/windows_tests $$targets"; \ + "$$ninja_bin" -C "$(OUT)/windows_tests" $$targets -j"$(JOBS)"; \ + for target in $$targets; do \ + echo "$(OUT)/windows_tests/$$target"; \ + "$(OUT)/windows_tests/$$target" $(EXTRA_ARGS); \ + done + +clean: require-src + rm -rf "$(OUT)" "$(PRODUCTS)" diff --git a/stream_build/gn/android.args b/stream_build/gn/android.args new file mode 100644 index 0000000000..8ffc5dc8f0 --- /dev/null +++ b/stream_build/gn/android.args @@ -0,0 +1 @@ +android_static_analysis = "off" diff --git a/stream_build/gn/apple.args b/stream_build/gn/apple.args new file mode 100644 index 0000000000..8475612194 --- /dev/null +++ b/stream_build/gn/apple.args @@ -0,0 +1,7 @@ +# Overlay for Apple framework slices (iOS, Catalyst, macOS). +rtc_enable_objc_symbol_export = true +ios_enable_code_signing = false +enable_dsyms = true +enable_stripping = true +rtc_libvpx_build_vp9 = true +use_rtti = false diff --git a/stream_build/gn/common.args b/stream_build/gn/common.args new file mode 100644 index 0000000000..6ae84c674a --- /dev/null +++ b/stream_build/gn/common.args @@ -0,0 +1,10 @@ +# Stream policy defaults applied to every gn gen. +rtc_allow_deprecated_namespaces = true +stream_enable_rendering_backend = true +is_component_build = false +rtc_include_tests = false +rtc_build_examples = false +treat_warnings_as_errors = false +use_siso = false +use_remoteexec = false +use_reclient = false diff --git a/stream_build/gn/ios-test.args b/stream_build/gn/ios-test.args new file mode 100644 index 0000000000..80e23379b8 --- /dev/null +++ b/stream_build/gn/ios-test.args @@ -0,0 +1,10 @@ +target_os = "ios" +target_environment = "simulator" +target_cpu = "arm64" +ios_enable_code_signing = false +ios_deployment_target = "13.0" +rtc_include_tests = true +is_debug = true +use_siso = false +use_remoteexec = false +use_reclient = false diff --git a/stream_build/gn/macos-test.args b/stream_build/gn/macos-test.args new file mode 100644 index 0000000000..8e90e8f672 --- /dev/null +++ b/stream_build/gn/macos-test.args @@ -0,0 +1,6 @@ +target_os = "mac" +rtc_include_tests = true +is_debug = true +use_siso = false +use_remoteexec = false +use_reclient = false diff --git a/stream_build/gn/slices.tsv b/stream_build/gn/slices.tsv new file mode 100644 index 0000000000..2610d1c344 --- /dev/null +++ b/stream_build/gn/slices.tsv @@ -0,0 +1,14 @@ +# name ninja_target gn_args (space-separated key=value) +ios-arm64-device framework_objc target_os="ios" target_environment="device" target_cpu="arm64" ios_deployment_target="13.0" +ios-arm64-simulator framework_objc target_os="ios" target_environment="simulator" target_cpu="arm64" ios_deployment_target="13.0" +ios-x64-simulator framework_objc target_os="ios" target_environment="simulator" target_cpu="x64" ios_deployment_target="13.0" +catalyst-arm64 framework_objc target_os="ios" target_environment="catalyst" target_cpu="arm64" ios_deployment_target="14.0" use_lld=false +catalyst-x64 framework_objc target_os="ios" target_environment="catalyst" target_cpu="x64" ios_deployment_target="14.0" use_lld=false +macos-arm64 mac_framework_objc target_os="mac" target_cpu="arm64" +macos-x64 mac_framework_objc target_os="mac" target_cpu="x64" +android-armeabi-v7a sdk/android:libwebrtc sdk/android:libjingle_peerconnection_so target_os="android" target_cpu="arm" arm_version=7 +android-arm64-v8a sdk/android:libwebrtc sdk/android:libjingle_peerconnection_so target_os="android" target_cpu="arm64" +android-x86 sdk/android:libwebrtc sdk/android:libjingle_peerconnection_so target_os="android" target_cpu="x86" +android-x86_64 sdk/android:libwebrtc sdk/android:libjingle_peerconnection_so target_os="android" target_cpu="x64" +windows-x64 webrtc target_os="win" target_cpu="x64" +windows-arm64 webrtc target_os="win" target_cpu="arm64" diff --git a/stream_build/gn/windows-test.args b/stream_build/gn/windows-test.args new file mode 100644 index 0000000000..69ba285fde --- /dev/null +++ b/stream_build/gn/windows-test.args @@ -0,0 +1,6 @@ +target_os = "win" +rtc_include_tests = true +is_debug = true +use_siso = false +use_remoteexec = false +use_reclient = false diff --git a/stream_build/gn/windows.args b/stream_build/gn/windows.args new file mode 100644 index 0000000000..d11596de9e --- /dev/null +++ b/stream_build/gn/windows.args @@ -0,0 +1 @@ +# Overlay for Windows ninja slices. diff --git a/stream_build/scripts/check.sh b/stream_build/scripts/check.sh new file mode 100755 index 0000000000..8eef8ce7c6 --- /dev/null +++ b/stream_build/scripts/check.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Sanity check for the Makefile wrapper (no WebRTC tree required). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +help_text="$(make -s help)" +[[ "$help_text" == *"make build|test|package"* ]] +printf '%s\n' "$help_text" | grep -q 'CONFIG=release (default' +printf '%s\n' "$help_text" | grep -q 'make combine' +printf '%s\n' "$help_text" | grep -q 'make rename apple' +printf '%s\n' "$help_text" | grep -q 'SKIP_MACCATALYST=1' +printf '%s\n' "$help_text" | grep -q 'make package ios' +printf '%s\n' "$help_text" | grep -q 'make package macos' +! printf '%s\n' "$help_text" | grep -q 'package apple' + +usage="$(make build 2>&1 || true)" +printf '%s\n' "$usage" | grep -q 'usage: make build' +! printf '%s\n' "$usage" | grep -q '|apple' + +text="$(make -s print-gn-args TARGET=ios-arm64-device CONFIG=release)" +printf '%s\n' "$text" | grep -q 'stream_enable_rendering_backend = true' +printf '%s\n' "$text" | grep -q 'target_os = "ios"' +printf '%s\n' "$text" | grep -q 'is_debug = false' + +debug="$(make -s print-gn-args TARGET=macos-arm64 CONFIG=debug GN_ARGS='rtc_use_h264=false')" +printf '%s\n' "$debug" | grep -q 'is_debug = true' +printf '%s\n' "$debug" | grep -q 'target_os = "mac"' +printf '%s\n' "$debug" | grep -q 'rtc_use_h264 = false' + +android="$(make -s print-gn-args TARGET=android-arm64-v8a)" +printf '%s\n' "$android" | grep -q 'target_os = "android"' +printf '%s\n' "$android" | grep -q 'target_cpu = "arm64"' + +ninja_target="$("$ROOT/scripts/gn-gen.sh" --ninja-target ios-arm64-device)" +[[ "$ninja_target" == framework_objc ]] + +banner="$(make --no-print-directory announce VERB=build PLATFORM=ios CONFIG=release)" +printf '%s\n' "$banner" | grep -q '==> build ios' +printf '%s\n' "$banner" | grep -q 'config: release' +printf '%s\n' "$banner" | grep -q 'deps_root:' +printf '%s\n' "$banner" | grep -q '/ios$' +printf '%s\n' "$banner" | grep 'slices:' | grep -q 'catalyst-arm64' +printf '%s\n' "$banner" | grep 'slices:' | grep -q 'catalyst-x64' + +ios_skip="$(make --no-print-directory announce VERB=package PLATFORM=ios SKIP_MACCATALYST=1)" +printf '%s\n' "$ios_skip" | grep -q 'skip_maccatalyst: 1' +printf '%s\n' "$ios_skip" | grep 'slices:' | grep -q 'ios-arm64-device' +! printf '%s\n' "$ios_skip" | grep 'slices:' | grep -q 'catalyst' + +macos_banner="$(make --no-print-directory announce VERB=package PLATFORM=macos SKIP_MACCATALYST=1)" +printf '%s\n' "$macos_banner" | grep -q '/macos$' +printf '%s\n' "$macos_banner" | grep 'slices:' | grep -q 'macos-arm64' +printf '%s\n' "$macos_banner" | grep 'slices:' | grep -q 'macos-x64' + +test_banner="$(make --no-print-directory announce VERB=test PLATFORM=macos CONFIG=release)" +printf '%s\n' "$test_banner" | grep -q '==> test macos' +printf '%s\n' "$test_banner" | grep -q 'config: debug (tests always debug)' + +empty="$(mktemp -d)" +combine_none="$(make combine PRODUCTS="$empty" SKIP_LICENSES=1 2>&1 || true)" +printf '%s\n' "$combine_none" | grep -q 'no WebRTC.xcframework' +rm -rf "$empty" + +one="$(mktemp -d)" +mkdir -p "$one/ios/WebRTC.xcframework" +printf 'stub\n' > "$one/ios/WebRTC.xcframework/Info.plist" +make combine PRODUCTS="$one" SKIP_LICENSES=1 +[[ -f "$one/WebRTC.xcframework/Info.plist" ]] +rm -rf "$one" + +rename_root="$(mktemp -d)" +rename_src="$rename_root/WebRTC.xcframework" +mkdir -p "$rename_src/ios-arm64/WebRTC.framework/Headers" +mkdir -p "$rename_src/ios-arm64/WebRTC.framework/Modules" +printf '%s\n' 'CFBundleNameWebRTC' \ + > "$rename_src/Info.plist" +printf '%s\n' 'framework module WebRTC { umbrella header "WebRTC.h" }' \ + > "$rename_src/ios-arm64/WebRTC.framework/Modules/module.modulemap" +printf '%s\n' '#import ' \ + > "$rename_src/ios-arm64/WebRTC.framework/Headers/WebRTC.h" +printf 'stub\n' > "$rename_src/ios-arm64/WebRTC.framework/WebRTC" +printf '%s\n' '' \ + > "$rename_src/ios-arm64/WebRTC.framework/Info.plist" +rename_out="$(mktemp -d)" +make rename apple XCFRAMEWORK="$rename_src" RENAMED="$rename_out" +[[ -d "$rename_src/ios-arm64/WebRTC.framework" ]] +[[ -d "$rename_out/StreamWebRTC.xcframework/ios-arm64/StreamWebRTC.framework" ]] +grep -q 'StreamWebRTC' "$rename_out/StreamWebRTC.xcframework/ios-arm64/StreamWebRTC.framework/Modules/module.modulemap" +grep -q 'import "$aar_dir/libwebrtc.aar" +make rename android AAR="$aar_dir/libwebrtc.aar" RENAMED="$aar_dir/renamed" +[[ -f "$aar_dir/libwebrtc.aar" ]] +[[ -f "$aar_dir/renamed/libwebrtc.aar" ]] +cmp -s "$aar_dir/libwebrtc.aar" "$aar_dir/renamed/libwebrtc.aar" +rm -rf "$aar_dir" + +deps_tmp="$(mktemp -d)" +fake_bin="$deps_tmp/bin" +mkdir -p "$fake_bin" "$deps_tmp/src_repo" "$deps_tmp/deps/.gclient-git-cache" +printf 'hooks = []\n' > "$deps_tmp/src_repo/DEPS" +cat > "$fake_bin/gclient" <<'FAKE' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "${FAKE_GCLIENT_LOG}" +if [[ "${1:-}" == sync ]]; then + printf '%s\n' "$*" > "${FAKE_GCLIENT_SYNC_ARGS}" + printf '%s\n' "${GIT_CACHE_PATH-}" > "${FAKE_GCLIENT_CACHE}" +fi +exit 0 +FAKE +chmod +x "$fake_bin/gclient" +export FAKE_GCLIENT_LOG="$deps_tmp/log" +export FAKE_GCLIENT_SYNC_ARGS="$deps_tmp/sync_args" +export FAKE_GCLIENT_CACHE="$deps_tmp/cache_env" +deps_out="$( + PATH="$fake_bin:$PATH" \ + DEPS_ROOT="$deps_tmp/deps" \ + WEBRTC_SRC="$deps_tmp/src_repo" \ + GIT_CACHE_PATH="$deps_tmp/deps/.gclient-git-cache" \ + RUN_HOOKS=0 JOBS=2 \ + "$ROOT/scripts/deps.sh" sync +)" +printf '%s\n' "$deps_out" | grep -q 'will not reset it' +printf '%s\n' "$deps_out" | grep -q 'running: gclient sync -j2 --nohooks' +! printf '%s\n' "$deps_out" | grep -q -- '--revision' +grep -q '"managed": False' "$deps_tmp/deps/.gclient" +! grep -q '"revision"' "$deps_tmp/deps/.gclient" +grep -q 'sync -j2 --nohooks' "$deps_tmp/sync_args" +! grep -q -- '--revision' "$deps_tmp/sync_args" +grep -q "$deps_tmp/deps/.gclient-git-cache" "$deps_tmp/cache_env" +[[ -L "$deps_tmp/deps/src" ]] +if PATH="$fake_bin:$PATH" \ + DEPS_ROOT="$deps_tmp/deps" \ + WEBRTC_SRC="$deps_tmp/src_repo" \ + WEBRTC_REVISION=eeff9252f32a40d1671974c31c096ce9fa776130 \ + "$ROOT/scripts/deps.sh" sync 2>"$deps_tmp/pin_err"; then + echo "expected pin to fail on symlink src" >&2 + exit 1 +fi +grep -q 'src is your git checkout; will not reset it' "$deps_tmp/pin_err" +rm -rf "$deps_tmp" + +echo "ok" diff --git a/stream_build/scripts/combine-apple.sh b/stream_build/scripts/combine-apple.sh new file mode 100755 index 0000000000..491ae2dcba --- /dev/null +++ b/stream_build/scripts/combine-apple.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Discover platform xcframeworks under PRODUCTS/*/ and emit one WebRTC.xcframework. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +require_darwin +require_cmd xcodebuild + +SRC="${WEBRTC_SRC:-}" +OUT="" +PRODUCTS="" +NAME="WebRTC" +ZIP=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --src) SRC="$2"; shift 2 ;; + --out) OUT="$2"; shift 2 ;; + --products) PRODUCTS="$2"; shift 2 ;; + --name) NAME="$2"; shift 2 ;; + --zip) ZIP=1; shift ;; + *) die "combine-apple.sh: unknown flag $1" ;; + esac +done + +[[ -n "$PRODUCTS" ]] || die "combine-apple.sh requires --products" + +shopt -s nullglob +found=() +for xcf in "$PRODUCTS"/*/"${NAME}.xcframework"; do + [[ -d "$xcf" ]] && found+=("$xcf") +done +shopt -u nullglob + +if [[ ${#found[@]} -eq 0 ]]; then + die "no ${NAME}.xcframework under ${PRODUCTS}/*/ — run make package ios and/or make package macos" +fi + +echo "combine: found ${#found[@]} platform xcframework(s):" +for xcf in "${found[@]}"; do + echo " $xcf" +done + +dest="${PRODUCTS}/${NAME}.xcframework" +rm -rf "$dest" + +if [[ ${#found[@]} -eq 1 ]]; then + echo "combine: one platform — copying ${found[0]} -> $dest" + cp -R "${found[0]}" "$dest" +else + require_cmd find + xc_args=(-create-xcframework) + added=0 + while IFS= read -r fw; do + [[ -d "$fw" ]] || continue + xc_args+=(-framework "$fw") + dsym="" + if [[ -d "${fw}.dSYM" ]]; then + dsym="${fw}.dSYM" + elif [[ -d "$(dirname "$fw")/dSYMs/$(basename "$fw").dSYM" ]]; then + dsym="$(dirname "$fw")/dSYMs/$(basename "$fw").dSYM" + fi + if [[ -n "$dsym" ]]; then + xc_args+=(-debug-symbols "$dsym") + fi + added=1 + done < <(find "${found[@]}" -name '*.framework' -type d | sort) + [[ "$added" -eq 1 ]] || die "no .framework slices inside: ${found[*]}" + xc_args+=(-output "$dest") + echo "xcodebuild ${xc_args[*]}" + xcodebuild "${xc_args[@]}" +fi + +if [[ "${SKIP_LICENSES:-0}" == 1 ]]; then + echo "skipping license generation (SKIP_LICENSES=1)" +else + [[ -n "$SRC" ]] || die "combine-apple.sh requires --src (or WEBRTC_SRC) to generate licenses" + [[ -n "$OUT" ]] || die "combine-apple.sh requires --out to generate licenses" + require_cmd python3 + license_script="$SRC/tools_webrtc/libs/generate_licenses.py" + [[ -f "$license_script" ]] || die "missing $license_script" + + gn_targets=() + build_dirs=() + seen_ios=0 + seen_macos=0 + for xcf in "${found[@]}"; do + platform="$(basename "$(dirname "$xcf")")" + case "$platform" in + macos) + if [[ "$seen_macos" -eq 0 ]]; then + gn_targets+=(--target "//sdk:mac_framework_objc") + seen_macos=1 + fi + for dir in "$OUT"/macos-*; do + [[ -d "$dir" ]] && build_dirs+=("$dir") + done + ;; + *) + if [[ "$seen_ios" -eq 0 ]]; then + gn_targets+=(--target "//sdk:framework_objc") + seen_ios=1 + fi + if [[ "$platform" == ios ]]; then + for dir in "$OUT"/ios-* "$OUT"/catalyst-*; do + [[ -d "$dir" ]] && build_dirs+=("$dir") + done + else + for dir in "$OUT/${platform}-"*; do + [[ -d "$dir" ]] && build_dirs+=("$dir") + done + fi + ;; + esac + done + [[ ${#gn_targets[@]} -gt 0 ]] || die "no license GN targets for: ${found[*]}" + [[ ${#build_dirs[@]} -gt 0 ]] || die "no slice out dirs under $OUT for license generation" + echo "python3 $license_script ${gn_targets[*]} $dest ${build_dirs[*]}" + python3 "$license_script" "${gn_targets[@]}" "$dest" "${build_dirs[@]}" + echo "wrote ${dest}/LICENSE.md" +fi + +if [[ "$ZIP" -eq 1 ]]; then + if command -v ditto >/dev/null 2>&1; then + ditto -c -k --sequesterRsrc --keepParent \ + "$dest" \ + "${PRODUCTS}/${NAME}.xcframework.zip" + else + ( + cd "$PRODUCTS" + zip --symlinks -r "${NAME}.xcframework.zip" "${NAME}.xcframework" + ) + fi +fi + +echo "wrote $dest" diff --git a/stream_build/scripts/common.sh b/stream_build/scripts/common.sh new file mode 100755 index 0000000000..4701e47a73 --- /dev/null +++ b/stream_build/scripts/common.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# Shared helpers for the WebRTC Makefile wrapper. +set -euo pipefail + +PIPELINE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GN_DIR="${PIPELINE_DIR}/gn" + +die() { + echo "error: $*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "required tool '$1' not found in PATH" +} + +host_uname() { + uname -s +} + +require_darwin() { + [[ "$(host_uname)" == Darwin ]] || die "Apple targets require macOS" +} + +require_linux() { + [[ "$(host_uname)" == Linux ]] || die "Android AAR builds require Linux" +} + +require_windows() { + case "$(host_uname)" in + MINGW*|MSYS*|CYGWIN*) return 0 ;; + esac + [[ "${OS:-}" == Windows_NT ]] || die "Windows targets require Windows" +} + +require_webrtc_src() { + local src="${1:-}" + [[ -n "$src" ]] || die "WEBRTC_SRC or WEBRTC_ROOT is required" + [[ -f "$src/DEPS" ]] || die "No WebRTC checkout at $src (missing DEPS)" +} + +abspath() { + local path="$1" + (cd "$(dirname "$path")" && printf '%s/%s\n' "$(pwd)" "$(basename "$path")") +} + +resolve_gn() { + local src="${1:-}" + local os + os="$(host_uname)" + local candidate="" + case "$os" in + Darwin) candidate="$src/buildtools/mac/gn" ;; + Linux) candidate="$src/buildtools/linux64/gn" ;; + MINGW*|MSYS*|CYGWIN*) candidate="$src/buildtools/win/gn.exe" ;; + esac + if [[ -n "$candidate" && -x "$candidate" ]]; then + printf '%s\n' "$candidate" + return + fi + command -v gn +} + +resolve_ninja() { + local src="${1:-}" + local bundled="$src/third_party/ninja/ninja" + if [[ -x "$bundled" ]]; then + printf '%s\n' "$bundled" + return + fi + command -v ninja +} + +is_apple_slice() { + local name="$1" + [[ "$name" == ios-* || "$name" == catalyst-* || "$name" == macos-* ]] +} + +is_android_slice() { + local name="$1" + [[ "$name" == android-* ]] +} + +is_windows_slice() { + local name="$1" + [[ "$name" == windows-* ]] +} + +slice_line() { + local name="$1" + local line + line="$(awk -F'\t' -v n="$name" '$1 == n { print; exit }' "${GN_DIR}/slices.tsv")" + [[ -n "$line" ]] || die "unknown slice '$name' (see gn/slices.tsv)" + printf '%s\n' "$line" +} + +slice_ninja_target() { + local name="$1" + slice_line "$name" | awk -F'\t' '{ print $2 }' +} + +slice_gn_args() { + local name="$1" + slice_line "$name" | awk -F'\t' '{ print $3 }' +} + +# Convert "key=value" / "key = value" tokens into args.gn lines. +gn_tokens_to_lines() { + local token key value + for token in "$@"; do + [[ -z "$token" ]] && continue + key="${token%%=*}" + value="${token#*=}" + key="${key%"${key##*[![:space:]]}"}" + key="${key#"${key%%[![:space:]]*}"}" + value="${value#"${value%%[![:space:]]*}"}" + printf '%s = %s\n' "$key" "$value" + done +} + +cat_gn_file() { + local path="$1" + [[ -f "$path" ]] || die "GN args file not found: $path" + grep -v '^[[:space:]]*#' "$path" | grep -v '^[[:space:]]*$' || true +} + +# Compose args.gn content. Reads env/flags via positional: +# compose_gn_args --config release --slice NAME --overlay FILE --extra TOKENS +compose_gn_args() { + local config="release" + local slice="" + local extra="" + local overlays=() + + while [[ $# -gt 0 ]]; do + case "$1" in + --config) config="$2"; shift 2 ;; + --slice) slice="$2"; shift 2 ;; + --overlay) + overlays+=("$2") + shift 2 + ;; + --extra) extra="${2:-}"; shift 2 ;; + *) die "compose_gn_args: unknown flag $1" ;; + esac + done + + cat_gn_file "${GN_DIR}/common.args" + if [[ -n "$slice" ]] && is_apple_slice "$slice"; then + cat_gn_file "${GN_DIR}/apple.args" + fi + if [[ -n "$slice" ]] && is_android_slice "$slice"; then + cat_gn_file "${GN_DIR}/android.args" + fi + if [[ -n "$slice" ]] && is_windows_slice "$slice"; then + cat_gn_file "${GN_DIR}/windows.args" + fi + local overlay + for overlay in "${overlays[@]+"${overlays[@]}"}"; do + [[ -z "$overlay" ]] && continue + if [[ -f "$overlay" ]]; then + cat_gn_file "$overlay" + elif [[ -f "${GN_DIR}/${overlay}.args" ]]; then + cat_gn_file "${GN_DIR}/${overlay}.args" + else + die "unknown GN overlay '$overlay'" + fi + done + if [[ "$config" == debug ]]; then + echo 'is_debug = true' + else + echo 'is_debug = false' + fi + if [[ -n "$slice" ]]; then + # shellcheck disable=SC2086 + gn_tokens_to_lines $(slice_gn_args "$slice") + fi + if [[ -n "$extra" ]]; then + # shellcheck disable=SC2086 + gn_tokens_to_lines $extra + fi +} + +flatten_gn_args() { + awk ' + /^[[:space:]]*$/ { next } + /^[[:space:]]*#/ { next } + { + line = $0 + sub(/^[[:space:]]+/, "", line) + sub(/[[:space:]]+$/, "", line) + split(line, parts, " = ") + if (length(parts) >= 2) { + value = substr(line, index(line, " = ") + 3) + printf "%s=%s\n", parts[1], value + } + } + ' +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + cmd="${1:-}" + shift || true + case "$cmd" in + gn) resolve_gn "${1:-}" ;; + ninja) resolve_ninja "${1:-}" ;; + slice-ninja) slice_ninja_target "${1:-}" ;; + *) die "usage: common.sh gn|ninja|slice-ninja ..." ;; + esac +fi diff --git a/stream_build/scripts/deps.sh b/stream_build/scripts/deps.sh new file mode 100755 index 0000000000..50e48a807c --- /dev/null +++ b/stream_build/scripts/deps.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# gclient config + sync for GetStream/webrtc. +# DEPS_ROOT is the gclient parent. src there is a symlink to WEBRTC_SRC (the git root). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +usage() { + cat <<'EOF' +usage: deps.sh sync|runhooks + +env: + DEPS_ROOT gclient parent (required). Default from make: /.gclient_deps + WEBRTC_SRC git root that contains DEPS (required) + TARGET_OS comma/space list, default: ios + JOBS gclient -j, default: 8 + RUN_HOOKS 1 to run hooks during sync, 0 for --nohooks + WEBRTC_REPO default: git@github.com:GetStream/webrtc.git + WEBRTC_REVISION pin src SHA only when src is a real clone gclient owns + WEBRTC_REF resolve SHA via git ls-remote when REVISION is empty + (refused when src is a symlink to this git checkout) +EOF +} + +DEPS_ROOT="${DEPS_ROOT:-${WEBRTC_ROOT:-}}" +WEBRTC_SRC="${WEBRTC_SRC:-}" +TARGET_OS="${TARGET_OS:-ios}" +JOBS="${JOBS:-8}" +RUN_HOOKS="${RUN_HOOKS:-1}" +WEBRTC_REPO="${WEBRTC_REPO:-git@github.com:GetStream/webrtc.git}" +WEBRTC_REVISION="${WEBRTC_REVISION:-}" +WEBRTC_REF="${WEBRTC_REF:-}" + +git_sha() { + [[ "$1" =~ ^[0-9a-fA-F]{7,40}$ ]] +} + +resolve_revision() { + if [[ -n "$WEBRTC_REVISION" ]]; then + printf '%s\n' "$WEBRTC_REVISION" + return + fi + if [[ -n "$WEBRTC_REF" ]]; then + if git_sha "$WEBRTC_REF"; then + printf '%s\n' "$WEBRTC_REF" + return + fi + local sha="" + local pattern + for pattern in "$WEBRTC_REF" "refs/heads/${WEBRTC_REF}" "refs/tags/${WEBRTC_REF}"; do + sha="$(git ls-remote --exit-code "$WEBRTC_REPO" "$pattern" 2>/dev/null | awk '{ print $1; exit }' || true)" + if [[ -n "$sha" ]]; then + printf '%s\n' "$sha" + return + fi + done + die "unable to resolve WEBRTC_REF='$WEBRTC_REF' from $WEBRTC_REPO" + fi + if [[ -n "$WEBRTC_SRC" && -d "$WEBRTC_SRC/.git" ]]; then + git -C "$WEBRTC_SRC" rev-parse HEAD + return + fi + printf '\n' +} + +quote_target_os() { + local raw="$1" + local os first=1 + printf '[' + # shellcheck disable=SC2086 + for os in ${raw//,/ }; do + [[ -z "$os" ]] && continue + if [[ $first -eq 1 ]]; then + first=0 + else + printf ', ' + fi + printf '"%s"' "$os" + done + printf ']\n' +} + +write_gclient() { + local dest="$1" + local revision="$2" + local revision_line="" + if [[ -n "$revision" ]]; then + revision_line=$'\n "revision": "'"${revision}"'",' + fi + cat >"${dest}/.gclient" </.gclient_deps; otherwise an absolute path. +src_link_target() { + local deps_abs src_abs + deps_abs="$(cd "$DEPS_ROOT" && pwd)" + src_abs="$(cd "$WEBRTC_SRC" && pwd)" + if [[ "$(cd "$DEPS_ROOT/.." && pwd)" == "$src_abs" && "$(basename "$DEPS_ROOT")" == ".gclient_deps" ]]; then + printf '..\n' + else + printf '%s\n' "$src_abs" + fi +} + +ensure_src_symlink() { + [[ -n "$DEPS_ROOT" ]] || die "DEPS_ROOT is required" + [[ -n "$WEBRTC_SRC" ]] || die "WEBRTC_SRC is required" + [[ -f "$WEBRTC_SRC/DEPS" ]] || die "No WebRTC checkout at $WEBRTC_SRC (missing DEPS)" + mkdir -p "$DEPS_ROOT" + local link="$DEPS_ROOT/src" + local target + target="$(src_link_target)" + if [[ -L "$link" ]]; then + ln -sfn "$target" "$link" + elif [[ -e "$link" ]]; then + die "$link exists and is not a symlink (refusing a nested webrtc checkout)" + else + ln -sfn "$target" "$link" + fi + [[ -f "$link/DEPS" ]] || die "symlink $link does not point at a WebRTC tree" +} + +# True when DEPS_ROOT/src is a symlink to this git checkout, not a clone +# gclient owns. gclient must not checkout/reset/clean that tree. +src_is_developer_symlink() { + local link="${DEPS_ROOT}/src" + [[ -L "$link" ]] || return 1 + [[ -n "$WEBRTC_SRC" ]] || return 0 + local link_abs src_abs + link_abs="$(cd "$link" && pwd -P)" + src_abs="$(cd "$WEBRTC_SRC" && pwd -P)" + [[ "$link_abs" == "$src_abs" ]] +} + +cmd_sync() { + require_cmd gclient + require_cmd git + require_cmd python3 + ensure_src_symlink + + local revision="" + if src_is_developer_symlink; then + if [[ -n "$WEBRTC_REVISION" || -n "$WEBRTC_REF" ]]; then + die "src is your git checkout; will not reset it" + fi + echo "src is a symlink to the git checkout; will not reset it" + else + revision="$(resolve_revision)" + if [[ -n "$revision" ]]; then + echo "pinning gclient src revision: $revision" + fi + fi + # DEPS checkouts use this cache as their git alternate/origin. Unsetting + # it makes gclient retarget them from the cache URL to googlesource. + if [[ -d "$DEPS_ROOT/.gclient-git-cache" ]]; then + export GIT_CACHE_PATH="$DEPS_ROOT/.gclient-git-cache" + fi + write_gclient "$DEPS_ROOT" "$revision" + + ( + cd "$DEPS_ROOT" + gclient root >/dev/null || true + local sync=(gclient sync -j"${JOBS}") + if [[ -n "$revision" ]]; then + sync+=(--revision "src@${revision}") + fi + if [[ "$RUN_HOOKS" == "0" || "$RUN_HOOKS" == "false" ]]; then + sync+=(--nohooks) + fi + echo "running: ${sync[*]}" + "${sync[@]}" + ) +} + +cmd_runhooks() { + require_cmd gclient + ensure_src_symlink + [[ -f "$DEPS_ROOT/.gclient" ]] || die "gclient config not found at $DEPS_ROOT/.gclient" + ( + cd "$DEPS_ROOT" + echo "running: gclient runhooks" + gclient runhooks + ) +} + +case "${1:-}" in + sync) cmd_sync ;; + runhooks) cmd_runhooks ;; + -h|--help|help) usage ;; + *) usage >&2; exit 1 ;; +esac diff --git a/stream_build/scripts/gn-gen.sh b/stream_build/scripts/gn-gen.sh new file mode 100755 index 0000000000..413b8a084d --- /dev/null +++ b/stream_build/scripts/gn-gen.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Write args.gn and run gn gen. Also prints composed args or a slice ninja target. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +usage() { + cat <<'EOF' +usage: gn-gen.sh --src DIR --out DIR [options] + gn-gen.sh --print [options] + gn-gen.sh --ninja-target SLICE + +options: + --config release|debug default: release + --slice NAME lookup gn/slices.tsv + --overlay NAME gn/NAME.args (repeatable) + --extra "k=v k2=v2" extra GN args + --flat with --print, emit key=value tokens +EOF +} + +SRC="" +OUT_DIR="" +CONFIG="release" +SLICE="" +EXTRA="" +PRINT=0 +FLAT=0 +NINJA_ONLY="" +OVERLAYS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --src) SRC="$2"; shift 2 ;; + --out) OUT_DIR="$2"; shift 2 ;; + --config) CONFIG="$2"; shift 2 ;; + --slice) SLICE="$2"; shift 2 ;; + --overlay) OVERLAYS+=("$2"); shift 2 ;; + --extra) EXTRA="${2:-}"; shift 2 ;; + --print) PRINT=1; shift ;; + --flat) FLAT=1; shift ;; + --ninja-target) NINJA_ONLY="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) die "gn-gen.sh: unknown flag $1" ;; + esac +done + +if [[ -n "$NINJA_ONLY" ]]; then + slice_ninja_target "$NINJA_ONLY" + exit 0 +fi + +compose_flags=(--config "$CONFIG") +[[ -n "$SLICE" ]] && compose_flags+=(--slice "$SLICE") +[[ -n "$EXTRA" ]] && compose_flags+=(--extra "$EXTRA") +for overlay in "${OVERLAYS[@]+"${OVERLAYS[@]}"}"; do + compose_flags+=(--overlay "$overlay") +done + +args_text="$(compose_gn_args "${compose_flags[@]}")" + +if [[ "$PRINT" -eq 1 ]]; then + if [[ "$FLAT" -eq 1 ]]; then + printf '%s\n' "$args_text" | flatten_gn_args + else + printf '%s\n' "$args_text" + fi + exit 0 +fi + +[[ -n "$SRC" && -n "$OUT_DIR" ]] || die "gn-gen.sh requires --src and --out (or --print)" +require_webrtc_src "$SRC" + +mkdir -p "$OUT_DIR" +printf '%s\n' "$args_text" >"${OUT_DIR}/args.gn" + +gn_bin="$(resolve_gn "$SRC")" +[[ -n "$gn_bin" ]] || die "gn not found" +echo "gn gen ${OUT_DIR}" +( + cd "$SRC" + "$gn_bin" gen "$OUT_DIR" +) diff --git a/stream_build/scripts/macos-gtest-filter.txt b/stream_build/scripts/macos-gtest-filter.txt new file mode 100644 index 0000000000..4651c92f6a --- /dev/null +++ b/stream_build/scripts/macos-gtest-filter.txt @@ -0,0 +1 @@ +*DeathTest*:ThreadTest.TwoThreadsInvokeDeathTest:ThreadTest.ThreeThreadsInvokeDeathTest:BitstreamReaderTest.InDebugModeRequiresToCheckOkStatusBeforeDestruction:BitstreamReaderTest.InDebugModeMayCheckRemainingBitsInsteadOfOkStatus:UnitBaseTest.CrashesWhenCreatedFromNan:AlwaysValidPointerTest.NoDefaultObjectPassNullPointer:AlwaysValidPointerTest.NoDefaultObjectPassNullUniquePointer:FieldTrialsTest.FieldTrialsDoesNotSupportSimultaneousInstances:CustomAudioProcessingTest.NullptrAudioProcessingIsUnsupported:FlatMap.AtFunction:*/FlatTreeTest/*.EraseEndDeath:NetworkTest.DefaultLocalAddress diff --git a/stream_build/scripts/package-android.sh b/stream_build/scripts/package-android.sh new file mode 100755 index 0000000000..acf52dc005 --- /dev/null +++ b/stream_build/scripts/package-android.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Pack already-built Android ABI dirs into libwebrtc.aar. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +require_linux +require_cmd python3 + +SRC="" +OUT="" +PRODUCTS="" +SLICES="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --src) SRC="$2"; shift 2 ;; + --out) OUT="$2"; shift 2 ;; + --products) PRODUCTS="$2"; shift 2 ;; + --slices) SLICES="$2"; shift 2 ;; + *) die "package-android.sh: unknown flag $1" ;; + esac +done + +[[ -n "$SRC" && -n "$OUT" && -n "$PRODUCTS" && -n "$SLICES" ]] || \ + die "package-android.sh requires --src --out --products --slices" + +manifest="$SRC/sdk/android/AndroidManifest.xml" +[[ -f "$manifest" ]] || die "missing $manifest" + +arch_from_slice() { + printf '%s\n' "${1#android-}" +} + +first="" +# shellcheck disable=SC2086 +for slice in $SLICES; do + dir="$OUT/$slice" + [[ -d "$dir" ]] || die "missing build output $dir (run: make build android)" + if [[ -z "$first" ]]; then + first="$slice" + fi +done + +jar="$OUT/$first/lib.java/sdk/android/libwebrtc.jar" +[[ -f "$jar" ]] || die "missing classes jar at $jar" + +mkdir -p "$PRODUCTS" +out_aar="$PRODUCTS/libwebrtc.aar" +rm -f "$out_aar" + +python3 - "$out_aar" "$manifest" "$jar" "$OUT" $SLICES <<'PY' +import os +import sys +import zipfile + +out_aar, manifest, jar, out_root, *slices = sys.argv[1:] +so_name = "libjingle_peerconnection_so.so" + +with zipfile.ZipFile(out_aar, "w") as aar: + aar.write(manifest, "AndroidManifest.xml") + aar.write(jar, "classes.jar") + for slice in slices: + arch = slice[len("android-"):] + so = os.path.join(out_root, slice, so_name) + if not os.path.isfile(so): + so = os.path.join(out_root, slice, "lib.unstripped", so_name) + if not os.path.isfile(so): + raise SystemExit(f"missing {so_name} in {out_root}/{slice}") + aar.write(so, f"jni/{arch}/{so_name}") +print(f"wrote {out_aar}") +PY + +if [[ "${SKIP_LICENSES:-0}" == 1 ]]; then + echo "skipping license generation (SKIP_LICENSES=1)" +else + require_cmd python3 + license_script="$SRC/tools_webrtc/libs/generate_licenses.py" + [[ -f "$license_script" ]] || die "missing $license_script" + build_dirs=() + # shellcheck disable=SC2086 + for slice in $SLICES; do + [[ -d "$OUT/$slice" ]] && build_dirs+=("$OUT/$slice") + done + [[ ${#build_dirs[@]} -gt 0 ]] || die "no slice out dirs for license generation" + echo "python3 $license_script --target sdk/android:libwebrtc --target sdk/android:libjingle_peerconnection_so $PRODUCTS ${build_dirs[*]}" + python3 "$license_script" \ + --target sdk/android:libwebrtc \ + --target sdk/android:libjingle_peerconnection_so \ + "$PRODUCTS" \ + "${build_dirs[@]}" + echo "wrote ${PRODUCTS}/LICENSE.md" +fi diff --git a/stream_build/scripts/package-apple.sh b/stream_build/scripts/package-apple.sh new file mode 100755 index 0000000000..ccaf5e10a2 --- /dev/null +++ b/stream_build/scripts/package-apple.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# lipo Apple framework slices and emit WebRTC.xcframework. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +require_darwin +require_cmd lipo +require_cmd xcodebuild + +SRC="${WEBRTC_SRC:-}" +OUT="" +PRODUCTS="" +SLICES="" +NAME="WebRTC" +ZIP=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --src) SRC="$2"; shift 2 ;; + --out) OUT="$2"; shift 2 ;; + --products) PRODUCTS="$2"; shift 2 ;; + --slices) SLICES="$2"; shift 2 ;; + --name) NAME="$2"; shift 2 ;; + --zip) ZIP=1; shift ;; + *) die "package-apple.sh: unknown flag $1" ;; + esac +done + +[[ -n "$OUT" && -n "$PRODUCTS" && -n "$SLICES" ]] || die "package-apple.sh requires --out --products --slices" + +framework_in_slice() { + printf '%s/%s/%s.framework\n' "$OUT" "$1" "$NAME" +} + +framework_binary() { + local fw="$1" + local bin="$fw/$NAME" + if [[ -e "$fw/Versions/A/$NAME" ]]; then + printf '%s\n' "$fw/Versions/A/$NAME" + return + fi + if [[ -L "$bin" ]]; then + printf '%s/%s\n' "$fw" "$(readlink "$bin")" + return + fi + printf '%s\n' "$bin" +} + +dsym_binary() { + printf '%s.dSYM/Contents/Resources/DWARF/%s\n' "$1" "$NAME" +} + +lipo_group() { + local dest="$1" + shift + local slices=("$@") + local present=() + local slice fw + for slice in "${slices[@]}"; do + fw="$(framework_in_slice "$slice")" + if [[ -d "$fw" ]]; then + present+=("$slice") + fi + done + if [[ ${#present[@]} -eq 0 ]]; then + return 1 + fi + + mkdir -p "$(dirname "$dest")" + rm -rf "$dest" + cp -R "$(framework_in_slice "${present[0]}")" "$dest" + + local binaries=() + for slice in "${present[@]}"; do + binaries+=("$(framework_binary "$(framework_in_slice "$slice")")") + done + local out_bin + out_bin="$(framework_binary "$dest")" + rm -f "$out_bin" + lipo -create "${binaries[@]}" -output "$out_bin" + + local first_dsym="${OUT}/${present[0]}/${NAME}.dSYM" + if [[ -d "$first_dsym" ]]; then + rm -rf "${dest}.dSYM" + cp -R "$first_dsym" "${dest}.dSYM" + local dsym_bins=() + for slice in "${present[@]}"; do + local dsym="${OUT}/${slice}/${NAME}.dSYM" + [[ -d "$dsym" ]] || continue + dsym_bins+=("$(dsym_binary "$dsym")") + done + if [[ ${#dsym_bins[@]} -gt 0 ]]; then + local out_dsym + out_dsym="$(dsym_binary "${dest}.dSYM")" + rm -f "$out_dsym" + mkdir -p "$(dirname "$out_dsym")" + lipo -create "${dsym_bins[@]}" -output "$out_dsym" + fi + fi + return 0 +} + +contains_slice() { + local needle="$1" + local s + # shellcheck disable=SC2086 + for s in $SLICES; do + [[ "$s" == "$needle" ]] && return 0 + done + return 1 +} + +work="${OUT}/_apple_universal" +rm -rf "$work" +mkdir -p "$work" + +xc_args=(-create-xcframework) +added=0 + +add_framework() { + local fw="$1" + [[ -d "$fw" ]] || return 0 + xc_args+=(-framework "$fw") + if [[ -d "${fw}.dSYM" ]]; then + xc_args+=(-debug-symbols "${fw}.dSYM") + fi + added=1 +} + +if contains_slice ios-arm64-device && lipo_group "${work}/ios-device/${NAME}.framework" ios-arm64-device; then + add_framework "${work}/ios-device/${NAME}.framework" +fi +if { contains_slice ios-arm64-simulator || contains_slice ios-x64-simulator; } && \ + lipo_group "${work}/ios-simulator/${NAME}.framework" ios-arm64-simulator ios-x64-simulator; then + add_framework "${work}/ios-simulator/${NAME}.framework" +fi +if { contains_slice catalyst-arm64 || contains_slice catalyst-x64; } && \ + lipo_group "${work}/catalyst/${NAME}.framework" catalyst-arm64 catalyst-x64; then + add_framework "${work}/catalyst/${NAME}.framework" +fi +if { contains_slice macos-arm64 || contains_slice macos-x64; } && \ + lipo_group "${work}/macos/${NAME}.framework" macos-arm64 macos-x64; then + add_framework "${work}/macos/${NAME}.framework" +fi + +[[ "$added" -eq 1 ]] || die "no Apple frameworks found under $OUT for slices: $SLICES" + +mkdir -p "$PRODUCTS" +rm -rf "${PRODUCTS}/${NAME}.xcframework" +xc_args+=(-output "${PRODUCTS}/${NAME}.xcframework") +echo "xcodebuild ${xc_args[*]}" +xcodebuild "${xc_args[@]}" + +xcframework="${PRODUCTS}/${NAME}.xcframework" +if [[ "${SKIP_LICENSES:-0}" == 1 ]]; then + echo "skipping license generation (SKIP_LICENSES=1)" +else + [[ -n "$SRC" ]] || die "package-apple.sh requires --src (or WEBRTC_SRC) to generate licenses" + require_cmd python3 + license_script="$SRC/tools_webrtc/libs/generate_licenses.py" + [[ -f "$license_script" ]] || die "missing $license_script" + build_dirs=() + # shellcheck disable=SC2086 + for slice in $SLICES; do + [[ -d "$OUT/$slice" ]] && build_dirs+=("$OUT/$slice") + done + [[ ${#build_dirs[@]} -gt 0 ]] || die "no slice out dirs for license generation" + gn_targets=() + has_ios=0 + has_macos=0 + # shellcheck disable=SC2086 + for slice in $SLICES; do + if [[ "$slice" == macos-* ]]; then + has_macos=1 + else + has_ios=1 + fi + done + [[ "$has_ios" -eq 1 ]] && gn_targets+=(--target "//sdk:framework_objc") + [[ "$has_macos" -eq 1 ]] && gn_targets+=(--target "//sdk:mac_framework_objc") + [[ ${#gn_targets[@]} -gt 0 ]] || die "no license GN targets for slices: $SLICES" + echo "python3 $license_script ${gn_targets[*]} $xcframework ${build_dirs[*]}" + python3 "$license_script" "${gn_targets[@]}" "$xcframework" "${build_dirs[@]}" + echo "wrote ${xcframework}/LICENSE.md" +fi + +if [[ "$ZIP" -eq 1 ]]; then + if command -v ditto >/dev/null 2>&1; then + ditto -c -k --sequesterRsrc --keepParent \ + "${PRODUCTS}/${NAME}.xcframework" \ + "${PRODUCTS}/${NAME}.xcframework.zip" + else + ( + cd "$PRODUCTS" + zip --symlinks -r "${NAME}.xcframework.zip" "${NAME}.xcframework" + ) + fi +fi + +echo "wrote ${PRODUCTS}/${NAME}.xcframework" diff --git a/stream_build/scripts/package-windows.sh b/stream_build/scripts/package-windows.sh new file mode 100755 index 0000000000..813fd10761 --- /dev/null +++ b/stream_build/scripts/package-windows.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Copy built Windows libs into PRODUCTS (optional zip). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +require_windows + +OUT="" +PRODUCTS="" +SLICES="" +ZIP=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --out) OUT="$2"; shift 2 ;; + --products) PRODUCTS="$2"; shift 2 ;; + --slices) SLICES="$2"; shift 2 ;; + --zip) ZIP=1; shift ;; + *) die "package-windows.sh: unknown flag $1" ;; + esac +done + +[[ -n "$OUT" && -n "$PRODUCTS" && -n "$SLICES" ]] || \ + die "package-windows.sh requires --out --products --slices" + +mkdir -p "$PRODUCTS/windows" +copied=0 +# shellcheck disable=SC2086 +for slice in $SLICES; do + dir="$OUT/$slice" + [[ -d "$dir" ]] || die "missing build output $dir (run: make build windows)" + dest="$PRODUCTS/windows/$slice" + mkdir -p "$dest" + local_copied=0 + for name in webrtc.lib libwebrtc.a webrtc.dll webrtc.dll.lib; do + if [[ -e "$dir/$name" ]]; then + cp -R "$dir/$name" "$dest/" + local_copied=1 + fi + done + if [[ "$local_copied" -eq 0 ]]; then + die "no webrtc lib in $dir" + fi + copied=1 +done + +[[ "$copied" -eq 1 ]] || die "nothing to package" + +if [[ "$ZIP" -eq 1 ]]; then + ( + cd "$PRODUCTS" + zip -r windows.zip windows + ) +fi + +echo "wrote $PRODUCTS/windows" diff --git a/stream_build/scripts/rename-android.sh b/stream_build/scripts/rename-android.sh new file mode 100755 index 0000000000..59090642a2 --- /dev/null +++ b/stream_build/scripts/rename-android.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Copy libwebrtc.aar into PRODUCTS/renamed/. The Android wrapper publishes +# that filename as-is (Maven coords live in stream-video-android-webrtc). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +NAME="libwebrtc.aar" +SRC="" +DEST_DIR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --src) SRC="$2"; shift 2 ;; + --dest) DEST_DIR="$2"; shift 2 ;; + *) die "rename-android.sh: unknown flag $1" ;; + esac +done + +[[ -n "$SRC" && -n "$DEST_DIR" ]] || die "rename-android.sh requires --src --dest" +[[ -f "$SRC" ]] || die "no AAR at $SRC" +base="$(basename "$SRC")" +[[ "$base" == "$NAME" ]] || die "expected $NAME, got $base" + +mkdir -p "$DEST_DIR" +src_abs="$(cd "$(dirname "$SRC")" && pwd)/$(basename "$SRC")" +dest_abs="$(cd "$DEST_DIR" && pwd)/$NAME" +[[ "$src_abs" != "$dest_abs" ]] || die "refusing to overwrite input $SRC" + +cp "$src_abs" "$dest_abs" +echo "copied $src_abs -> $dest_abs" +echo "original preserved at $src_abs" diff --git a/stream_build/scripts/rename-apple.sh b/stream_build/scripts/rename-apple.sh new file mode 100755 index 0000000000..c2c1ce3fda --- /dev/null +++ b/stream_build/scripts/rename-apple.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Copy WebRTC.xcframework → StreamWebRTC.xcframework (original untouched). +# Matches GetStream/webrtc fastlane rename_product and +# stream-video-swift-webrtc clone_and_modify_xcframework. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +require_darwin +require_cmd find +require_cmd file +require_cmd plutil +require_cmd install_name_tool + +OLD="WebRTC" +NEW="StreamWebRTC" +SRC="" +DEST_DIR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --src) SRC="$2"; shift 2 ;; + --dest) DEST_DIR="$2"; shift 2 ;; + *) die "rename-apple.sh: unknown flag $1" ;; + esac +done + +[[ -n "$SRC" && -n "$DEST_DIR" ]] || die "rename-apple.sh requires --src --dest" +[[ -d "$SRC" ]] || die "no xcframework at $SRC" +base="$(basename "$SRC")" +[[ "$base" == "${OLD}.xcframework" ]] || die "expected ${OLD}.xcframework, got $base" + +dest="${DEST_DIR}/${NEW}.xcframework" +src_abs="$(cd "$SRC" && pwd)" +mkdir -p "$DEST_DIR" +dest_parent="$(cd "$DEST_DIR" && pwd)" +dest_abs="${dest_parent}/${NEW}.xcframework" +[[ "$src_abs" != "$dest_abs" ]] || die "refusing to overwrite input $SRC" + +rm -rf "$dest_abs" +cp -R "$src_abs" "$dest_abs" +echo "copied $src_abs -> $dest_abs" + +while IFS= read -r -d '' path; do + name="$(basename "$path")" + case "$name" in + "${OLD}.framework"|"${OLD}.dSYM"|"${OLD}.h"|"$OLD") + dir="$(dirname "$path")" + mv "$path" "${dir}/${name/${OLD}/${NEW}}" + ;; + esac +done < <(find "$dest_abs" -depth -print0) + +while IFS= read -r -d '' file; do + if [[ "$file" == *.plist ]]; then + plutil -convert xml1 "$file" + fi + old_text="$(cat "$file")" + new_text="${old_text//${OLD}/${NEW}}" + if [[ "$old_text" != "$new_text" ]]; then + printf '%s\n' "$new_text" > "$file" + fi +done < <(find "$dest_abs" \( -name Info.plist -o -name module.modulemap \) -print0) + +while IFS= read -r -d '' file; do + old_text="$(cat "$file")" + new_text="${old_text//import <${OLD}/import <${NEW}}" + if [[ "$old_text" != "$new_text" ]]; then + printf '%s\n' "$new_text" > "$file" + fi +done < <(find "$dest_abs" -name '*.h' -print0) + +while IFS= read -r -d '' fw; do + ( + cd "$fw" + if [[ -L "$NEW" ]]; then + old_link="$(readlink "$NEW")" + new_link="${old_link//${OLD}/${NEW}}" + if [[ "$old_link" != "$new_link" ]]; then + rm -f "$NEW" + ln -s "$new_link" "$NEW" + fi + fi + bin="$NEW" + [[ -e "$bin" ]] || continue + if file -b "$bin" | grep -q 'Mach-O'; then + install_name_tool -id "@rpath/${NEW}.framework/${NEW}" "$bin" + fi + ) +done < <(find "$dest_abs" -name "${NEW}.framework" -type d -print0) + +echo "wrote $dest_abs" +echo "original preserved at $src_abs" diff --git a/stream_build/scripts/run-ios-tests.sh b/stream_build/scripts/run-ios-tests.sh new file mode 100755 index 0000000000..55ad2900a8 --- /dev/null +++ b/stream_build/scripts/run-ios-tests.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Run iOS XCTest wrappers produced by ninja sdk_*unittests. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +require_darwin +require_cmd xcrun +require_cmd xcodebuild +require_cmd python3 + +BUILD_DIR="" +TARGETS="" +SIMULATOR_PLATFORM="${SIMULATOR_PLATFORM:-}" +SIMULATOR_VERSION="${SIMULATOR_VERSION:-}" +EXTRA_ARGS="${EXTRA_ARGS:-}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --build-dir) BUILD_DIR="$2"; shift 2 ;; + --targets) TARGETS="$2"; shift 2 ;; + --platform) SIMULATOR_PLATFORM="$2"; shift 2 ;; + --version) SIMULATOR_VERSION="$2"; shift 2 ;; + --extra) EXTRA_ARGS="${2:-}"; shift 2 ;; + *) die "run-ios-tests.sh: unknown flag $1" ;; + esac +done + +[[ -n "$BUILD_DIR" && -n "$TARGETS" ]] || die "run-ios-tests.sh requires --build-dir and --targets" + +pick_simulator() { + python3 - "$SIMULATOR_PLATFORM" "$SIMULATOR_VERSION" <<'PY' +import json, subprocess, sys + +want_name, want_version = sys.argv[1], sys.argv[2] +payload = json.loads( + subprocess.check_output( + ["xcrun", "simctl", "list", "devices", "available", "--json"], + text=True, + ) +) +candidates = [] +for runtime, devices in payload.get("devices", {}).items(): + if "iOS" not in runtime: + continue + version = runtime.split("iOS-")[-1].replace("-", ".") + for device in devices: + if device.get("isAvailable") is False: + continue + name = device.get("name") or "" + if want_name and name != want_name: + continue + if want_version and version != want_version: + continue + candidates.append( + ( + 1 if device.get("state") == "Booted" else 0, + 1 if name.startswith("iPhone") else 0, + tuple(int(p) for p in version.split(".") if p.isdigit()), + name, + version, + ) + ) +if not candidates: + sys.exit("no available iOS simulator matched the request") +best = max(candidates) +print(f"{best[3]}\t{best[4]}") +PY +} + +if [[ -z "$SIMULATOR_PLATFORM" || -z "$SIMULATOR_VERSION" ]]; then + selected="$(pick_simulator)" + SIMULATOR_PLATFORM="${selected%%$'\t'*}" + SIMULATOR_VERSION="${selected#*$'\t'}" + echo "auto-selected simulator: ${SIMULATOR_PLATFORM} (iOS ${SIMULATOR_VERSION})" +fi + +xcode_build_version="$(xcodebuild -version | awk '/Build version/{print $3; exit}')" +xcode_build_version="${xcode_build_version:-local}" +out_dir="${BUILD_DIR}/test_output" +rm -rf "$out_dir" +mkdir -p "$out_dir" + +run_target() { + local target="$1" + local wrapper="${BUILD_DIR}/bin/run_${target}" + [[ -x "$wrapper" ]] || die "run script not found for ${target} at ${wrapper}" + local args=( + --xctest + --out-dir "$out_dir" + --xcode-build-version "$xcode_build_version" + --platform "$SIMULATOR_PLATFORM" + --version "$SIMULATOR_VERSION" + ) + # shellcheck disable=SC2086 + if [[ -n "$EXTRA_ARGS" ]]; then + # shellcheck disable=SC2206 + args+=($EXTRA_ARGS) + fi + echo "running ${wrapper} ${args[*]}" + if "$wrapper" "${args[@]}"; then + return 0 + fi + if grep -Rqs "Test Suite 'All tests' passed\|Test Suite 'Selected tests' passed" "$out_dir"; then + echo "iOS test wrapper reported failure after XCTest already passed; treating as success" + return 0 + fi + return 1 +} + +# shellcheck disable=SC2086 +for target in $TARGETS; do + run_target "$target" +done From 3abc38bd91ea2a5b5019de3635ad906f203d66f7 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Thu, 10 Sep 2026 21:31:49 +0300 Subject: [PATCH 02/24] Improve caching --- .github/actions/cache-webrtc/action.yml | 2 +- .github/actions/deps-android/action.yml | 2 +- .github/actions/deps-apple/action.yml | 2 +- .github/actions/prepare-common/action.yml | 30 +++++++++++----- .github/workflows/_make.yml | 43 +++++++++++++---------- 5 files changed, 50 insertions(+), 29 deletions(-) diff --git a/.github/actions/cache-webrtc/action.yml b/.github/actions/cache-webrtc/action.yml index 5ee050df48..12d81ffbf2 100644 --- a/.github/actions/cache-webrtc/action.yml +++ b/.github/actions/cache-webrtc/action.yml @@ -7,7 +7,7 @@ inputs: description: restore or save target_os_cache_key: required: true - description: Cache-safe target_os suffix (ios-mac, android-unix, win). + description: Cache-safe target_os suffix (apple, android-unix, win). config: required: true description: release, debug, or deps-only label included in the key. diff --git a/.github/actions/deps-android/action.yml b/.github/actions/deps-android/action.yml index d6d45d6ab2..525a3979aa 100644 --- a/.github/actions/deps-android/action.yml +++ b/.github/actions/deps-android/action.yml @@ -41,7 +41,7 @@ runs: env: DEPS_ROOT: ${{ github.workspace }}/.gclient_deps TARGET_OS: ${{ inputs.target_os }} - JOBS: "2" + JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps diff --git a/.github/actions/deps-apple/action.yml b/.github/actions/deps-apple/action.yml index f505ae8977..731bdf64ee 100644 --- a/.github/actions/deps-apple/action.yml +++ b/.github/actions/deps-apple/action.yml @@ -40,7 +40,7 @@ runs: env: DEPS_ROOT: ${{ github.workspace }}/.gclient_deps TARGET_OS: ${{ inputs.target_os }} - JOBS: "2" + JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps diff --git a/.github/actions/prepare-common/action.yml b/.github/actions/prepare-common/action.yml index b8b4400714..efa9c24adf 100644 --- a/.github/actions/prepare-common/action.yml +++ b/.github/actions/prepare-common/action.yml @@ -15,6 +15,10 @@ inputs: required: false default: "false" description: Whether Windows is selected. + skip_maccatalyst: + required: false + default: "false" + description: Omit Mac Catalyst from the Apple deps label. outputs: run_ios: @@ -54,19 +58,29 @@ runs: macos='${{ inputs.platform_macos }}' android='${{ inputs.platform_android }}' windows='${{ inputs.platform_windows }}' + skip_maccatalyst='${{ inputs.skip_maccatalyst }}' - apple_os_list=() - if [[ "${ios}" == "true" ]]; then apple_os_list+=("ios"); fi - if [[ "${macos}" == "true" ]]; then apple_os_list+=("mac"); fi + # Display follows the selected platforms. gclient TARGET_OS is the + # Apple superset so iOS-only and macOS-only share one cache tree. + apple_label_list=() + if [[ "${ios}" == "true" ]]; then apple_label_list+=("ios"); fi + if [[ "${macos}" == "true" ]]; then apple_label_list+=("macos"); fi + if [[ "${ios}" == "true" && "${skip_maccatalyst}" != "true" ]]; then + apple_label_list+=("maccatalyst") + fi run_apple=false apple_target_os="" + apple_target_os_cache_key="" apple_target_os_label="" - if [[ ${#apple_os_list[@]} -gt 0 ]]; then + if [[ "${ios}" == "true" || "${macos}" == "true" ]]; then run_apple=true - IFS=',' - apple_target_os="${apple_os_list[*]}" - apple_target_os_label="${apple_target_os//,/, }" + apple_target_os="ios,mac" + apple_target_os_cache_key="apple" + old_ifs="$IFS" + IFS=', ' + apple_target_os_label="${apple_label_list[*]}" + IFS="$old_ifs" fi android_target_os="" @@ -85,7 +99,7 @@ runs: echo "run_windows=${windows}" >> "${GITHUB_OUTPUT}" echo "run_apple=${run_apple}" >> "${GITHUB_OUTPUT}" echo "apple_target_os=${apple_target_os}" >> "${GITHUB_OUTPUT}" - echo "apple_target_os_cache_key=${apple_target_os//,/-}" >> "${GITHUB_OUTPUT}" + echo "apple_target_os_cache_key=${apple_target_os_cache_key}" >> "${GITHUB_OUTPUT}" echo "apple_target_os_label=${apple_target_os_label}" >> "${GITHUB_OUTPUT}" echo "android_target_os=${android_target_os}" >> "${GITHUB_OUTPUT}" echo "android_target_os_cache_key=${android_target_os//,/-}" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 90a36bfb33..c834e0ea82 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -100,6 +100,13 @@ jobs: needs: validate_inputs runs-on: ubuntu-latest outputs: + config: ${{ inputs.config }} + mode: ${{ inputs.mode }} + run_ios: ${{ steps.plan.outputs.run_ios }} + run_macos: ${{ steps.plan.outputs.run_macos }} + run_android: ${{ steps.plan.outputs.run_android }} + run_windows: ${{ steps.plan.outputs.run_windows }} + run_apple: ${{ steps.plan.outputs.run_apple }} apple_target_os: ${{ steps.plan.outputs.apple_target_os }} apple_target_os_cache_key: ${{ steps.plan.outputs.apple_target_os_cache_key }} apple_target_os_label: ${{ steps.plan.outputs.apple_target_os_label }} @@ -117,7 +124,7 @@ jobs: platform_windows: ${{ inputs.windows }} deps_apple: - name: Deps macOS (${{ needs.plan.outputs.apple_target_os_label }}) + name: Deps Apple (${{ needs.plan.outputs.run_apple == 'true' && needs.plan.outputs.apple_target_os_label || 'skipped' }}) needs: plan if: ${{ inputs.ios || inputs.macos }} runs-on: macos-26 @@ -134,7 +141,7 @@ jobs: ignore_cache: ${{ inputs.ignore_cache }} deps_android: - name: Deps Android + name: ${{ needs.plan.outputs.run_android == 'true' && 'Deps Android' || 'Deps Android (skipped)' }} needs: plan if: ${{ inputs.android }} runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} @@ -151,7 +158,7 @@ jobs: ignore_cache: ${{ inputs.ignore_cache }} build_ios: - name: Build iOS (${{ inputs.config }}) + name: Build iOS (${{ (needs.plan.outputs.mode == 'build' && needs.plan.outputs.run_ios == 'true') && needs.plan.outputs.config || 'skipped' }}) needs: [plan, deps_apple] if: ${{ inputs.mode == 'build' && inputs.ios }} runs-on: macos-26 @@ -178,7 +185,7 @@ jobs: config: ${{ inputs.config }} build_macos: - name: Build macOS (${{ inputs.config }}) + name: Build macOS (${{ (needs.plan.outputs.mode == 'build' && needs.plan.outputs.run_macos == 'true') && needs.plan.outputs.config || 'skipped' }}) needs: [plan, deps_apple] if: ${{ inputs.mode == 'build' && inputs.macos }} runs-on: macos-26 @@ -205,7 +212,7 @@ jobs: config: ${{ inputs.config }} build_android: - name: Build Android (${{ inputs.config }}) + name: Build Android (${{ (needs.plan.outputs.mode == 'build' && needs.plan.outputs.run_android == 'true') && needs.plan.outputs.config || 'skipped' }}) needs: [plan, deps_android] if: ${{ inputs.mode == 'build' && inputs.android }} runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} @@ -240,7 +247,7 @@ jobs: config: ${{ inputs.config }} test_ios: - name: Test iOS (build + debug test) + name: Test iOS (${{ ((needs.plan.outputs.mode == 'test' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_ios == 'true') && needs.plan.outputs.config || 'skipped' }}) needs: [plan, deps_apple] if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.ios }} runs-on: macos-26 @@ -273,7 +280,7 @@ jobs: config: ${{ inputs.config }} test_macos: - name: Test macOS (build + debug test) + name: Test macOS (${{ ((needs.plan.outputs.mode == 'test' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_macos == 'true') && needs.plan.outputs.config || 'skipped' }}) needs: [plan, deps_apple] if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.macos }} runs-on: macos-26 @@ -306,7 +313,7 @@ jobs: config: ${{ inputs.config }} test_windows: - name: Test Windows (build + debug test) + name: Test Windows (${{ ((needs.plan.outputs.mode == 'test' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_windows == 'true') && needs.plan.outputs.config || 'skipped' }}) needs: plan if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.windows }} runs-on: windows-latest @@ -335,7 +342,7 @@ jobs: env: DEPS_ROOT: ${{ github.workspace }}/.gclient_deps TARGET_OS: win - JOBS: "2" + JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps - name: Build Windows @@ -355,8 +362,8 @@ jobs: run: make test windows tests_passed: - name: Tests passed - needs: [test_ios, test_macos, test_windows] + name: ${{ (needs.plan.outputs.mode == 'package' || needs.plan.outputs.mode == 'release') && 'Tests passed' || 'Tests passed (skipped)' }} + needs: [plan, test_ios, test_macos, test_windows] if: ${{ always() && !cancelled() && (inputs.mode == 'package' || inputs.mode == 'release') }} runs-on: ubuntu-latest steps: @@ -395,7 +402,7 @@ jobs: echo "Selected-platform tests passed (Android tests are unwired and skipped)." package_ios: - name: Package iOS (${{ inputs.config }}) + name: Package iOS (${{ ((needs.plan.outputs.mode == 'package' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_ios == 'true') && needs.plan.outputs.config || 'skipped' }}) needs: [plan, deps_apple, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.ios && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: macos-26 @@ -438,7 +445,7 @@ jobs: retention-days: 7 package_macos: - name: Package macOS (${{ inputs.config }}) + name: Package macOS (${{ ((needs.plan.outputs.mode == 'package' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_macos == 'true') && needs.plan.outputs.config || 'skipped' }}) needs: [plan, deps_apple, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.macos && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: macos-26 @@ -481,7 +488,7 @@ jobs: retention-days: 7 package_android: - name: Package Android (${{ inputs.config }}) + name: Package Android (${{ ((needs.plan.outputs.mode == 'package' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_android == 'true') && needs.plan.outputs.config || 'skipped' }}) needs: [plan, deps_android, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.android && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} @@ -539,7 +546,7 @@ jobs: retention-days: 7 build_windows: - name: Build Windows (${{ inputs.config }}) + name: Build Windows (${{ (needs.plan.outputs.mode == 'build' && needs.plan.outputs.run_windows == 'true') && needs.plan.outputs.config || 'skipped' }}) needs: plan if: ${{ inputs.mode == 'build' && inputs.windows }} runs-on: windows-latest @@ -568,7 +575,7 @@ jobs: env: DEPS_ROOT: ${{ github.workspace }}/.gclient_deps TARGET_OS: win - JOBS: "2" + JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps - name: Build Windows @@ -581,7 +588,7 @@ jobs: run: make build windows package_windows: - name: Package Windows (${{ inputs.config }}) + name: Package Windows (${{ ((needs.plan.outputs.mode == 'package' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_windows == 'true') && needs.plan.outputs.config || 'skipped' }}) needs: [plan, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.windows && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: windows-latest @@ -610,7 +617,7 @@ jobs: env: DEPS_ROOT: ${{ github.workspace }}/.gclient_deps TARGET_OS: win - JOBS: "2" + JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps - name: Build Windows From e5d66227e378ed9399ad03786aa8ecd4ce717ad9 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Thu, 10 Sep 2026 22:27:30 +0300 Subject: [PATCH 03/24] Imporve job titles --- .github/actions/cache-webrtc/action.yml | 4 +- .github/actions/setup-webrtc/action.yml | 4 +- .github/workflows/_make.yml | 198 ++++++++++++++++++------ 3 files changed, 155 insertions(+), 51 deletions(-) diff --git a/.github/actions/cache-webrtc/action.yml b/.github/actions/cache-webrtc/action.yml index 12d81ffbf2..e2a0759c68 100644 --- a/.github/actions/cache-webrtc/action.yml +++ b/.github/actions/cache-webrtc/action.yml @@ -32,7 +32,7 @@ runs: - name: Restore deps and out cache if: ${{ inputs.operation == 'restore' && inputs.ignore_cache != 'true' }} - uses: actions/cache/restore@v4 + uses: actions/cache/restore@v6 with: path: | .gclient_deps @@ -56,7 +56,7 @@ runs: - name: Save deps and out cache if: ${{ inputs.operation == 'save' }} - uses: actions/cache/save@v4 + uses: actions/cache/save@v6 with: path: | .gclient_deps diff --git a/.github/actions/setup-webrtc/action.yml b/.github/actions/setup-webrtc/action.yml index 47c9e067cf..25317ccc47 100644 --- a/.github/actions/setup-webrtc/action.yml +++ b/.github/actions/setup-webrtc/action.yml @@ -27,7 +27,7 @@ runs: cp -R "${GITHUB_WORKSPACE}/.github/actions" "${RUNNER_TEMP}/pipeline/actions" - name: Check out GetStream/webrtc - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: ref: ${{ inputs.webrtc_ref }} @@ -51,7 +51,7 @@ runs: sudo apt-get install -y python3-pip openjdk-8-jdk lsb-release software-properties-common - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: "3.x" diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index c834e0ea82..16e725661b 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -112,8 +112,25 @@ jobs: apple_target_os_label: ${{ steps.plan.outputs.apple_target_os_label }} android_target_os: ${{ steps.plan.outputs.android_target_os }} android_target_os_cache_key: ${{ steps.plan.outputs.android_target_os_cache_key }} + deps_apple_name: ${{ steps.names.outputs.deps_apple_name }} + deps_android_name: ${{ steps.names.outputs.deps_android_name }} + build_ios_name: ${{ steps.names.outputs.build_ios_name }} + build_macos_name: ${{ steps.names.outputs.build_macos_name }} + build_android_name: ${{ steps.names.outputs.build_android_name }} + build_windows_name: ${{ steps.names.outputs.build_windows_name }} + test_ios_name: ${{ steps.names.outputs.test_ios_name }} + test_macos_name: ${{ steps.names.outputs.test_macos_name }} + test_windows_name: ${{ steps.names.outputs.test_windows_name }} + tests_passed_name: ${{ steps.names.outputs.tests_passed_name }} + package_ios_name: ${{ steps.names.outputs.package_ios_name }} + package_macos_name: ${{ steps.names.outputs.package_macos_name }} + package_android_name: ${{ steps.names.outputs.package_android_name }} + package_windows_name: ${{ steps.names.outputs.package_windows_name }} + finalise_package_name: ${{ steps.names.outputs.finalise_package_name }} + github_release_name: ${{ steps.names.outputs.github_release_name }} + trigger_downstream_name: ${{ steps.names.outputs.trigger_downstream_name }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - id: plan name: Plan target_os uses: ./.github/actions/prepare-common @@ -122,15 +139,102 @@ jobs: platform_macos: ${{ inputs.macos }} platform_android: ${{ inputs.android }} platform_windows: ${{ inputs.windows }} + - id: names + name: Plan job display names + env: + MODE: ${{ inputs.mode }} + CONFIG: ${{ inputs.config }} + VERSION: ${{ inputs.version }} + RUN_IOS: ${{ steps.plan.outputs.run_ios }} + RUN_MACOS: ${{ steps.plan.outputs.run_macos }} + RUN_ANDROID: ${{ steps.plan.outputs.run_android }} + RUN_WINDOWS: ${{ steps.plan.outputs.run_windows }} + RUN_APPLE: ${{ steps.plan.outputs.run_apple }} + APPLE_LABEL: ${{ steps.plan.outputs.apple_target_os_label }} + run: | + set -euo pipefail + # Job `name:` does not evaluate nested ==/&&/||. Emit plain strings. + + active_or_skipped() { + local prefix="$1" + local is_active="$2" + local detail="${3:-}" + if [[ "${is_active}" == "true" ]]; then + if [[ -n "${detail}" ]]; then + printf '%s (%s)\n' "${prefix}" "${detail}" + else + printf '%s\n' "${prefix}" + fi + else + printf '%s (skipped)\n' "${prefix}" + fi + } + + apple_label="${APPLE_LABEL//, /,}" + apple_label="${apple_label//,/, }" + + is_build=false + is_test=false + is_package=false + is_release=false + [[ "${MODE}" == "build" ]] && is_build=true + [[ "${MODE}" == "test" || "${MODE}" == "release" ]] && is_test=true + [[ "${MODE}" == "package" || "${MODE}" == "release" ]] && is_package=true + [[ "${MODE}" == "release" ]] && is_release=true + + run_build_ios=false + run_build_macos=false + run_build_android=false + run_build_windows=false + run_test_ios=false + run_test_macos=false + run_test_windows=false + run_package_ios=false + run_package_macos=false + run_package_android=false + run_package_windows=false + [[ "${is_build}" == "true" && "${RUN_IOS}" == "true" ]] && run_build_ios=true + [[ "${is_build}" == "true" && "${RUN_MACOS}" == "true" ]] && run_build_macos=true + [[ "${is_build}" == "true" && "${RUN_ANDROID}" == "true" ]] && run_build_android=true + [[ "${is_build}" == "true" && "${RUN_WINDOWS}" == "true" ]] && run_build_windows=true + [[ "${is_test}" == "true" && "${RUN_IOS}" == "true" ]] && run_test_ios=true + [[ "${is_test}" == "true" && "${RUN_MACOS}" == "true" ]] && run_test_macos=true + [[ "${is_test}" == "true" && "${RUN_WINDOWS}" == "true" ]] && run_test_windows=true + [[ "${is_package}" == "true" && "${RUN_IOS}" == "true" ]] && run_package_ios=true + [[ "${is_package}" == "true" && "${RUN_MACOS}" == "true" ]] && run_package_macos=true + [[ "${is_package}" == "true" && "${RUN_ANDROID}" == "true" ]] && run_package_android=true + [[ "${is_package}" == "true" && "${RUN_WINDOWS}" == "true" ]] && run_package_windows=true + + { + echo "deps_apple_name=$(active_or_skipped "Deps Apple" "${RUN_APPLE}" "${apple_label}")" + echo "deps_android_name=$(active_or_skipped "Deps Android" "${RUN_ANDROID}")" + echo "build_ios_name=$(active_or_skipped "Build iOS" "${run_build_ios}" "${CONFIG}")" + echo "build_macos_name=$(active_or_skipped "Build macOS" "${run_build_macos}" "${CONFIG}")" + echo "build_android_name=$(active_or_skipped "Build Android" "${run_build_android}" "${CONFIG}")" + echo "build_windows_name=$(active_or_skipped "Build Windows" "${run_build_windows}" "${CONFIG}")" + echo "test_ios_name=$(active_or_skipped "Test iOS" "${run_test_ios}" "${CONFIG}")" + echo "test_macos_name=$(active_or_skipped "Test macOS" "${run_test_macos}" "${CONFIG}")" + echo "test_windows_name=$(active_or_skipped "Test Windows" "${run_test_windows}" "${CONFIG}")" + echo "tests_passed_name=$(active_or_skipped "Tests passed" "${is_package}")" + echo "package_ios_name=$(active_or_skipped "Package iOS" "${run_package_ios}" "${CONFIG}")" + echo "package_macos_name=$(active_or_skipped "Package macOS" "${run_package_macos}" "${CONFIG}")" + echo "package_android_name=$(active_or_skipped "Package Android" "${run_package_android}" "${CONFIG}")" + echo "package_windows_name=$(active_or_skipped "Package Windows" "${run_package_windows}" "${CONFIG}")" + echo "finalise_package_name=$(active_or_skipped "Finalise package" "${is_package}")" + release_title="Release (skipped)" + [[ "${is_release}" == "true" ]] && release_title="Release ${VERSION}" + echo "github_release_name=${release_title}" + echo "trigger_downstream_name=$(active_or_skipped "Trigger downstream WebRTC releases" "${is_release}")" + } >> "${GITHUB_OUTPUT}" deps_apple: - name: Deps Apple (${{ needs.plan.outputs.run_apple == 'true' && needs.plan.outputs.apple_target_os_label || 'skipped' }}) + name: ${{ needs.plan.outputs.deps_apple_name }} needs: plan if: ${{ inputs.ios || inputs.macos }} runs-on: macos-26 timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Deps Apple uses: ./.github/actions/deps-apple with: @@ -141,13 +245,13 @@ jobs: ignore_cache: ${{ inputs.ignore_cache }} deps_android: - name: ${{ needs.plan.outputs.run_android == 'true' && 'Deps Android' || 'Deps Android (skipped)' }} + name: ${{ needs.plan.outputs.deps_android_name }} needs: plan if: ${{ inputs.android }} runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Deps Android uses: ./.github/actions/deps-android with: @@ -158,13 +262,13 @@ jobs: ignore_cache: ${{ inputs.ignore_cache }} build_ios: - name: Build iOS (${{ (needs.plan.outputs.mode == 'build' && needs.plan.outputs.run_ios == 'true') && needs.plan.outputs.config || 'skipped' }}) + name: ${{ needs.plan.outputs.build_ios_name }} needs: [plan, deps_apple] if: ${{ inputs.mode == 'build' && inputs.ios }} runs-on: macos-26 timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} @@ -185,13 +289,13 @@ jobs: config: ${{ inputs.config }} build_macos: - name: Build macOS (${{ (needs.plan.outputs.mode == 'build' && needs.plan.outputs.run_macos == 'true') && needs.plan.outputs.config || 'skipped' }}) + name: ${{ needs.plan.outputs.build_macos_name }} needs: [plan, deps_apple] if: ${{ inputs.mode == 'build' && inputs.macos }} runs-on: macos-26 timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} @@ -212,13 +316,13 @@ jobs: config: ${{ inputs.config }} build_android: - name: Build Android (${{ (needs.plan.outputs.mode == 'build' && needs.plan.outputs.run_android == 'true') && needs.plan.outputs.config || 'skipped' }}) + name: ${{ needs.plan.outputs.build_android_name }} needs: [plan, deps_android] if: ${{ inputs.mode == 'build' && inputs.android }} runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} @@ -247,13 +351,13 @@ jobs: config: ${{ inputs.config }} test_ios: - name: Test iOS (${{ ((needs.plan.outputs.mode == 'test' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_ios == 'true') && needs.plan.outputs.config || 'skipped' }}) + name: ${{ needs.plan.outputs.test_ios_name }} needs: [plan, deps_apple] if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.ios }} runs-on: macos-26 timeout-minutes: 180 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} @@ -280,13 +384,13 @@ jobs: config: ${{ inputs.config }} test_macos: - name: Test macOS (${{ ((needs.plan.outputs.mode == 'test' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_macos == 'true') && needs.plan.outputs.config || 'skipped' }}) + name: ${{ needs.plan.outputs.test_macos_name }} needs: [plan, deps_apple] if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.macos }} runs-on: macos-26 timeout-minutes: 180 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} @@ -313,13 +417,13 @@ jobs: config: ${{ inputs.config }} test_windows: - name: Test Windows (${{ ((needs.plan.outputs.mode == 'test' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_windows == 'true') && needs.plan.outputs.config || 'skipped' }}) + name: ${{ needs.plan.outputs.test_windows_name }} needs: plan if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.windows }} runs-on: windows-latest timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Check Windows WebRTC host shell: bash run: | @@ -362,7 +466,7 @@ jobs: run: make test windows tests_passed: - name: ${{ (needs.plan.outputs.mode == 'package' || needs.plan.outputs.mode == 'release') && 'Tests passed' || 'Tests passed (skipped)' }} + name: ${{ needs.plan.outputs.tests_passed_name }} needs: [plan, test_ios, test_macos, test_windows] if: ${{ always() && !cancelled() && (inputs.mode == 'package' || inputs.mode == 'release') }} runs-on: ubuntu-latest @@ -402,13 +506,13 @@ jobs: echo "Selected-platform tests passed (Android tests are unwired and skipped)." package_ios: - name: Package iOS (${{ ((needs.plan.outputs.mode == 'package' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_ios == 'true') && needs.plan.outputs.config || 'skipped' }}) + name: ${{ needs.plan.outputs.package_ios_name }} needs: [plan, deps_apple, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.ios && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: macos-26 timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} @@ -437,7 +541,7 @@ jobs: target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} config: ${{ inputs.config }} - name: Upload products-ios - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: products-ios path: products/ios @@ -445,13 +549,13 @@ jobs: retention-days: 7 package_macos: - name: Package macOS (${{ ((needs.plan.outputs.mode == 'package' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_macos == 'true') && needs.plan.outputs.config || 'skipped' }}) + name: ${{ needs.plan.outputs.package_macos_name }} needs: [plan, deps_apple, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.macos && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: macos-26 timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} @@ -480,7 +584,7 @@ jobs: target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} config: ${{ inputs.config }} - name: Upload products-macos - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: products-macos path: products/macos @@ -488,13 +592,13 @@ jobs: retention-days: 7 package_android: - name: Package Android (${{ ((needs.plan.outputs.mode == 'package' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_android == 'true') && needs.plan.outputs.config || 'skipped' }}) + name: ${{ needs.plan.outputs.package_android_name }} needs: [plan, deps_android, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.android && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} @@ -538,7 +642,7 @@ jobs: target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} config: ${{ inputs.config }} - name: Upload products-android - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: products-android path: products @@ -546,13 +650,13 @@ jobs: retention-days: 7 build_windows: - name: Build Windows (${{ (needs.plan.outputs.mode == 'build' && needs.plan.outputs.run_windows == 'true') && needs.plan.outputs.config || 'skipped' }}) + name: ${{ needs.plan.outputs.build_windows_name }} needs: plan if: ${{ inputs.mode == 'build' && inputs.windows }} runs-on: windows-latest timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Check Windows WebRTC host shell: bash run: | @@ -588,13 +692,13 @@ jobs: run: make build windows package_windows: - name: Package Windows (${{ ((needs.plan.outputs.mode == 'package' || needs.plan.outputs.mode == 'release') && needs.plan.outputs.run_windows == 'true') && needs.plan.outputs.config || 'skipped' }}) + name: ${{ needs.plan.outputs.package_windows_name }} needs: [plan, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.windows && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: windows-latest timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Check Windows WebRTC host shell: bash run: | @@ -639,7 +743,7 @@ jobs: SKIP_DEPS: "1" run: make package windows - name: Upload products-windows - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: products-windows path: products/windows @@ -647,16 +751,16 @@ jobs: retention-days: 7 finalise_package: - name: Finalise package - needs: [tests_passed, package_ios, package_macos, package_android, package_windows] + name: ${{ needs.plan.outputs.finalise_package_name }} + needs: [plan, tests_passed, package_ios, package_macos, package_android, package_windows] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && !cancelled() && needs.tests_passed.result == 'success' && !failure() }} runs-on: ${{ (inputs.ios || inputs.macos) && 'macos-26' || 'ubuntu-latest' }} timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Download product artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: pattern: products-* path: combine-in @@ -727,7 +831,7 @@ jobs: - name: Upload WebRTC.xcframework.zip if: ${{ inputs.ios || inputs.macos }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: final-apple path: WebRTC.xcframework.zip @@ -736,7 +840,7 @@ jobs: - name: Upload libwebrtc.aar if: ${{ inputs.android }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: final-android path: final-android @@ -745,7 +849,7 @@ jobs: - name: Upload Windows libs if: ${{ inputs.windows }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: final-windows path: windows-libs @@ -754,7 +858,7 @@ jobs: - name: Upload StreamWebRTC.xcframework.zip if: ${{ inputs.mode == 'release' && (inputs.ios || inputs.macos) }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: final-apple-renamed path: StreamWebRTC.xcframework.zip @@ -763,7 +867,7 @@ jobs: - name: Upload renamed Android AAR if: ${{ inputs.mode == 'release' && inputs.android }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: final-android-renamed path: libwebrtc-renamed.aar @@ -771,17 +875,17 @@ jobs: retention-days: 7 github_release: - name: Release ${{ inputs.version }} - needs: finalise_package + name: ${{ needs.plan.outputs.github_release_name }} + needs: [plan, finalise_package] if: ${{ inputs.mode == 'release' && !cancelled() && !failure() }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: ref: ${{ inputs.webrtc_ref }} - name: Download finalised artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: pattern: final-* merge-multiple: true @@ -824,8 +928,8 @@ jobs: gh release create "${release_args[@]}" trigger_downstream_releases: - name: Trigger downstream WebRTC releases - needs: github_release + name: ${{ needs.plan.outputs.trigger_downstream_name }} + needs: [plan, github_release] if: ${{ inputs.mode == 'release' && !cancelled() && !failure() }} runs-on: ubuntu-latest steps: From ec3aa7b00625a9b51119c282ae0ff87bd41b51e4 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Thu, 10 Sep 2026 22:47:22 +0300 Subject: [PATCH 04/24] Update job names and caching keys --- .github/actions/cache-webrtc/action.yml | 27 +-- .github/actions/deps-android/action.yml | 9 +- .github/actions/deps-apple/action.yml | 9 +- .github/actions/prepare-common/action.yml | 8 +- .github/actions/restore-tree/action.yml | 21 +- .github/workflows/_make.yml | 212 ++++---------------- .github/workflows/manual-platform-tests.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/release.yml | 2 +- 9 files changed, 73 insertions(+), 219 deletions(-) diff --git a/.github/actions/cache-webrtc/action.yml b/.github/actions/cache-webrtc/action.yml index e2a0759c68..c88c9e11ea 100644 --- a/.github/actions/cache-webrtc/action.yml +++ b/.github/actions/cache-webrtc/action.yml @@ -1,5 +1,5 @@ name: Cache WebRTC deps and out -description: OS-keyed cache of gclient trees plus DEPS_ROOT/out. Never share across OS. +description: Warm gclient/third_party base keyed only by OS family. inputs: operation: @@ -7,10 +7,7 @@ inputs: description: restore or save target_os_cache_key: required: true - description: Cache-safe target_os suffix (apple, android-unix, win). - config: - required: true - description: release, debug, or deps-only label included in the key. + description: apple, android, or windows. ignore_cache: required: false default: "false" @@ -19,16 +16,12 @@ inputs: runs: using: composite steps: - - id: key - name: Compute cache key + # GHA caches are immutable: a second save of webrtc-apple|android|windows + # is a no-op. That is intended (warm base until eviction). Do not add + # run_id, DEPS hash, config, or github.job. + - name: Cache key shell: bash - run: | - set -euo pipefail - deps_hash="${{ hashFiles('DEPS') }}" - prefix="webrtc-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ inputs.config }}-${deps_hash}" - echo "prefix=${prefix}" >> "${GITHUB_OUTPUT}" - echo "key=${prefix}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}" >> "${GITHUB_OUTPUT}" - echo "Cache prefix: ${prefix}" + run: echo "Cache key: webrtc-${{ inputs.target_os_cache_key }}" - name: Restore deps and out cache if: ${{ inputs.operation == 'restore' && inputs.ignore_cache != 'true' }} @@ -42,9 +35,7 @@ runs: testing ios tools - key: ${{ steps.key.outputs.key }} - restore-keys: | - ${{ steps.key.outputs.prefix }}- + key: webrtc-${{ inputs.target_os_cache_key }} - name: Re-link DEPS_ROOT src after cache restore if: ${{ inputs.operation == 'restore' }} @@ -66,4 +57,4 @@ runs: testing ios tools - key: ${{ steps.key.outputs.key }} + key: webrtc-${{ inputs.target_os_cache_key }} diff --git a/.github/actions/deps-android/action.yml b/.github/actions/deps-android/action.yml index 525a3979aa..0e02ea0f73 100644 --- a/.github/actions/deps-android/action.yml +++ b/.github/actions/deps-android/action.yml @@ -1,5 +1,5 @@ name: Deps Android -description: Linux gclient sync for Android. Caches deps trees and out/. +description: Linux gclient sync for Android. Caches the warm android deps tree. inputs: webrtc_ref: @@ -10,10 +10,7 @@ inputs: description: Comma-separated gclient target_os (android,unix). target_os_cache_key: required: true - description: Cache-safe target_os key suffix. - config: - required: true - description: Cache key config (release or debug). + description: android ignore_cache: required: true description: Skip restoring caches and force a fresh sync. @@ -32,7 +29,6 @@ runs: with: operation: restore target_os_cache_key: ${{ inputs.target_os_cache_key }} - config: ${{ inputs.config }} ignore_cache: ${{ inputs.ignore_cache }} - name: gclient sync (Android) @@ -50,5 +46,4 @@ runs: with: operation: save target_os_cache_key: ${{ inputs.target_os_cache_key }} - config: ${{ inputs.config }} ignore_cache: ${{ inputs.ignore_cache }} diff --git a/.github/actions/deps-apple/action.yml b/.github/actions/deps-apple/action.yml index 731bdf64ee..324932eb68 100644 --- a/.github/actions/deps-apple/action.yml +++ b/.github/actions/deps-apple/action.yml @@ -1,5 +1,5 @@ name: Deps Apple -description: Darwin gclient sync for iOS/macOS. Caches deps trees and out/. +description: Darwin gclient sync for iOS/macOS. Caches the warm apple deps tree. inputs: webrtc_ref: @@ -10,10 +10,7 @@ inputs: description: Comma-separated gclient target_os (ios and/or mac). target_os_cache_key: required: true - description: Cache-safe target_os key suffix. - config: - required: true - description: Cache key config (release or debug). + description: apple ignore_cache: required: true description: Skip restoring caches and force a fresh sync. @@ -31,7 +28,6 @@ runs: with: operation: restore target_os_cache_key: ${{ inputs.target_os_cache_key }} - config: ${{ inputs.config }} ignore_cache: ${{ inputs.ignore_cache }} - name: gclient sync (Apple) @@ -49,5 +45,4 @@ runs: with: operation: save target_os_cache_key: ${{ inputs.target_os_cache_key }} - config: ${{ inputs.config }} ignore_cache: ${{ inputs.ignore_cache }} diff --git a/.github/actions/prepare-common/action.yml b/.github/actions/prepare-common/action.yml index efa9c24adf..bc5c5cdb9e 100644 --- a/.github/actions/prepare-common/action.yml +++ b/.github/actions/prepare-common/action.yml @@ -84,13 +84,17 @@ runs: fi android_target_os="" + android_target_os_cache_key="" if [[ "${android}" == "true" ]]; then android_target_os="android,unix" + android_target_os_cache_key="android" fi windows_target_os="" + windows_target_os_cache_key="" if [[ "${windows}" == "true" ]]; then windows_target_os="win" + windows_target_os_cache_key="windows" fi echo "run_ios=${ios}" >> "${GITHUB_OUTPUT}" @@ -102,6 +106,6 @@ runs: echo "apple_target_os_cache_key=${apple_target_os_cache_key}" >> "${GITHUB_OUTPUT}" echo "apple_target_os_label=${apple_target_os_label}" >> "${GITHUB_OUTPUT}" echo "android_target_os=${android_target_os}" >> "${GITHUB_OUTPUT}" - echo "android_target_os_cache_key=${android_target_os//,/-}" >> "${GITHUB_OUTPUT}" + echo "android_target_os_cache_key=${android_target_os_cache_key}" >> "${GITHUB_OUTPUT}" echo "windows_target_os=${windows_target_os}" >> "${GITHUB_OUTPUT}" - echo "windows_target_os_cache_key=${windows_target_os}" >> "${GITHUB_OUTPUT}" + echo "windows_target_os_cache_key=${windows_target_os_cache_key}" >> "${GITHUB_OUTPUT}" diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index 2975b86abd..d3a36e8483 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -1,16 +1,16 @@ name: Restore WebRTC tree -description: Same-OS cache restore, then SKIP_DEPS=1 consumers can make. +description: Restore the warm OS cache, then gclient sync so SKIP_DEPS=1 can follow. inputs: webrtc_ref: required: true description: Branch, tag, or SHA for GetStream/webrtc. - target_os_cache_key: + target_os: required: true - description: Cache-safe target_os key suffix. - config: + description: gclient TARGET_OS (ios,mac / android,unix / win). + target_os_cache_key: required: true - description: Cache key config (release or debug). + description: apple, android, or windows. install_android_packages: required: false default: "false" @@ -30,5 +30,14 @@ runs: with: operation: restore target_os_cache_key: ${{ inputs.target_os_cache_key }} - config: ${{ inputs.config }} ignore_cache: "false" + + - name: gclient sync + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + TARGET_OS: ${{ inputs.target_os }} + JOBS: "8" + WEBRTC_REPO: https://github.com/GetStream/webrtc.git + run: make deps diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 16e725661b..76011598f1 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -109,26 +109,10 @@ jobs: run_apple: ${{ steps.plan.outputs.run_apple }} apple_target_os: ${{ steps.plan.outputs.apple_target_os }} apple_target_os_cache_key: ${{ steps.plan.outputs.apple_target_os_cache_key }} - apple_target_os_label: ${{ steps.plan.outputs.apple_target_os_label }} android_target_os: ${{ steps.plan.outputs.android_target_os }} android_target_os_cache_key: ${{ steps.plan.outputs.android_target_os_cache_key }} - deps_apple_name: ${{ steps.names.outputs.deps_apple_name }} - deps_android_name: ${{ steps.names.outputs.deps_android_name }} - build_ios_name: ${{ steps.names.outputs.build_ios_name }} - build_macos_name: ${{ steps.names.outputs.build_macos_name }} - build_android_name: ${{ steps.names.outputs.build_android_name }} - build_windows_name: ${{ steps.names.outputs.build_windows_name }} - test_ios_name: ${{ steps.names.outputs.test_ios_name }} - test_macos_name: ${{ steps.names.outputs.test_macos_name }} - test_windows_name: ${{ steps.names.outputs.test_windows_name }} - tests_passed_name: ${{ steps.names.outputs.tests_passed_name }} - package_ios_name: ${{ steps.names.outputs.package_ios_name }} - package_macos_name: ${{ steps.names.outputs.package_macos_name }} - package_android_name: ${{ steps.names.outputs.package_android_name }} - package_windows_name: ${{ steps.names.outputs.package_windows_name }} - finalise_package_name: ${{ steps.names.outputs.finalise_package_name }} - github_release_name: ${{ steps.names.outputs.github_release_name }} - trigger_downstream_name: ${{ steps.names.outputs.trigger_downstream_name }} + windows_target_os: ${{ steps.plan.outputs.windows_target_os }} + windows_target_os_cache_key: ${{ steps.plan.outputs.windows_target_os_cache_key }} steps: - uses: actions/checkout@v7 - id: plan @@ -139,96 +123,9 @@ jobs: platform_macos: ${{ inputs.macos }} platform_android: ${{ inputs.android }} platform_windows: ${{ inputs.windows }} - - id: names - name: Plan job display names - env: - MODE: ${{ inputs.mode }} - CONFIG: ${{ inputs.config }} - VERSION: ${{ inputs.version }} - RUN_IOS: ${{ steps.plan.outputs.run_ios }} - RUN_MACOS: ${{ steps.plan.outputs.run_macos }} - RUN_ANDROID: ${{ steps.plan.outputs.run_android }} - RUN_WINDOWS: ${{ steps.plan.outputs.run_windows }} - RUN_APPLE: ${{ steps.plan.outputs.run_apple }} - APPLE_LABEL: ${{ steps.plan.outputs.apple_target_os_label }} - run: | - set -euo pipefail - # Job `name:` does not evaluate nested ==/&&/||. Emit plain strings. - - active_or_skipped() { - local prefix="$1" - local is_active="$2" - local detail="${3:-}" - if [[ "${is_active}" == "true" ]]; then - if [[ -n "${detail}" ]]; then - printf '%s (%s)\n' "${prefix}" "${detail}" - else - printf '%s\n' "${prefix}" - fi - else - printf '%s (skipped)\n' "${prefix}" - fi - } - - apple_label="${APPLE_LABEL//, /,}" - apple_label="${apple_label//,/, }" - - is_build=false - is_test=false - is_package=false - is_release=false - [[ "${MODE}" == "build" ]] && is_build=true - [[ "${MODE}" == "test" || "${MODE}" == "release" ]] && is_test=true - [[ "${MODE}" == "package" || "${MODE}" == "release" ]] && is_package=true - [[ "${MODE}" == "release" ]] && is_release=true - - run_build_ios=false - run_build_macos=false - run_build_android=false - run_build_windows=false - run_test_ios=false - run_test_macos=false - run_test_windows=false - run_package_ios=false - run_package_macos=false - run_package_android=false - run_package_windows=false - [[ "${is_build}" == "true" && "${RUN_IOS}" == "true" ]] && run_build_ios=true - [[ "${is_build}" == "true" && "${RUN_MACOS}" == "true" ]] && run_build_macos=true - [[ "${is_build}" == "true" && "${RUN_ANDROID}" == "true" ]] && run_build_android=true - [[ "${is_build}" == "true" && "${RUN_WINDOWS}" == "true" ]] && run_build_windows=true - [[ "${is_test}" == "true" && "${RUN_IOS}" == "true" ]] && run_test_ios=true - [[ "${is_test}" == "true" && "${RUN_MACOS}" == "true" ]] && run_test_macos=true - [[ "${is_test}" == "true" && "${RUN_WINDOWS}" == "true" ]] && run_test_windows=true - [[ "${is_package}" == "true" && "${RUN_IOS}" == "true" ]] && run_package_ios=true - [[ "${is_package}" == "true" && "${RUN_MACOS}" == "true" ]] && run_package_macos=true - [[ "${is_package}" == "true" && "${RUN_ANDROID}" == "true" ]] && run_package_android=true - [[ "${is_package}" == "true" && "${RUN_WINDOWS}" == "true" ]] && run_package_windows=true - - { - echo "deps_apple_name=$(active_or_skipped "Deps Apple" "${RUN_APPLE}" "${apple_label}")" - echo "deps_android_name=$(active_or_skipped "Deps Android" "${RUN_ANDROID}")" - echo "build_ios_name=$(active_or_skipped "Build iOS" "${run_build_ios}" "${CONFIG}")" - echo "build_macos_name=$(active_or_skipped "Build macOS" "${run_build_macos}" "${CONFIG}")" - echo "build_android_name=$(active_or_skipped "Build Android" "${run_build_android}" "${CONFIG}")" - echo "build_windows_name=$(active_or_skipped "Build Windows" "${run_build_windows}" "${CONFIG}")" - echo "test_ios_name=$(active_or_skipped "Test iOS" "${run_test_ios}" "${CONFIG}")" - echo "test_macos_name=$(active_or_skipped "Test macOS" "${run_test_macos}" "${CONFIG}")" - echo "test_windows_name=$(active_or_skipped "Test Windows" "${run_test_windows}" "${CONFIG}")" - echo "tests_passed_name=$(active_or_skipped "Tests passed" "${is_package}")" - echo "package_ios_name=$(active_or_skipped "Package iOS" "${run_package_ios}" "${CONFIG}")" - echo "package_macos_name=$(active_or_skipped "Package macOS" "${run_package_macos}" "${CONFIG}")" - echo "package_android_name=$(active_or_skipped "Package Android" "${run_package_android}" "${CONFIG}")" - echo "package_windows_name=$(active_or_skipped "Package Windows" "${run_package_windows}" "${CONFIG}")" - echo "finalise_package_name=$(active_or_skipped "Finalise package" "${is_package}")" - release_title="Release (skipped)" - [[ "${is_release}" == "true" ]] && release_title="Release ${VERSION}" - echo "github_release_name=${release_title}" - echo "trigger_downstream_name=$(active_or_skipped "Trigger downstream WebRTC releases" "${is_release}")" - } >> "${GITHUB_OUTPUT}" deps_apple: - name: ${{ needs.plan.outputs.deps_apple_name }} + name: Deps Apple needs: plan if: ${{ inputs.ios || inputs.macos }} runs-on: macos-26 @@ -241,14 +138,13 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} ignore_cache: ${{ inputs.ignore_cache }} deps_android: - name: ${{ needs.plan.outputs.deps_android_name }} + name: Deps Android needs: plan if: ${{ inputs.android }} - runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} + runs-on: ubuntu-latest timeout-minutes: 360 steps: - uses: actions/checkout@v7 @@ -258,11 +154,10 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.android_target_os }} target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} - config: ${{ inputs.config }} ignore_cache: ${{ inputs.ignore_cache }} build_ios: - name: ${{ needs.plan.outputs.build_ios_name }} + name: Build iOS needs: [plan, deps_apple] if: ${{ inputs.mode == 'build' && inputs.ios }} runs-on: macos-26 @@ -272,8 +167,8 @@ jobs: - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.plan.outputs.apple_target_os }} target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} - name: Build iOS working-directory: stream_build env: @@ -286,10 +181,9 @@ jobs: with: operation: save target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} build_macos: - name: ${{ needs.plan.outputs.build_macos_name }} + name: Build macOS needs: [plan, deps_apple] if: ${{ inputs.mode == 'build' && inputs.macos }} runs-on: macos-26 @@ -299,8 +193,8 @@ jobs: - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.plan.outputs.apple_target_os }} target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} - name: Build macOS working-directory: stream_build env: @@ -313,21 +207,20 @@ jobs: with: operation: save target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} build_android: - name: ${{ needs.plan.outputs.build_android_name }} + name: Build Android needs: [plan, deps_android] if: ${{ inputs.mode == 'build' && inputs.android }} - runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} + runs-on: ubuntu-latest timeout-minutes: 360 steps: - uses: actions/checkout@v7 - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.plan.outputs.android_target_os }} target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} - config: ${{ inputs.config }} install_android_packages: "true" - name: Build Android working-directory: stream_build @@ -348,10 +241,9 @@ jobs: with: operation: save target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} - config: ${{ inputs.config }} test_ios: - name: ${{ needs.plan.outputs.test_ios_name }} + name: Test iOS needs: [plan, deps_apple] if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.ios }} runs-on: macos-26 @@ -361,8 +253,8 @@ jobs: - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.plan.outputs.apple_target_os }} target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} - name: Build iOS working-directory: stream_build env: @@ -381,10 +273,9 @@ jobs: with: operation: save target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} test_macos: - name: ${{ needs.plan.outputs.test_macos_name }} + name: Test macOS needs: [plan, deps_apple] if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.macos }} runs-on: macos-26 @@ -394,8 +285,8 @@ jobs: - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.plan.outputs.apple_target_os }} target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} - name: Build macOS working-directory: stream_build env: @@ -414,10 +305,9 @@ jobs: with: operation: save target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} test_windows: - name: ${{ needs.plan.outputs.test_windows_name }} + name: Test Windows needs: plan if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.windows }} runs-on: windows-latest @@ -438,17 +328,8 @@ jobs: - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} - target_os_cache_key: win - config: ${{ inputs.config }} - - name: Windows deps - working-directory: stream_build - shell: bash - env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps - TARGET_OS: win - JOBS: "8" - WEBRTC_REPO: https://github.com/GetStream/webrtc.git - run: make deps + target_os: ${{ needs.plan.outputs.windows_target_os }} + target_os_cache_key: ${{ needs.plan.outputs.windows_target_os_cache_key }} - name: Build Windows working-directory: stream_build shell: bash @@ -466,7 +347,7 @@ jobs: run: make test windows tests_passed: - name: ${{ needs.plan.outputs.tests_passed_name }} + name: Tests passed needs: [plan, test_ios, test_macos, test_windows] if: ${{ always() && !cancelled() && (inputs.mode == 'package' || inputs.mode == 'release') }} runs-on: ubuntu-latest @@ -506,7 +387,7 @@ jobs: echo "Selected-platform tests passed (Android tests are unwired and skipped)." package_ios: - name: ${{ needs.plan.outputs.package_ios_name }} + name: Package iOS needs: [plan, deps_apple, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.ios && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: macos-26 @@ -516,8 +397,8 @@ jobs: - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.plan.outputs.apple_target_os }} target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} - name: Build iOS working-directory: stream_build env: @@ -539,7 +420,6 @@ jobs: with: operation: save target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} - name: Upload products-ios uses: actions/upload-artifact@v7 with: @@ -549,7 +429,7 @@ jobs: retention-days: 7 package_macos: - name: ${{ needs.plan.outputs.package_macos_name }} + name: Package macOS needs: [plan, deps_apple, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.macos && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: macos-26 @@ -559,8 +439,8 @@ jobs: - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.plan.outputs.apple_target_os }} target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} - name: Build macOS working-directory: stream_build env: @@ -582,7 +462,6 @@ jobs: with: operation: save target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - config: ${{ inputs.config }} - name: Upload products-macos uses: actions/upload-artifact@v7 with: @@ -592,18 +471,18 @@ jobs: retention-days: 7 package_android: - name: ${{ needs.plan.outputs.package_android_name }} + name: Package Android needs: [plan, deps_android, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.android && !cancelled() && needs.tests_passed.result == 'success' }} - runs-on: ${{ vars.WEBRTC_ANDROID_RUNNER != '' && vars.WEBRTC_ANDROID_RUNNER || 'runner-extra-capacity' }} + runs-on: ubuntu-latest timeout-minutes: 360 steps: - uses: actions/checkout@v7 - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.plan.outputs.android_target_os }} target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} - config: ${{ inputs.config }} install_android_packages: "true" - name: Build Android working-directory: stream_build @@ -640,7 +519,6 @@ jobs: with: operation: save target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} - config: ${{ inputs.config }} - name: Upload products-android uses: actions/upload-artifact@v7 with: @@ -650,7 +528,7 @@ jobs: retention-days: 7 build_windows: - name: ${{ needs.plan.outputs.build_windows_name }} + name: Build Windows needs: plan if: ${{ inputs.mode == 'build' && inputs.windows }} runs-on: windows-latest @@ -671,17 +549,8 @@ jobs: - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} - target_os_cache_key: win - config: ${{ inputs.config }} - - name: Windows deps - working-directory: stream_build - shell: bash - env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps - TARGET_OS: win - JOBS: "8" - WEBRTC_REPO: https://github.com/GetStream/webrtc.git - run: make deps + target_os: ${{ needs.plan.outputs.windows_target_os }} + target_os_cache_key: ${{ needs.plan.outputs.windows_target_os_cache_key }} - name: Build Windows working-directory: stream_build shell: bash @@ -692,7 +561,7 @@ jobs: run: make build windows package_windows: - name: ${{ needs.plan.outputs.package_windows_name }} + name: Package Windows needs: [plan, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.windows && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: windows-latest @@ -713,17 +582,8 @@ jobs: - uses: ./.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} - target_os_cache_key: win - config: ${{ inputs.config }} - - name: Windows deps - working-directory: stream_build - shell: bash - env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps - TARGET_OS: win - JOBS: "8" - WEBRTC_REPO: https://github.com/GetStream/webrtc.git - run: make deps + target_os: ${{ needs.plan.outputs.windows_target_os }} + target_os_cache_key: ${{ needs.plan.outputs.windows_target_os_cache_key }} - name: Build Windows working-directory: stream_build shell: bash @@ -751,7 +611,7 @@ jobs: retention-days: 7 finalise_package: - name: ${{ needs.plan.outputs.finalise_package_name }} + name: Finalise package needs: [plan, tests_passed, package_ios, package_macos, package_android, package_windows] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && !cancelled() && needs.tests_passed.result == 'success' && !failure() }} runs-on: ${{ (inputs.ios || inputs.macos) && 'macos-26' || 'ubuntu-latest' }} @@ -875,7 +735,7 @@ jobs: retention-days: 7 github_release: - name: ${{ needs.plan.outputs.github_release_name }} + name: Release needs: [plan, finalise_package] if: ${{ inputs.mode == 'release' && !cancelled() && !failure() }} runs-on: ubuntu-latest @@ -928,7 +788,7 @@ jobs: gh release create "${release_args[@]}" trigger_downstream_releases: - name: ${{ needs.plan.outputs.trigger_downstream_name }} + name: Trigger downstream WebRTC releases needs: [plan, github_release] if: ${{ inputs.mode == 'release' && !cancelled() && !failure() }} runs-on: ubuntu-latest diff --git a/.github/workflows/manual-platform-tests.yml b/.github/workflows/manual-platform-tests.yml index 74e7f96263..eea9c41497 100644 --- a/.github/workflows/manual-platform-tests.yml +++ b/.github/workflows/manual-platform-tests.yml @@ -4,7 +4,7 @@ name: Build run-name: >- - Build config:${{ inputs.config }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true') && ', Android' || ' Android') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} + Build config:${{ inputs.config }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && format('{0} Android{1}', (inputs.ios == 'true' || inputs.macos == 'true') && ',' || '', inputs.android_arch != '' && format(' ({0})', inputs.android_arch) || '') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} permissions: contents: read diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 34b65860a7..e8da3fce7f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,7 +4,7 @@ name: Package run-name: >- - Package config:${{ inputs.config }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true') && ', Android' || ' Android') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} + Package config:${{ inputs.config }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && format('{0} Android{1}', (inputs.ios == 'true' || inputs.macos == 'true') && ',' || '', inputs.android_arch != '' && format(' ({0})', inputs.android_arch) || '') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} permissions: contents: read diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 27959538af..9b9e49289e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,7 +4,7 @@ name: Release run-name: >- - Release config:${{ inputs.config }}${{ inputs.prerelease == 'true' && ' Pre-Release' || '' }} ${{ inputs.version }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true') && ', Android' || ' Android') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} + Release config:${{ inputs.config }}${{ inputs.prerelease == 'true' && ' Pre-Release' || '' }} ${{ inputs.version }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && format('{0} Android{1}', (inputs.ios == 'true' || inputs.macos == 'true') && ',' || '', inputs.android_arch != '' && format(' ({0})', inputs.android_arch) || '') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} permissions: contents: write From dd02f53c64029e857a49ded6b633d6b6bb3f118b Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Thu, 10 Sep 2026 23:05:26 +0300 Subject: [PATCH 05/24] Fix failing cache restoration --- .github/actions/cache-webrtc/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/cache-webrtc/action.yml b/.github/actions/cache-webrtc/action.yml index c88c9e11ea..53a6951e2e 100644 --- a/.github/actions/cache-webrtc/action.yml +++ b/.github/actions/cache-webrtc/action.yml @@ -21,7 +21,8 @@ runs: # run_id, DEPS hash, config, or github.job. - name: Cache key shell: bash - run: echo "Cache key: webrtc-${{ inputs.target_os_cache_key }}" + run: | + echo "Cache key: webrtc-${{ inputs.target_os_cache_key }}" - name: Restore deps and out cache if: ${{ inputs.operation == 'restore' && inputs.ignore_cache != 'true' }} From c99e6a4fb52a5443cfe5719bd3c8f36dc07140cd Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Fri, 11 Sep 2026 00:30:53 +0300 Subject: [PATCH 06/24] Improve caching mechanism --- .github/actions/cache-webrtc/action.yml | 100 +++++++++--- .github/actions/deps-android/action.yml | 49 ------ .github/actions/deps-apple/action.yml | 48 ------ .github/actions/restore-tree/action.yml | 36 ++++- .github/actions/setup-webrtc/action.yml | 3 +- .github/workflows/_make.yml | 194 ++++++++++++++++-------- 6 files changed, 239 insertions(+), 191 deletions(-) delete mode 100644 .github/actions/deps-android/action.yml delete mode 100644 .github/actions/deps-apple/action.yml diff --git a/.github/actions/cache-webrtc/action.yml b/.github/actions/cache-webrtc/action.yml index 53a6951e2e..5c53200591 100644 --- a/.github/actions/cache-webrtc/action.yml +++ b/.github/actions/cache-webrtc/action.yml @@ -1,5 +1,5 @@ -name: Cache WebRTC deps and out -description: Warm gclient/third_party base keyed only by OS family. +name: Cache gclient object cache +description: Cross-run actions/cache for gclient objects only. Same-run tree handoff is the deps-* artifact. inputs: operation: @@ -16,46 +16,96 @@ inputs: runs: using: composite steps: - # GHA caches are immutable: a second save of webrtc-apple|android|windows - # is a no-op. That is intended (warm base until eviction). Do not add - # run_id, DEPS hash, config, or github.job. + # Objects-only: .gclient_deps/.gclient-git-cache. gclient sync refills + # working trees (third_party, build, ios, tools, …). Do not cache those. + # Keys are immutable warm bases (webrtc-apple|android|windows). Do not + # add run_id, DEPS hash, config, or github.job. - name: Cache key shell: bash run: | echo "Cache key: webrtc-${{ inputs.target_os_cache_key }}" - - name: Restore deps and out cache + - name: Restore gclient object cache + id: cache-restore if: ${{ inputs.operation == 'restore' && inputs.ignore_cache != 'true' }} uses: actions/cache/restore@v6 with: - path: | - .gclient_deps - third_party - build - buildtools - testing - ios - tools + path: .gclient_deps/.gclient-git-cache key: webrtc-${{ inputs.target_os_cache_key }} - - name: Re-link DEPS_ROOT src after cache restore + - name: Prepare DEPS_ROOT after object-cache restore if: ${{ inputs.operation == 'restore' }} shell: bash + env: + CACHE_HIT: ${{ steps.cache-restore.outputs.cache-hit }} run: | set -euo pipefail - mkdir -p "${GITHUB_WORKSPACE}/.gclient_deps" + echo "cache/restore finished cache-hit=${CACHE_HIT:-miss-or-skipped}" + mkdir -p "${GITHUB_WORKSPACE}/.gclient_deps/.gclient-git-cache" ln -sfn .. "${GITHUB_WORKSPACE}/.gclient_deps/src" - - name: Save deps and out cache + # Keys are immutable. A second save of webrtc-apple|android|windows + # fails; that is the warm base until eviction. + # GitHub repo cache quota is 10 GB total. Cap saves at 10 GiB + # (10737418240 bytes) so we do not spend minutes uploading a blob + # that will be evicted. Same-run deps-* artifacts still upload. + - name: Measure git-cache size + id: measure if: ${{ inputs.operation == 'save' }} + shell: bash + env: + CACHE_DIR: ${{ github.workspace }}/.gclient_deps/.gclient-git-cache + run: | + set -euo pipefail + # Conservative vs GitHub's 10 GB (10*1000^3) repo cache quota. + cap_bytes=$((10 * 1024 * 1024 * 1024)) + + if [[ ! -d "${CACHE_DIR}" ]]; then + echo "git-cache missing: ${CACHE_DIR}" + exit 0 + fi + + human="unknown" + if h="$(du -sh "${CACHE_DIR}" 2>/dev/null | awk '{print $1}')"; then + human="${h:-unknown}" + fi + + bytes="" + if out="$(du -sb "${CACHE_DIR}" 2>/dev/null)"; then + bytes="$(awk '{print $1}' <<<"${out}")" + fi + if [[ ! "${bytes}" =~ ^[0-9]+$ ]]; then + if out="$(du -sk "${CACHE_DIR}" 2>/dev/null)"; then + kb="$(awk '{print $1}' <<<"${out}")" + if [[ "${kb}" =~ ^[0-9]+$ ]]; then + bytes=$((kb * 1024)) + fi + fi + fi + if [[ ! "${bytes}" =~ ^[0-9]+$ ]]; then + echo "::warning title=Actions cache size::Could not measure ${CACHE_DIR} (du failed). Cache save will still be attempted." + exit 0 + fi + + gib="$(awk -v b="${bytes}" 'BEGIN { printf "%.2f", b / 1073741824 }')" + echo "git-cache size: ${human} (${bytes} bytes, ${gib} GiB)" + echo "cap: 10 GiB = ${cap_bytes} bytes (GitHub quota is 10 GB = 10000000000 bytes; 10 GiB is the conservative save cap)" + echo "bytes=${bytes}" >> "${GITHUB_OUTPUT}" + + quota_note="Apple and Android caches (webrtc-apple|android; also windows) both count against the same 10 GB repository Actions cache quota." + if (( bytes > cap_bytes )); then + echo "skip_save=true" >> "${GITHUB_OUTPUT}" + echo "::warning title=Actions cache size::git-cache is ${human} (${bytes} bytes, ${gib} GiB), over the 10 GiB cap (${cap_bytes} bytes). Skipping actions/cache save — GitHub would evict it after a multi-minute upload that still prints Cache saved. ${quota_note}" + echo "::warning title=Deps artifact size::git-cache is ${human} (${bytes} bytes, ${gib} GiB). Same-run artifact upload still proceeds (separate quota from Actions cache)." + echo "Skipping actions/cache save (over 10 GiB cap)." + else + echo "Under cap; actions/cache save will run." + fi + + - name: Save gclient object cache + if: ${{ inputs.operation == 'save' && steps.measure.outputs.skip_save != 'true' }} + continue-on-error: true uses: actions/cache/save@v6 with: - path: | - .gclient_deps - third_party - build - buildtools - testing - ios - tools + path: .gclient_deps/.gclient-git-cache key: webrtc-${{ inputs.target_os_cache_key }} diff --git a/.github/actions/deps-android/action.yml b/.github/actions/deps-android/action.yml deleted file mode 100644 index 0e02ea0f73..0000000000 --- a/.github/actions/deps-android/action.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Deps Android -description: Linux gclient sync for Android. Caches the warm android deps tree. - -inputs: - webrtc_ref: - required: true - description: Branch, tag, or SHA for GetStream/webrtc. - target_os: - required: true - description: Comma-separated gclient target_os (android,unix). - target_os_cache_key: - required: true - description: android - ignore_cache: - required: true - description: Skip restoring caches and force a fresh sync. - -runs: - using: composite - steps: - - name: Setup WebRTC checkout - uses: ./.github/actions/setup-webrtc - with: - webrtc_ref: ${{ inputs.webrtc_ref }} - install_android_packages: "true" - - - name: Restore Linux deps and out - uses: ./.github/actions/cache-webrtc - with: - operation: restore - target_os_cache_key: ${{ inputs.target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} - - - name: gclient sync (Android) - working-directory: stream_build - shell: bash - env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps - TARGET_OS: ${{ inputs.target_os }} - JOBS: "8" - WEBRTC_REPO: https://github.com/GetStream/webrtc.git - run: make deps - - - name: Save Linux deps and out - uses: ./.github/actions/cache-webrtc - with: - operation: save - target_os_cache_key: ${{ inputs.target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} diff --git a/.github/actions/deps-apple/action.yml b/.github/actions/deps-apple/action.yml deleted file mode 100644 index 324932eb68..0000000000 --- a/.github/actions/deps-apple/action.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Deps Apple -description: Darwin gclient sync for iOS/macOS. Caches the warm apple deps tree. - -inputs: - webrtc_ref: - required: true - description: Branch, tag, or SHA for GetStream/webrtc. - target_os: - required: true - description: Comma-separated gclient target_os (ios and/or mac). - target_os_cache_key: - required: true - description: apple - ignore_cache: - required: true - description: Skip restoring caches and force a fresh sync. - -runs: - using: composite - steps: - - name: Setup WebRTC checkout - uses: ./.github/actions/setup-webrtc - with: - webrtc_ref: ${{ inputs.webrtc_ref }} - - - name: Restore Darwin deps and out - uses: ./.github/actions/cache-webrtc - with: - operation: restore - target_os_cache_key: ${{ inputs.target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} - - - name: gclient sync (Apple) - working-directory: stream_build - shell: bash - env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps - TARGET_OS: ${{ inputs.target_os }} - JOBS: "8" - WEBRTC_REPO: https://github.com/GetStream/webrtc.git - run: make deps - - - name: Save Darwin deps and out - uses: ./.github/actions/cache-webrtc - with: - operation: save - target_os_cache_key: ${{ inputs.target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index d3a36e8483..a82b2b2cb0 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -1,5 +1,5 @@ name: Restore WebRTC tree -description: Restore the warm OS cache, then gclient sync so SKIP_DEPS=1 can follow. +description: Download the same-run git-cache artifact, then gclient sync from local objects so SKIP_DEPS=1 can follow. inputs: webrtc_ref: @@ -8,9 +8,9 @@ inputs: target_os: required: true description: gclient TARGET_OS (ios,mac / android,unix / win). - target_os_cache_key: + deps_artifact: required: true - description: apple, android, or windows. + description: Same-run artifact (deps-apple / deps-android / deps-windows). install_android_packages: required: false default: "false" @@ -25,18 +25,38 @@ runs: webrtc_ref: ${{ inputs.webrtc_ref }} install_android_packages: ${{ inputs.install_android_packages }} - - name: Restore same-OS deps and out - uses: ./.github/actions/cache-webrtc + # Same-run handoff. Do not restore actions/cache here: the warm OS + # cache is evicted (10 GB quota) even when Deps just saved it. + # Artifact is git-cache only; the full tree exceeds the per-artifact cap. + - name: Download same-run deps artifact + uses: actions/download-artifact@v8 with: - operation: restore - target_os_cache_key: ${{ inputs.target_os_cache_key }} - ignore_cache: "false" + name: ${{ inputs.deps_artifact }} + path: ${{ runner.temp }}/deps-git-cache + + - name: Install same-run gclient object cache + shell: bash + run: | + set -euo pipefail + dest="${GITHUB_WORKSPACE}/.gclient_deps/.gclient-git-cache" + src="${RUNNER_TEMP}/deps-git-cache" + if [[ -d "${src}/.gclient_deps/.gclient-git-cache" ]]; then + src="${src}/.gclient_deps/.gclient-git-cache" + elif [[ -d "${src}/.gclient-git-cache" ]]; then + src="${src}/.gclient-git-cache" + fi + mkdir -p "${GITHUB_WORKSPACE}/.gclient_deps" + rm -rf "${dest}" + mv "${src}" "${dest}" + test -n "$(ls -A "${dest}")" + ln -sfn .. "${GITHUB_WORKSPACE}/.gclient_deps/src" - name: gclient sync working-directory: stream_build shell: bash env: DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient_deps/.gclient-git-cache TARGET_OS: ${{ inputs.target_os }} JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git diff --git a/.github/actions/setup-webrtc/action.yml b/.github/actions/setup-webrtc/action.yml index 25317ccc47..b8614701fe 100644 --- a/.github/actions/setup-webrtc/action.yml +++ b/.github/actions/setup-webrtc/action.yml @@ -88,6 +88,7 @@ runs: shell: bash run: | set -euo pipefail - mkdir -p "${GITHUB_WORKSPACE}/.gclient_deps" + mkdir -p "${GITHUB_WORKSPACE}/.gclient_deps/.gclient-git-cache" ln -sfn .. "${GITHUB_WORKSPACE}/.gclient_deps/src" + echo "GIT_CACHE_PATH=${GITHUB_WORKSPACE}/.gclient_deps/.gclient-git-cache" >> "${GITHUB_ENV}" test -f "${GITHUB_WORKSPACE}/.gclient_deps/src/DEPS" diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 76011598f1..d7f5d05b4d 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -132,13 +132,44 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 - - name: Deps Apple - uses: ./.github/actions/deps-apple + - name: Setup WebRTC checkout + uses: ./.github/actions/setup-webrtc with: webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.plan.outputs.apple_target_os }} + - name: Restore Darwin object cache + uses: ./.github/actions/cache-webrtc + with: + operation: restore + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + ignore_cache: ${{ inputs.ignore_cache }} + - name: gclient sync (Apple) + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient_deps/.gclient-git-cache + TARGET_OS: ${{ needs.plan.outputs.apple_target_os }} + JOBS: "8" + WEBRTC_REPO: https://github.com/GetStream/webrtc.git + run: make deps + - name: Save Darwin object cache + uses: ./.github/actions/cache-webrtc + with: + operation: save target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} ignore_cache: ${{ inputs.ignore_cache }} + # Same-run handoff. The full tree is ~20 GB (over the artifact cap); + # git-cache only. Build downloads this and runs make deps locally. + # Oversize git-cache skips actions/cache save (10 GiB cap in + # cache-webrtc); this artifact still uploads (separate quota). + - name: Upload deps-apple + uses: actions/upload-artifact@v7 + with: + name: deps-apple + path: .gclient_deps/.gclient-git-cache + include-hidden-files: true + if-no-files-found: error + retention-days: 1 + compression-level: 0 deps_android: name: Deps Android @@ -148,13 +179,96 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 - - name: Deps Android - uses: ./.github/actions/deps-android + - name: Setup WebRTC checkout + uses: ./.github/actions/setup-webrtc with: webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.plan.outputs.android_target_os }} + install_android_packages: "true" + - name: Restore Linux object cache + uses: ./.github/actions/cache-webrtc + with: + operation: restore + target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} + ignore_cache: ${{ inputs.ignore_cache }} + - name: gclient sync (Android) + working-directory: stream_build + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient_deps/.gclient-git-cache + TARGET_OS: ${{ needs.plan.outputs.android_target_os }} + JOBS: "8" + WEBRTC_REPO: https://github.com/GetStream/webrtc.git + run: make deps + - name: Save Linux object cache + uses: ./.github/actions/cache-webrtc + with: + operation: save target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} ignore_cache: ${{ inputs.ignore_cache }} + - name: Upload deps-android + uses: actions/upload-artifact@v7 + with: + name: deps-android + path: .gclient_deps/.gclient-git-cache + include-hidden-files: true + if-no-files-found: error + retention-days: 1 + compression-level: 0 + + deps_windows: + name: Deps Windows + needs: plan + if: ${{ inputs.windows }} + runs-on: windows-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v7 + - name: Check Windows WebRTC host + shell: bash + run: | + set -euo pipefail + missing=() + command -v make >/dev/null || missing+=("GNU make") + command -v python3 >/dev/null || missing+=("python3") + if [[ ${#missing[@]} -gt 0 ]]; then + echo "::error::Windows WebRTC host is not ready on windows-latest (missing: ${missing[*]}). This job is wired for make deps windows, but the runner image cannot run it yet." + exit 1 + fi + - name: Setup WebRTC checkout + uses: ./.github/actions/setup-webrtc + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + - name: Restore Windows object cache + uses: ./.github/actions/cache-webrtc + with: + operation: restore + target_os_cache_key: ${{ needs.plan.outputs.windows_target_os_cache_key }} + ignore_cache: ${{ inputs.ignore_cache }} + - name: gclient sync (Windows) + working-directory: stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient_deps/.gclient-git-cache + TARGET_OS: ${{ needs.plan.outputs.windows_target_os }} + JOBS: "8" + WEBRTC_REPO: https://github.com/GetStream/webrtc.git + run: make deps + - name: Save Windows object cache + uses: ./.github/actions/cache-webrtc + with: + operation: save + target_os_cache_key: ${{ needs.plan.outputs.windows_target_os_cache_key }} + ignore_cache: ${{ inputs.ignore_cache }} + - name: Upload deps-windows + uses: actions/upload-artifact@v7 + with: + name: deps-windows + path: .gclient_deps/.gclient-git-cache + include-hidden-files: true + if-no-files-found: error + retention-days: 1 + compression-level: 0 build_ios: name: Build iOS @@ -168,7 +282,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + deps_artifact: deps-apple - name: Build iOS working-directory: stream_build env: @@ -176,11 +290,6 @@ jobs: CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build ios - - uses: ./.github/actions/cache-webrtc - continue-on-error: true - with: - operation: save - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} build_macos: name: Build macOS @@ -194,7 +303,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + deps_artifact: deps-apple - name: Build macOS working-directory: stream_build env: @@ -202,11 +311,6 @@ jobs: CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build macos - - uses: ./.github/actions/cache-webrtc - continue-on-error: true - with: - operation: save - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} build_android: name: Build Android @@ -220,7 +324,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.android_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} + deps_artifact: deps-android install_android_packages: "true" - name: Build Android working-directory: stream_build @@ -236,11 +340,6 @@ jobs: extra+=(ARCHS="${ANDROID_ARCH}") fi make build android "${extra[@]}" - - uses: ./.github/actions/cache-webrtc - continue-on-error: true - with: - operation: save - target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} test_ios: name: Test iOS @@ -254,7 +353,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + deps_artifact: deps-apple - name: Build iOS working-directory: stream_build env: @@ -268,11 +367,6 @@ jobs: DEPS_ROOT: ${{ github.workspace }}/.gclient_deps SKIP_DEPS: "1" run: make test ios - - uses: ./.github/actions/cache-webrtc - continue-on-error: true - with: - operation: save - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} test_macos: name: Test macOS @@ -286,7 +380,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + deps_artifact: deps-apple - name: Build macOS working-directory: stream_build env: @@ -300,15 +394,10 @@ jobs: DEPS_ROOT: ${{ github.workspace }}/.gclient_deps SKIP_DEPS: "1" run: make test macos - - uses: ./.github/actions/cache-webrtc - continue-on-error: true - with: - operation: save - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} test_windows: name: Test Windows - needs: plan + needs: [plan, deps_windows] if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.windows }} runs-on: windows-latest timeout-minutes: 360 @@ -329,7 +418,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.windows_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.windows_target_os_cache_key }} + deps_artifact: deps-windows - name: Build Windows working-directory: stream_build shell: bash @@ -398,7 +487,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + deps_artifact: deps-apple - name: Build iOS working-directory: stream_build env: @@ -415,11 +504,6 @@ jobs: CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make package ios - - uses: ./.github/actions/cache-webrtc - continue-on-error: true - with: - operation: save - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - name: Upload products-ios uses: actions/upload-artifact@v7 with: @@ -440,7 +524,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + deps_artifact: deps-apple - name: Build macOS working-directory: stream_build env: @@ -457,11 +541,6 @@ jobs: CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make package macos - - uses: ./.github/actions/cache-webrtc - continue-on-error: true - with: - operation: save - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - name: Upload products-macos uses: actions/upload-artifact@v7 with: @@ -482,7 +561,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.android_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} + deps_artifact: deps-android install_android_packages: "true" - name: Build Android working-directory: stream_build @@ -514,11 +593,6 @@ jobs: extra+=(ARCHS="${ANDROID_ARCH}") fi make package android "${extra[@]}" - - uses: ./.github/actions/cache-webrtc - continue-on-error: true - with: - operation: save - target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} - name: Upload products-android uses: actions/upload-artifact@v7 with: @@ -529,7 +603,7 @@ jobs: build_windows: name: Build Windows - needs: plan + needs: [plan, deps_windows] if: ${{ inputs.mode == 'build' && inputs.windows }} runs-on: windows-latest timeout-minutes: 360 @@ -550,7 +624,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.windows_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.windows_target_os_cache_key }} + deps_artifact: deps-windows - name: Build Windows working-directory: stream_build shell: bash @@ -562,7 +636,7 @@ jobs: package_windows: name: Package Windows - needs: [plan, tests_passed] + needs: [plan, deps_windows, tests_passed] if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.windows && !cancelled() && needs.tests_passed.result == 'success' }} runs-on: windows-latest timeout-minutes: 360 @@ -583,7 +657,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.windows_target_os }} - target_os_cache_key: ${{ needs.plan.outputs.windows_target_os_cache_key }} + deps_artifact: deps-windows - name: Build Windows working-directory: stream_build shell: bash From 41df5b05191d75953b5efa5fe1ac7b5036b12a86 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Fri, 11 Sep 2026 14:47:29 +0300 Subject: [PATCH 07/24] Update structure --- .github/actions/cache-webrtc/action.yml | 111 -------- .github/actions/prepare-common/action.yml | 19 +- .github/actions/restore-tree/action.yml | 26 +- .github/actions/setup-webrtc/action.yml | 59 ++-- .github/workflows/_make.yml | 235 ++++++++-------- .github/workflows/manual-platform-tests.yml | 5 - .github/workflows/publish.yml | 5 - .github/workflows/release.yml | 5 - .github/workflows/test.yml | 5 - .gitignore | 2 + stream_build/AGENTS.md | 51 +++- stream_build/Makefile | 61 +++-- stream_build/gn/ios-test.args | 1 + stream_build/scripts/bootstrap.sh | 204 ++++++++++++++ stream_build/scripts/check.sh | 288 ++++++++++++++++++-- stream_build/scripts/common.sh | 120 ++++++++ stream_build/scripts/deps.sh | 164 ++--------- stream_build/scripts/run-ios-tests.sh | 10 +- stream_build/webrtc.mk | 20 ++ 19 files changed, 894 insertions(+), 497 deletions(-) delete mode 100644 .github/actions/cache-webrtc/action.yml create mode 100755 stream_build/scripts/bootstrap.sh create mode 100644 stream_build/webrtc.mk diff --git a/.github/actions/cache-webrtc/action.yml b/.github/actions/cache-webrtc/action.yml deleted file mode 100644 index 5c53200591..0000000000 --- a/.github/actions/cache-webrtc/action.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: Cache gclient object cache -description: Cross-run actions/cache for gclient objects only. Same-run tree handoff is the deps-* artifact. - -inputs: - operation: - required: true - description: restore or save - target_os_cache_key: - required: true - description: apple, android, or windows. - ignore_cache: - required: false - default: "false" - description: Skip restore (save still runs). - -runs: - using: composite - steps: - # Objects-only: .gclient_deps/.gclient-git-cache. gclient sync refills - # working trees (third_party, build, ios, tools, …). Do not cache those. - # Keys are immutable warm bases (webrtc-apple|android|windows). Do not - # add run_id, DEPS hash, config, or github.job. - - name: Cache key - shell: bash - run: | - echo "Cache key: webrtc-${{ inputs.target_os_cache_key }}" - - - name: Restore gclient object cache - id: cache-restore - if: ${{ inputs.operation == 'restore' && inputs.ignore_cache != 'true' }} - uses: actions/cache/restore@v6 - with: - path: .gclient_deps/.gclient-git-cache - key: webrtc-${{ inputs.target_os_cache_key }} - - - name: Prepare DEPS_ROOT after object-cache restore - if: ${{ inputs.operation == 'restore' }} - shell: bash - env: - CACHE_HIT: ${{ steps.cache-restore.outputs.cache-hit }} - run: | - set -euo pipefail - echo "cache/restore finished cache-hit=${CACHE_HIT:-miss-or-skipped}" - mkdir -p "${GITHUB_WORKSPACE}/.gclient_deps/.gclient-git-cache" - ln -sfn .. "${GITHUB_WORKSPACE}/.gclient_deps/src" - - # Keys are immutable. A second save of webrtc-apple|android|windows - # fails; that is the warm base until eviction. - # GitHub repo cache quota is 10 GB total. Cap saves at 10 GiB - # (10737418240 bytes) so we do not spend minutes uploading a blob - # that will be evicted. Same-run deps-* artifacts still upload. - - name: Measure git-cache size - id: measure - if: ${{ inputs.operation == 'save' }} - shell: bash - env: - CACHE_DIR: ${{ github.workspace }}/.gclient_deps/.gclient-git-cache - run: | - set -euo pipefail - # Conservative vs GitHub's 10 GB (10*1000^3) repo cache quota. - cap_bytes=$((10 * 1024 * 1024 * 1024)) - - if [[ ! -d "${CACHE_DIR}" ]]; then - echo "git-cache missing: ${CACHE_DIR}" - exit 0 - fi - - human="unknown" - if h="$(du -sh "${CACHE_DIR}" 2>/dev/null | awk '{print $1}')"; then - human="${h:-unknown}" - fi - - bytes="" - if out="$(du -sb "${CACHE_DIR}" 2>/dev/null)"; then - bytes="$(awk '{print $1}' <<<"${out}")" - fi - if [[ ! "${bytes}" =~ ^[0-9]+$ ]]; then - if out="$(du -sk "${CACHE_DIR}" 2>/dev/null)"; then - kb="$(awk '{print $1}' <<<"${out}")" - if [[ "${kb}" =~ ^[0-9]+$ ]]; then - bytes=$((kb * 1024)) - fi - fi - fi - if [[ ! "${bytes}" =~ ^[0-9]+$ ]]; then - echo "::warning title=Actions cache size::Could not measure ${CACHE_DIR} (du failed). Cache save will still be attempted." - exit 0 - fi - - gib="$(awk -v b="${bytes}" 'BEGIN { printf "%.2f", b / 1073741824 }')" - echo "git-cache size: ${human} (${bytes} bytes, ${gib} GiB)" - echo "cap: 10 GiB = ${cap_bytes} bytes (GitHub quota is 10 GB = 10000000000 bytes; 10 GiB is the conservative save cap)" - echo "bytes=${bytes}" >> "${GITHUB_OUTPUT}" - - quota_note="Apple and Android caches (webrtc-apple|android; also windows) both count against the same 10 GB repository Actions cache quota." - if (( bytes > cap_bytes )); then - echo "skip_save=true" >> "${GITHUB_OUTPUT}" - echo "::warning title=Actions cache size::git-cache is ${human} (${bytes} bytes, ${gib} GiB), over the 10 GiB cap (${cap_bytes} bytes). Skipping actions/cache save — GitHub would evict it after a multi-minute upload that still prints Cache saved. ${quota_note}" - echo "::warning title=Deps artifact size::git-cache is ${human} (${bytes} bytes, ${gib} GiB). Same-run artifact upload still proceeds (separate quota from Actions cache)." - echo "Skipping actions/cache save (over 10 GiB cap)." - else - echo "Under cap; actions/cache save will run." - fi - - - name: Save gclient object cache - if: ${{ inputs.operation == 'save' && steps.measure.outputs.skip_save != 'true' }} - continue-on-error: true - uses: actions/cache/save@v6 - with: - path: .gclient_deps/.gclient-git-cache - key: webrtc-${{ inputs.target_os_cache_key }} diff --git a/.github/actions/prepare-common/action.yml b/.github/actions/prepare-common/action.yml index bc5c5cdb9e..3abaf7316b 100644 --- a/.github/actions/prepare-common/action.yml +++ b/.github/actions/prepare-common/action.yml @@ -1,5 +1,5 @@ name: Plan platform deps -description: Map selected platforms to gclient target_os and cache key suffixes. +description: Map selected platforms to gclient target_os. inputs: platform_ios: @@ -33,18 +33,12 @@ outputs: value: ${{ steps.flags.outputs.run_apple }} apple_target_os: value: ${{ steps.flags.outputs.apple_target_os }} - apple_target_os_cache_key: - value: ${{ steps.flags.outputs.apple_target_os_cache_key }} apple_target_os_label: value: ${{ steps.flags.outputs.apple_target_os_label }} android_target_os: value: ${{ steps.flags.outputs.android_target_os }} - android_target_os_cache_key: - value: ${{ steps.flags.outputs.android_target_os_cache_key }} windows_target_os: value: ${{ steps.flags.outputs.windows_target_os }} - windows_target_os_cache_key: - value: ${{ steps.flags.outputs.windows_target_os_cache_key }} runs: using: composite @@ -61,7 +55,7 @@ runs: skip_maccatalyst='${{ inputs.skip_maccatalyst }}' # Display follows the selected platforms. gclient TARGET_OS is the - # Apple superset so iOS-only and macOS-only share one cache tree. + # Apple superset so iOS-only and macOS-only share one deps job. apple_label_list=() if [[ "${ios}" == "true" ]]; then apple_label_list+=("ios"); fi if [[ "${macos}" == "true" ]]; then apple_label_list+=("macos"); fi @@ -71,12 +65,10 @@ runs: run_apple=false apple_target_os="" - apple_target_os_cache_key="" apple_target_os_label="" if [[ "${ios}" == "true" || "${macos}" == "true" ]]; then run_apple=true apple_target_os="ios,mac" - apple_target_os_cache_key="apple" old_ifs="$IFS" IFS=', ' apple_target_os_label="${apple_label_list[*]}" @@ -84,17 +76,13 @@ runs: fi android_target_os="" - android_target_os_cache_key="" if [[ "${android}" == "true" ]]; then android_target_os="android,unix" - android_target_os_cache_key="android" fi windows_target_os="" - windows_target_os_cache_key="" if [[ "${windows}" == "true" ]]; then windows_target_os="win" - windows_target_os_cache_key="windows" fi echo "run_ios=${ios}" >> "${GITHUB_OUTPUT}" @@ -103,9 +91,6 @@ runs: echo "run_windows=${windows}" >> "${GITHUB_OUTPUT}" echo "run_apple=${run_apple}" >> "${GITHUB_OUTPUT}" echo "apple_target_os=${apple_target_os}" >> "${GITHUB_OUTPUT}" - echo "apple_target_os_cache_key=${apple_target_os_cache_key}" >> "${GITHUB_OUTPUT}" echo "apple_target_os_label=${apple_target_os_label}" >> "${GITHUB_OUTPUT}" echo "android_target_os=${android_target_os}" >> "${GITHUB_OUTPUT}" - echo "android_target_os_cache_key=${android_target_os_cache_key}" >> "${GITHUB_OUTPUT}" echo "windows_target_os=${windows_target_os}" >> "${GITHUB_OUTPUT}" - echo "windows_target_os_cache_key=${windows_target_os_cache_key}" >> "${GITHUB_OUTPUT}" diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index a82b2b2cb0..26106be854 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -4,7 +4,7 @@ description: Download the same-run git-cache artifact, then gclient sync from lo inputs: webrtc_ref: required: true - description: Branch, tag, or SHA for GetStream/webrtc. + description: Branch, tag, or SHA already checked out at path src. target_os: required: true description: gclient TARGET_OS (ios,mac / android,unix / win). @@ -20,14 +20,13 @@ runs: using: composite steps: - name: Setup WebRTC checkout - uses: ./.github/actions/setup-webrtc + uses: ./src/.github/actions/setup-webrtc with: webrtc_ref: ${{ inputs.webrtc_ref }} install_android_packages: ${{ inputs.install_android_packages }} - # Same-run handoff. Do not restore actions/cache here: the warm OS - # cache is evicted (10 GB quota) even when Deps just saved it. - # Artifact is git-cache only; the full tree exceeds the per-artifact cap. + # Same-run handoff. Artifact is git-cache only; the full tree exceeds + # the per-artifact cap. - name: Download same-run deps artifact uses: actions/download-artifact@v8 with: @@ -38,25 +37,24 @@ runs: shell: bash run: | set -euo pipefail - dest="${GITHUB_WORKSPACE}/.gclient_deps/.gclient-git-cache" + dest="${GITHUB_WORKSPACE}/.gclient-git-cache" src="${RUNNER_TEMP}/deps-git-cache" - if [[ -d "${src}/.gclient_deps/.gclient-git-cache" ]]; then - src="${src}/.gclient_deps/.gclient-git-cache" - elif [[ -d "${src}/.gclient-git-cache" ]]; then + if [[ -d "${src}/.gclient-git-cache" ]]; then src="${src}/.gclient-git-cache" fi - mkdir -p "${GITHUB_WORKSPACE}/.gclient_deps" rm -rf "${dest}" mv "${src}" "${dest}" test -n "$(ls -A "${dest}")" - ln -sfn .. "${GITHUB_WORKSPACE}/.gclient_deps/src" + test -f "${GITHUB_WORKSPACE}/src/DEPS" + test ! -L "${GITHUB_WORKSPACE}/src" - name: gclient sync - working-directory: stream_build + working-directory: src/stream_build shell: bash env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps - GIT_CACHE_PATH: ${{ github.workspace }}/.gclient_deps/.gclient-git-cache + DEPS_ROOT: ${{ github.workspace }} + WEBRTC_SRC: ${{ github.workspace }}/src + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache TARGET_OS: ${{ inputs.target_os }} JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git diff --git a/.github/actions/setup-webrtc/action.yml b/.github/actions/setup-webrtc/action.yml index b8614701fe..1f7b33738b 100644 --- a/.github/actions/setup-webrtc/action.yml +++ b/.github/actions/setup-webrtc/action.yml @@ -1,10 +1,10 @@ name: Setup WebRTC checkout -description: Check out webrtc_ref, keep this workflow's stream_build, install depot_tools. +description: Install depot_tools against GITHUB_WORKSPACE/src (this checkout). No second clone. inputs: webrtc_ref: required: true - description: Branch, tag, or SHA for GetStream/webrtc. + description: Branch, tag, or SHA already checked out at path src. install_android_packages: required: false default: "false" @@ -13,35 +13,10 @@ inputs: runs: using: composite steps: - - name: Preserve pipeline from the workflow revision - shell: bash - run: | - set -euo pipefail - test -d "${GITHUB_WORKSPACE}/stream_build" || { - echo "::error::stream_build is missing from the workflow checkout." - exit 1 - } - mkdir -p "${RUNNER_TEMP}/pipeline" - rm -rf "${RUNNER_TEMP}/pipeline/stream_build" "${RUNNER_TEMP}/pipeline/actions" - cp -R "${GITHUB_WORKSPACE}/stream_build" "${RUNNER_TEMP}/pipeline/stream_build" - cp -R "${GITHUB_WORKSPACE}/.github/actions" "${RUNNER_TEMP}/pipeline/actions" - - - name: Check out GetStream/webrtc - uses: actions/checkout@v7 - with: - ref: ${{ inputs.webrtc_ref }} - - - name: Restore pipeline from the workflow revision - shell: bash - run: | - set -euo pipefail - rm -rf "${GITHUB_WORKSPACE}/stream_build" "${GITHUB_WORKSPACE}/.github/actions" - mkdir -p "${GITHUB_WORKSPACE}/.github" - cp -R "${RUNNER_TEMP}/pipeline/stream_build" "${GITHUB_WORKSPACE}/stream_build" - cp -R "${RUNNER_TEMP}/pipeline/actions" "${GITHUB_WORKSPACE}/.github/actions" - test -f "${GITHUB_WORKSPACE}/DEPS" - test -f "${GITHUB_WORKSPACE}/stream_build/Makefile" - + # Caller checks out webrtc_ref with path: src. GITHUB_WORKSPACE is the + # webrtc-named gclient parent. + # gclient realpath()s gclient_gn_args_file (src/build/config/gclient_args.gni) + # and rejects a src that resolves outside DEPS_ROOT (depot_tools a6ba01c). - name: Install Android host packages if: ${{ inputs.install_android_packages == 'true' }} shell: bash @@ -80,15 +55,23 @@ runs: fi echo "${HOME}/depot_tools" >> "${GITHUB_PATH}" echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" - echo "DEPS_ROOT=${GITHUB_WORKSPACE}/.gclient_deps" >> "${GITHUB_ENV}" - echo "WEBRTC_SRC=${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}" + echo "DEPS_ROOT=${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}" + echo "WEBRTC_SRC=${GITHUB_WORKSPACE}/src" >> "${GITHUB_ENV}" echo "WEBRTC_REPO=https://github.com/GetStream/webrtc.git" >> "${GITHUB_ENV}" - - name: Ensure DEPS_ROOT src symlink + - name: Ensure git-cache directory shell: bash run: | set -euo pipefail - mkdir -p "${GITHUB_WORKSPACE}/.gclient_deps/.gclient-git-cache" - ln -sfn .. "${GITHUB_WORKSPACE}/.gclient_deps/src" - echo "GIT_CACHE_PATH=${GITHUB_WORKSPACE}/.gclient_deps/.gclient-git-cache" >> "${GITHUB_ENV}" - test -f "${GITHUB_WORKSPACE}/.gclient_deps/src/DEPS" + mkdir -p "${GITHUB_WORKSPACE}/.gclient-git-cache" + echo "GIT_CACHE_PATH=${GITHUB_WORKSPACE}/.gclient-git-cache" >> "${GITHUB_ENV}" + test -f "${GITHUB_WORKSPACE}/src/DEPS" + test -f "${GITHUB_WORKSPACE}/src/stream_build/Makefile" + test ! -L "${GITHUB_WORKSPACE}/src" + + - name: Chromium webrtc/src layout + working-directory: src/stream_build + shell: bash + env: + CONFIRM: "1" + run: make bootstrap diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index d7f5d05b4d..eeb743b0e5 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -29,9 +29,6 @@ on: config: required: true type: string - ignore_cache: - required: true - type: boolean android_arch: required: false type: string @@ -49,6 +46,11 @@ on: type: string default: "" +env: + DEPS_ROOT: ${{ github.workspace }} + WEBRTC_SRC: ${{ github.workspace }}/src + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache + jobs: validate_inputs: name: Validate inputs @@ -108,16 +110,15 @@ jobs: run_windows: ${{ steps.plan.outputs.run_windows }} run_apple: ${{ steps.plan.outputs.run_apple }} apple_target_os: ${{ steps.plan.outputs.apple_target_os }} - apple_target_os_cache_key: ${{ steps.plan.outputs.apple_target_os_cache_key }} android_target_os: ${{ steps.plan.outputs.android_target_os }} - android_target_os_cache_key: ${{ steps.plan.outputs.android_target_os_cache_key }} windows_target_os: ${{ steps.plan.outputs.windows_target_os }} - windows_target_os_cache_key: ${{ steps.plan.outputs.windows_target_os_cache_key }} steps: - uses: actions/checkout@v7 + with: + path: src - id: plan name: Plan target_os - uses: ./.github/actions/prepare-common + uses: ./src/.github/actions/prepare-common with: platform_ios: ${{ inputs.ios }} platform_macos: ${{ inputs.macos }} @@ -132,40 +133,30 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 + with: + path: src + ref: ${{ inputs.webrtc_ref }} - name: Setup WebRTC checkout - uses: ./.github/actions/setup-webrtc + uses: ./src/.github/actions/setup-webrtc with: webrtc_ref: ${{ inputs.webrtc_ref }} - - name: Restore Darwin object cache - uses: ./.github/actions/cache-webrtc - with: - operation: restore - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} - name: gclient sync (Apple) - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps - GIT_CACHE_PATH: ${{ github.workspace }}/.gclient_deps/.gclient-git-cache + DEPS_ROOT: ${{ github.workspace }} + WEBRTC_SRC: ${{ github.workspace }}/src + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache TARGET_OS: ${{ needs.plan.outputs.apple_target_os }} JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps - - name: Save Darwin object cache - uses: ./.github/actions/cache-webrtc - with: - operation: save - target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} # Same-run handoff. The full tree is ~20 GB (over the artifact cap); # git-cache only. Build downloads this and runs make deps locally. - # Oversize git-cache skips actions/cache save (10 GiB cap in - # cache-webrtc); this artifact still uploads (separate quota). - name: Upload deps-apple uses: actions/upload-artifact@v7 with: name: deps-apple - path: .gclient_deps/.gclient-git-cache + path: .gclient-git-cache include-hidden-files: true if-no-files-found: error retention-days: 1 @@ -179,37 +170,29 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 + with: + path: src + ref: ${{ inputs.webrtc_ref }} - name: Setup WebRTC checkout - uses: ./.github/actions/setup-webrtc + uses: ./src/.github/actions/setup-webrtc with: webrtc_ref: ${{ inputs.webrtc_ref }} install_android_packages: "true" - - name: Restore Linux object cache - uses: ./.github/actions/cache-webrtc - with: - operation: restore - target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} - name: gclient sync (Android) - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps - GIT_CACHE_PATH: ${{ github.workspace }}/.gclient_deps/.gclient-git-cache + DEPS_ROOT: ${{ github.workspace }} + WEBRTC_SRC: ${{ github.workspace }}/src + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache TARGET_OS: ${{ needs.plan.outputs.android_target_os }} JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps - - name: Save Linux object cache - uses: ./.github/actions/cache-webrtc - with: - operation: save - target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} - name: Upload deps-android uses: actions/upload-artifact@v7 with: name: deps-android - path: .gclient_deps/.gclient-git-cache + path: .gclient-git-cache include-hidden-files: true if-no-files-found: error retention-days: 1 @@ -223,6 +206,9 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 + with: + path: src + ref: ${{ inputs.webrtc_ref }} - name: Check Windows WebRTC host shell: bash run: | @@ -235,36 +221,25 @@ jobs: exit 1 fi - name: Setup WebRTC checkout - uses: ./.github/actions/setup-webrtc + uses: ./src/.github/actions/setup-webrtc with: webrtc_ref: ${{ inputs.webrtc_ref }} - - name: Restore Windows object cache - uses: ./.github/actions/cache-webrtc - with: - operation: restore - target_os_cache_key: ${{ needs.plan.outputs.windows_target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} - name: gclient sync (Windows) - working-directory: stream_build + working-directory: src/stream_build shell: bash env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps - GIT_CACHE_PATH: ${{ github.workspace }}/.gclient_deps/.gclient-git-cache + DEPS_ROOT: ${{ github.workspace }} + WEBRTC_SRC: ${{ github.workspace }}/src + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache TARGET_OS: ${{ needs.plan.outputs.windows_target_os }} JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps - - name: Save Windows object cache - uses: ./.github/actions/cache-webrtc - with: - operation: save - target_os_cache_key: ${{ needs.plan.outputs.windows_target_os_cache_key }} - ignore_cache: ${{ inputs.ignore_cache }} - name: Upload deps-windows uses: actions/upload-artifact@v7 with: name: deps-windows - path: .gclient_deps/.gclient-git-cache + path: .gclient-git-cache include-hidden-files: true if-no-files-found: error retention-days: 1 @@ -278,15 +253,18 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 - - uses: ./.github/actions/restore-tree + with: + path: src + ref: ${{ inputs.webrtc_ref }} + - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-apple - name: Build iOS - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build ios @@ -299,15 +277,18 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 - - uses: ./.github/actions/restore-tree + with: + path: src + ref: ${{ inputs.webrtc_ref }} + - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-apple - name: Build macOS - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build macos @@ -320,16 +301,19 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 - - uses: ./.github/actions/restore-tree + with: + path: src + ref: ${{ inputs.webrtc_ref }} + - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.android_target_os }} deps_artifact: deps-android install_android_packages: "true" - name: Build Android - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" ANDROID_ARCH: ${{ inputs.android_arch }} @@ -349,22 +333,25 @@ jobs: timeout-minutes: 180 steps: - uses: actions/checkout@v7 - - uses: ./.github/actions/restore-tree + with: + path: src + ref: ${{ inputs.webrtc_ref }} + - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-apple - name: Build iOS - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build ios - name: Test iOS - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} SKIP_DEPS: "1" run: make test ios @@ -376,22 +363,25 @@ jobs: timeout-minutes: 180 steps: - uses: actions/checkout@v7 - - uses: ./.github/actions/restore-tree + with: + path: src + ref: ${{ inputs.webrtc_ref }} + - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-apple - name: Build macOS - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build macos - name: Test macOS - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} SKIP_DEPS: "1" run: make test macos @@ -403,6 +393,9 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 + with: + path: src + ref: ${{ inputs.webrtc_ref }} - name: Check Windows WebRTC host shell: bash run: | @@ -414,24 +407,24 @@ jobs: echo "::error::Windows WebRTC host is not ready on windows-latest (missing: ${missing[*]}). This job is wired for make test windows, but the runner image cannot run it yet." exit 1 fi - - uses: ./.github/actions/restore-tree + - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.windows_target_os }} deps_artifact: deps-windows - name: Build Windows - working-directory: stream_build + working-directory: src/stream_build shell: bash env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build windows - name: Test Windows - working-directory: stream_build + working-directory: src/stream_build shell: bash env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} SKIP_DEPS: "1" run: make test windows @@ -483,23 +476,26 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 - - uses: ./.github/actions/restore-tree + with: + path: src + ref: ${{ inputs.webrtc_ref }} + - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-apple - name: Build iOS - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} PRODUCTS: ${{ github.workspace }}/products CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build ios - name: Package iOS - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} PRODUCTS: ${{ github.workspace }}/products CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" @@ -520,23 +516,26 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 - - uses: ./.github/actions/restore-tree + with: + path: src + ref: ${{ inputs.webrtc_ref }} + - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-apple - name: Build macOS - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} PRODUCTS: ${{ github.workspace }}/products CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build macos - name: Package macOS - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} PRODUCTS: ${{ github.workspace }}/products CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" @@ -557,16 +556,19 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 - - uses: ./.github/actions/restore-tree + with: + path: src + ref: ${{ inputs.webrtc_ref }} + - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.android_target_os }} deps_artifact: deps-android install_android_packages: "true" - name: Build Android - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} PRODUCTS: ${{ github.workspace }}/products CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" @@ -579,9 +581,9 @@ jobs: fi make build android "${extra[@]}" - name: Package Android - working-directory: stream_build + working-directory: src/stream_build env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} PRODUCTS: ${{ github.workspace }}/products CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" @@ -609,6 +611,9 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 + with: + path: src + ref: ${{ inputs.webrtc_ref }} - name: Check Windows WebRTC host shell: bash run: | @@ -620,16 +625,16 @@ jobs: echo "::error::Windows WebRTC host is not ready on windows-latest (missing: ${missing[*]}). This job is wired for make build windows, but the runner image cannot run it yet." exit 1 fi - - uses: ./.github/actions/restore-tree + - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.windows_target_os }} deps_artifact: deps-windows - name: Build Windows - working-directory: stream_build + working-directory: src/stream_build shell: bash env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build windows @@ -642,6 +647,9 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v7 + with: + path: src + ref: ${{ inputs.webrtc_ref }} - name: Check Windows WebRTC host shell: bash run: | @@ -653,25 +661,25 @@ jobs: echo "::error::Windows WebRTC host is not ready on windows-latest (missing: ${missing[*]}). This job is wired for make package windows, but the runner image cannot run it yet." exit 1 fi - - uses: ./.github/actions/restore-tree + - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.windows_target_os }} deps_artifact: deps-windows - name: Build Windows - working-directory: stream_build + working-directory: src/stream_build shell: bash env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} PRODUCTS: ${{ github.workspace }}/products CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build windows - name: Package Windows - working-directory: stream_build + working-directory: src/stream_build shell: bash env: - DEPS_ROOT: ${{ github.workspace }}/.gclient_deps + DEPS_ROOT: ${{ github.workspace }} PRODUCTS: ${{ github.workspace }}/products CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" @@ -692,6 +700,14 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v7 + with: + path: src + + - name: Chromium webrtc/src layout + working-directory: src/stream_build + env: + CONFIRM: "1" + run: make bootstrap - name: Download product artifacts uses: actions/download-artifact@v8 @@ -701,7 +717,7 @@ jobs: - name: Combine Apple xcframeworks if: ${{ inputs.ios || inputs.macos }} - working-directory: stream_build + working-directory: src/stream_build env: PRODUCTS: ${{ github.workspace }}/combine-in SKIP_LICENSES: "1" @@ -746,7 +762,7 @@ jobs: - name: Rename copies for wrapper SDKs if: ${{ inputs.mode == 'release' }} - working-directory: stream_build + working-directory: src/stream_build env: PRODUCTS: ${{ github.workspace }}/combine-in run: | @@ -816,6 +832,7 @@ jobs: steps: - uses: actions/checkout@v7 with: + path: src ref: ${{ inputs.webrtc_ref }} - name: Download finalised artifacts diff --git a/.github/workflows/manual-platform-tests.yml b/.github/workflows/manual-platform-tests.yml index eea9c41497..0e4f884bb4 100644 --- a/.github/workflows/manual-platform-tests.yml +++ b/.github/workflows/manual-platform-tests.yml @@ -43,10 +43,6 @@ on: description: Optional Android ABI, for example arm64-v8a. Empty = default ABI set. required: false default: "" - ignore_cache: - description: Skip restoring dependency caches and force a fresh sync - type: boolean - default: false jobs: build: @@ -60,5 +56,4 @@ jobs: android: ${{ inputs.android }} windows: ${{ inputs.windows }} config: ${{ inputs.config }} - ignore_cache: ${{ inputs.ignore_cache }} android_arch: ${{ inputs.android_arch }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e8da3fce7f..983e6c12ad 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,10 +43,6 @@ on: description: Optional Android ABI, for example arm64-v8a. Empty = default ABI set. required: false default: "" - ignore_cache: - description: Skip restoring dependency caches and force a fresh sync - type: boolean - default: false jobs: package: @@ -60,5 +56,4 @@ jobs: android: ${{ inputs.android }} windows: ${{ inputs.windows }} config: ${{ inputs.config }} - ignore_cache: ${{ inputs.ignore_cache }} android_arch: ${{ inputs.android_arch }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b9e49289e..c0ffce711b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,10 +54,6 @@ on: description: Optional Android ABI, for example arm64-v8a. Empty = default ABI set. required: false default: "" - ignore_cache: - description: Skip restoring dependency caches and force a fresh sync - type: boolean - default: false jobs: release: @@ -71,7 +67,6 @@ jobs: android: ${{ inputs.android }} windows: ${{ inputs.windows }} config: ${{ inputs.config }} - ignore_cache: ${{ inputs.ignore_cache }} android_arch: ${{ inputs.android_arch }} version: ${{ inputs.version }} prerelease: ${{ inputs.prerelease }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b50e0dd79..1549f607c1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,10 +27,6 @@ on: description: Run Windows tests (fails clearly if the runner is not a WebRTC host) type: boolean default: false - ignore_cache: - description: Skip restoring dependency caches and force a fresh sync - type: boolean - default: false jobs: test: @@ -44,4 +40,3 @@ jobs: android: false windows: ${{ inputs.windows }} config: debug - ignore_cache: ${{ inputs.ignore_cache }} diff --git a/.gitignore b/.gitignore index 08d318a6b1..3e45fbb3f2 100644 --- a/.gitignore +++ b/.gitignore @@ -88,5 +88,7 @@ out_ios_libs .output .products out_macos_libs +# Leftover in-repo gclient parent (pre-webrtc/src bootstrap). .gclient_deps/ .gclient-git-cache +.idea diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index 12c7107f91..ac15b8fc31 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -4,6 +4,7 @@ Public API: ``` cd stream_build +make bootstrap # once; CONFIRM=1 in CI make build|test|package ios|android|macos|windows [VAR=value ...] make combine make rename apple|android @@ -35,7 +36,9 @@ Renamed copies feed stream-video-swift-webrtc and stream-video-android-webrtc. - `Makefile` — verb + platform dispatch - `gn/common.args` — Stream policy GN args - `gn/slices.tsv` — slice → ninja target + GN overlay -- `scripts/deps.sh` — gclient parent at `DEPS_ROOT` with `src` → git root +- `scripts/bootstrap.sh` — wrap this checkout as Chromium `webrtc/src` +- `webrtc.mk` — catch-all parent `webrtc/Makefile` template (copied if missing) +- `scripts/deps.sh` — `gclient sync` at `DEPS_ROOT`; uses this `src` (no second clone) - `scripts/gn-gen.sh` — args.gn + gn gen - `scripts/package-apple.sh` — lipo + create-xcframework - `scripts/combine-apple.sh` — discover platform xcframeworks and merge @@ -46,12 +49,42 @@ Renamed copies feed stream-video-swift-webrtc and stream-video-android-webrtc. - `scripts/run-ios-tests.sh` - `scripts/check.sh` -Default `DEPS_ROOT` is `/.gclient_deps` (gitignored). `src` there is a -symlink, not a nested clone; `make deps` does not reset that git checkout. -Ninja output is `DEPS_ROOT/out`. +Required tree (gclient parent **must** be named `webrtc`, checkout **must** +be named `src`): + +``` +webrtc/ # DEPS_ROOT / gclient root + Makefile # bootstrap copies webrtc.mk if missing + .gclient + .gclient-git-cache/ # GIT_CACHE_PATH + src/ # this git checkout (WEBRTC_SRC) + DEPS + stream_build/ + third_party/ # gclient writes here + out/ # ninja (sibling of src) +``` + +`solutions.name = src`, `managed: False`. `src` is this worktree, not a +symlink and not a second clone. Official DEPS stays (`src/build`, +`src/third_party`, `gclient_gn_args_file = src/build/config/gclient_args.gni`). + +`make bootstrap` (interactive; `CONFIRM=1` in CI) renames/wraps into that +layout. Other verbs +(except `help` / `check` / `bootstrap`) go through +`scripts/bootstrap.sh --check` and fail with `run: make bootstrap` unless +the tree is `webrtc/src` (real directory, not a symlink). `deps` / +`build` / `test` / `package` / `runhooks` also require parent `.gclient` +(`--check --gclient`). `make bootstrap` copies `webrtc.mk` to +`$(DEPS_ROOT)/Makefile` if that file is missing (parent is outside git). + +CI checks out with `path: src` so `GITHUB_WORKSPACE` is the webrtc-named +folder. `DEPS_ROOT=$GITHUB_WORKSPACE`. gclient objects live at +`$DEPS_ROOT/.gclient-git-cache`; same-run jobs hand that directory off as +the `deps-*` artifact (no GitHub Actions cache). ## Host gates +- bootstrap: any host (writes layout + `.gclient`; no depot_tools) - ios / macos / combine / rename apple: Darwin - android / rename android: Linux for build/package; rename android is a file copy on any host - windows: Windows @@ -63,11 +96,13 @@ Ninja output is `DEPS_ROOT/out`. |------|------| | `CONFIG` | `release` (default, `is_debug=false`) or `debug`. `make test` always uses debug. | | `GN_ARGS` | extra `key=value` tokens, applied last | -| `DEPS_ROOT` | gclient parent (`.gclient` + `src` symlink + `out/`) | -| `WEBRTC_SRC` | git root containing `DEPS` (default: parent of `stream_build/`) | -| `OUT` / `PRODUCTS` | ninja dirs / packaged output (default under `DEPS_ROOT`) | +| `DEPS_ROOT` | gclient parent (`webrtc/`; `.gclient` + `src/` + `out/`) | +| `WEBRTC_SRC` | this git checkout (`webrtc/src`; default: parent of `stream_build/`) | +| `OUT` / `PRODUCTS` | ninja dirs / packaged output (default under `DEPS_ROOT`, sibling of `src`) | +| `GIT_CACHE_PATH` | gclient object cache (default `DEPS_ROOT/.gclient-git-cache`) | | `ARCHS` | android ABI or windows cpu (`arm64-v8a`, `x64`, …) | | `JOBS` | ninja/gclient parallelism | +| `SHALLOW` | `1` (default) `gclient sync --no-history --shallow`. `0` = full history. | | `ZIP` | `1` to zip Apple/Windows products | | `XCFRAMEWORK` | input for `make rename apple` (default `$(PRODUCTS)/WebRTC.xcframework`) | | `AAR` | input for `make rename android` (default `$(PRODUCTS)/libwebrtc.aar`) | @@ -77,9 +112,9 @@ Ninja output is `DEPS_ROOT/out`. | `SKIP_MACCATALYST` | `1` drops `catalyst-arm64` and `catalyst-x64` from ios build+package only. Default `0`. | ```bash +make bootstrap CONFIRM=1 make build ios make build ios SKIP_DEPS=1 -make build ios DEPS_ROOT=/tmp/webrtc-deps make build ios SKIP_MACCATALYST=1 make build android ARCHS=arm64-v8a SKIP_DEPS=1 make package ios SKIP_DEPS=1 SKIP_LICENSES=1 diff --git a/stream_build/Makefile b/stream_build/Makefile index e007587734..739ef83c09 100644 --- a/stream_build/Makefile +++ b/stream_build/Makefile @@ -1,10 +1,14 @@ # Public API: +# make bootstrap # make build|test|package ios|android|macos|windows [VAR=value ...] # make combine # make rename apple|android # GN overrides: CONFIG=debug GN_ARGS='key=value key2=value' # +# Layout: Chromium-style webrtc/src (this git checkout). Run make bootstrap +# once. Other verbs assume that layout. # From the git root: cd stream_build && make build ios +# From webrtc/: make build ios (bootstrap writes parent Makefile) # SKIP_DEPS=1 skips gclient sync only; gn/ninja still run. # SKIP_LICENSES=1 skips license generation only; lipo/zip still run. # SKIP_MACCATALYST=1 drops catalyst-* from ios build+package only. @@ -21,21 +25,25 @@ SCRIPTS := $(PIPELINE)scripts CONFIG ?= release TARGET_OS ?= ios JOBS ?= $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 8) +SHALLOW ?= 1 SKIP_DEPS ?= 0 SKIP_LICENSES ?= 0 SKIP_MACCATALYST ?= 0 RUN_HOOKS ?= 1 +CONFIRM ?= 0 GN_ARGS ?= EXTRA_GN_ARGS ?= WEBRTC_REPO ?= git@github.com:GetStream/webrtc.git WEBRTC_ROOT ?= -WEBRTC_SRC ?= $(abspath $(PIPELINE)/..) +REPO_ROOT := $(abspath $(PIPELINE)/..) +WEBRTC_SRC ?= $(REPO_ROOT) ifneq ($(WEBRTC_ROOT),) DEPS_ROOT ?= $(WEBRTC_ROOT) endif -DEPS_ROOT ?= $(WEBRTC_SRC)/.gclient_deps +DEPS_ROOT ?= $(abspath $(REPO_ROOT)/..) OUT ?= $(DEPS_ROOT)/out PRODUCTS ?= $(DEPS_ROOT)/products +GIT_CACHE_PATH ?= $(DEPS_ROOT)/.gclient-git-cache TARGET ?= NINJA_TARGET ?= ARCHS ?= @@ -68,13 +76,14 @@ VERBS := build test package PLATFORM := $(firstword $(filter $(PLATFORMS),$(MAKECMDGOALS))) export DEPS_ROOT WEBRTC_SRC WEBRTC_ROOT WEBRTC_REPO WEBRTC_REVISION WEBRTC_REF -export TARGET_OS JOBS RUN_HOOKS CONFIG OUT PRODUCTS ARCHS SKIP_LICENSES -export GN_ARGS EXTRA_GN_ARGS ALL_GN_ARGS +export TARGET_OS JOBS SHALLOW RUN_HOOKS CONFIG OUT PRODUCTS ARCHS SKIP_LICENSES +export GN_ARGS EXTRA_GN_ARGS ALL_GN_ARGS GIT_CACHE_PATH CONFIRM BOOTSTRAP_SRC .DEFAULT_GOAL := help -.PHONY: help check deps runhooks gen ninja slice clean print-gn-args \ - announce require-src require-darwin require-linux require-windows maybe-deps \ +.PHONY: help check bootstrap deps runhooks gen ninja slice clean print-gn-args \ + announce require-layout require-gclient require-src require-darwin \ + require-linux require-windows maybe-deps \ combine apple rename rename-apple rename-android \ $(PLATFORMS) $(VERBS) \ build-ios build-macos build-android build-windows \ @@ -89,10 +98,12 @@ ifneq ($(filter build,$(MAKECMDGOALS)),) endif help: + @echo "make bootstrap" @echo "make build|test|package ios|android|macos|windows [VAR=value ...]" @echo "make combine" @echo "make rename apple|android" @echo + @echo " bootstrap wrap this checkout as webrtc/src (CONFIRM=1 in CI)" @echo " build gn gen + ninja" @echo " test build and run tests" @echo " package assemble artifacts (xcframework, aar, zip)" @@ -101,40 +112,53 @@ help: @echo @echo " CONFIG=release (default, is_debug=false) or CONFIG=debug" @echo " tests always gn-gen with debug, ignoring CONFIG" + @echo " SHALLOW=1 (default) gclient --no-history --shallow; SHALLOW=0 full history" @echo " SKIP_DEPS=1 skips gclient sync only; build/test/package still run" @echo " SKIP_LICENSES=1 skips license generation only; lipo/zip still run" @echo " SKIP_MACCATALYST=1 drops catalyst-arm64 catalyst-x64 from ios" @echo @echo " make package ios" @echo " make package macos" + @echo " from webrtc/: make build ios" @echo " make combine # merges whatever platform dirs exist under PRODUCTS" @echo " make rename apple XCFRAMEWORK=path/to/WebRTC.xcframework" @echo " make rename android AAR=path/to/libwebrtc.aar" @echo " make build package ios compile then pack" @echo " make build android ARCHS=arm64-v8a" @echo " make build ios GN_ARGS='rtc_use_h264=false' CONFIG=debug" - @echo " make build ios DEPS_ROOT=/tmp/webrtc-deps" + @echo " make bootstrap CONFIRM=1" @echo " make build ios SKIP_MACCATALYST=1" @echo @echo "Also: deps runhooks check clean print-gn-args TARGET=slice" - @echo "Vars: DEPS_ROOT WEBRTC_SRC OUT PRODUCTS CONFIG GN_ARGS JOBS ARCHS ZIP XCFRAMEWORK AAR RENAMED SKIP_DEPS SKIP_LICENSES SKIP_MACCATALYST" + @echo "Vars: DEPS_ROOT WEBRTC_SRC OUT PRODUCTS GIT_CACHE_PATH CONFIG GN_ARGS JOBS SHALLOW ARCHS ZIP XCFRAMEWORK AAR RENAMED SKIP_DEPS SKIP_LICENSES SKIP_MACCATALYST CONFIRM" + @echo "OUT defaults to DEPS_ROOT/out (sibling of src)." check: @$(SCRIPTS)/check.sh -deps: +bootstrap: + @$(SCRIPTS)/bootstrap.sh + +deps: require-gclient @$(SCRIPTS)/deps.sh sync -runhooks: +runhooks: require-gclient @$(SCRIPTS)/deps.sh runhooks -print-gn-args: +print-gn-args: require-layout @test -n "$(TARGET)" || { echo "TARGET is required (slice name)"; exit 1; } @$(SCRIPTS)/gn-gen.sh --print --slice "$(TARGET)" --config "$(CONFIG)" --extra "$(ALL_GN_ARGS)" -require-src: - @test -n "$(WEBRTC_SRC)" || { echo "WEBRTC_SRC is required"; exit 1; } - @test -f "$(WEBRTC_SRC)/DEPS" || { echo "No WebRTC checkout at $(WEBRTC_SRC)"; exit 1; } +require-layout: + @$(SCRIPTS)/bootstrap.sh --check + +require-gclient: + @$(SCRIPTS)/bootstrap.sh --check --gclient + +require-src: require-layout + @test -n "$(WEBRTC_SRC)" || { echo "run: make bootstrap"; exit 1; } + @test ! -L "$(WEBRTC_SRC)" || { echo "run: make bootstrap"; exit 1; } + @test -f "$(WEBRTC_SRC)/DEPS" || { echo "run: make bootstrap"; exit 1; } require-darwin: @test "$$(uname -s)" = Darwin || { echo "Apple targets require macOS"; exit 1; } @@ -146,7 +170,7 @@ require-windows: @case "$$(uname -s)" in MINGW*|MSYS*|CYGWIN*) exit 0 ;; esac; \ test "$${OS}" = Windows_NT || { echo "Windows targets require Windows"; exit 1; } -maybe-deps: +maybe-deps: require-gclient ifneq ($(SKIP_DEPS),1) $(MAKE) deps endif @@ -173,6 +197,7 @@ announce: esac; \ echo " products: $$products"; \ echo " jobs: $(JOBS)"; \ + echo " shallow: $(SHALLOW)"; \ echo " skip_deps: $(SKIP_DEPS)"; \ echo " skip_licenses: $(SKIP_LICENSES)"; \ echo " skip_maccatalyst: $(SKIP_MACCATALYST)"; \ @@ -206,7 +231,7 @@ announce: echo " developer: $${DEVELOPER_DIR:-$$(xcode-select -p 2>/dev/null)}"; \ fi -build test package: +build test package: require-gclient @test -n "$(PLATFORM)" || { echo "usage: make $@ ios|android|macos|windows"; exit 1; } @$(MAKE) --no-print-directory announce VERB=$@ PLATFORM="$(PLATFORM)" @$(MAKE) --no-print-directory $@-$(PLATFORM) @@ -219,7 +244,7 @@ combine: require-darwin require-src --products "$(PRODUCTS)" \ $(if $(filter 1,$(ZIP)),--zip,) -rename: +rename: require-layout @target="$(firstword $(filter apple android,$(MAKECMDGOALS)))"; \ test -n "$$target" || { echo "usage: make rename apple|android"; exit 1; }; \ $(MAKE) --no-print-directory announce VERB=rename PLATFORM="$$target"; \ @@ -350,5 +375,5 @@ test-windows: require-windows maybe-deps require-src "$(OUT)/windows_tests/$$target" $(EXTRA_ARGS); \ done -clean: require-src +clean: require-layout rm -rf "$(OUT)" "$(PRODUCTS)" diff --git a/stream_build/gn/ios-test.args b/stream_build/gn/ios-test.args index 80e23379b8..90e4e665eb 100644 --- a/stream_build/gn/ios-test.args +++ b/stream_build/gn/ios-test.args @@ -4,6 +4,7 @@ target_cpu = "arm64" ios_enable_code_signing = false ios_deployment_target = "13.0" rtc_include_tests = true +enable_run_ios_unittests_with_xctest = true is_debug = true use_siso = false use_remoteexec = false diff --git a/stream_build/scripts/bootstrap.sh b/stream_build/scripts/bootstrap.sh new file mode 100755 index 0000000000..2b1adf2b3b --- /dev/null +++ b/stream_build/scripts/bootstrap.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# Put this checkout at webrtc/src and write the parent .gclient. +# src is this worktree (no second clone). Refuses a symlink src. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +CONFIRM="${CONFIRM:-0}" + +usage() { + cat <<'EOF' +usage: bootstrap.sh [--check] [--gclient] + + make bootstrap interactive wrap to webrtc/src + make bootstrap CONFIRM=1 non-interactive (CI) + +env: + CONFIRM 1 to skip prompts + BOOTSTRAP_SRC git checkout to wrap (overrides WEBRTC_SRC) + WEBRTC_SRC git checkout (default: stream_build/..) + DEPS_ROOT gclient parent (default: parent of src) + WEBRTC_REPO written into .gclient + TARGET_OS written into .gclient +EOF +} + +logical_pwd() { + (cd "$1" && pwd) +} + +ask() { + local prompt="$1" + if [[ "$CONFIRM" == "1" ]]; then + echo "$prompt (CONFIRM=1: yes)" + return 0 + fi + if [[ ! -t 0 ]]; then + echo "error: non-interactive bootstrap requires CONFIRM=1" >&2 + return 1 + fi + local ans="" + read -r -p "$prompt [y/N] " ans || true + [[ "$ans" == "y" || "$ans" == "Y" || "$ans" == "yes" ]] +} + +refuse_with_commands() { + echo "error: refused. run:" >&2 + local cmd + for cmd in "$@"; do + echo " $cmd" >&2 + done + echo "then: make bootstrap" >&2 + exit 1 +} + +layout_fail() { + echo "run: make bootstrap" >&2 + return 1 +} + +# 1-3 always: basename src, parent webrtc, src is a real directory. +# --gclient also requires DEPS_ROOT/.gclient (deps/build/test/package). +layout_check() { + local src="$1" + local deps_root="$2" + local need_gclient="${3:-0}" + if [[ ! -d "$src" || -L "$src" ]]; then + layout_fail + return 1 + fi + if [[ "$(basename "$src")" != "src" ]]; then + layout_fail + return 1 + fi + if [[ "$(basename "$deps_root")" != "webrtc" ]]; then + layout_fail + return 1 + fi + if [[ ! -d "$deps_root/src" || -L "$deps_root/src" ]]; then + layout_fail + return 1 + fi + local src_real deps_src_real + src_real="$(cd "$src" && pwd -P)" + deps_src_real="$(cd "$deps_root/src" && pwd -P)" + if [[ "$src_real" != "$deps_src_real" ]]; then + layout_fail + return 1 + fi + if [[ "$need_gclient" == "1" && ! -f "$deps_root/.gclient" ]]; then + layout_fail + return 1 + fi + return 0 +} + +install_parent_makefile() { + local dest="$1/Makefile" + local tmpl="$SCRIPT_DIR/../webrtc.mk" + if [[ -e "$dest" ]]; then + echo "keeping $dest" + return 0 + fi + [[ -f "$tmpl" ]] || die "missing template $tmpl" + cp "$tmpl" "$dest" + echo "wrote $dest" +} + +finish_layout() { + local src="$1" + local deps_root="$2" + local git_cache="${GIT_CACHE_PATH:-$deps_root/.gclient-git-cache}" + mkdir -p "$git_cache" + write_gclient "$deps_root" + install_parent_makefile "$deps_root" + rewrite_git_cache_alternates "$src" "$git_cache" + echo "gclient root: $deps_root" + echo "src: $src" + echo "git-cache: $git_cache" + echo "out: $deps_root/out" +} + +cmd_check() { + local src deps_root need_gclient=0 arg + for arg in "$@"; do + case "$arg" in + --gclient) need_gclient=1 ;; + esac + done + src="${BOOTSTRAP_SRC:-${WEBRTC_SRC:-$(logical_pwd "$SCRIPT_DIR/..")}}" + deps_root="${DEPS_ROOT:-$(logical_pwd "$src/..")}" + layout_check "$src" "$deps_root" "$need_gclient" || exit 1 +} + +cmd_bootstrap() { + local src + src="${BOOTSTRAP_SRC:-${WEBRTC_SRC:-$(logical_pwd "$SCRIPT_DIR/..")}}" + [[ -d "$src" ]] || die "checkout not found: $src" + if [[ -L "$src" ]]; then + die "bootstrap refuses symlink src ($src -> $(readlink "$src"))" + fi + src="$(logical_pwd "$src")" + [[ -f "$src/DEPS" ]] || die "no DEPS at $src" + + if [[ "$(basename "$src")" != "src" ]]; then + local parent rename_dest + parent="$(dirname "$src")" + rename_dest="$parent/src" + echo "checkout is named '$(basename "$src")'; gclient requires 'src'." + echo "plan:" + echo " mv $src $rename_dest" + if [[ -e "$rename_dest" ]]; then + die "refusing to overwrite $rename_dest" + fi + if ! ask "rename to src?"; then + refuse_with_commands "mv $src $rename_dest" + fi + echo "mv $src $rename_dest" + mv "$src" "$rename_dest" + src="$rename_dest" + fi + + local parent deps_root + parent="$(dirname "$src")" + if [[ "$(basename "$parent")" != "webrtc" ]]; then + deps_root="$parent/webrtc" + echo "parent is named '$(basename "$parent")'; gclient root must be named webrtc." + echo "plan:" + echo " mkdir $deps_root" + echo " mv $src $deps_root/src" + echo "result: $deps_root/src" + if [[ -e "$deps_root" ]]; then + die "refusing to overwrite $deps_root" + fi + if ! ask "wrap as $deps_root/src?"; then + refuse_with_commands "mkdir $deps_root" "mv $src $deps_root/src" + fi + mkdir "$deps_root" + echo "mv $src $deps_root/src" + mv "$src" "$deps_root/src" + src="$deps_root/src" + parent="$deps_root" + fi + + deps_root="$(logical_pwd "$parent")" + src="$(logical_pwd "$src")" + if [[ -L "$src" || -L "$deps_root/src" ]]; then + die "bootstrap refuses symlink src" + fi + + finish_layout "$src" "$deps_root" +} + +case "${1:-}" in + --check) + shift + cmd_check "$@" + ;; + -h|--help|help) usage ;; + "") cmd_bootstrap ;; + *) usage >&2; exit 1 ;; +esac diff --git a/stream_build/scripts/check.sh b/stream_build/scripts/check.sh index 8eef8ce7c6..e7f51095e1 100755 --- a/stream_build/scripts/check.sh +++ b/stream_build/scripts/check.sh @@ -1,12 +1,37 @@ #!/usr/bin/env bash -# Sanity check for the Makefile wrapper (no WebRTC tree required). +# Sanity check for the Makefile wrapper (no real gclient tree required). set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" +# make check exports WEBRTC_SRC/DEPS_ROOT. Fixtures must not inherit them. +unset WEBRTC_SRC DEPS_ROOT WEBRTC_ROOT BOOTSTRAP_SRC + +init_git_tree() { + local dir="$1" + mkdir -p "$dir" + printf 'hooks = []\n' > "$dir/DEPS" + git -C "$dir" init -q + git -C "$dir" config user.email "check@example.com" + git -C "$dir" config user.name "check" + git -C "$dir" config commit.gpgsign false + git -C "$dir" add DEPS + git -C "$dir" commit -q -m init +} + +fake_layout() { + local parent + parent="$(mktemp -d)/webrtc" + mkdir -p "$parent/src" + printf 'hooks = []\n' > "$parent/src/DEPS" + printf '%s\n' 'solutions = [{"name": "src", "managed": False}]' \ + > "$parent/.gclient" + printf '%s\n' "$parent" +} help_text="$(make -s help)" [[ "$help_text" == *"make build|test|package"* ]] +printf '%s\n' "$help_text" | grep -q 'make bootstrap' printf '%s\n' "$help_text" | grep -q 'CONFIG=release (default' printf '%s\n' "$help_text" | grep -q 'make combine' printf '%s\n' "$help_text" | grep -q 'make rename apple' @@ -15,21 +40,24 @@ printf '%s\n' "$help_text" | grep -q 'make package ios' printf '%s\n' "$help_text" | grep -q 'make package macos' ! printf '%s\n' "$help_text" | grep -q 'package apple' -usage="$(make build 2>&1 || true)" +layout="$(fake_layout)" +layout_make=(make DEPS_ROOT="$layout" WEBRTC_SRC="$layout/src") + +usage="$("${layout_make[@]}" build 2>&1 || true)" printf '%s\n' "$usage" | grep -q 'usage: make build' ! printf '%s\n' "$usage" | grep -q '|apple' -text="$(make -s print-gn-args TARGET=ios-arm64-device CONFIG=release)" +text="$("${layout_make[@]}" -s print-gn-args TARGET=ios-arm64-device CONFIG=release)" printf '%s\n' "$text" | grep -q 'stream_enable_rendering_backend = true' printf '%s\n' "$text" | grep -q 'target_os = "ios"' printf '%s\n' "$text" | grep -q 'is_debug = false' -debug="$(make -s print-gn-args TARGET=macos-arm64 CONFIG=debug GN_ARGS='rtc_use_h264=false')" +debug="$("${layout_make[@]}" -s print-gn-args TARGET=macos-arm64 CONFIG=debug GN_ARGS='rtc_use_h264=false')" printf '%s\n' "$debug" | grep -q 'is_debug = true' printf '%s\n' "$debug" | grep -q 'target_os = "mac"' printf '%s\n' "$debug" | grep -q 'rtc_use_h264 = false' -android="$(make -s print-gn-args TARGET=android-arm64-v8a)" +android="$("${layout_make[@]}" -s print-gn-args TARGET=android-arm64-v8a)" printf '%s\n' "$android" | grep -q 'target_os = "android"' printf '%s\n' "$android" | grep -q 'target_cpu = "arm64"' @@ -59,14 +87,18 @@ printf '%s\n' "$test_banner" | grep -q '==> test macos' printf '%s\n' "$test_banner" | grep -q 'config: debug (tests always debug)' empty="$(mktemp -d)" -combine_none="$(make combine PRODUCTS="$empty" SKIP_LICENSES=1 2>&1 || true)" +combine_none="$( + make combine PRODUCTS="$empty" SKIP_LICENSES=1 \ + DEPS_ROOT="$layout" WEBRTC_SRC="$layout/src" 2>&1 || true +)" printf '%s\n' "$combine_none" | grep -q 'no WebRTC.xcframework' rm -rf "$empty" one="$(mktemp -d)" mkdir -p "$one/ios/WebRTC.xcframework" printf 'stub\n' > "$one/ios/WebRTC.xcframework/Info.plist" -make combine PRODUCTS="$one" SKIP_LICENSES=1 +make combine PRODUCTS="$one" SKIP_LICENSES=1 \ + DEPS_ROOT="$layout" WEBRTC_SRC="$layout/src" [[ -f "$one/WebRTC.xcframework/Info.plist" ]] rm -rf "$one" @@ -84,7 +116,8 @@ printf 'stub\n' > "$rename_src/ios-arm64/WebRTC.framework/WebRTC" printf '%s\n' '' \ > "$rename_src/ios-arm64/WebRTC.framework/Info.plist" rename_out="$(mktemp -d)" -make rename apple XCFRAMEWORK="$rename_src" RENAMED="$rename_out" +make rename apple XCFRAMEWORK="$rename_src" RENAMED="$rename_out" \ + DEPS_ROOT="$layout" WEBRTC_SRC="$layout/src" [[ -d "$rename_src/ios-arm64/WebRTC.framework" ]] [[ -d "$rename_out/StreamWebRTC.xcframework/ios-arm64/StreamWebRTC.framework" ]] grep -q 'StreamWebRTC' "$rename_out/StreamWebRTC.xcframework/ios-arm64/StreamWebRTC.framework/Modules/module.modulemap" @@ -94,16 +127,164 @@ rm -rf "$rename_root" "$rename_out" aar_dir="$(mktemp -d)" printf 'aar-stub\n' > "$aar_dir/libwebrtc.aar" -make rename android AAR="$aar_dir/libwebrtc.aar" RENAMED="$aar_dir/renamed" +make rename android AAR="$aar_dir/libwebrtc.aar" RENAMED="$aar_dir/renamed" \ + DEPS_ROOT="$layout" WEBRTC_SRC="$layout/src" [[ -f "$aar_dir/libwebrtc.aar" ]] [[ -f "$aar_dir/renamed/libwebrtc.aar" ]] cmp -s "$aar_dir/libwebrtc.aar" "$aar_dir/renamed/libwebrtc.aar" -rm -rf "$aar_dir" +rm -rf "$aar_dir" "$(dirname "$layout")" + +wrong="$(mktemp -d)/not-src" +init_git_tree "$wrong" +wrong_root="$(dirname "$wrong")" +if WEBRTC_SRC="$wrong" DEPS_ROOT="$wrong_root" \ + "$ROOT/scripts/bootstrap.sh" --check 2>"$wrong.err"; then + echo "expected --check to fail on non-src checkout" >&2 + exit 1 +fi +grep -q 'run: make bootstrap' "$wrong.err" +[[ "$(cat "$wrong.err")" == "run: make bootstrap" ]] + +# help/check/bootstrap stay ungated; every other user verb hits the guard. +help_wrong="$(WEBRTC_SRC="$wrong" DEPS_ROOT="$wrong_root" make -s help)" +printf '%s\n' "$help_wrong" | grep -q 'make bootstrap' + +expect_make_bootstrap() { + local err="$wrong.err" + if WEBRTC_SRC="$wrong" DEPS_ROOT="$wrong_root" \ + make --no-print-directory "$@" 2>"$err"; then + echo "expected make $* to fail on non-src checkout" >&2 + exit 1 + fi + grep -q 'run: make bootstrap' "$err" +} + +expect_make_bootstrap build ios +expect_make_bootstrap test macos +expect_make_bootstrap package ios +expect_make_bootstrap deps +expect_make_bootstrap runhooks +expect_make_bootstrap combine +expect_make_bootstrap rename apple +expect_make_bootstrap clean +expect_make_bootstrap print-gn-args TARGET=ios-arm64-device +rm -rf "$wrong_root" "$wrong.err" + +# 1-3 pass without .gclient; deps/build/test/package still need it. +bare="$(mktemp -d)/webrtc" +mkdir -p "$bare/src" +printf 'hooks = []\n' > "$bare/src/DEPS" +WEBRTC_SRC="$bare/src" DEPS_ROOT="$bare" \ + "$ROOT/scripts/bootstrap.sh" --check +if WEBRTC_SRC="$bare/src" DEPS_ROOT="$bare" \ + "$ROOT/scripts/bootstrap.sh" --check --gclient 2>"$bare.err"; then + echo "expected --check --gclient to fail without .gclient" >&2 + exit 1 +fi +grep -q 'run: make bootstrap' "$bare.err" +if WEBRTC_SRC="$bare/src" DEPS_ROOT="$bare" \ + make --no-print-directory build ios 2>"$bare.err"; then + echo "expected make build to fail without .gclient" >&2 + exit 1 +fi +grep -q 'run: make bootstrap' "$bare.err" +bare_rename="$( + WEBRTC_SRC="$bare/src" DEPS_ROOT="$bare" \ + make --no-print-directory rename 2>&1 || true +)" +printf '%s\n' "$bare_rename" | grep -q 'usage: make rename' +! printf '%s\n' "$bare_rename" | grep -q 'run: make bootstrap' +rm -rf "$(dirname "$bare")" "$bare.err" + +# Named webrtc (git) -> webrtc/src. Parent .gclient + wrapper Makefile. +parent="$(mktemp -d)" +repo="$parent/webrtc" +init_git_tree "$repo" +CONFIRM=1 BOOTSTRAP_SRC="$repo" "$ROOT/scripts/bootstrap.sh" >/dev/null +[[ -f "$parent/webrtc/src/DEPS" ]] +[[ -f "$parent/webrtc/.gclient" ]] +[[ -f "$parent/webrtc/Makefile" ]] +grep -q 'created by bootstrap if missing' "$parent/webrtc/Makefile" +grep -q 'src/stream_build' "$parent/webrtc/Makefile" + +mkdir -p "$parent/webrtc/src/stream_build" +cat > "$parent/webrtc/src/stream_build/Makefile" <<'STUB' +.DEFAULT_GOAL := help +FIRST := $(firstword $(MAKECMDGOALS)) +REST := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) +.PHONY: help $(MAKECMDGOALS) +help: + @echo "stub-help CONFIG=$(CONFIG) JOBS=$(JOBS)" +ifneq ($(FIRST),) +ifneq ($(FIRST),help) +$(FIRST): + @echo "stub-goals $(MAKECMDGOALS) CONFIG=$(CONFIG) JOBS=$(JOBS)" +endif +endif +ifneq ($(REST),) +$(REST): + @: +endif +STUB +fwd="$(make -C "$parent/webrtc" --no-print-directory build ios CONFIG=debug JOBS=8)" +printf '%s\n' "$fwd" | grep -q 'stub-goals build ios' +printf '%s\n' "$fwd" | grep -q 'CONFIG=debug' +printf '%s\n' "$fwd" | grep -q 'JOBS=8' +[[ "$(printf '%s\n' "$fwd" | grep -c 'stub-goals')" == 1 ]] +fwd_all="$(make -C "$parent/webrtc" --no-print-directory)" +printf '%s\n' "$fwd_all" | grep -q 'stub-help' + +printf '%s\n' 'all:' $'\t@echo foreign' > "$parent/webrtc/Makefile" +grep -q '"managed": False' "$parent/webrtc/.gclient" +! grep -q '"revision"' "$parent/webrtc/.gclient" +[[ ! -L "$parent/webrtc/src" ]] +[[ -d "$parent/webrtc/src/.git" ]] +git -C "$parent/webrtc/src" rev-parse --is-inside-work-tree >/dev/null +CONFIRM=1 BOOTSTRAP_SRC="$parent/webrtc/src" "$ROOT/scripts/bootstrap.sh" >/dev/null +[[ -f "$parent/webrtc/.gclient" ]] +grep -q foreign "$parent/webrtc/Makefile" +rm -f "$parent/webrtc/Makefile" +CONFIRM=1 BOOTSTRAP_SRC="$parent/webrtc/src" "$ROOT/scripts/bootstrap.sh" >/dev/null +grep -q 'created by bootstrap if missing' "$parent/webrtc/Makefile" +WEBRTC_SRC="$parent/webrtc/src" DEPS_ROOT="$parent/webrtc" \ + "$ROOT/scripts/bootstrap.sh" --check +WEBRTC_SRC="$parent/webrtc/src" DEPS_ROOT="$parent/webrtc" \ + "$ROOT/scripts/bootstrap.sh" --check --gclient + +symlink_parent="$(mktemp -d)/webrtc" +mkdir -p "$symlink_parent" +real_src="$(mktemp -d)/real" +init_git_tree "$real_src" +ln -sfn "$real_src" "$symlink_parent/src" +if WEBRTC_SRC="$symlink_parent/src" DEPS_ROOT="$symlink_parent" \ + "$ROOT/scripts/bootstrap.sh" --check 2>"$symlink_parent.err"; then + echo "expected --check to fail on symlink src" >&2 + exit 1 +fi +grep -q 'run: make bootstrap' "$symlink_parent.err" +if CONFIRM=1 BOOTSTRAP_SRC="$symlink_parent/src" \ + "$ROOT/scripts/bootstrap.sh" 2>"$symlink_parent.err"; then + echo "expected bootstrap to refuse symlink src" >&2 + exit 1 +fi +grep -q 'refuses symlink src' "$symlink_parent.err" +rm -rf "$parent" "$(dirname "$symlink_parent")" "$(dirname "$real_src")" \ + "$symlink_parent.err" + +# Non-interactive without CONFIRM=1 prints the plan and exits. +ni_parent="$(mktemp -d)" +ni_repo="$ni_parent/webrtc" +init_git_tree "$ni_repo" +ni_out="$(BOOTSTRAP_SRC="$ni_repo" "$ROOT/scripts/bootstrap.sh" 2>&1 || true)" +printf '%s\n' "$ni_out" | grep -q 'CONFIRM=1' +printf '%s\n' "$ni_out" | grep -q "mv $ni_repo" +[[ -d "$ni_repo/.git" ]] +rm -rf "$ni_parent" deps_tmp="$(mktemp -d)" fake_bin="$deps_tmp/bin" -mkdir -p "$fake_bin" "$deps_tmp/src_repo" "$deps_tmp/deps/.gclient-git-cache" -printf 'hooks = []\n' > "$deps_tmp/src_repo/DEPS" +mkdir -p "$fake_bin" "$deps_tmp/webrtc/src" "$deps_tmp/webrtc/.gclient-git-cache" +init_git_tree "$deps_tmp/webrtc/src" cat > "$fake_bin/gclient" <<'FAKE' #!/usr/bin/env bash printf '%s\n' "$*" >> "${FAKE_GCLIENT_LOG}" @@ -117,32 +298,87 @@ chmod +x "$fake_bin/gclient" export FAKE_GCLIENT_LOG="$deps_tmp/log" export FAKE_GCLIENT_SYNC_ARGS="$deps_tmp/sync_args" export FAKE_GCLIENT_CACHE="$deps_tmp/cache_env" +wt_before="$(git -C "$deps_tmp/webrtc/src" worktree list | wc -l | tr -d ' ')" deps_out="$( PATH="$fake_bin:$PATH" \ - DEPS_ROOT="$deps_tmp/deps" \ - WEBRTC_SRC="$deps_tmp/src_repo" \ - GIT_CACHE_PATH="$deps_tmp/deps/.gclient-git-cache" \ + DEPS_ROOT="$deps_tmp/webrtc" \ + WEBRTC_SRC="$deps_tmp/webrtc/src" \ + GIT_CACHE_PATH="$deps_tmp/webrtc/.gclient-git-cache" \ RUN_HOOKS=0 JOBS=2 \ "$ROOT/scripts/deps.sh" sync )" printf '%s\n' "$deps_out" | grep -q 'will not reset it' -printf '%s\n' "$deps_out" | grep -q 'running: gclient sync -j2 --nohooks' +printf '%s\n' "$deps_out" | grep -q 'running: gclient sync -j2 --no-history --shallow --nohooks' ! printf '%s\n' "$deps_out" | grep -q -- '--revision' -grep -q '"managed": False' "$deps_tmp/deps/.gclient" -! grep -q '"revision"' "$deps_tmp/deps/.gclient" -grep -q 'sync -j2 --nohooks' "$deps_tmp/sync_args" +grep -q '"managed": False' "$deps_tmp/webrtc/.gclient" +! grep -q '"revision"' "$deps_tmp/webrtc/.gclient" +grep -q 'sync -j2 --no-history --shallow --nohooks' "$deps_tmp/sync_args" ! grep -q -- '--revision' "$deps_tmp/sync_args" -grep -q "$deps_tmp/deps/.gclient-git-cache" "$deps_tmp/cache_env" -[[ -L "$deps_tmp/deps/src" ]] +grep -q "$deps_tmp/webrtc/.gclient-git-cache" "$deps_tmp/cache_env" +mkdir -p "$deps_tmp/webrtc/src/third_party/.git/objects/info" +printf '%s\n' \ + "$deps_tmp/webrtc/.gclient_deps/.gclient-git-cache/fake-repo/objects" \ + > "$deps_tmp/webrtc/src/third_party/.git/objects/info/alternates" +PATH="$fake_bin:$PATH" \ + DEPS_ROOT="$deps_tmp/webrtc" \ + WEBRTC_SRC="$deps_tmp/webrtc/src" \ + GIT_CACHE_PATH="$deps_tmp/webrtc/.gclient-git-cache" \ + RUN_HOOKS=0 JOBS=2 \ + "$ROOT/scripts/deps.sh" sync >/dev/null +grep -qx "$deps_tmp/webrtc/.gclient-git-cache/fake-repo/objects" \ + "$deps_tmp/webrtc/src/third_party/.git/objects/info/alternates" +! grep -q '.gclient_deps/.gclient-git-cache' \ + "$deps_tmp/webrtc/src/third_party/.git/objects/info/alternates" +PATH="$fake_bin:$PATH" \ + DEPS_ROOT="$deps_tmp/webrtc" \ + WEBRTC_SRC="$deps_tmp/webrtc/src" \ + GIT_CACHE_PATH="$deps_tmp/webrtc/.gclient-git-cache" \ + RUN_HOOKS=0 JOBS=2 SHALLOW=0 \ + "$ROOT/scripts/deps.sh" sync >/dev/null +grep -q 'sync -j2 --nohooks' "$deps_tmp/sync_args" +! grep -q -- '--no-history' "$deps_tmp/sync_args" +! grep -q -- '--shallow' "$deps_tmp/sync_args" +[[ ! -L "$deps_tmp/webrtc/src" ]] +[[ -f "$deps_tmp/webrtc/src/DEPS" ]] +wt_after="$(git -C "$deps_tmp/webrtc/src" worktree list | wc -l | tr -d ' ')" +[[ "$wt_before" == "$wt_after" ]] +python3 - "$deps_tmp/webrtc" <<'PY' +import os +import sys +prefix = os.path.realpath(sys.argv[1]) +gn = os.path.abspath(os.path.join(prefix, "src/build/config/gclient_args.gni")) +real_gn = os.path.realpath(gn) +if os.path.commonpath([prefix, real_gn]) != prefix: + raise SystemExit("gclient_gn_args_file would escape %r -> %r" % (prefix, real_gn)) +PY if PATH="$fake_bin:$PATH" \ - DEPS_ROOT="$deps_tmp/deps" \ - WEBRTC_SRC="$deps_tmp/src_repo" \ + DEPS_ROOT="$deps_tmp/webrtc" \ + WEBRTC_SRC="$deps_tmp/webrtc/src" \ WEBRTC_REVISION=eeff9252f32a40d1671974c31c096ce9fa776130 \ "$ROOT/scripts/deps.sh" sync 2>"$deps_tmp/pin_err"; then - echo "expected pin to fail on symlink src" >&2 + echo "expected pin to fail on seeded src" >&2 + exit 1 +fi +grep -q 'src is seeded from your git checkout; will not reset it' "$deps_tmp/pin_err" + +missing="$(mktemp -d)/webrtc" +mkdir -p "$missing" +if PATH="$fake_bin:$PATH" DEPS_ROOT="$missing" \ + "$ROOT/scripts/deps.sh" sync 2>"$deps_tmp/missing_err"; then + echo "expected deps.sh to fail without src" >&2 + exit 1 +fi +grep -q 'run: make bootstrap' "$deps_tmp/missing_err" + +rm -rf "$deps_tmp/linkroot" +mkdir -p "$deps_tmp/linkroot" +ln -sfn "$deps_tmp/webrtc/src" "$deps_tmp/linkroot/src" +if PATH="$fake_bin:$PATH" DEPS_ROOT="$deps_tmp/linkroot" \ + "$ROOT/scripts/deps.sh" sync 2>"$deps_tmp/link_err"; then + echo "expected deps.sh to refuse symlink src" >&2 exit 1 fi -grep -q 'src is your git checkout; will not reset it' "$deps_tmp/pin_err" -rm -rf "$deps_tmp" +grep -q 'src is a symlink' "$deps_tmp/link_err" +rm -rf "$deps_tmp" "$missing" echo "ok" diff --git a/stream_build/scripts/common.sh b/stream_build/scripts/common.sh index 4701e47a73..c3d506214a 100755 --- a/stream_build/scripts/common.sh +++ b/stream_build/scripts/common.sh @@ -39,6 +39,126 @@ require_webrtc_src() { [[ -f "$src/DEPS" ]] || die "No WebRTC checkout at $src (missing DEPS)" } +quote_target_os() { + local raw="$1" + local os first=1 + printf '[' + # shellcheck disable=SC2086 + for os in ${raw//,/ }; do + [[ -z "$os" ]] && continue + if [[ $first -eq 1 ]]; then + first=0 + else + printf ', ' + fi + printf '"%s"' "$os" + done + printf ']\n' +} + +# Write Chromium .gclient at the gclient parent. managed: False, no revision. +write_gclient() { + local dest="$1" + local repo="${2:-${WEBRTC_REPO:-git@github.com:GetStream/webrtc.git}}" + local target_os="${3:-${TARGET_OS:-ios}}" + cat >"${dest}/.gclient" < str: + idx = line.find(frag) + if idx < 0: + return line + start = idx + while start > 0 and line[start - 1] not in stops: + start -= 1 + return line[:start] + new_cache + line[idx + len(frag) :] + + +repos = set() +for path in listing.splitlines(): + if not path: + continue + try: + with open(path, encoding="utf-8", errors="replace") as fh: + text = fh.read() + except OSError: + continue + if frag not in text: + continue + rewritten = "".join(rewrite_line(line) for line in text.splitlines(True)) + if rewritten == text: + continue + with open(path, "w", encoding="utf-8") as fh: + fh.write(rewritten) + marker = "/.git/" + i = path.find(marker) + repos.add(path[:i] if i >= 0 else path) + +if repos: + print("rewrote git-cache paths in %d repos" % len(repos)) +PY +} + abspath() { local path="$1" (cd "$(dirname "$path")" && printf '%s/%s\n' "$(pwd)" "$(basename "$path")") diff --git a/stream_build/scripts/deps.sh b/stream_build/scripts/deps.sh index 50e48a807c..ce84d948e4 100755 --- a/stream_build/scripts/deps.sh +++ b/stream_build/scripts/deps.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # gclient config + sync for GetStream/webrtc. -# DEPS_ROOT is the gclient parent. src there is a symlink to WEBRTC_SRC (the git root). +# DEPS_ROOT is the gclient parent (named webrtc). src there is this git +# checkout, not a second clone/worktree and not a symlink to the git root. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -12,15 +13,16 @@ usage() { usage: deps.sh sync|runhooks env: - DEPS_ROOT gclient parent (required). Default from make: /.gclient_deps - WEBRTC_SRC git root that contains DEPS (required) + DEPS_ROOT gclient parent (required). Default from make: parent of src + WEBRTC_SRC this git checkout (DEPS_ROOT/src). Not cloned again. TARGET_OS comma/space list, default: ios JOBS gclient -j, default: 8 + SHALLOW 1 (default) adds --no-history --shallow; 0 for full history RUN_HOOKS 1 to run hooks during sync, 0 for --nohooks WEBRTC_REPO default: git@github.com:GetStream/webrtc.git - WEBRTC_REVISION pin src SHA only when src is a real clone gclient owns - WEBRTC_REF resolve SHA via git ls-remote when REVISION is empty - (refused when src is a symlink to this git checkout) + GIT_CACHE_PATH default: DEPS_ROOT/.gclient-git-cache + WEBRTC_REVISION refused (src is this worktree; gclient must not reset it) + WEBRTC_REF refused (same as WEBRTC_REVISION) EOF } @@ -28,154 +30,46 @@ DEPS_ROOT="${DEPS_ROOT:-${WEBRTC_ROOT:-}}" WEBRTC_SRC="${WEBRTC_SRC:-}" TARGET_OS="${TARGET_OS:-ios}" JOBS="${JOBS:-8}" +SHALLOW="${SHALLOW:-1}" RUN_HOOKS="${RUN_HOOKS:-1}" WEBRTC_REPO="${WEBRTC_REPO:-git@github.com:GetStream/webrtc.git}" WEBRTC_REVISION="${WEBRTC_REVISION:-}" WEBRTC_REF="${WEBRTC_REF:-}" -git_sha() { - [[ "$1" =~ ^[0-9a-fA-F]{7,40}$ ]] -} - -resolve_revision() { - if [[ -n "$WEBRTC_REVISION" ]]; then - printf '%s\n' "$WEBRTC_REVISION" - return - fi - if [[ -n "$WEBRTC_REF" ]]; then - if git_sha "$WEBRTC_REF"; then - printf '%s\n' "$WEBRTC_REF" - return - fi - local sha="" - local pattern - for pattern in "$WEBRTC_REF" "refs/heads/${WEBRTC_REF}" "refs/tags/${WEBRTC_REF}"; do - sha="$(git ls-remote --exit-code "$WEBRTC_REPO" "$pattern" 2>/dev/null | awk '{ print $1; exit }' || true)" - if [[ -n "$sha" ]]; then - printf '%s\n' "$sha" - return - fi - done - die "unable to resolve WEBRTC_REF='$WEBRTC_REF' from $WEBRTC_REPO" - fi - if [[ -n "$WEBRTC_SRC" && -d "$WEBRTC_SRC/.git" ]]; then - git -C "$WEBRTC_SRC" rev-parse HEAD - return - fi - printf '\n' -} - -quote_target_os() { - local raw="$1" - local os first=1 - printf '[' - # shellcheck disable=SC2086 - for os in ${raw//,/ }; do - [[ -z "$os" ]] && continue - if [[ $first -eq 1 ]]; then - first=0 - else - printf ', ' - fi - printf '"%s"' "$os" - done - printf ']\n' -} - -write_gclient() { - local dest="$1" - local revision="$2" - local revision_line="" - if [[ -n "$revision" ]]; then - revision_line=$'\n "revision": "'"${revision}"'",' - fi - cat >"${dest}/.gclient" </.gclient_deps; otherwise an absolute path. -src_link_target() { - local deps_abs src_abs - deps_abs="$(cd "$DEPS_ROOT" && pwd)" - src_abs="$(cd "$WEBRTC_SRC" && pwd)" - if [[ "$(cd "$DEPS_ROOT/.." && pwd)" == "$src_abs" && "$(basename "$DEPS_ROOT")" == ".gclient_deps" ]]; then - printf '..\n' - else - printf '%s\n' "$src_abs" - fi -} - -ensure_src_symlink() { +ensure_src() { [[ -n "$DEPS_ROOT" ]] || die "DEPS_ROOT is required" - [[ -n "$WEBRTC_SRC" ]] || die "WEBRTC_SRC is required" - [[ -f "$WEBRTC_SRC/DEPS" ]] || die "No WebRTC checkout at $WEBRTC_SRC (missing DEPS)" - mkdir -p "$DEPS_ROOT" - local link="$DEPS_ROOT/src" - local target - target="$(src_link_target)" - if [[ -L "$link" ]]; then - ln -sfn "$target" "$link" - elif [[ -e "$link" ]]; then - die "$link exists and is not a symlink (refusing a nested webrtc checkout)" - else - ln -sfn "$target" "$link" + local dest="${DEPS_ROOT}/src" + if [[ -L "$dest" ]]; then + die "src is a symlink ($dest). run: make bootstrap" fi - [[ -f "$link/DEPS" ]] || die "symlink $link does not point at a WebRTC tree" -} - -# True when DEPS_ROOT/src is a symlink to this git checkout, not a clone -# gclient owns. gclient must not checkout/reset/clean that tree. -src_is_developer_symlink() { - local link="${DEPS_ROOT}/src" - [[ -L "$link" ]] || return 1 - [[ -n "$WEBRTC_SRC" ]] || return 0 - local link_abs src_abs - link_abs="$(cd "$link" && pwd -P)" - src_abs="$(cd "$WEBRTC_SRC" && pwd -P)" - [[ "$link_abs" == "$src_abs" ]] + if [[ -f "$dest/DEPS" ]] && git -C "$dest" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + return 0 + fi + die "$dest is not this git checkout. run: make bootstrap" } cmd_sync() { require_cmd gclient require_cmd git require_cmd python3 - ensure_src_symlink + ensure_src - local revision="" - if src_is_developer_symlink; then - if [[ -n "$WEBRTC_REVISION" || -n "$WEBRTC_REF" ]]; then - die "src is your git checkout; will not reset it" - fi - echo "src is a symlink to the git checkout; will not reset it" - else - revision="$(resolve_revision)" - if [[ -n "$revision" ]]; then - echo "pinning gclient src revision: $revision" - fi - fi - # DEPS checkouts use this cache as their git alternate/origin. Unsetting - # it makes gclient retarget them from the cache URL to googlesource. - if [[ -d "$DEPS_ROOT/.gclient-git-cache" ]]; then - export GIT_CACHE_PATH="$DEPS_ROOT/.gclient-git-cache" + if [[ -n "$WEBRTC_REVISION" || -n "$WEBRTC_REF" ]]; then + die "src is seeded from your git checkout; will not reset it" fi - write_gclient "$DEPS_ROOT" "$revision" + echo "src is already present; will not reset it" + GIT_CACHE_PATH="${GIT_CACHE_PATH:-$DEPS_ROOT/.gclient-git-cache}" + mkdir -p "$GIT_CACHE_PATH" + export GIT_CACHE_PATH + rewrite_git_cache_alternates "${WEBRTC_SRC:-$DEPS_ROOT/src}" "$GIT_CACHE_PATH" + write_gclient "$DEPS_ROOT" "$WEBRTC_REPO" "$TARGET_OS" ( cd "$DEPS_ROOT" gclient root >/dev/null || true local sync=(gclient sync -j"${JOBS}") - if [[ -n "$revision" ]]; then - sync+=(--revision "src@${revision}") + if [[ "$SHALLOW" == "1" || "$SHALLOW" == "true" ]]; then + sync+=(--no-history --shallow) fi if [[ "$RUN_HOOKS" == "0" || "$RUN_HOOKS" == "false" ]]; then sync+=(--nohooks) @@ -187,7 +81,7 @@ cmd_sync() { cmd_runhooks() { require_cmd gclient - ensure_src_symlink + ensure_src [[ -f "$DEPS_ROOT/.gclient" ]] || die "gclient config not found at $DEPS_ROOT/.gclient" ( cd "$DEPS_ROOT" diff --git a/stream_build/scripts/run-ios-tests.sh b/stream_build/scripts/run-ios-tests.sh index 55ad2900a8..23d72803c7 100755 --- a/stream_build/scripts/run-ios-tests.sh +++ b/stream_build/scripts/run-ios-tests.sh @@ -10,6 +10,14 @@ require_darwin require_cmd xcrun require_cmd xcodebuild require_cmd python3 +require_cmd vpython3 + +# Chromium's generated wrappers are `#!/usr/bin/env vpython3` and probe +# upward for .vpython3. That works when out/ lives under src/. Stream's +# out/ is a sibling of src/, so the probe never reaches src/.vpython3 +# (psutil / cipd wheels). Point vpython at Chromium's spec explicitly. +vpython_spec="${WEBRTC_SRC:-$(cd "${PIPELINE_DIR}/.." && pwd)}/.vpython3" +[[ -f "$vpython_spec" ]] || die "missing vpython spec: ${vpython_spec}" BUILD_DIR="" TARGETS="" @@ -100,7 +108,7 @@ run_target() { args+=($EXTRA_ARGS) fi echo "running ${wrapper} ${args[*]}" - if "$wrapper" "${args[@]}"; then + if vpython3 -vpython-spec "$vpython_spec" "$wrapper" "${args[@]}"; then return 0 fi if grep -Rqs "Test Suite 'All tests' passed\|Test Suite 'Selected tests' passed" "$out_dir"; then diff --git a/stream_build/webrtc.mk b/stream_build/webrtc.mk new file mode 100644 index 0000000000..070d265f91 --- /dev/null +++ b/stream_build/webrtc.mk @@ -0,0 +1,20 @@ +# webrtc/Makefile — created by bootstrap if missing +# Catch-all forwarder. Recipes live in src/stream_build (cwd must be there). + +MAKECMDGOALS ?= +STREAM_BUILD := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))src/stream_build +GOALS := $(filter-out all,$(MAKECMDGOALS)) +FIRST := $(firstword $(or $(MAKECMDGOALS),all)) +REST := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) + +.DEFAULT_GOAL := all +.PHONY: all $(MAKECMDGOALS) + +# One recursive make for `make build ios`; extra goals are no-ops. +$(FIRST): + @$(MAKE) -C "$(STREAM_BUILD)" $(GOALS) + +ifneq ($(REST),) +$(REST): + @: +endif From aa2347206711f75ae64d2832b75ff9c13a1d8c19 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Fri, 11 Sep 2026 15:03:42 +0300 Subject: [PATCH 08/24] Fix ios tests execution --- stream_build/scripts/run-ios-tests.sh | 104 ++++++++++++++++++-------- 1 file changed, 74 insertions(+), 30 deletions(-) diff --git a/stream_build/scripts/run-ios-tests.sh b/stream_build/scripts/run-ios-tests.sh index 23d72803c7..c815a9d3cd 100755 --- a/stream_build/scripts/run-ios-tests.sh +++ b/stream_build/scripts/run-ios-tests.sh @@ -38,55 +38,98 @@ done [[ -n "$BUILD_DIR" && -n "$TARGETS" ]] || die "run-ios-tests.sh requires --build-dir and --targets" +# Match the SDK the .app was compiled with. Newest-device auto-select +# picks iOS 27.0 on Xcode 26.6 (SDK 26.5), then Chromium creates a +# simulator on that runtime and xcodebuild cannot launch the runner. +compiled_ios_sdk() { + local target plist ver + # shellcheck disable=SC2086 + for target in $TARGETS; do + plist="${BUILD_DIR}/${target}.app/Info.plist" + [[ -f "$plist" ]] || continue + ver="$(plutil -extract DTPlatformVersion raw "$plist" 2>/dev/null || true)" + if [[ -n "$ver" ]]; then + printf '%s\n' "$ver" + return 0 + fi + done + xcrun --sdk iphonesimulator --show-sdk-version +} + +# Chromium's wrapper bakes --xcode-path ../../src/Xcode.app (CIPD). +# argparse last-wins; point at the selected Xcode so local runs do not +# look for a hermetic tree. install_xcode() no-ops without LUCI_CONTEXT. +selected_xcode_app() { + (cd "$(xcode-select -p)/../.." && pwd) +} + pick_simulator() { python3 - "$SIMULATOR_PLATFORM" "$SIMULATOR_VERSION" <<'PY' import json, subprocess, sys want_name, want_version = sys.argv[1], sys.argv[2] payload = json.loads( - subprocess.check_output( - ["xcrun", "simctl", "list", "devices", "available", "--json"], - text=True, - ) + subprocess.check_output(["xcrun", "simctl", "list", "--json"], text=True) ) -candidates = [] -for runtime, devices in payload.get("devices", {}).items(): - if "iOS" not in runtime: + + +def matches(runtime_version): + rv, want = runtime_version.strip(), want_version.strip() + return rv == want or rv.startswith(want + ".") or want.startswith(rv + ".") + + +def is_iphone(devicetype): + if devicetype.get("productFamily") == "iPhone": + return True + return (devicetype.get("name") or "").startswith("iPhone") + + +for runtime in payload.get("runtimes") or []: + ident = runtime.get("identifier") or "" + name = runtime.get("name") or "" + if "iOS" not in ident and "iOS" not in name: continue - version = runtime.split("iOS-")[-1].replace("-", ".") - for device in devices: - if device.get("isAvailable") is False: - continue - name = device.get("name") or "" - if want_name and name != want_name: - continue - if want_version and version != want_version: + if runtime.get("isAvailable") is False: + continue + version = (runtime.get("version") or "").strip() + if want_version and not matches(version): + continue + types = [ + dt.get("name") or "" + for dt in (runtime.get("supportedDeviceTypes") or []) + if is_iphone(dt) + ] + types = [t for t in types if t] + if want_name: + if want_name not in types: continue - candidates.append( - ( - 1 if device.get("state") == "Booted" else 0, - 1 if name.startswith("iPhone") else 0, - tuple(int(p) for p in version.split(".") if p.isdigit()), - name, - version, - ) - ) -if not candidates: - sys.exit("no available iOS simulator matched the request") -best = max(candidates) -print(f"{best[3]}\t{best[4]}") + print(f"{want_name}\t{version}") + raise SystemExit(0) + if types: + # ponytail: Apple lists newest iPhones first on Xcode 26.x. + # If that order flips, first-iPhone still matches the SDK. + print(f"{types[0]}\t{version}") + raise SystemExit(0) + +sys.exit( + f"no available iOS simulator runtime matched SDK {want_version or '?'}" +) PY } -if [[ -z "$SIMULATOR_PLATFORM" || -z "$SIMULATOR_VERSION" ]]; then +if [[ -z "$SIMULATOR_VERSION" ]]; then + SIMULATOR_VERSION="$(compiled_ios_sdk)" +fi +if [[ -z "$SIMULATOR_PLATFORM" ]]; then selected="$(pick_simulator)" SIMULATOR_PLATFORM="${selected%%$'\t'*}" SIMULATOR_VERSION="${selected#*$'\t'}" - echo "auto-selected simulator: ${SIMULATOR_PLATFORM} (iOS ${SIMULATOR_VERSION})" fi +echo "using simulator: ${SIMULATOR_PLATFORM} (iOS ${SIMULATOR_VERSION})" xcode_build_version="$(xcodebuild -version | awk '/Build version/{print $3; exit}')" xcode_build_version="${xcode_build_version:-local}" +xcode_app="$(selected_xcode_app)" out_dir="${BUILD_DIR}/test_output" rm -rf "$out_dir" mkdir -p "$out_dir" @@ -99,6 +142,7 @@ run_target() { --xctest --out-dir "$out_dir" --xcode-build-version "$xcode_build_version" + --xcode-path "$xcode_app" --platform "$SIMULATOR_PLATFORM" --version "$SIMULATOR_VERSION" ) From 717f607f768e9676d0e1ff3b2dc5f28066ef9679 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Fri, 11 Sep 2026 17:23:19 +0300 Subject: [PATCH 09/24] Improve pipeline shape --- .github/actions/prepare-common/action.yml | 37 +- .github/actions/restore-tree/action.yml | 4 +- .github/workflows/_make.yml | 372 ++++++-------------- .github/workflows/manual-platform-tests.yml | 1 + .github/workflows/publish.yml | 2 + .github/workflows/release.yml | 1 + .github/workflows/test.yml | 1 + stream_build/AGENTS.md | 10 +- 8 files changed, 151 insertions(+), 277 deletions(-) diff --git a/.github/actions/prepare-common/action.yml b/.github/actions/prepare-common/action.yml index 3abaf7316b..73185153f9 100644 --- a/.github/actions/prepare-common/action.yml +++ b/.github/actions/prepare-common/action.yml @@ -39,6 +39,8 @@ outputs: value: ${{ steps.flags.outputs.android_target_os }} windows_target_os: value: ${{ steps.flags.outputs.windows_target_os }} + linux_target_os: + value: ${{ steps.flags.outputs.linux_target_os }} runs: using: composite @@ -54,27 +56,49 @@ runs: windows='${{ inputs.platform_windows }}' skip_maccatalyst='${{ inputs.skip_maccatalyst }}' - # Display follows the selected platforms. gclient TARGET_OS is the - # Apple superset so iOS-only and macOS-only share one deps job. + # TARGET_OS is only tokens selected this run (ios, mac, android,unix, win). + apple_os_list=() apple_label_list=() - if [[ "${ios}" == "true" ]]; then apple_label_list+=("ios"); fi - if [[ "${macos}" == "true" ]]; then apple_label_list+=("macos"); fi + linux_os_list=() + if [[ "${ios}" == "true" ]]; then + apple_os_list+=("ios") + apple_label_list+=("ios") + linux_os_list+=("ios") + fi + if [[ "${macos}" == "true" ]]; then + apple_os_list+=("mac") + apple_label_list+=("macos") + linux_os_list+=("mac") + fi if [[ "${ios}" == "true" && "${skip_maccatalyst}" != "true" ]]; then apple_label_list+=("maccatalyst") fi + if [[ "${android}" == "true" ]]; then + linux_os_list+=("android" "unix") + fi + + join_csv() { + local IFS=, + printf '%s' "$*" + } run_apple=false apple_target_os="" apple_target_os_label="" - if [[ "${ios}" == "true" || "${macos}" == "true" ]]; then + if [[ ${#apple_os_list[@]} -gt 0 ]]; then run_apple=true - apple_target_os="ios,mac" + apple_target_os="$(join_csv "${apple_os_list[@]}")" old_ifs="$IFS" IFS=', ' apple_target_os_label="${apple_label_list[*]}" IFS="$old_ifs" fi + linux_target_os="" + if [[ ${#linux_os_list[@]} -gt 0 ]]; then + linux_target_os="$(join_csv "${linux_os_list[@]}")" + fi + android_target_os="" if [[ "${android}" == "true" ]]; then android_target_os="android,unix" @@ -94,3 +118,4 @@ runs: echo "apple_target_os_label=${apple_target_os_label}" >> "${GITHUB_OUTPUT}" echo "android_target_os=${android_target_os}" >> "${GITHUB_OUTPUT}" echo "windows_target_os=${windows_target_os}" >> "${GITHUB_OUTPUT}" + echo "linux_target_os=${linux_target_os}" >> "${GITHUB_OUTPUT}" diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index 26106be854..8f958b08bf 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -7,10 +7,10 @@ inputs: description: Branch, tag, or SHA already checked out at path src. target_os: required: true - description: gclient TARGET_OS (ios,mac / android,unix / win). + description: gclient TARGET_OS (selected tokens only). deps_artifact: required: true - description: Same-run artifact (deps-apple / deps-android / deps-windows). + description: Same-run artifact (deps-git / deps-windows). install_android_packages: required: false default: "false" diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index eeb743b0e5..7651726e5b 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -1,6 +1,12 @@ # Called by Build / Test / Package / Release dispatch workflows. # Not listed in the Actions dispatch UI (workflow_call only). # Callers pass mode: build | test | package | release. +# +# I/O: Linux Deps (RUN_HOOKS=0) uploads deps-git for selected ios/mac/android,unix. +# Windows Deps stays separate (deps-windows). Build mode is ninja only (no +# products). Package/Release Build jobs also make package and upload products-*. +# Package combine consumes products-* (no third ninja) and uploads final-*. +# Release attaches final-*. Tests need Deps only (make test, no make build). name: WebRTC make @@ -112,6 +118,7 @@ jobs: apple_target_os: ${{ steps.plan.outputs.apple_target_os }} android_target_os: ${{ steps.plan.outputs.android_target_os }} windows_target_os: ${{ steps.plan.outputs.windows_target_os }} + linux_target_os: ${{ steps.plan.outputs.linux_target_os }} steps: - uses: actions/checkout@v7 with: @@ -125,47 +132,10 @@ jobs: platform_android: ${{ inputs.android }} platform_windows: ${{ inputs.windows }} - deps_apple: - name: Deps Apple + deps: + name: Deps needs: plan - if: ${{ inputs.ios || inputs.macos }} - runs-on: macos-26 - timeout-minutes: 360 - steps: - - uses: actions/checkout@v7 - with: - path: src - ref: ${{ inputs.webrtc_ref }} - - name: Setup WebRTC checkout - uses: ./src/.github/actions/setup-webrtc - with: - webrtc_ref: ${{ inputs.webrtc_ref }} - - name: gclient sync (Apple) - working-directory: src/stream_build - env: - DEPS_ROOT: ${{ github.workspace }} - WEBRTC_SRC: ${{ github.workspace }}/src - GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache - TARGET_OS: ${{ needs.plan.outputs.apple_target_os }} - JOBS: "8" - WEBRTC_REPO: https://github.com/GetStream/webrtc.git - run: make deps - # Same-run handoff. The full tree is ~20 GB (over the artifact cap); - # git-cache only. Build downloads this and runs make deps locally. - - name: Upload deps-apple - uses: actions/upload-artifact@v7 - with: - name: deps-apple - path: .gclient-git-cache - include-hidden-files: true - if-no-files-found: error - retention-days: 1 - compression-level: 0 - - deps_android: - name: Deps Android - needs: plan - if: ${{ inputs.android }} + if: ${{ inputs.ios || inputs.macos || inputs.android }} runs-on: ubuntu-latest timeout-minutes: 360 steps: @@ -177,21 +147,24 @@ jobs: uses: ./src/.github/actions/setup-webrtc with: webrtc_ref: ${{ inputs.webrtc_ref }} - install_android_packages: "true" - - name: gclient sync (Android) + - name: gclient sync working-directory: src/stream_build env: DEPS_ROOT: ${{ github.workspace }} WEBRTC_SRC: ${{ github.workspace }}/src GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache - TARGET_OS: ${{ needs.plan.outputs.android_target_os }} + TARGET_OS: ${{ needs.plan.outputs.linux_target_os }} + RUN_HOOKS: "0" JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps - - name: Upload deps-android + # Same-run handoff. The full tree is ~20 GB (over the artifact cap); + # git-cache only. Build/Test download this and run make deps locally. + # RUN_HOOKS=0: no hermetic Xcode CIPD on Linux. + - name: Upload deps-git uses: actions/upload-artifact@v7 with: - name: deps-android + name: deps-git path: .gclient-git-cache include-hidden-files: true if-no-files-found: error @@ -247,8 +220,8 @@ jobs: build_ios: name: Build iOS - needs: [plan, deps_apple] - if: ${{ inputs.mode == 'build' && inputs.ios }} + needs: [plan, deps] + if: ${{ (inputs.mode == 'build' || inputs.mode == 'package' || inputs.mode == 'release') && inputs.ios }} runs-on: macos-26 timeout-minutes: 360 steps: @@ -260,19 +233,37 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-apple + deps_artifact: deps-git - name: Build iOS working-directory: src/stream_build env: DEPS_ROOT: ${{ github.workspace }} + PRODUCTS: ${{ github.workspace }}/products CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build ios + - name: Package iOS + if: ${{ inputs.mode == 'package' || inputs.mode == 'release' }} + working-directory: src/stream_build + env: + DEPS_ROOT: ${{ github.workspace }} + PRODUCTS: ${{ github.workspace }}/products + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make package ios + - name: Upload products-ios + if: ${{ inputs.mode == 'package' || inputs.mode == 'release' }} + uses: actions/upload-artifact@v7 + with: + name: products-ios + path: products/ios + if-no-files-found: error + retention-days: 7 build_macos: name: Build macOS - needs: [plan, deps_apple] - if: ${{ inputs.mode == 'build' && inputs.macos }} + needs: [plan, deps] + if: ${{ (inputs.mode == 'build' || inputs.mode == 'package' || inputs.mode == 'release') && inputs.macos }} runs-on: macos-26 timeout-minutes: 360 steps: @@ -284,19 +275,37 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-apple + deps_artifact: deps-git - name: Build macOS working-directory: src/stream_build env: DEPS_ROOT: ${{ github.workspace }} + PRODUCTS: ${{ github.workspace }}/products CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build macos + - name: Package macOS + if: ${{ inputs.mode == 'package' || inputs.mode == 'release' }} + working-directory: src/stream_build + env: + DEPS_ROOT: ${{ github.workspace }} + PRODUCTS: ${{ github.workspace }}/products + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + run: make package macos + - name: Upload products-macos + if: ${{ inputs.mode == 'package' || inputs.mode == 'release' }} + uses: actions/upload-artifact@v7 + with: + name: products-macos + path: products/macos + if-no-files-found: error + retention-days: 7 build_android: name: Build Android - needs: [plan, deps_android] - if: ${{ inputs.mode == 'build' && inputs.android }} + needs: [plan, deps] + if: ${{ (inputs.mode == 'build' || inputs.mode == 'package' || inputs.mode == 'release') && inputs.android }} runs-on: ubuntu-latest timeout-minutes: 360 steps: @@ -308,12 +317,13 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.android_target_os }} - deps_artifact: deps-android + deps_artifact: deps-git install_android_packages: "true" - name: Build Android working-directory: src/stream_build env: DEPS_ROOT: ${{ github.workspace }} + PRODUCTS: ${{ github.workspace }}/products CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" ANDROID_ARCH: ${{ inputs.android_arch }} @@ -324,10 +334,34 @@ jobs: extra+=(ARCHS="${ANDROID_ARCH}") fi make build android "${extra[@]}" + - name: Package Android + if: ${{ inputs.mode == 'package' || inputs.mode == 'release' }} + working-directory: src/stream_build + env: + DEPS_ROOT: ${{ github.workspace }} + PRODUCTS: ${{ github.workspace }}/products + CONFIG: ${{ inputs.config }} + SKIP_DEPS: "1" + ANDROID_ARCH: ${{ inputs.android_arch }} + run: | + set -euo pipefail + extra=() + if [[ -n "${ANDROID_ARCH}" ]]; then + extra+=(ARCHS="${ANDROID_ARCH}") + fi + make package android "${extra[@]}" + - name: Upload products-android + if: ${{ inputs.mode == 'package' || inputs.mode == 'release' }} + uses: actions/upload-artifact@v7 + with: + name: products-android + path: products + if-no-files-found: error + retention-days: 7 test_ios: name: Test iOS - needs: [plan, deps_apple] + needs: [plan, deps] if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.ios }} runs-on: macos-26 timeout-minutes: 180 @@ -340,14 +374,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-apple - - name: Build iOS - working-directory: src/stream_build - env: - DEPS_ROOT: ${{ github.workspace }} - CONFIG: ${{ inputs.config }} - SKIP_DEPS: "1" - run: make build ios + deps_artifact: deps-git - name: Test iOS working-directory: src/stream_build env: @@ -357,7 +384,7 @@ jobs: test_macos: name: Test macOS - needs: [plan, deps_apple] + needs: [plan, deps] if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.macos }} runs-on: macos-26 timeout-minutes: 180 @@ -370,14 +397,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-apple - - name: Build macOS - working-directory: src/stream_build - env: - DEPS_ROOT: ${{ github.workspace }} - CONFIG: ${{ inputs.config }} - SKIP_DEPS: "1" - run: make build macos + deps_artifact: deps-git - name: Test macOS working-directory: src/stream_build env: @@ -412,14 +432,6 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.windows_target_os }} deps_artifact: deps-windows - - name: Build Windows - working-directory: src/stream_build - shell: bash - env: - DEPS_ROOT: ${{ github.workspace }} - CONFIG: ${{ inputs.config }} - SKIP_DEPS: "1" - run: make build windows - name: Test Windows working-directory: src/stream_build shell: bash @@ -431,12 +443,11 @@ jobs: tests_passed: name: Tests passed needs: [plan, test_ios, test_macos, test_windows] - if: ${{ always() && !cancelled() && (inputs.mode == 'package' || inputs.mode == 'release') }} + if: ${{ always() && !cancelled() && inputs.mode == 'release' }} runs-on: ubuntu-latest steps: - - name: Require tests before package on Release + - name: Require tests before publish env: - MODE: ${{ inputs.mode }} WANT_IOS: ${{ inputs.ios }} WANT_MACOS: ${{ inputs.macos }} WANT_WINDOWS: ${{ inputs.windows }} @@ -445,10 +456,6 @@ jobs: WINDOWS_RESULT: ${{ needs.test_windows.result }} run: | set -euo pipefail - if [[ "${MODE}" != "release" ]]; then - echo "Package mode: tests are not required." - exit 0 - fi failed=0 if [[ "${WANT_IOS}" == "true" && "${IOS_RESULT}" != "success" ]]; then echo "::error::Release is blocked: iOS tests ${IOS_RESULT}." @@ -463,150 +470,15 @@ jobs: failed=1 fi if [[ "${failed}" -ne 0 ]]; then - echo "If tests fail, do not package or publish." + echo "If tests fail, do not publish." exit 1 fi echo "Selected-platform tests passed (Android tests are unwired and skipped)." - package_ios: - name: Package iOS - needs: [plan, deps_apple, tests_passed] - if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.ios && !cancelled() && needs.tests_passed.result == 'success' }} - runs-on: macos-26 - timeout-minutes: 360 - steps: - - uses: actions/checkout@v7 - with: - path: src - ref: ${{ inputs.webrtc_ref }} - - uses: ./src/.github/actions/restore-tree - with: - webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-apple - - name: Build iOS - working-directory: src/stream_build - env: - DEPS_ROOT: ${{ github.workspace }} - PRODUCTS: ${{ github.workspace }}/products - CONFIG: ${{ inputs.config }} - SKIP_DEPS: "1" - run: make build ios - - name: Package iOS - working-directory: src/stream_build - env: - DEPS_ROOT: ${{ github.workspace }} - PRODUCTS: ${{ github.workspace }}/products - CONFIG: ${{ inputs.config }} - SKIP_DEPS: "1" - run: make package ios - - name: Upload products-ios - uses: actions/upload-artifact@v7 - with: - name: products-ios - path: products/ios - if-no-files-found: error - retention-days: 7 - - package_macos: - name: Package macOS - needs: [plan, deps_apple, tests_passed] - if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.macos && !cancelled() && needs.tests_passed.result == 'success' }} - runs-on: macos-26 - timeout-minutes: 360 - steps: - - uses: actions/checkout@v7 - with: - path: src - ref: ${{ inputs.webrtc_ref }} - - uses: ./src/.github/actions/restore-tree - with: - webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-apple - - name: Build macOS - working-directory: src/stream_build - env: - DEPS_ROOT: ${{ github.workspace }} - PRODUCTS: ${{ github.workspace }}/products - CONFIG: ${{ inputs.config }} - SKIP_DEPS: "1" - run: make build macos - - name: Package macOS - working-directory: src/stream_build - env: - DEPS_ROOT: ${{ github.workspace }} - PRODUCTS: ${{ github.workspace }}/products - CONFIG: ${{ inputs.config }} - SKIP_DEPS: "1" - run: make package macos - - name: Upload products-macos - uses: actions/upload-artifact@v7 - with: - name: products-macos - path: products/macos - if-no-files-found: error - retention-days: 7 - - package_android: - name: Package Android - needs: [plan, deps_android, tests_passed] - if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.android && !cancelled() && needs.tests_passed.result == 'success' }} - runs-on: ubuntu-latest - timeout-minutes: 360 - steps: - - uses: actions/checkout@v7 - with: - path: src - ref: ${{ inputs.webrtc_ref }} - - uses: ./src/.github/actions/restore-tree - with: - webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.plan.outputs.android_target_os }} - deps_artifact: deps-android - install_android_packages: "true" - - name: Build Android - working-directory: src/stream_build - env: - DEPS_ROOT: ${{ github.workspace }} - PRODUCTS: ${{ github.workspace }}/products - CONFIG: ${{ inputs.config }} - SKIP_DEPS: "1" - ANDROID_ARCH: ${{ inputs.android_arch }} - run: | - set -euo pipefail - extra=() - if [[ -n "${ANDROID_ARCH}" ]]; then - extra+=(ARCHS="${ANDROID_ARCH}") - fi - make build android "${extra[@]}" - - name: Package Android - working-directory: src/stream_build - env: - DEPS_ROOT: ${{ github.workspace }} - PRODUCTS: ${{ github.workspace }}/products - CONFIG: ${{ inputs.config }} - SKIP_DEPS: "1" - ANDROID_ARCH: ${{ inputs.android_arch }} - run: | - set -euo pipefail - extra=() - if [[ -n "${ANDROID_ARCH}" ]]; then - extra+=(ARCHS="${ANDROID_ARCH}") - fi - make package android "${extra[@]}" - - name: Upload products-android - uses: actions/upload-artifact@v7 - with: - name: products-android - path: products - if-no-files-found: error - retention-days: 7 - build_windows: name: Build Windows needs: [plan, deps_windows] - if: ${{ inputs.mode == 'build' && inputs.windows }} + if: ${{ (inputs.mode == 'build' || inputs.mode == 'package' || inputs.mode == 'release') && inputs.windows }} runs-on: windows-latest timeout-minutes: 360 steps: @@ -630,42 +502,6 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.windows_target_os }} deps_artifact: deps-windows - - name: Build Windows - working-directory: src/stream_build - shell: bash - env: - DEPS_ROOT: ${{ github.workspace }} - CONFIG: ${{ inputs.config }} - SKIP_DEPS: "1" - run: make build windows - - package_windows: - name: Package Windows - needs: [plan, deps_windows, tests_passed] - if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && inputs.windows && !cancelled() && needs.tests_passed.result == 'success' }} - runs-on: windows-latest - timeout-minutes: 360 - steps: - - uses: actions/checkout@v7 - with: - path: src - ref: ${{ inputs.webrtc_ref }} - - name: Check Windows WebRTC host - shell: bash - run: | - set -euo pipefail - missing=() - command -v make >/dev/null || missing+=("GNU make") - command -v python3 >/dev/null || missing+=("python3") - if [[ ${#missing[@]} -gt 0 ]]; then - echo "::error::Windows WebRTC host is not ready on windows-latest (missing: ${missing[*]}). This job is wired for make package windows, but the runner image cannot run it yet." - exit 1 - fi - - uses: ./src/.github/actions/restore-tree - with: - webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.plan.outputs.windows_target_os }} - deps_artifact: deps-windows - name: Build Windows working-directory: src/stream_build shell: bash @@ -676,6 +512,7 @@ jobs: SKIP_DEPS: "1" run: make build windows - name: Package Windows + if: ${{ inputs.mode == 'package' || inputs.mode == 'release' }} working-directory: src/stream_build shell: bash env: @@ -685,6 +522,7 @@ jobs: SKIP_DEPS: "1" run: make package windows - name: Upload products-windows + if: ${{ inputs.mode == 'package' || inputs.mode == 'release' }} uses: actions/upload-artifact@v7 with: name: products-windows @@ -692,10 +530,10 @@ jobs: if-no-files-found: error retention-days: 7 - finalise_package: - name: Finalise package - needs: [plan, tests_passed, package_ios, package_macos, package_android, package_windows] - if: ${{ (inputs.mode == 'package' || inputs.mode == 'release') && !cancelled() && needs.tests_passed.result == 'success' && !failure() }} + package: + name: Package + needs: [plan, build_ios, build_macos, build_android, build_windows] + if: ${{ always() && !cancelled() && (inputs.mode == 'package' || inputs.mode == 'release') && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && (needs.build_ios.result == 'success' || needs.build_macos.result == 'success' || needs.build_android.result == 'success' || needs.build_windows.result == 'success') }} runs-on: ${{ (inputs.ios || inputs.macos) && 'macos-26' || 'ubuntu-latest' }} timeout-minutes: 60 steps: @@ -826,8 +664,8 @@ jobs: github_release: name: Release - needs: [plan, finalise_package] - if: ${{ inputs.mode == 'release' && !cancelled() && !failure() }} + needs: [plan, tests_passed, package] + if: ${{ always() && !cancelled() && inputs.mode == 'release' && needs.tests_passed.result == 'success' && needs.package.result == 'success' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -881,7 +719,7 @@ jobs: trigger_downstream_releases: name: Trigger downstream WebRTC releases needs: [plan, github_release] - if: ${{ inputs.mode == 'release' && !cancelled() && !failure() }} + if: ${{ always() && !cancelled() && inputs.mode == 'release' && needs.github_release.result == 'success' }} runs-on: ubuntu-latest steps: - name: Trigger stream-video-swift-webrtc release diff --git a/.github/workflows/manual-platform-tests.yml b/.github/workflows/manual-platform-tests.yml index 0e4f884bb4..2827d17b97 100644 --- a/.github/workflows/manual-platform-tests.yml +++ b/.github/workflows/manual-platform-tests.yml @@ -1,5 +1,6 @@ # Manual WebRTC build. Dispatchable from a PR because this path exists on the # default branch. Implementation: .github/workflows/_make.yml +# Deps → Build (restore-tree + make build). No package combine, no products. name: Build diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 983e6c12ad..3bfb353c49 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,5 +1,7 @@ # Package WebRTC artifacts. Same path as today's Publish so a PR can dispatch it. # Does not create a GitHub release. +# _make.yml runs Build then Package combine (products-* in, final-* out). +# Package does not rebuild from git-cache. name: Package diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c0ffce711b..de9040094c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,6 @@ # GitHub release of packaged WebRTC artifacts, then downstream wrapper publishes. # Not dispatchable until this file exists on the default branch. +# Tests run parallel with Build (needs Deps only). Publish waits on Test + Package. name: Release diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1549f607c1..11582233cd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,5 @@ # WebRTC tests. Not dispatchable until this file exists on the default branch. +# _make.yml: Deps then make test only (no extra framework-slice make build). name: Test diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index ac15b8fc31..709949ff07 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -79,8 +79,14 @@ the tree is `webrtc/src` (real directory, not a symlink). `deps` / CI checks out with `path: src` so `GITHUB_WORKSPACE` is the webrtc-named folder. `DEPS_ROOT=$GITHUB_WORKSPACE`. gclient objects live at -`$DEPS_ROOT/.gclient-git-cache`; same-run jobs hand that directory off as -the `deps-*` artifact (no GitHub Actions cache). +`$DEPS_ROOT/.gclient-git-cache` (no GitHub Actions cache). Linux Deps +(`RUN_HOOKS=0`) uploads one `deps-git` artifact for the selected +`ios` / `mac` / `android,unix` tokens; Windows Deps uploads `deps-windows`. +Build restores that cache then `make deps` + `make build`. Package/Release +Build jobs also `make package` and upload `products-*`. Package combine +consumes `products-*` (no third ninja) and uploads `final-*`. Release +attaches `final-*`. Tests need Deps only and run `make test` (no extra +framework-slice build). `TARGET_OS` is only the tokens selected this run. ## Host gates From 3b330d5ef3648ce4b7cae6438fb06b0aeab14e45 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Fri, 11 Sep 2026 21:02:56 +0300 Subject: [PATCH 10/24] Update Hetzner caching --- .github/actions/artifact-download/action.yml | 84 +++++++++++++++++++ .github/actions/artifact-upload/action.yml | 85 ++++++++++++++++++++ .github/actions/restore-tree/action.yml | 43 +++++++++- .github/workflows/_make.yml | 47 +++++++---- stream_build/AGENTS.md | 35 ++++++-- 5 files changed, 268 insertions(+), 26 deletions(-) create mode 100644 .github/actions/artifact-download/action.yml create mode 100644 .github/actions/artifact-upload/action.yml diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml new file mode 100644 index 0000000000..e0ab5054e6 --- /dev/null +++ b/.github/actions/artifact-download/action.yml @@ -0,0 +1,84 @@ +# Hetzner-only. Callers pass org secrets: +# ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} +# ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} +# ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} +name: Download WebRTC deps from Hetzner +description: Stream a Hetzner deps tarball onto $GITHUB_WORKSPACE and untar. + +inputs: + path: + description: Object stem. Downloads artifacts///.tar + required: true + hetzner_access_key: + description: Pass ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }}. + required: true + hetzner_secret_access_key: + description: Pass ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }}. + required: true + hetzner_bucket: + description: Pass ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }}. + required: true + +runs: + using: composite + steps: + - name: Ensure aws CLI + shell: bash + run: | + set -euo pipefail + if command -v aws >/dev/null; then + exit 0 + fi + if [[ "$(uname -s)" == Darwin ]]; then + brew install awscli + else + sudo apt-get update + sudo apt-get install -y awscli + fi + + - name: Download and extract deps tarball + shell: bash + working-directory: ${{ github.workspace }} + env: + HETZNER_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_access_key }} + HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_secret_access_key }} + HETZNER_BUCKET_CI_ARTIFACTS: ${{ inputs.hetzner_bucket }} + OBJECT_STEM: ${{ inputs.path }} + run: | + set -euo pipefail + export AWS_ACCESS_KEY_ID="${HETZNER_ACCESS_KEY_CI_ARTIFACTS:?}" + export AWS_SECRET_ACCESS_KEY="${HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS:?}" + export AWS_DEFAULT_REGION=hel1 + export AWS_EC2_METADATA_DISABLED=true + endpoint="--endpoint-url https://hel1.your-objectstorage.com" + BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" + object="artifacts/${{ github.repository }}/${{ github.run_id }}/${OBJECT_STEM}.tar" + echo "extracting ${object} -> ${GITHUB_WORKSPACE}" + # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. + aws ${endpoint} s3 cp "s3://${BUCKET}/${object}" - | tar xf - + test -f .gclient + test -d .gclient-git-cache + test -d src/third_party + test -f src/DEPS + test ! -L src + for p in \ + .gclient \ + .gclient_entries \ + .gclient_previous_sync_commits \ + .gcs_entries \ + .cipd \ + .gclient-git-cache \ + src/.landmines \ + src/third_party \ + src/build \ + src/buildtools \ + src/testing \ + src/tools \ + src/ios + do + if [[ -d "${p}" ]]; then + find "${p}" -type f -exec touch {} + + elif [[ -f "${p}" ]]; then + touch "${p}" + fi + done diff --git a/.github/actions/artifact-upload/action.yml b/.github/actions/artifact-upload/action.yml new file mode 100644 index 0000000000..628ba2b227 --- /dev/null +++ b/.github/actions/artifact-upload/action.yml @@ -0,0 +1,85 @@ +# Hetzner-only. Callers pass org secrets: +# ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} +# ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} +# ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} +name: Upload WebRTC deps to Hetzner +description: Stream-tar gclient deps at $GITHUB_WORKSPACE to one Hetzner object. + +inputs: + path: + description: Object stem. Uploads artifacts///.tar + required: true + hetzner_access_key: + description: Pass ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }}. + required: true + hetzner_secret_access_key: + description: Pass ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }}. + required: true + hetzner_bucket: + description: Pass ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }}. + required: true + +runs: + using: composite + steps: + - name: Ensure aws CLI + shell: bash + run: | + set -euo pipefail + if command -v aws >/dev/null; then + exit 0 + fi + if [[ "$(uname -s)" == Darwin ]]; then + brew install awscli + else + sudo apt-get update + sudo apt-get install -y awscli + fi + + - name: Upload deps tarball + shell: bash + working-directory: ${{ github.workspace }} + env: + HETZNER_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_access_key }} + HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_secret_access_key }} + HETZNER_BUCKET_CI_ARTIFACTS: ${{ inputs.hetzner_bucket }} + OBJECT_STEM: ${{ inputs.path }} + run: | + set -euo pipefail + export AWS_ACCESS_KEY_ID="${HETZNER_ACCESS_KEY_CI_ARTIFACTS:?}" + export AWS_SECRET_ACCESS_KEY="${HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS:?}" + export AWS_DEFAULT_REGION=hel1 + export AWS_EC2_METADATA_DISABLED=true + endpoint="--endpoint-url https://hel1.your-objectstorage.com" + members=() + for p in \ + .gclient \ + .gclient_entries \ + .gclient_previous_sync_commits \ + .gcs_entries \ + .cipd \ + .gclient-git-cache \ + src/.landmines \ + src/third_party \ + src/build \ + src/buildtools \ + src/testing \ + src/tools \ + src/ios + do + if [[ -e "${p}" ]]; then + members+=("${p}") + fi + done + if [[ ${#members[@]} -eq 0 ]]; then + echo "::error::no webrtc deps members to pack under ${GITHUB_WORKSPACE}" + exit 1 + fi + test -f .gclient + test -d .gclient-git-cache + test -d src/third_party + BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" + object="artifacts/${{ github.repository }}/${{ github.run_id }}/${OBJECT_STEM}.tar" + echo "packing ${members[*]} -> ${object}" + # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. + tar cf - "${members[@]}" | aws ${endpoint} s3 cp - "s3://${BUCKET}/${object}" diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index 8f958b08bf..e237f48c52 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -1,5 +1,5 @@ name: Restore WebRTC tree -description: Download the same-run git-cache artifact, then gclient sync from local objects so SKIP_DEPS=1 can follow. +description: Restore same-run deps (Hetzner tarball for deps-git, GH artifact for deps-windows). inputs: webrtc_ref: @@ -15,6 +15,18 @@ inputs: required: false default: "false" description: Install apt packages needed for Android builds. + hetzner_access_key: + required: false + default: "" + description: Pass ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} (deps-git). + hetzner_secret_access_key: + required: false + default: "" + description: Pass ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} (deps-git). + hetzner_bucket: + required: false + default: "" + description: Pass ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} (deps-git). runs: using: composite @@ -25,15 +37,39 @@ runs: webrtc_ref: ${{ inputs.webrtc_ref }} install_android_packages: ${{ inputs.install_android_packages }} - # Same-run handoff. Artifact is git-cache only; the full tree exceeds - # the per-artifact cap. + # Linux Deps packed the gclient working tree. Extract in place and + # run hooks only (no second gclient sync). + - name: Download same-run deps from Hetzner + if: ${{ inputs.deps_artifact == 'deps-git' }} + uses: ./src/.github/actions/artifact-download + with: + path: ${{ inputs.deps_artifact }} + hetzner_access_key: ${{ inputs.hetzner_access_key }} + hetzner_secret_access_key: ${{ inputs.hetzner_secret_access_key }} + hetzner_bucket: ${{ inputs.hetzner_bucket }} + + - name: Run gclient hooks + if: ${{ inputs.deps_artifact == 'deps-git' }} + working-directory: src/stream_build + shell: bash + env: + DEPS_ROOT: ${{ github.workspace }} + WEBRTC_SRC: ${{ github.workspace }}/src + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache + TARGET_OS: ${{ inputs.target_os }} + WEBRTC_REPO: https://github.com/GetStream/webrtc.git + run: make runhooks + + # Windows is still a git-cache-only GitHub artifact. - name: Download same-run deps artifact + if: ${{ inputs.deps_artifact != 'deps-git' }} uses: actions/download-artifact@v8 with: name: ${{ inputs.deps_artifact }} path: ${{ runner.temp }}/deps-git-cache - name: Install same-run gclient object cache + if: ${{ inputs.deps_artifact != 'deps-git' }} shell: bash run: | set -euo pipefail @@ -49,6 +85,7 @@ runs: test ! -L "${GITHUB_WORKSPACE}/src" - name: gclient sync + if: ${{ inputs.deps_artifact != 'deps-git' }} working-directory: src/stream_build shell: bash env: diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 7651726e5b..8f19d42f43 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -2,11 +2,12 @@ # Not listed in the Actions dispatch UI (workflow_call only). # Callers pass mode: build | test | package | release. # -# I/O: Linux Deps (RUN_HOOKS=0) uploads deps-git for selected ios/mac/android,unix. -# Windows Deps stays separate (deps-windows). Build mode is ninja only (no -# products). Package/Release Build jobs also make package and upload products-*. -# Package combine consumes products-* (no third ninja) and uploads final-*. -# Release attaches final-*. Tests need Deps only (make test, no make build). +# I/O: Linux Deps (RUN_HOOKS=0) stream-tars gclient deps to Hetzner as +# deps-git.tar (not a GitHub artifact). Windows Deps stays deps-windows on +# GitHub. Build mode is ninja only (no products). Package/Release Build jobs +# also make package and upload products-* (GitHub). Package combine consumes +# products-* (no third ninja) and uploads final-*. Release attaches final-*. +# Tests need Deps only (make test, no make build). name: WebRTC make @@ -56,6 +57,9 @@ env: DEPS_ROOT: ${{ github.workspace }} WEBRTC_SRC: ${{ github.workspace }}/src GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache + HETZNER_ACCESS_KEY_CI_ARTIFACTS: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + HETZNER_BUCKET_CI_ARTIFACTS: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} jobs: validate_inputs: @@ -158,18 +162,16 @@ jobs: JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps - # Same-run handoff. The full tree is ~20 GB (over the artifact cap); - # git-cache only. Build/Test download this and run make deps locally. - # RUN_HOOKS=0: no hermetic Xcode CIPD on Linux. + # Same-run handoff. Stream-tar the gclient working tree to Hetzner + # (no zip on disk, no GitHub artifact quota). RUN_HOOKS=0: no + # hermetic Xcode CIPD on Linux; Build/Test run make runhooks. - name: Upload deps-git - uses: actions/upload-artifact@v7 + uses: ./src/.github/actions/artifact-upload with: - name: deps-git - path: .gclient-git-cache - include-hidden-files: true - if-no-files-found: error - retention-days: 1 - compression-level: 0 + path: deps-git + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} deps_windows: name: Deps Windows @@ -234,6 +236,9 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-git + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Build iOS working-directory: src/stream_build env: @@ -276,6 +281,9 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-git + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Build macOS working-directory: src/stream_build env: @@ -319,6 +327,9 @@ jobs: target_os: ${{ needs.plan.outputs.android_target_os }} deps_artifact: deps-git install_android_packages: "true" + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Build Android working-directory: src/stream_build env: @@ -375,6 +386,9 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-git + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Test iOS working-directory: src/stream_build env: @@ -398,6 +412,9 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-git + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Test macOS working-directory: src/stream_build env: diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index 709949ff07..a042e4985d 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -79,14 +79,33 @@ the tree is `webrtc/src` (real directory, not a symlink). `deps` / CI checks out with `path: src` so `GITHUB_WORKSPACE` is the webrtc-named folder. `DEPS_ROOT=$GITHUB_WORKSPACE`. gclient objects live at -`$DEPS_ROOT/.gclient-git-cache` (no GitHub Actions cache). Linux Deps -(`RUN_HOOKS=0`) uploads one `deps-git` artifact for the selected -`ios` / `mac` / `android,unix` tokens; Windows Deps uploads `deps-windows`. -Build restores that cache then `make deps` + `make build`. Package/Release -Build jobs also `make package` and upload `products-*`. Package combine -consumes `products-*` (no third ninja) and uploads `final-*`. Release -attaches `final-*`. Tests need Deps only and run `make test` (no extra -framework-slice build). `TARGET_OS` is only the tokens selected this run. +`$DEPS_ROOT/.gclient-git-cache`. Linux Deps (`RUN_HOOKS=0`) stream-tars +the gclient working tree to Hetzner as `deps-git.tar` (not a GitHub +artifact or Actions cache). Windows Deps still uploads `deps-windows` to +GitHub. Build/Test restore `deps-git` via `artifact-download`, then +`make runhooks` (no second `gclient sync`) and `SKIP_DEPS=1` on +build/test. Package/Release Build jobs also `make package` and upload +`products-*` (GitHub). Package combine consumes `products-*` (no third +ninja) and uploads `final-*`. Release attaches `final-*`. Tests need +Deps only and run `make test` (no extra framework-slice build). +`TARGET_OS` is only the tokens selected this run. + +Hetzner deps handoff: +`.github/actions/artifact-upload` and `artifact-download`. +`aws s3 cp` talks to Hetzner's S3-compatible API, not AWS. +Pipe `tar cf - … | aws s3 cp - "s3://${BUCKET}/…"` and +`aws s3 cp "s3://${BUCKET}/…" - | tar xf -` (no tarball on disk). Object: +`/artifacts///.tar` +with `--endpoint-url https://hel1.your-objectstorage.com` and region +`hel1`. Packs `.gclient*`, `.cipd`, `.gclient-git-cache`, and gclient +checkouts under `src/` (`third_party`, `build`, `buildtools`, `testing`, +`tools`, `ios`) — not the GetStream/webrtc `src` git worktree, `out/`, +or `products/`. Callers pass org secrets +`${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }}`, +`${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }}`, and +`${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }}`. After extract, files are +`touch`ed so make/ninja do not rebuild from mtime. `products-*` / +`final-*` stay on `actions/upload-artifact`. ## Host gates From 74900481061c838d75583b5fc58dd450a3caa115 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Fri, 11 Sep 2026 21:42:22 +0300 Subject: [PATCH 11/24] Fix secrets access --- .github/actions/artifact-download/action.yml | 14 +++++++------- .github/actions/artifact-upload/action.yml | 14 +++++++------- .github/actions/restore-tree/action.yml | 6 +++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml index e0ab5054e6..8dff872e25 100644 --- a/.github/actions/artifact-download/action.yml +++ b/.github/actions/artifact-download/action.yml @@ -1,7 +1,7 @@ -# Hetzner-only. Callers pass org secrets: -# ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} -# ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} -# ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} +# Hetzner-only. Callers pass org secrets via with: +# hetzner_access_key +# hetzner_secret_access_key +# hetzner_bucket name: Download WebRTC deps from Hetzner description: Stream a Hetzner deps tarball onto $GITHUB_WORKSPACE and untar. @@ -10,13 +10,13 @@ inputs: description: Object stem. Downloads artifacts///.tar required: true hetzner_access_key: - description: Pass ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }}. + description: Hetzner object storage access key. required: true hetzner_secret_access_key: - description: Pass ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }}. + description: Hetzner object storage secret access key. required: true hetzner_bucket: - description: Pass ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }}. + description: Hetzner object storage bucket name. required: true runs: diff --git a/.github/actions/artifact-upload/action.yml b/.github/actions/artifact-upload/action.yml index 628ba2b227..bac8d77367 100644 --- a/.github/actions/artifact-upload/action.yml +++ b/.github/actions/artifact-upload/action.yml @@ -1,7 +1,7 @@ -# Hetzner-only. Callers pass org secrets: -# ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} -# ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} -# ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} +# Hetzner-only. Callers pass org secrets via with: +# hetzner_access_key +# hetzner_secret_access_key +# hetzner_bucket name: Upload WebRTC deps to Hetzner description: Stream-tar gclient deps at $GITHUB_WORKSPACE to one Hetzner object. @@ -10,13 +10,13 @@ inputs: description: Object stem. Uploads artifacts///.tar required: true hetzner_access_key: - description: Pass ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }}. + description: Hetzner object storage access key. required: true hetzner_secret_access_key: - description: Pass ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }}. + description: Hetzner object storage secret access key. required: true hetzner_bucket: - description: Pass ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }}. + description: Hetzner object storage bucket name. required: true runs: diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index e237f48c52..b8aa8d61ff 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -18,15 +18,15 @@ inputs: hetzner_access_key: required: false default: "" - description: Pass ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} (deps-git). + description: Hetzner access key for deps-git. hetzner_secret_access_key: required: false default: "" - description: Pass ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} (deps-git). + description: Hetzner secret access key for deps-git. hetzner_bucket: required: false default: "" - description: Pass ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} (deps-git). + description: Hetzner bucket for deps-git. runs: using: composite From 2c8246a052b46a219b7eca3268330c3198f4dca6 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Fri, 11 Sep 2026 23:12:18 +0300 Subject: [PATCH 12/24] Fix apple failures --- .github/actions/artifact-download/action.yml | 29 ++++++++++++++++++-- .github/actions/artifact-upload/action.yml | 9 ++++-- .github/actions/restore-tree/action.yml | 19 ++++++++----- .github/workflows/_make.yml | 28 ++++++++++++++----- stream_build/AGENTS.md | 13 +++++---- stream_build/scripts/check.sh | 10 +++++++ 6 files changed, 84 insertions(+), 24 deletions(-) diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml index 8dff872e25..dd6e009273 100644 --- a/.github/actions/artifact-download/action.yml +++ b/.github/actions/artifact-download/action.yml @@ -7,8 +7,15 @@ description: Stream a Hetzner deps tarball onto $GITHUB_WORKSPACE and untar. inputs: path: - description: Object stem. Downloads artifacts///.tar + description: Object stem. Downloads artifacts///.tar required: true + webrtc_ref: + description: Stable deps identity (branch, tag, or SHA). + required: true + if_missing: + description: error fails when the object is absent; skip continues. + required: false + default: error hetzner_access_key: description: Hetzner object storage access key. required: true @@ -44,6 +51,8 @@ runs: HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_secret_access_key }} HETZNER_BUCKET_CI_ARTIFACTS: ${{ inputs.hetzner_bucket }} OBJECT_STEM: ${{ inputs.path }} + WEBRTC_REF: ${{ inputs.webrtc_ref }} + IF_MISSING: ${{ inputs.if_missing }} run: | set -euo pipefail export AWS_ACCESS_KEY_ID="${HETZNER_ACCESS_KEY_CI_ARTIFACTS:?}" @@ -52,7 +61,23 @@ runs: export AWS_EC2_METADATA_DISABLED=true endpoint="--endpoint-url https://hel1.your-objectstorage.com" BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" - object="artifacts/${{ github.repository }}/${{ github.run_id }}/${OBJECT_STEM}.tar" + [[ -n "${WEBRTC_REF}" ]] + object="artifacts/${{ github.repository }}/${OBJECT_STEM}/${WEBRTC_REF}.tar" + set +e + head_out="$(aws ${endpoint} s3api head-object \ + --bucket "${BUCKET}" --key "${object}" 2>&1)" + head_rc=$? + set -e + if [[ "${head_rc}" -ne 0 ]]; then + if [[ "${IF_MISSING}" == "skip" ]] && \ + grep -qiE '404|Not Found|NoSuchKey|NotFound' <<<"${head_out}"; then + echo "missing ${object}; skipping extract" + exit 0 + fi + echo "::error::failed to access s3://${BUCKET}/${object}" + printf '%s\n' "${head_out}" + exit 1 + fi echo "extracting ${object} -> ${GITHUB_WORKSPACE}" # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. aws ${endpoint} s3 cp "s3://${BUCKET}/${object}" - | tar xf - diff --git a/.github/actions/artifact-upload/action.yml b/.github/actions/artifact-upload/action.yml index bac8d77367..38bf1ca527 100644 --- a/.github/actions/artifact-upload/action.yml +++ b/.github/actions/artifact-upload/action.yml @@ -7,7 +7,10 @@ description: Stream-tar gclient deps at $GITHUB_WORKSPACE to one Hetzner object. inputs: path: - description: Object stem. Uploads artifacts///.tar + description: Object stem. Uploads artifacts///.tar + required: true + webrtc_ref: + description: Stable deps identity (branch, tag, or SHA). required: true hetzner_access_key: description: Hetzner object storage access key. @@ -44,6 +47,7 @@ runs: HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_secret_access_key }} HETZNER_BUCKET_CI_ARTIFACTS: ${{ inputs.hetzner_bucket }} OBJECT_STEM: ${{ inputs.path }} + WEBRTC_REF: ${{ inputs.webrtc_ref }} run: | set -euo pipefail export AWS_ACCESS_KEY_ID="${HETZNER_ACCESS_KEY_CI_ARTIFACTS:?}" @@ -78,8 +82,9 @@ runs: test -f .gclient test -d .gclient-git-cache test -d src/third_party + [[ -n "${WEBRTC_REF}" ]] BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" - object="artifacts/${{ github.repository }}/${{ github.run_id }}/${OBJECT_STEM}.tar" + object="artifacts/${{ github.repository }}/${OBJECT_STEM}/${WEBRTC_REF}.tar" echo "packing ${members[*]} -> ${object}" # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. tar cf - "${members[@]}" | aws ${endpoint} s3 cp - "s3://${BUCKET}/${object}" diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index b8aa8d61ff..8e30d4ec83 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -1,5 +1,5 @@ name: Restore WebRTC tree -description: Restore same-run deps (Hetzner tarball for deps-git, GH artifact for deps-windows). +description: Restore deps-git from Hetzner or deps-windows from a GitHub artifact. inputs: webrtc_ref: @@ -10,7 +10,7 @@ inputs: description: gclient TARGET_OS (selected tokens only). deps_artifact: required: true - description: Same-run artifact (deps-git / deps-windows). + description: deps-git (Hetzner) or deps-windows (GitHub artifact). install_android_packages: required: false default: "false" @@ -37,18 +37,20 @@ runs: webrtc_ref: ${{ inputs.webrtc_ref }} install_android_packages: ${{ inputs.install_android_packages }} - # Linux Deps packed the gclient working tree. Extract in place and - # run hooks only (no second gclient sync). - - name: Download same-run deps from Hetzner + # Linux Deps packed the gclient working tree. Extract, then shallow + # gclient sync so host GCS (rust-toolchain) matches this runner. + # runhooks does not download GCS. Missing object fails (Deps uploaded). + - name: Download deps from Hetzner if: ${{ inputs.deps_artifact == 'deps-git' }} uses: ./src/.github/actions/artifact-download with: path: ${{ inputs.deps_artifact }} + webrtc_ref: ${{ inputs.webrtc_ref }} hetzner_access_key: ${{ inputs.hetzner_access_key }} hetzner_secret_access_key: ${{ inputs.hetzner_secret_access_key }} hetzner_bucket: ${{ inputs.hetzner_bucket }} - - name: Run gclient hooks + - name: gclient sync and hooks if: ${{ inputs.deps_artifact == 'deps-git' }} working-directory: src/stream_build shell: bash @@ -57,8 +59,11 @@ runs: WEBRTC_SRC: ${{ github.workspace }}/src GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache TARGET_OS: ${{ inputs.target_os }} + JOBS: "8" + SHALLOW: "1" + RUN_HOOKS: "1" WEBRTC_REPO: https://github.com/GetStream/webrtc.git - run: make runhooks + run: make deps # Windows is still a git-cache-only GitHub artifact. - name: Download same-run deps artifact diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 8f19d42f43..43fe076c06 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -2,10 +2,13 @@ # Not listed in the Actions dispatch UI (workflow_call only). # Callers pass mode: build | test | package | release. # -# I/O: Linux Deps (RUN_HOOKS=0) stream-tars gclient deps to Hetzner as -# deps-git.tar (not a GitHub artifact). Windows Deps stays deps-windows on -# GitHub. Build mode is ninja only (no products). Package/Release Build jobs -# also make package and upload products-* (GitHub). Package combine consumes +# I/O: Linux Deps restores Hetzner deps-git (skip if missing), shallow +# gclient sync (RUN_HOOKS=0), overwrites the same object. Key: +# artifacts//deps-git/.tar +# (not a GitHub artifact, not run_id). Windows Deps stays deps-windows on +# GitHub. Build/Test restore that key after Deps (missing = fail). Build +# mode is ninja only (no products). Package/Release Build jobs also make +# package and upload products-* (GitHub). Package combine consumes # products-* (no third ninja) and uploads final-*. Release attaches final-*. # Tests need Deps only (make test, no make build). @@ -151,6 +154,15 @@ jobs: uses: ./src/.github/actions/setup-webrtc with: webrtc_ref: ${{ inputs.webrtc_ref }} + - name: Download cached deps-git + uses: ./src/.github/actions/artifact-download + with: + path: deps-git + webrtc_ref: ${{ inputs.webrtc_ref }} + if_missing: skip + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: gclient sync working-directory: src/stream_build env: @@ -159,16 +171,18 @@ jobs: GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache TARGET_OS: ${{ needs.plan.outputs.linux_target_os }} RUN_HOOKS: "0" + SHALLOW: "1" JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps - # Same-run handoff. Stream-tar the gclient working tree to Hetzner - # (no zip on disk, no GitHub artifact quota). RUN_HOOKS=0: no - # hermetic Xcode CIPD on Linux; Build/Test run make runhooks. + # Overwrite the stable deps-git object. Stream-tar (no zip on disk, + # no GitHub artifact quota). RUN_HOOKS=0: no hermetic Xcode CIPD on + # Linux; Build/Test restore this key then make deps (host GCS + hooks). - name: Upload deps-git uses: ./src/.github/actions/artifact-upload with: path: deps-git + webrtc_ref: ${{ inputs.webrtc_ref }} hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index a042e4985d..186b1bfa82 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -79,11 +79,12 @@ the tree is `webrtc/src` (real directory, not a symlink). `deps` / CI checks out with `path: src` so `GITHUB_WORKSPACE` is the webrtc-named folder. `DEPS_ROOT=$GITHUB_WORKSPACE`. gclient objects live at -`$DEPS_ROOT/.gclient-git-cache`. Linux Deps (`RUN_HOOKS=0`) stream-tars -the gclient working tree to Hetzner as `deps-git.tar` (not a GitHub -artifact or Actions cache). Windows Deps still uploads `deps-windows` to -GitHub. Build/Test restore `deps-git` via `artifact-download`, then -`make runhooks` (no second `gclient sync`) and `SKIP_DEPS=1` on +`$DEPS_ROOT/.gclient-git-cache`. Linux Deps restores the Hetzner +`deps-git` tarball (skip if missing), `gclient sync --no-history +--shallow` (`RUN_HOOKS=0`), and overwrites the same object. Not a GitHub +artifact or Actions cache. Windows Deps still uploads `deps-windows` to +GitHub. Build/Test restore that same key via `artifact-download` (missing +fails), then `make deps` (host GCS + hooks) and `SKIP_DEPS=1` on build/test. Package/Release Build jobs also `make package` and upload `products-*` (GitHub). Package combine consumes `products-*` (no third ninja) and uploads `final-*`. Release attaches `final-*`. Tests need @@ -95,7 +96,7 @@ Hetzner deps handoff: `aws s3 cp` talks to Hetzner's S3-compatible API, not AWS. Pipe `tar cf - … | aws s3 cp - "s3://${BUCKET}/…"` and `aws s3 cp "s3://${BUCKET}/…" - | tar xf -` (no tarball on disk). Object: -`/artifacts///.tar` +`/artifacts//deps-git/.tar` with `--endpoint-url https://hel1.your-objectstorage.com` and region `hel1`. Packs `.gclient*`, `.cipd`, `.gclient-git-cache`, and gclient checkouts under `src/` (`third_party`, `build`, `buildtools`, `testing`, diff --git a/stream_build/scripts/check.sh b/stream_build/scripts/check.sh index e7f51095e1..80c4fd6048 100755 --- a/stream_build/scripts/check.sh +++ b/stream_build/scripts/check.sh @@ -381,4 +381,14 @@ fi grep -q 'src is a symlink' "$deps_tmp/link_err" rm -rf "$deps_tmp" "$missing" +gha="$ROOT/../.github" +! grep -q 'github.run_id' "$gha/actions/artifact-upload/action.yml" +! grep -q 'github.run_id' "$gha/actions/artifact-download/action.yml" +grep -q '\${OBJECT_STEM}/\${WEBRTC_REF}.tar' \ + "$gha/actions/artifact-upload/action.yml" +grep -q '\${OBJECT_STEM}/\${WEBRTC_REF}.tar' \ + "$gha/actions/artifact-download/action.yml" +grep -q 'if_missing: skip' "$gha/workflows/_make.yml" +grep -q 'SHALLOW: "1"' "$gha/workflows/_make.yml" + echo "ok" From 51909783d65197508e45303b6028119163a147a7 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Fri, 11 Sep 2026 23:25:39 +0300 Subject: [PATCH 13/24] Fix apple compilations --- .github/actions/artifact-download/action.yml | 10 ++----- .github/actions/artifact-upload/action.yml | 10 ++----- .github/actions/restore-tree/action.yml | 21 +++++++------- .github/workflows/_make.yml | 30 +++++++++----------- stream_build/AGENTS.md | 4 +-- stream_build/scripts/check.sh | 10 +++++-- 6 files changed, 40 insertions(+), 45 deletions(-) diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml index dd6e009273..8bc65a663c 100644 --- a/.github/actions/artifact-download/action.yml +++ b/.github/actions/artifact-download/action.yml @@ -7,10 +7,7 @@ description: Stream a Hetzner deps tarball onto $GITHUB_WORKSPACE and untar. inputs: path: - description: Object stem. Downloads artifacts///.tar - required: true - webrtc_ref: - description: Stable deps identity (branch, tag, or SHA). + description: Object name. Downloads artifacts//.tar required: true if_missing: description: error fails when the object is absent; skip continues. @@ -51,7 +48,6 @@ runs: HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_secret_access_key }} HETZNER_BUCKET_CI_ARTIFACTS: ${{ inputs.hetzner_bucket }} OBJECT_STEM: ${{ inputs.path }} - WEBRTC_REF: ${{ inputs.webrtc_ref }} IF_MISSING: ${{ inputs.if_missing }} run: | set -euo pipefail @@ -61,8 +57,8 @@ runs: export AWS_EC2_METADATA_DISABLED=true endpoint="--endpoint-url https://hel1.your-objectstorage.com" BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" - [[ -n "${WEBRTC_REF}" ]] - object="artifacts/${{ github.repository }}/${OBJECT_STEM}/${WEBRTC_REF}.tar" + [[ -n "${OBJECT_STEM}" ]] + object="artifacts/${{ github.repository }}/${OBJECT_STEM}.tar" set +e head_out="$(aws ${endpoint} s3api head-object \ --bucket "${BUCKET}" --key "${object}" 2>&1)" diff --git a/.github/actions/artifact-upload/action.yml b/.github/actions/artifact-upload/action.yml index 38bf1ca527..eddbb4e3fb 100644 --- a/.github/actions/artifact-upload/action.yml +++ b/.github/actions/artifact-upload/action.yml @@ -7,10 +7,7 @@ description: Stream-tar gclient deps at $GITHUB_WORKSPACE to one Hetzner object. inputs: path: - description: Object stem. Uploads artifacts///.tar - required: true - webrtc_ref: - description: Stable deps identity (branch, tag, or SHA). + description: Object name. Uploads artifacts//.tar required: true hetzner_access_key: description: Hetzner object storage access key. @@ -47,7 +44,6 @@ runs: HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_secret_access_key }} HETZNER_BUCKET_CI_ARTIFACTS: ${{ inputs.hetzner_bucket }} OBJECT_STEM: ${{ inputs.path }} - WEBRTC_REF: ${{ inputs.webrtc_ref }} run: | set -euo pipefail export AWS_ACCESS_KEY_ID="${HETZNER_ACCESS_KEY_CI_ARTIFACTS:?}" @@ -82,9 +78,9 @@ runs: test -f .gclient test -d .gclient-git-cache test -d src/third_party - [[ -n "${WEBRTC_REF}" ]] + [[ -n "${OBJECT_STEM}" ]] BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" - object="artifacts/${{ github.repository }}/${OBJECT_STEM}/${WEBRTC_REF}.tar" + object="artifacts/${{ github.repository }}/${OBJECT_STEM}.tar" echo "packing ${members[*]} -> ${object}" # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. tar cf - "${members[@]}" | aws ${endpoint} s3 cp - "s3://${BUCKET}/${object}" diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index 8e30d4ec83..f23ae3e048 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -1,5 +1,5 @@ name: Restore WebRTC tree -description: Restore deps-git from Hetzner or deps-windows from a GitHub artifact. +description: Restore deps-key from Hetzner or deps-windows from a GitHub artifact. inputs: webrtc_ref: @@ -10,7 +10,7 @@ inputs: description: gclient TARGET_OS (selected tokens only). deps_artifact: required: true - description: deps-git (Hetzner) or deps-windows (GitHub artifact). + description: deps-key (Hetzner) or deps-windows (GitHub artifact). install_android_packages: required: false default: "false" @@ -18,15 +18,15 @@ inputs: hetzner_access_key: required: false default: "" - description: Hetzner access key for deps-git. + description: Hetzner access key for deps-key. hetzner_secret_access_key: required: false default: "" - description: Hetzner secret access key for deps-git. + description: Hetzner secret access key for deps-key. hetzner_bucket: required: false default: "" - description: Hetzner bucket for deps-git. + description: Hetzner bucket for deps-key. runs: using: composite @@ -41,17 +41,16 @@ runs: # gclient sync so host GCS (rust-toolchain) matches this runner. # runhooks does not download GCS. Missing object fails (Deps uploaded). - name: Download deps from Hetzner - if: ${{ inputs.deps_artifact == 'deps-git' }} + if: ${{ inputs.deps_artifact == 'deps-key' }} uses: ./src/.github/actions/artifact-download with: path: ${{ inputs.deps_artifact }} - webrtc_ref: ${{ inputs.webrtc_ref }} hetzner_access_key: ${{ inputs.hetzner_access_key }} hetzner_secret_access_key: ${{ inputs.hetzner_secret_access_key }} hetzner_bucket: ${{ inputs.hetzner_bucket }} - name: gclient sync and hooks - if: ${{ inputs.deps_artifact == 'deps-git' }} + if: ${{ inputs.deps_artifact == 'deps-key' }} working-directory: src/stream_build shell: bash env: @@ -67,14 +66,14 @@ runs: # Windows is still a git-cache-only GitHub artifact. - name: Download same-run deps artifact - if: ${{ inputs.deps_artifact != 'deps-git' }} + if: ${{ inputs.deps_artifact != 'deps-key' }} uses: actions/download-artifact@v8 with: name: ${{ inputs.deps_artifact }} path: ${{ runner.temp }}/deps-git-cache - name: Install same-run gclient object cache - if: ${{ inputs.deps_artifact != 'deps-git' }} + if: ${{ inputs.deps_artifact != 'deps-key' }} shell: bash run: | set -euo pipefail @@ -90,7 +89,7 @@ runs: test ! -L "${GITHUB_WORKSPACE}/src" - name: gclient sync - if: ${{ inputs.deps_artifact != 'deps-git' }} + if: ${{ inputs.deps_artifact != 'deps-key' }} working-directory: src/stream_build shell: bash env: diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 43fe076c06..f81f63196b 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -2,11 +2,11 @@ # Not listed in the Actions dispatch UI (workflow_call only). # Callers pass mode: build | test | package | release. # -# I/O: Linux Deps restores Hetzner deps-git (skip if missing), shallow +# I/O: Linux Deps restores Hetzner deps-key (skip if missing), shallow # gclient sync (RUN_HOOKS=0), overwrites the same object. Key: -# artifacts//deps-git/.tar -# (not a GitHub artifact, not run_id). Windows Deps stays deps-windows on -# GitHub. Build/Test restore that key after Deps (missing = fail). Build +# artifacts//deps-key.tar +# (not a GitHub artifact, not run_id, not webrtc_ref). Windows Deps stays +# deps-windows on GitHub. Build/Test restore that key after Deps (missing = fail). Build # mode is ninja only (no products). Package/Release Build jobs also make # package and upload products-* (GitHub). Package combine consumes # products-* (no third ninja) and uploads final-*. Release attaches final-*. @@ -154,11 +154,10 @@ jobs: uses: ./src/.github/actions/setup-webrtc with: webrtc_ref: ${{ inputs.webrtc_ref }} - - name: Download cached deps-git + - name: Download cached deps-key uses: ./src/.github/actions/artifact-download with: - path: deps-git - webrtc_ref: ${{ inputs.webrtc_ref }} + path: deps-key if_missing: skip hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} @@ -175,14 +174,13 @@ jobs: JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps - # Overwrite the stable deps-git object. Stream-tar (no zip on disk, + # Overwrite the stable deps-key object. Stream-tar (no zip on disk, # no GitHub artifact quota). RUN_HOOKS=0: no hermetic Xcode CIPD on # Linux; Build/Test restore this key then make deps (host GCS + hooks). - - name: Upload deps-git + - name: Upload deps-key uses: ./src/.github/actions/artifact-upload with: - path: deps-git - webrtc_ref: ${{ inputs.webrtc_ref }} + path: deps-key hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} @@ -249,7 +247,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-git + deps_artifact: deps-key hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} @@ -294,7 +292,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-git + deps_artifact: deps-key hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} @@ -339,7 +337,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.android_target_os }} - deps_artifact: deps-git + deps_artifact: deps-key install_android_packages: "true" hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} @@ -399,7 +397,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-git + deps_artifact: deps-key hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} @@ -425,7 +423,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-git + deps_artifact: deps-key hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index 186b1bfa82..b52be26558 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -80,7 +80,7 @@ the tree is `webrtc/src` (real directory, not a symlink). `deps` / CI checks out with `path: src` so `GITHUB_WORKSPACE` is the webrtc-named folder. `DEPS_ROOT=$GITHUB_WORKSPACE`. gclient objects live at `$DEPS_ROOT/.gclient-git-cache`. Linux Deps restores the Hetzner -`deps-git` tarball (skip if missing), `gclient sync --no-history +`deps-key` tarball (skip if missing), `gclient sync --no-history --shallow` (`RUN_HOOKS=0`), and overwrites the same object. Not a GitHub artifact or Actions cache. Windows Deps still uploads `deps-windows` to GitHub. Build/Test restore that same key via `artifact-download` (missing @@ -96,7 +96,7 @@ Hetzner deps handoff: `aws s3 cp` talks to Hetzner's S3-compatible API, not AWS. Pipe `tar cf - … | aws s3 cp - "s3://${BUCKET}/…"` and `aws s3 cp "s3://${BUCKET}/…" - | tar xf -` (no tarball on disk). Object: -`/artifacts//deps-git/.tar` +`/artifacts//deps-key.tar` with `--endpoint-url https://hel1.your-objectstorage.com` and region `hel1`. Packs `.gclient*`, `.cipd`, `.gclient-git-cache`, and gclient checkouts under `src/` (`third_party`, `build`, `buildtools`, `testing`, diff --git a/stream_build/scripts/check.sh b/stream_build/scripts/check.sh index 80c4fd6048..fe122c5345 100755 --- a/stream_build/scripts/check.sh +++ b/stream_build/scripts/check.sh @@ -384,10 +384,16 @@ rm -rf "$deps_tmp" "$missing" gha="$ROOT/../.github" ! grep -q 'github.run_id' "$gha/actions/artifact-upload/action.yml" ! grep -q 'github.run_id' "$gha/actions/artifact-download/action.yml" -grep -q '\${OBJECT_STEM}/\${WEBRTC_REF}.tar' \ +! grep -q 'WEBRTC_REF' "$gha/actions/artifact-upload/action.yml" +! grep -q 'WEBRTC_REF' "$gha/actions/artifact-download/action.yml" +! grep -q 'webrtc_ref:' "$gha/actions/artifact-upload/action.yml" +! grep -q 'webrtc_ref:' "$gha/actions/artifact-download/action.yml" +grep -q 'object="artifacts/\${{ github.repository }}/\${OBJECT_STEM}.tar"' \ "$gha/actions/artifact-upload/action.yml" -grep -q '\${OBJECT_STEM}/\${WEBRTC_REF}.tar' \ +grep -q 'object="artifacts/\${{ github.repository }}/\${OBJECT_STEM}.tar"' \ "$gha/actions/artifact-download/action.yml" +grep -q 'path: deps-key' "$gha/workflows/_make.yml" +grep -q "deps_artifact == 'deps-key'" "$gha/actions/restore-tree/action.yml" grep -q 'if_missing: skip' "$gha/workflows/_make.yml" grep -q 'SHALLOW: "1"' "$gha/workflows/_make.yml" From 2b304a75df1166885af493c22b77876f158c2d00 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Sat, 12 Sep 2026 01:35:15 +0300 Subject: [PATCH 14/24] Increase caching scope --- .github/actions/artifact-download/action.yml | 3 ++- .github/actions/artifact-upload/action.yml | 3 ++- .github/actions/restore-tree/action.yml | 7 +++--- .github/workflows/_make.yml | 13 ++++++++++- stream_build/AGENTS.md | 24 ++++++++++++-------- stream_build/scripts/check.sh | 5 ++++ 6 files changed, 39 insertions(+), 16 deletions(-) diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml index 8bc65a663c..2b4cde80a7 100644 --- a/.github/actions/artifact-download/action.yml +++ b/.github/actions/artifact-download/action.yml @@ -95,7 +95,8 @@ runs: src/buildtools \ src/testing \ src/tools \ - src/ios + src/ios \ + src/resources do if [[ -d "${p}" ]]; then find "${p}" -type f -exec touch {} + diff --git a/.github/actions/artifact-upload/action.yml b/.github/actions/artifact-upload/action.yml index eddbb4e3fb..5b904f3203 100644 --- a/.github/actions/artifact-upload/action.yml +++ b/.github/actions/artifact-upload/action.yml @@ -65,7 +65,8 @@ runs: src/buildtools \ src/testing \ src/tools \ - src/ios + src/ios \ + src/resources do if [[ -e "${p}" ]]; then members+=("${p}") diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index f23ae3e048..b66dea83ee 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -37,9 +37,10 @@ runs: webrtc_ref: ${{ inputs.webrtc_ref }} install_android_packages: ${{ inputs.install_android_packages }} - # Linux Deps packed the gclient working tree. Extract, then shallow - # gclient sync so host GCS (rust-toolchain) matches this runner. - # runhooks does not download GCS. Missing object fails (Deps uploaded). + # Linux Deps packed the gclient working tree (including src/resources). + # Extract, then shallow gclient sync so host GCS (rust-toolchain) + # matches this runner. Keep RUN_HOOKS=1; packed resources make the + # webrtc-resources GCS hook a sha1 no-op. Missing object fails. - name: Download deps from Hetzner if: ${{ inputs.deps_artifact == 'deps-key' }} uses: ./src/.github/actions/artifact-download diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index f81f63196b..77eca4a7e2 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -3,7 +3,8 @@ # Callers pass mode: build | test | package | release. # # I/O: Linux Deps restores Hetzner deps-key (skip if missing), shallow -# gclient sync (RUN_HOOKS=0), overwrites the same object. Key: +# gclient sync (RUN_HOOKS=0), fetches chromium-webrtc-resources into +# src/resources, overwrites the same object. Key: # artifacts//deps-key.tar # (not a GitHub artifact, not run_id, not webrtc_ref). Windows Deps stays # deps-windows on GitHub. Build/Test restore that key after Deps (missing = fail). Build @@ -174,6 +175,16 @@ jobs: JOBS: "8" WEBRTC_REPO: https://github.com/GetStream/webrtc.git run: make deps + # Same command as src/DEPS chromium-webrtc-resources. Do not set + # RUN_HOOKS=1 here: that runs mac_toolchain.py CIPD when mac is in + # TARGET_OS. Build restore still RUN_HOOKS=1 (rust-toolchain). + - name: Fetch chromium-webrtc-resources + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + download_from_google_storage --directory --recursive \ + --num_threads=10 --no_auth --quiet \ + --bucket chromium-webrtc-resources src/resources # Overwrite the stable deps-key object. Stream-tar (no zip on disk, # no GitHub artifact quota). RUN_HOOKS=0: no hermetic Xcode CIPD on # Linux; Build/Test restore this key then make deps (host GCS + hooks). diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index b52be26558..72e89c3a23 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -81,14 +81,18 @@ CI checks out with `path: src` so `GITHUB_WORKSPACE` is the webrtc-named folder. `DEPS_ROOT=$GITHUB_WORKSPACE`. gclient objects live at `$DEPS_ROOT/.gclient-git-cache`. Linux Deps restores the Hetzner `deps-key` tarball (skip if missing), `gclient sync --no-history ---shallow` (`RUN_HOOKS=0`), and overwrites the same object. Not a GitHub -artifact or Actions cache. Windows Deps still uploads `deps-windows` to -GitHub. Build/Test restore that same key via `artifact-download` (missing -fails), then `make deps` (host GCS + hooks) and `SKIP_DEPS=1` on -build/test. Package/Release Build jobs also `make package` and upload -`products-*` (GitHub). Package combine consumes `products-*` (no third -ninja) and uploads `final-*`. Release attaches `final-*`. Tests need -Deps only and run `make test` (no extra framework-slice build). +--shallow` (`RUN_HOOKS=0`), fetches `chromium-webrtc-resources` into +`src/resources`, and overwrites the same object. Not a GitHub +artifact or Actions cache. Do not set `RUN_HOOKS=1` on Linux Deps +(`mac` in `TARGET_OS` would run hermetic Xcode CIPD). Windows Deps +still uploads `deps-windows` to GitHub. Build/Test restore that same +key via `artifact-download` (missing fails), then `make deps` (host +GCS + hooks; packed `src/resources` makes the webrtc-resources hook a +sha1 no-op) and `SKIP_DEPS=1` on build/test. Package/Release Build +jobs also `make package` and upload `products-*` (GitHub). Package +combine consumes `products-*` (no third ninja) and uploads `final-*`. +Release attaches `final-*`. Tests need Deps only and run `make test` +(no extra framework-slice build). `TARGET_OS` is only the tokens selected this run. Hetzner deps handoff: @@ -100,8 +104,8 @@ Pipe `tar cf - … | aws s3 cp - "s3://${BUCKET}/…"` and with `--endpoint-url https://hel1.your-objectstorage.com` and region `hel1`. Packs `.gclient*`, `.cipd`, `.gclient-git-cache`, and gclient checkouts under `src/` (`third_party`, `build`, `buildtools`, `testing`, -`tools`, `ios`) — not the GetStream/webrtc `src` git worktree, `out/`, -or `products/`. Callers pass org secrets +`tools`, `ios`, `resources`) — not the GetStream/webrtc `src` git +worktree, `out/`, or `products/`. Callers pass org secrets `${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }}`, `${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }}`, and `${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }}`. After extract, files are diff --git a/stream_build/scripts/check.sh b/stream_build/scripts/check.sh index fe122c5345..b1dd99c6e9 100755 --- a/stream_build/scripts/check.sh +++ b/stream_build/scripts/check.sh @@ -396,5 +396,10 @@ grep -q 'path: deps-key' "$gha/workflows/_make.yml" grep -q "deps_artifact == 'deps-key'" "$gha/actions/restore-tree/action.yml" grep -q 'if_missing: skip' "$gha/workflows/_make.yml" grep -q 'SHALLOW: "1"' "$gha/workflows/_make.yml" +grep -q 'RUN_HOOKS: "0"' "$gha/workflows/_make.yml" +! grep -q 'RUN_HOOKS: "1"' "$gha/workflows/_make.yml" +grep -q 'chromium-webrtc-resources' "$gha/workflows/_make.yml" +grep -q 'src/resources' "$gha/actions/artifact-upload/action.yml" +grep -q 'src/resources' "$gha/actions/artifact-download/action.yml" echo "ok" From cba613d054291b5e6982b07a6b1e81ba55d5b211 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Mon, 14 Sep 2026 12:24:02 +0300 Subject: [PATCH 15/24] Restructure GH artifact flow --- .../action.yml | 51 ++++------ .github/actions/restore-tree/action.yml | 70 +++++++++----- .github/workflows/_make.yml | 93 +++++++++++++------ .github/workflows/manual-platform-tests.yml | 4 +- .github/workflows/publish.yml | 3 +- .github/workflows/release.yml | 3 +- .github/workflows/test.yml | 3 +- stream_build/AGENTS.md | 52 ++++++----- stream_build/scripts/check.sh | 32 ++++++- 9 files changed, 192 insertions(+), 119 deletions(-) rename .github/actions/{artifact-upload => artifact-put}/action.yml (59%) diff --git a/.github/actions/artifact-upload/action.yml b/.github/actions/artifact-put/action.yml similarity index 59% rename from .github/actions/artifact-upload/action.yml rename to .github/actions/artifact-put/action.yml index 5b904f3203..2891883705 100644 --- a/.github/actions/artifact-upload/action.yml +++ b/.github/actions/artifact-put/action.yml @@ -2,12 +2,15 @@ # hetzner_access_key # hetzner_secret_access_key # hetzner_bucket -name: Upload WebRTC deps to Hetzner -description: Stream-tar gclient deps at $GITHUB_WORKSPACE to one Hetzner object. +name: Upload WebRTC deps tarball to Hetzner +description: Stream a tar of unpacked deps members to Hetzner. Does not zip. inputs: path: - description: Object name. Uploads artifacts//.tar + description: Object stem. Uploads artifacts//.tar + required: true + source: + description: Directory of unpacked deps members (archive root). required: true hetzner_access_key: description: Hetzner object storage access key. @@ -36,14 +39,14 @@ runs: sudo apt-get install -y awscli fi - - name: Upload deps tarball + - name: Stream deps tarball to Hetzner shell: bash - working-directory: ${{ github.workspace }} env: HETZNER_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_access_key }} HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_secret_access_key }} HETZNER_BUCKET_CI_ARTIFACTS: ${{ inputs.hetzner_bucket }} OBJECT_STEM: ${{ inputs.path }} + SOURCE: ${{ inputs.source }} run: | set -euo pipefail export AWS_ACCESS_KEY_ID="${HETZNER_ACCESS_KEY_CI_ARTIFACTS:?}" @@ -51,37 +54,15 @@ runs: export AWS_DEFAULT_REGION=hel1 export AWS_EC2_METADATA_DISABLED=true endpoint="--endpoint-url https://hel1.your-objectstorage.com" - members=() - for p in \ - .gclient \ - .gclient_entries \ - .gclient_previous_sync_commits \ - .gcs_entries \ - .cipd \ - .gclient-git-cache \ - src/.landmines \ - src/third_party \ - src/build \ - src/buildtools \ - src/testing \ - src/tools \ - src/ios \ - src/resources - do - if [[ -e "${p}" ]]; then - members+=("${p}") - fi - done - if [[ ${#members[@]} -eq 0 ]]; then - echo "::error::no webrtc deps members to pack under ${GITHUB_WORKSPACE}" - exit 1 - fi - test -f .gclient - test -d .gclient-git-cache - test -d src/third_party [[ -n "${OBJECT_STEM}" ]] + [[ -n "${SOURCE}" ]] + test -d "${SOURCE}" + test -f "${SOURCE}/.gclient" + test -d "${SOURCE}/.gclient-git-cache" + test -d "${SOURCE}/src/third_party" BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" object="artifacts/${{ github.repository }}/${OBJECT_STEM}.tar" - echo "packing ${members[*]} -> ${object}" + echo "streaming ${SOURCE} -> ${object}" # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. - tar cf - "${members[@]}" | aws ${endpoint} s3 cp - "s3://${BUCKET}/${object}" + tar cf - -C "${SOURCE}" . | aws ${endpoint} s3 cp - \ + "s3://${BUCKET}/${object}" diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index b66dea83ee..c7c1df08be 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -1,5 +1,5 @@ name: Restore WebRTC tree -description: Restore deps-key from Hetzner or deps-windows from a GitHub artifact. +description: Restore deps-key or deps-windows from a GitHub artifact. inputs: webrtc_ref: @@ -10,23 +10,11 @@ inputs: description: gclient TARGET_OS (selected tokens only). deps_artifact: required: true - description: deps-key (Hetzner) or deps-windows (GitHub artifact). + description: deps-key or deps-windows (GitHub artifact). install_android_packages: required: false default: "false" description: Install apt packages needed for Android builds. - hetzner_access_key: - required: false - default: "" - description: Hetzner access key for deps-key. - hetzner_secret_access_key: - required: false - default: "" - description: Hetzner secret access key for deps-key. - hetzner_bucket: - required: false - default: "" - description: Hetzner bucket for deps-key. runs: using: composite @@ -37,18 +25,52 @@ runs: webrtc_ref: ${{ inputs.webrtc_ref }} install_android_packages: ${{ inputs.install_android_packages }} - # Linux Deps packed the gclient working tree (including src/resources). - # Extract, then shallow gclient sync so host GCS (rust-toolchain) - # matches this runner. Keep RUN_HOOKS=1; packed resources make the - # webrtc-resources GCS hook a sha1 no-op. Missing object fails. - - name: Download deps from Hetzner + # Linux Deps uploaded the gclient working tree (including + # src/resources) as one GitHub artifact. download-artifact unzips + # that zip into $GITHUB_WORKSPACE; that unzip is the extract. Then + # shallow gclient sync so host GCS (rust-toolchain) matches this + # runner. Keep RUN_HOOKS=1; packed resources make the + # webrtc-resources GCS hook a sha1 no-op. + - name: Download deps-key if: ${{ inputs.deps_artifact == 'deps-key' }} - uses: ./src/.github/actions/artifact-download + uses: actions/download-artifact@v8 with: - path: ${{ inputs.deps_artifact }} - hetzner_access_key: ${{ inputs.hetzner_access_key }} - hetzner_secret_access_key: ${{ inputs.hetzner_secret_access_key }} - hetzner_bucket: ${{ inputs.hetzner_bucket }} + name: ${{ inputs.deps_artifact }} + path: ${{ github.workspace }} + + - name: Verify deps-key + if: ${{ inputs.deps_artifact == 'deps-key' }} + working-directory: ${{ github.workspace }} + shell: bash + run: | + set -euo pipefail + test -f .gclient + test -d .gclient-git-cache + test -d src/third_party + test -f src/DEPS + test ! -L src + for p in \ + .gclient \ + .gclient_entries \ + .gclient_previous_sync_commits \ + .gcs_entries \ + .cipd \ + .gclient-git-cache \ + src/.landmines \ + src/third_party \ + src/build \ + src/buildtools \ + src/testing \ + src/tools \ + src/ios \ + src/resources + do + if [[ -d "${p}" ]]; then + find "${p}" -type f -exec touch {} + + elif [[ -f "${p}" ]]; then + touch "${p}" + fi + done - name: gclient sync and hooks if: ${{ inputs.deps_artifact == 'deps-key' }} diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 77eca4a7e2..8c21b3d331 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -2,16 +2,19 @@ # Not listed in the Actions dispatch UI (workflow_call only). # Callers pass mode: build | test | package | release. # -# I/O: Linux Deps restores Hetzner deps-key (skip if missing), shallow +# I/O: Linux Deps restores Hetzner deps-key.tar (skip if missing), shallow # gclient sync (RUN_HOOKS=0), fetches chromium-webrtc-resources into -# src/resources, overwrites the same object. Key: -# artifacts//deps-key.tar -# (not a GitHub artifact, not run_id, not webrtc_ref). Windows Deps stays -# deps-windows on GitHub. Build/Test restore that key after Deps (missing = fail). Build -# mode is ninja only (no products). Package/Release Build jobs also make -# package and upload products-* (GitHub). Package combine consumes -# products-* (no third ninja) and uploads final-*. Release attaches final-*. -# Tests need Deps only (make test, no make build). +# src/resources, uploads GitHub artifact deps-key of the deps members +# (compression-level: 0; GitHub zips once). No deps-key.tar on GitHub. +# Does not upload to Hetzner. Build/Test download-artifact deps-key +# (unzip is the extract), make deps (host GCS). Hetzner backfill +# (needs: deps only) download-artifact then tar cf - | aws s3 cp - to +# artifacts//deps-key.tar in parallel with Build. +# Windows Deps stays deps-windows on GitHub. Build mode is ninja only +# (no products). Package/Release Build jobs also make package and upload +# products-* (GitHub). Package combine consumes products-* (no third +# ninja) and uploads final-*. Release attaches final-*. Tests need Deps +# only (make test, no make build). name: WebRTC make @@ -185,13 +188,62 @@ jobs: download_from_google_storage --directory --recursive \ --num_threads=10 --no_auth --quiet \ --bucket chromium-webrtc-resources src/resources - # Overwrite the stable deps-key object. Stream-tar (no zip on disk, - # no GitHub artifact quota). RUN_HOOKS=0: no hermetic Xcode CIPD on - # Linux; Build/Test restore this key then make deps (host GCS + hooks). + # GitHub artifact is the deps members (one zip). No deps-key.tar. + # RUN_HOOKS=0: no hermetic Xcode CIPD on Linux; Build/Test restore + # this key then make deps (host GCS). Hetzner backfill tars the + # same tree in parallel; this job does not. + - name: Check deps-key members + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + test -f .gclient + test -d .gclient-git-cache + test -d src/third_party - name: Upload deps-key - uses: ./src/.github/actions/artifact-upload + uses: actions/upload-artifact@v7 + with: + name: deps-key + path: | + .gclient + .gclient_entries + .gclient_previous_sync_commits + .gcs_entries + .cipd + .gclient-git-cache + src/.landmines + src/third_party + src/build + src/buildtools + src/testing + src/tools + src/ios + src/resources + include-hidden-files: true + if-no-files-found: error + retention-days: 1 + compression-level: 0 + + hetzner_backfill: + name: Hetzner backfill + needs: [deps] + if: ${{ inputs.ios || inputs.macos || inputs.android }} + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v7 + with: + path: src + ref: ${{ inputs.webrtc_ref }} + - name: Download deps-key + uses: actions/download-artifact@v8 + with: + name: deps-key + path: ${{ github.workspace }}/deps-tree + - name: Upload deps-key to Hetzner + uses: ./src/.github/actions/artifact-put with: path: deps-key + source: ${{ github.workspace }}/deps-tree hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} @@ -259,9 +311,6 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-key - hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Build iOS working-directory: src/stream_build env: @@ -304,9 +353,6 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-key - hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Build macOS working-directory: src/stream_build env: @@ -350,9 +396,6 @@ jobs: target_os: ${{ needs.plan.outputs.android_target_os }} deps_artifact: deps-key install_android_packages: "true" - hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Build Android working-directory: src/stream_build env: @@ -409,9 +452,6 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-key - hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Test iOS working-directory: src/stream_build env: @@ -435,9 +475,6 @@ jobs: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.apple_target_os }} deps_artifact: deps-key - hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Test macOS working-directory: src/stream_build env: diff --git a/.github/workflows/manual-platform-tests.yml b/.github/workflows/manual-platform-tests.yml index 2827d17b97..a661e99643 100644 --- a/.github/workflows/manual-platform-tests.yml +++ b/.github/workflows/manual-platform-tests.yml @@ -1,6 +1,8 @@ # Manual WebRTC build. Dispatchable from a PR because this path exists on the # default branch. Implementation: .github/workflows/_make.yml -# Deps → Build (restore-tree + make build). No package combine, no products. +# Deps → Build (GitHub deps-key + restore-tree + make build). Hetzner +# backfill runs parallel with Build (needs: deps only). No package +# combine, no products. name: Build diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3bfb353c49..a999b8793b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,7 @@ # Package WebRTC artifacts. Same path as today's Publish so a PR can dispatch it. # Does not create a GitHub release. -# _make.yml runs Build then Package combine (products-* in, final-* out). +# _make.yml: Deps → Build → Package combine (products-* in, final-* out). +# Hetzner backfill runs parallel with Build (needs: deps only). # Package does not rebuild from git-cache. name: Package diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de9040094c..11047959a1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,7 @@ # GitHub release of packaged WebRTC artifacts, then downstream wrapper publishes. # Not dispatchable until this file exists on the default branch. -# Tests run parallel with Build (needs Deps only). Publish waits on Test + Package. +# Tests run parallel with Build (needs Deps only). Hetzner backfill +# also needs Deps only. Publish waits on Test + Package. name: Release diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 11582233cd..46a64e4c89 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,6 @@ # WebRTC tests. Not dispatchable until this file exists on the default branch. -# _make.yml: Deps then make test only (no extra framework-slice make build). +# _make.yml: Deps then make test only (no extra framework-slice make +# build). Hetzner backfill runs parallel with Test (needs: deps only). name: Test diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index 72e89c3a23..7dc585340a 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -80,32 +80,38 @@ the tree is `webrtc/src` (real directory, not a symlink). `deps` / CI checks out with `path: src` so `GITHUB_WORKSPACE` is the webrtc-named folder. `DEPS_ROOT=$GITHUB_WORKSPACE`. gclient objects live at `$DEPS_ROOT/.gclient-git-cache`. Linux Deps restores the Hetzner -`deps-key` tarball (skip if missing), `gclient sync --no-history +`deps-key.tar` (skip if missing), `gclient sync --no-history --shallow` (`RUN_HOOKS=0`), fetches `chromium-webrtc-resources` into -`src/resources`, and overwrites the same object. Not a GitHub -artifact or Actions cache. Do not set `RUN_HOOKS=1` on Linux Deps -(`mac` in `TARGET_OS` would run hermetic Xcode CIPD). Windows Deps -still uploads `deps-windows` to GitHub. Build/Test restore that same -key via `artifact-download` (missing fails), then `make deps` (host -GCS + hooks; packed `src/resources` makes the webrtc-resources hook a -sha1 no-op) and `SKIP_DEPS=1` on build/test. Package/Release Build -jobs also `make package` and upload `products-*` (GitHub). Package -combine consumes `products-*` (no third ninja) and uploads `final-*`. -Release attaches `final-*`. Tests need Deps only and run `make test` -(no extra framework-slice build). +`src/resources`, and uploads GitHub artifact `deps-key` of the deps +members (`compression-level: 0`; GitHub zips once). No `deps-key.tar` +on GitHub. Do not set `RUN_HOOKS=1` on Linux Deps (`mac` in +`TARGET_OS` would run hermetic Xcode CIPD). Do not upload to Hetzner +from Deps. Build/Test `needs: deps` only, `download-artifact` +`deps-key` (unzip is the extract), then `make deps` (host GCS + +hooks; packed `src/resources` makes the webrtc-resources hook a sha1 +no-op) and `SKIP_DEPS=1` on build/test. `Hetzner backfill` (`needs: +deps` only) `download-artifact` then `tar cf - | aws s3 cp -` in +parallel with Build. Windows Deps still uploads `deps-windows` to +GitHub. Package/Release Build jobs also `make package` and upload +`products-*` (GitHub). Package combine consumes `products-*` (no +third ninja) and uploads `final-*`. Release attaches `final-*`. Tests +need Deps only and run `make test` (no extra framework-slice build). `TARGET_OS` is only the tokens selected this run. -Hetzner deps handoff: -`.github/actions/artifact-upload` and `artifact-download`. -`aws s3 cp` talks to Hetzner's S3-compatible API, not AWS. -Pipe `tar cf - … | aws s3 cp - "s3://${BUCKET}/…"` and -`aws s3 cp "s3://${BUCKET}/…" - | tar xf -` (no tarball on disk). Object: -`/artifacts//deps-key.tar` -with `--endpoint-url https://hel1.your-objectstorage.com` and region -`hel1`. Packs `.gclient*`, `.cipd`, `.gclient-git-cache`, and gclient -checkouts under `src/` (`third_party`, `build`, `buildtools`, `testing`, -`tools`, `ios`, `resources`) — not the GetStream/webrtc `src` git -worktree, `out/`, or `products/`. Callers pass org secrets +Deps packing / Hetzner: +Member list: `.gclient*`, `.cipd`, `.gclient-git-cache`, and gclient +checkouts under `src/` (`third_party`, `build`, `buildtools`, +`testing`, `tools`, `ios`, `resources`). Not the GetStream/webrtc +`src` git worktree, `out/`, or `products/`. Same-run handoff is +`actions/upload-artifact` name `deps-key` of those members +(`include-hidden-files: true`, `compression-level: 0`). GitHub zips +once. No `deps-key.tar` for GH. `artifact-download` is Hetzner-only +(Deps warm cache, `if_missing: skip`). `artifact-put` is Hetzner-only +(`tar cf - -C . | aws s3 cp -`) to +`/artifacts//deps-key.tar` with +`--endpoint-url https://hel1.your-objectstorage.com` and region +`hel1`. `aws s3 cp` talks to Hetzner's S3-compatible API, not AWS. +Callers pass org secrets via `with:` `${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }}`, `${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }}`, and `${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }}`. After extract, files are diff --git a/stream_build/scripts/check.sh b/stream_build/scripts/check.sh index b1dd99c6e9..868baf2804 100755 --- a/stream_build/scripts/check.sh +++ b/stream_build/scripts/check.sh @@ -382,24 +382,46 @@ grep -q 'src is a symlink' "$deps_tmp/link_err" rm -rf "$deps_tmp" "$missing" gha="$ROOT/../.github" -! grep -q 'github.run_id' "$gha/actions/artifact-upload/action.yml" +[[ ! -e "$gha/actions/artifact-upload/action.yml" ]] ! grep -q 'github.run_id' "$gha/actions/artifact-download/action.yml" -! grep -q 'WEBRTC_REF' "$gha/actions/artifact-upload/action.yml" +! grep -q 'github.run_id' "$gha/actions/artifact-put/action.yml" ! grep -q 'WEBRTC_REF' "$gha/actions/artifact-download/action.yml" -! grep -q 'webrtc_ref:' "$gha/actions/artifact-upload/action.yml" +! grep -q 'WEBRTC_REF' "$gha/actions/artifact-put/action.yml" ! grep -q 'webrtc_ref:' "$gha/actions/artifact-download/action.yml" +! grep -q 'webrtc_ref:' "$gha/actions/artifact-put/action.yml" +! grep -q 'secrets\.' "$gha/actions/artifact-download/action.yml" +! grep -q 'secrets\.' "$gha/actions/artifact-put/action.yml" +! grep -q 'secrets\.' "$gha/actions/restore-tree/action.yml" +grep -q 'tar cf -' "$gha/actions/artifact-put/action.yml" +grep -q 's3 cp -' "$gha/actions/artifact-put/action.yml" +! grep -q 's3 cp "${OBJECT_STEM}.tar"' "$gha/actions/artifact-put/action.yml" grep -q 'object="artifacts/\${{ github.repository }}/\${OBJECT_STEM}.tar"' \ - "$gha/actions/artifact-upload/action.yml" + "$gha/actions/artifact-put/action.yml" grep -q 'object="artifacts/\${{ github.repository }}/\${OBJECT_STEM}.tar"' \ "$gha/actions/artifact-download/action.yml" grep -q 'path: deps-key' "$gha/workflows/_make.yml" +! grep -q 'path: deps-key.tar' "$gha/workflows/_make.yml" +grep -q 'name: deps-key' "$gha/workflows/_make.yml" +grep -q 'include-hidden-files: true' "$gha/workflows/_make.yml" +grep -q 'compression-level: 0' "$gha/workflows/_make.yml" +grep -q 'name: Hetzner backfill' "$gha/workflows/_make.yml" +grep -A5 'name: Hetzner backfill' "$gha/workflows/_make.yml" | grep -q 'needs: \[deps\]' +grep -A5 'name: Build iOS' "$gha/workflows/_make.yml" | grep -q 'needs: \[plan, deps\]' +! grep -A8 'name: Build iOS' "$gha/workflows/_make.yml" | grep -q hetzner_backfill +grep -A5 'name: Test iOS' "$gha/workflows/_make.yml" | grep -q 'needs: \[plan, deps\]' +! grep -A8 'name: Test iOS' "$gha/workflows/_make.yml" | grep -q hetzner_backfill grep -q "deps_artifact == 'deps-key'" "$gha/actions/restore-tree/action.yml" +! grep -q 'artifact-download' "$gha/actions/restore-tree/action.yml" +! grep -q 'hetzner_access_key' "$gha/actions/restore-tree/action.yml" +grep -q 'actions/download-artifact' "$gha/actions/restore-tree/action.yml" +! grep -q 'tar xf' "$gha/actions/restore-tree/action.yml" grep -q 'if_missing: skip' "$gha/workflows/_make.yml" grep -q 'SHALLOW: "1"' "$gha/workflows/_make.yml" grep -q 'RUN_HOOKS: "0"' "$gha/workflows/_make.yml" ! grep -q 'RUN_HOOKS: "1"' "$gha/workflows/_make.yml" grep -q 'chromium-webrtc-resources' "$gha/workflows/_make.yml" -grep -q 'src/resources' "$gha/actions/artifact-upload/action.yml" +grep -q 'src/resources' "$gha/workflows/_make.yml" grep -q 'src/resources' "$gha/actions/artifact-download/action.yml" +grep -q 'uses: ./src/.github/actions/artifact-put' "$gha/workflows/_make.yml" echo "ok" From 28382894c61a872eaf3bc2970aa7152be1ce6f00 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Mon, 14 Sep 2026 13:46:59 +0300 Subject: [PATCH 16/24] Fix build failuires --- .github/actions/artifact-download/action.yml | 29 ++------- .github/actions/artifact-put/action.yml | 18 +++--- .github/actions/restore-tree/action.yml | 65 +++++++++----------- .github/workflows/_make.yml | 40 +++++------- stream_build/AGENTS.md | 51 ++++++++------- stream_build/scripts/check.sh | 10 ++- 6 files changed, 99 insertions(+), 114 deletions(-) diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml index 2b4cde80a7..f1757a879f 100644 --- a/.github/actions/artifact-download/action.yml +++ b/.github/actions/artifact-download/action.yml @@ -77,30 +77,11 @@ runs: echo "extracting ${object} -> ${GITHUB_WORKSPACE}" # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. aws ${endpoint} s3 cp "s3://${BUCKET}/${object}" - | tar xf - - test -f .gclient test -d .gclient-git-cache - test -d src/third_party + test -n "$(ls -A .gclient-git-cache)" test -f src/DEPS test ! -L src - for p in \ - .gclient \ - .gclient_entries \ - .gclient_previous_sync_commits \ - .gcs_entries \ - .cipd \ - .gclient-git-cache \ - src/.landmines \ - src/third_party \ - src/build \ - src/buildtools \ - src/testing \ - src/tools \ - src/ios \ - src/resources - do - if [[ -d "${p}" ]]; then - find "${p}" -type f -exec touch {} + - elif [[ -f "${p}" ]]; then - touch "${p}" - fi - done + find .gclient-git-cache -type f -exec touch {} + + if [[ -d src/resources ]]; then + find src/resources -type f -exec touch {} + + fi diff --git a/.github/actions/artifact-put/action.yml b/.github/actions/artifact-put/action.yml index 2891883705..15b672aafb 100644 --- a/.github/actions/artifact-put/action.yml +++ b/.github/actions/artifact-put/action.yml @@ -3,14 +3,14 @@ # hetzner_secret_access_key # hetzner_bucket name: Upload WebRTC deps tarball to Hetzner -description: Stream a tar of unpacked deps members to Hetzner. Does not zip. +description: Stream a tar of .gclient-git-cache and src/resources to Hetzner. Does not zip. inputs: path: description: Object stem. Uploads artifacts//.tar required: true source: - description: Directory of unpacked deps members (archive root). + description: Directory with .gclient-git-cache and src/resources. required: true hetzner_access_key: description: Hetzner object storage access key. @@ -57,12 +57,16 @@ runs: [[ -n "${OBJECT_STEM}" ]] [[ -n "${SOURCE}" ]] test -d "${SOURCE}" - test -f "${SOURCE}/.gclient" test -d "${SOURCE}/.gclient-git-cache" - test -d "${SOURCE}/src/third_party" + test -n "$(ls -A "${SOURCE}/.gclient-git-cache")" + if [[ ! -d "${SOURCE}/src/resources" ]] || \ + [[ -z "$(ls -A "${SOURCE}/src/resources")" ]]; then + echo "::error::deps-key missing src/resources (chromium-webrtc-resources)" + exit 1 + fi BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" object="artifacts/${{ github.repository }}/${OBJECT_STEM}.tar" - echo "streaming ${SOURCE} -> ${object}" + echo "streaming ${SOURCE}/{.gclient-git-cache,src/resources} -> ${object}" # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. - tar cf - -C "${SOURCE}" . | aws ${endpoint} s3 cp - \ - "s3://${BUCKET}/${object}" + tar cf - -C "${SOURCE}" .gclient-git-cache src/resources | \ + aws ${endpoint} s3 cp - "s3://${BUCKET}/${object}" diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index c7c1df08be..71c1d2db2b 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -25,52 +25,45 @@ runs: webrtc_ref: ${{ inputs.webrtc_ref }} install_android_packages: ${{ inputs.install_android_packages }} - # Linux Deps uploaded the gclient working tree (including - # src/resources) as one GitHub artifact. download-artifact unzips - # that zip into $GITHUB_WORKSPACE; that unzip is the extract. Then - # shallow gclient sync so host GCS (rust-toolchain) matches this - # runner. Keep RUN_HOOKS=1; packed resources make the - # webrtc-resources GCS hook a sha1 no-op. + # Linux Deps uploaded .gclient-git-cache + src/resources. Unzip to + # a temp dir (do not overlay onto src/), install both, then shallow + # gclient sync from that cache. Packed resources make the + # webrtc-resources hook a sha1 no-op. RUN_HOOKS=1 still fetches + # host GCS (rust-toolchain on Apple). rewrite_git_cache_alternates + # runs inside make deps. Do not transplant a Linux working tree. - name: Download deps-key if: ${{ inputs.deps_artifact == 'deps-key' }} uses: actions/download-artifact@v8 with: name: ${{ inputs.deps_artifact }} - path: ${{ github.workspace }} + path: ${{ runner.temp }}/deps-key - - name: Verify deps-key + - name: Install deps-key cache and resources if: ${{ inputs.deps_artifact == 'deps-key' }} - working-directory: ${{ github.workspace }} shell: bash run: | set -euo pipefail - test -f .gclient - test -d .gclient-git-cache - test -d src/third_party - test -f src/DEPS - test ! -L src - for p in \ - .gclient \ - .gclient_entries \ - .gclient_previous_sync_commits \ - .gcs_entries \ - .cipd \ - .gclient-git-cache \ - src/.landmines \ - src/third_party \ - src/build \ - src/buildtools \ - src/testing \ - src/tools \ - src/ios \ - src/resources - do - if [[ -d "${p}" ]]; then - find "${p}" -type f -exec touch {} + - elif [[ -f "${p}" ]]; then - touch "${p}" - fi - done + src="${RUNNER_TEMP}/deps-key" + cache="${src}/.gclient-git-cache" + resources="${src}/src/resources" + if [[ ! -d "${cache}" ]]; then + echo "::error::deps-key missing .gclient-git-cache" + exit 1 + fi + if [[ ! -d "${resources}" ]]; then + echo "::error::deps-key missing src/resources (chromium-webrtc-resources)" + exit 1 + fi + dest_cache="${GITHUB_WORKSPACE}/.gclient-git-cache" + dest_res="${GITHUB_WORKSPACE}/src/resources" + rm -rf "${dest_cache}" "${dest_res}" + mv "${cache}" "${dest_cache}" + mkdir -p "${GITHUB_WORKSPACE}/src" + mv "${resources}" "${dest_res}" + test -n "$(ls -A "${dest_cache}")" + test -n "$(ls -A "${dest_res}")" + test -f "${GITHUB_WORKSPACE}/src/DEPS" + test ! -L "${GITHUB_WORKSPACE}/src" - name: gclient sync and hooks if: ${{ inputs.deps_artifact == 'deps-key' }} diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 8c21b3d331..902654ae26 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -4,11 +4,14 @@ # # I/O: Linux Deps restores Hetzner deps-key.tar (skip if missing), shallow # gclient sync (RUN_HOOKS=0), fetches chromium-webrtc-resources into -# src/resources, uploads GitHub artifact deps-key of the deps members -# (compression-level: 0; GitHub zips once). No deps-key.tar on GitHub. -# Does not upload to Hetzner. Build/Test download-artifact deps-key -# (unzip is the extract), make deps (host GCS). Hetzner backfill -# (needs: deps only) download-artifact then tar cf - | aws s3 cp - to +# src/resources, uploads GitHub artifact deps-key of .gclient-git-cache +# and src/resources (compression-level: 0; GitHub zips once). No +# gclient checkout trees (third_party/build/buildtools/…). No +# deps-key.tar on GitHub. Does not upload to Hetzner. Build/Test +# download-artifact deps-key, install git-cache + src/resources, make +# deps (gclient from cache; host GCS). Hetzner backfill (needs: deps +# only) download-artifact then +# tar cf - .gclient-git-cache src/resources | aws s3 cp - to # artifacts//deps-key.tar in parallel with Build. # Windows Deps stays deps-windows on GitHub. Build mode is ninja only # (no products). Package/Release Build jobs also make package and upload @@ -188,35 +191,26 @@ jobs: download_from_google_storage --directory --recursive \ --num_threads=10 --no_auth --quiet \ --bucket chromium-webrtc-resources src/resources - # GitHub artifact is the deps members (one zip). No deps-key.tar. - # RUN_HOOKS=0: no hermetic Xcode CIPD on Linux; Build/Test restore - # this key then make deps (host GCS). Hetzner backfill tars the - # same tree in parallel; this job does not. + # GitHub artifact is .gclient-git-cache + src/resources (one zip). + # Do not pack the gclient working tree: zip cannot store git + # hardlinks, so cache+checkout doubles objects (~33GB) and + # transplants Linux checkouts onto Mac. RUN_HOOKS=0: no hermetic + # Xcode CIPD on Linux; Build/Test restore this key then make deps + # (host GCS). Hetzner backfill tars the same members in parallel. - name: Check deps-key members working-directory: ${{ github.workspace }} run: | set -euo pipefail - test -f .gclient test -d .gclient-git-cache - test -d src/third_party + test -n "$(ls -A .gclient-git-cache)" + test -d src/resources + test -n "$(ls -A src/resources)" - name: Upload deps-key uses: actions/upload-artifact@v7 with: name: deps-key path: | - .gclient - .gclient_entries - .gclient_previous_sync_commits - .gcs_entries - .cipd .gclient-git-cache - src/.landmines - src/third_party - src/build - src/buildtools - src/testing - src/tools - src/ios src/resources include-hidden-files: true if-no-files-found: error diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index 7dc585340a..c9817e5ffc 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -82,41 +82,48 @@ folder. `DEPS_ROOT=$GITHUB_WORKSPACE`. gclient objects live at `$DEPS_ROOT/.gclient-git-cache`. Linux Deps restores the Hetzner `deps-key.tar` (skip if missing), `gclient sync --no-history --shallow` (`RUN_HOOKS=0`), fetches `chromium-webrtc-resources` into -`src/resources`, and uploads GitHub artifact `deps-key` of the deps -members (`compression-level: 0`; GitHub zips once). No `deps-key.tar` +`src/resources`, and uploads GitHub artifact `deps-key` of +`.gclient-git-cache` and `src/resources` (`compression-level: 0`; +GitHub zips once). Do not pack gclient working trees +(`src/third_party`, `src/build`, `src/buildtools`, …) next to the +cache: zip cannot store git hardlinks, so that dual pack doubles +objects and transplants Linux checkouts onto Mac. No `deps-key.tar` on GitHub. Do not set `RUN_HOOKS=1` on Linux Deps (`mac` in `TARGET_OS` would run hermetic Xcode CIPD). Do not upload to Hetzner from Deps. Build/Test `needs: deps` only, `download-artifact` -`deps-key` (unzip is the extract), then `make deps` (host GCS + -hooks; packed `src/resources` makes the webrtc-resources hook a sha1 -no-op) and `SKIP_DEPS=1` on build/test. `Hetzner backfill` (`needs: -deps` only) `download-artifact` then `tar cf - | aws s3 cp -` in -parallel with Build. Windows Deps still uploads `deps-windows` to -GitHub. Package/Release Build jobs also `make package` and upload -`products-*` (GitHub). Package combine consumes `products-*` (no -third ninja) and uploads `final-*`. Release attaches `final-*`. Tests -need Deps only and run `make test` (no extra framework-slice build). -`TARGET_OS` is only the tokens selected this run. +`deps-key`, install the git-cache and `src/resources`, then +`make deps` (gclient from cache; packed resources make the +webrtc-resources hook a sha1 no-op; host GCS still fetches Apple +`rust-toolchain`; `rewrite_git_cache_alternates` retargets cache +paths) and `SKIP_DEPS=1` on build/test. `Hetzner backfill` (`needs: +deps` only) `download-artifact` then +`tar cf - .gclient-git-cache src/resources | aws s3 cp -` in +parallel with Build. +Windows Deps still uploads `deps-windows` to GitHub. Package/Release +Build jobs also `make package` and upload `products-*` (GitHub). +Package combine consumes `products-*` (no third ninja) and uploads +`final-*`. Release attaches `final-*`. Tests need Deps only and run +`make test` (no extra framework-slice build). `TARGET_OS` is only the +tokens selected this run. Deps packing / Hetzner: -Member list: `.gclient*`, `.cipd`, `.gclient-git-cache`, and gclient -checkouts under `src/` (`third_party`, `build`, `buildtools`, -`testing`, `tools`, `ios`, `resources`). Not the GetStream/webrtc -`src` git worktree, `out/`, or `products/`. Same-run handoff is -`actions/upload-artifact` name `deps-key` of those members +Member list: `.gclient-git-cache` and `src/resources` only. Not +gclient checkouts (`third_party`, `build`, `buildtools`, …), not the +GetStream/webrtc `src` git worktree, `out/`, or `products/`. Same-run +handoff is `actions/upload-artifact` name `deps-key` of those paths (`include-hidden-files: true`, `compression-level: 0`). GitHub zips once. No `deps-key.tar` for GH. `artifact-download` is Hetzner-only (Deps warm cache, `if_missing: skip`). `artifact-put` is Hetzner-only -(`tar cf - -C . | aws s3 cp -`) to -`/artifacts//deps-key.tar` with +(`tar cf - -C .gclient-git-cache src/resources | aws s3 cp -`) +to `/artifacts//deps-key.tar` with `--endpoint-url https://hel1.your-objectstorage.com` and region `hel1`. `aws s3 cp` talks to Hetzner's S3-compatible API, not AWS. Callers pass org secrets via `with:` `${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }}`, `${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }}`, and -`${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }}`. After extract, files are -`touch`ed so make/ninja do not rebuild from mtime. `products-*` / -`final-*` stay on `actions/upload-artifact`. +`${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }}`. After extract, packed +files are `touch`ed so make/ninja do not rebuild from mtime. +`products-*` / `final-*` stay on `actions/upload-artifact`. ## Host gates diff --git a/stream_build/scripts/check.sh b/stream_build/scripts/check.sh index 868baf2804..c1712d4cc3 100755 --- a/stream_build/scripts/check.sh +++ b/stream_build/scripts/check.sh @@ -419,9 +419,15 @@ grep -q 'if_missing: skip' "$gha/workflows/_make.yml" grep -q 'SHALLOW: "1"' "$gha/workflows/_make.yml" grep -q 'RUN_HOOKS: "0"' "$gha/workflows/_make.yml" ! grep -q 'RUN_HOOKS: "1"' "$gha/workflows/_make.yml" +! grep -A20 'name: Upload deps-key' "$gha/workflows/_make.yml" | grep -q 'src/third_party' +! grep -A20 'name: Upload deps-key' "$gha/workflows/_make.yml" | grep -q 'src/buildtools' +grep -A20 'name: Upload deps-key' "$gha/workflows/_make.yml" | grep -q '.gclient-git-cache' +grep -A20 'name: Upload deps-key' "$gha/workflows/_make.yml" | grep -q 'src/resources' grep -q 'chromium-webrtc-resources' "$gha/workflows/_make.yml" -grep -q 'src/resources' "$gha/workflows/_make.yml" -grep -q 'src/resources' "$gha/actions/artifact-download/action.yml" +! grep -q 'src/third_party' "$gha/actions/artifact-download/action.yml" +! grep -q 'src/third_party' "$gha/actions/restore-tree/action.yml" +grep -q 'tar cf - -C "${SOURCE}" .gclient-git-cache src/resources' \ + "$gha/actions/artifact-put/action.yml" grep -q 'uses: ./src/.github/actions/artifact-put' "$gha/workflows/_make.yml" echo "ok" From df3a02085db4c7084a2a61a15b11059ce1152cde Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Mon, 14 Sep 2026 14:29:46 +0300 Subject: [PATCH 17/24] Add skip cache step --- .github/workflows/_make.yml | 9 ++++++++- .github/workflows/manual-platform-tests.yml | 5 +++++ stream_build/AGENTS.md | 6 ++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 902654ae26..3478114c50 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -2,7 +2,8 @@ # Not listed in the Actions dispatch UI (workflow_call only). # Callers pass mode: build | test | package | release. # -# I/O: Linux Deps restores Hetzner deps-key.tar (skip if missing), shallow +# I/O: Linux Deps restores Hetzner deps-key.tar (skip if missing; +# skip_deps_cache skips the download for a cold make deps), shallow # gclient sync (RUN_HOOKS=0), fetches chromium-webrtc-resources into # src/resources, uploads GitHub artifact deps-key of .gclient-git-cache # and src/resources (compression-level: 0; GitHub zips once). No @@ -62,6 +63,11 @@ on: required: false type: string default: "" + skip_deps_cache: + description: Skip Hetzner deps-key.tar download (cold make deps) + required: false + type: boolean + default: false env: DEPS_ROOT: ${{ github.workspace }} @@ -162,6 +168,7 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} - name: Download cached deps-key + if: ${{ !inputs.skip_deps_cache }} uses: ./src/.github/actions/artifact-download with: path: deps-key diff --git a/.github/workflows/manual-platform-tests.yml b/.github/workflows/manual-platform-tests.yml index a661e99643..814f87a2eb 100644 --- a/.github/workflows/manual-platform-tests.yml +++ b/.github/workflows/manual-platform-tests.yml @@ -46,6 +46,10 @@ on: description: Optional Android ABI, for example arm64-v8a. Empty = default ABI set. required: false default: "" + skip_deps_cache: + description: Skip Hetzner deps cache (fresh gclient) + type: boolean + default: false jobs: build: @@ -60,3 +64,4 @@ jobs: windows: ${{ inputs.windows }} config: ${{ inputs.config }} android_arch: ${{ inputs.android_arch }} + skip_deps_cache: ${{ inputs.skip_deps_cache }} diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index c9817e5ffc..9b02c4eb86 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -80,7 +80,8 @@ the tree is `webrtc/src` (real directory, not a symlink). `deps` / CI checks out with `path: src` so `GITHUB_WORKSPACE` is the webrtc-named folder. `DEPS_ROOT=$GITHUB_WORKSPACE`. gclient objects live at `$DEPS_ROOT/.gclient-git-cache`. Linux Deps restores the Hetzner -`deps-key.tar` (skip if missing), `gclient sync --no-history +`deps-key.tar` (skip if missing; Build dispatch `skip_deps_cache` +skips the download so Deps does a cold `make deps`), `gclient sync --no-history --shallow` (`RUN_HOOKS=0`), fetches `chromium-webrtc-resources` into `src/resources`, and uploads GitHub artifact `deps-key` of `.gclient-git-cache` and `src/resources` (`compression-level: 0`; @@ -113,7 +114,8 @@ GetStream/webrtc `src` git worktree, `out/`, or `products/`. Same-run handoff is `actions/upload-artifact` name `deps-key` of those paths (`include-hidden-files: true`, `compression-level: 0`). GitHub zips once. No `deps-key.tar` for GH. `artifact-download` is Hetzner-only -(Deps warm cache, `if_missing: skip`). `artifact-put` is Hetzner-only +(Deps warm cache, `if_missing: skip`; skipped when `skip_deps_cache` +is true). `artifact-put` is Hetzner-only (`tar cf - -C .gclient-git-cache src/resources | aws s3 cp -`) to `/artifacts//deps-key.tar` with `--endpoint-url https://hel1.your-objectstorage.com` and region From 67bad9c9248a5aded374c56477dd8d152034f8bb Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Mon, 14 Sep 2026 16:15:33 +0300 Subject: [PATCH 18/24] Per platform cache --- .github/actions/artifact-download/action.yml | 25 ++- .github/actions/artifact-put/action.yml | 32 ++- .github/actions/restore-tree/action.yml | 92 ++++---- .github/workflows/_make.yml | 214 ++++++++----------- .github/workflows/manual-platform-tests.yml | 8 +- .github/workflows/publish.yml | 6 +- .github/workflows/release.yml | 4 +- .github/workflows/test.yml | 4 +- stream_build/AGENTS.md | 78 +++---- stream_build/scripts/check.sh | 50 ++--- 10 files changed, 246 insertions(+), 267 deletions(-) diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml index f1757a879f..fef2acaf78 100644 --- a/.github/actions/artifact-download/action.yml +++ b/.github/actions/artifact-download/action.yml @@ -2,8 +2,8 @@ # hetzner_access_key # hetzner_secret_access_key # hetzner_bucket -name: Download WebRTC deps from Hetzner -description: Stream a Hetzner deps tarball onto $GITHUB_WORKSPACE and untar. +name: Download WebRTC build cache from Hetzner +description: Stream a Hetzner build tarball onto $GITHUB_WORKSPACE and untar. inputs: path: @@ -40,7 +40,7 @@ runs: sudo apt-get install -y awscli fi - - name: Download and extract deps tarball + - name: Download and extract build tarball shell: bash working-directory: ${{ github.workspace }} env: @@ -74,14 +74,21 @@ runs: printf '%s\n' "${head_out}" exit 1 fi + test -d src/.git echo "extracting ${object} -> ${GITHUB_WORKSPACE}" # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. + # Members are gclient dirs + out/; not src/.git. aws ${endpoint} s3 cp "s3://${BUCKET}/${object}" - | tar xf - - test -d .gclient-git-cache - test -n "$(ls -A .gclient-git-cache)" + test -d src/.git test -f src/DEPS test ! -L src - find .gclient-git-cache -type f -exec touch {} + - if [[ -d src/resources ]]; then - find src/resources -type f -exec touch {} + - fi + test -d src/third_party + test -n "$(ls -A src/third_party)" + # Checkout set src tracked-file mtimes to now. Bump restored + # tree + out/ so ninja does not see objects as stale. + for p in src/third_party src/build src/buildtools \ + src/testing src/tools src/ios src/resources out; do + if [[ -d "${p}" ]]; then + find "${p}" -exec touch {} + + fi + done diff --git a/.github/actions/artifact-put/action.yml b/.github/actions/artifact-put/action.yml index 15b672aafb..9a86ebc73b 100644 --- a/.github/actions/artifact-put/action.yml +++ b/.github/actions/artifact-put/action.yml @@ -2,15 +2,15 @@ # hetzner_access_key # hetzner_secret_access_key # hetzner_bucket -name: Upload WebRTC deps tarball to Hetzner -description: Stream a tar of .gclient-git-cache and src/resources to Hetzner. Does not zip. +name: Upload WebRTC build tarball to Hetzner +description: Stream an uncompressed tar of the host gclient tree and out/ to Hetzner. inputs: path: description: Object stem. Uploads artifacts//.tar required: true source: - description: Directory with .gclient-git-cache and src/resources. + description: DEPS_ROOT with gclient checkouts, src/resources, and out/. required: true hetzner_access_key: description: Hetzner object storage access key. @@ -39,7 +39,7 @@ runs: sudo apt-get install -y awscli fi - - name: Stream deps tarball to Hetzner + - name: Stream build tarball to Hetzner shell: bash env: HETZNER_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_access_key }} @@ -57,16 +57,30 @@ runs: [[ -n "${OBJECT_STEM}" ]] [[ -n "${SOURCE}" ]] test -d "${SOURCE}" - test -d "${SOURCE}/.gclient-git-cache" - test -n "$(ls -A "${SOURCE}/.gclient-git-cache")" + test -d "${SOURCE}/src/third_party" + test -n "$(ls -A "${SOURCE}/src/third_party")" + test -d "${SOURCE}/out" + test -n "$(ls -A "${SOURCE}/out")" if [[ ! -d "${SOURCE}/src/resources" ]] || \ [[ -z "$(ls -A "${SOURCE}/src/resources")" ]]; then - echo "::error::deps-key missing src/resources (chromium-webrtc-resources)" + echo "::error::build cache missing src/resources" exit 1 fi + members=() + # Host tree + ninja. Do not pack .gclient-git-cache (objects + # twice) or src/.git / Stream-tracked src files. + for p in .gclient .gclient_entries \ + .gclient_previous_sync_commits .cipd \ + src/third_party src/build src/buildtools \ + src/testing src/tools src/ios src/resources out; do + if [[ -e "${SOURCE}/${p}" ]]; then + members+=("${p}") + fi + done + [[ ${#members[@]} -gt 0 ]] BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" object="artifacts/${{ github.repository }}/${OBJECT_STEM}.tar" - echo "streaming ${SOURCE}/{.gclient-git-cache,src/resources} -> ${object}" + echo "streaming ${SOURCE}/{${members[*]}} -> ${object}" # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. - tar cf - -C "${SOURCE}" .gclient-git-cache src/resources | \ + tar cf - -C "${SOURCE}" "${members[@]}" | \ aws ${endpoint} s3 cp - "s3://${BUCKET}/${object}" diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index 71c1d2db2b..6962565c91 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -1,5 +1,5 @@ name: Restore WebRTC tree -description: Restore deps-key or deps-windows from a GitHub artifact. +description: HIT host build cache (or Windows GitHub deps), then make deps. inputs: webrtc_ref: @@ -8,13 +8,34 @@ inputs: target_os: required: true description: gclient TARGET_OS (selected tokens only). + cache_key: + required: false + default: "" + description: Hetzner stem (build-ios, build-macos, build-android). + skip_cache: + required: false + default: "false" + description: Skip Hetzner download (still make deps; caller uploads). deps_artifact: - required: true - description: deps-key or deps-windows (GitHub artifact). + required: false + default: "" + description: GitHub artifact for Windows (deps-windows). install_android_packages: required: false default: "false" description: Install apt packages needed for Android builds. + hetzner_access_key: + required: false + default: "" + description: Hetzner object storage access key. + hetzner_secret_access_key: + required: false + default: "" + description: Hetzner object storage secret access key. + hetzner_bucket: + required: false + default: "" + description: Hetzner object storage bucket name. runs: using: composite @@ -25,48 +46,37 @@ runs: webrtc_ref: ${{ inputs.webrtc_ref }} install_android_packages: ${{ inputs.install_android_packages }} - # Linux Deps uploaded .gclient-git-cache + src/resources. Unzip to - # a temp dir (do not overlay onto src/), install both, then shallow - # gclient sync from that cache. Packed resources make the - # webrtc-resources hook a sha1 no-op. RUN_HOOKS=1 still fetches - # host GCS (rust-toolchain on Apple). rewrite_git_cache_alternates - # runs inside make deps. Do not transplant a Linux working tree. - - name: Download deps-key - if: ${{ inputs.deps_artifact == 'deps-key' }} - uses: actions/download-artifact@v8 + # Checkout src first (caller). Extract overlays gclient dirs + out/ + # without replacing src/.git. Missing object = cold make deps. + - name: Download host build cache + if: ${{ inputs.cache_key != '' && inputs.skip_cache != 'true' }} + uses: ./src/.github/actions/artifact-download with: - name: ${{ inputs.deps_artifact }} - path: ${{ runner.temp }}/deps-key + path: ${{ inputs.cache_key }} + if_missing: skip + hetzner_access_key: ${{ inputs.hetzner_access_key }} + hetzner_secret_access_key: ${{ inputs.hetzner_secret_access_key }} + hetzner_bucket: ${{ inputs.hetzner_bucket }} - - name: Install deps-key cache and resources - if: ${{ inputs.deps_artifact == 'deps-key' }} + - name: Drop git-cache alternates + if: ${{ inputs.cache_key != '' }} shell: bash run: | set -euo pipefail - src="${RUNNER_TEMP}/deps-key" - cache="${src}/.gclient-git-cache" - resources="${src}/src/resources" - if [[ ! -d "${cache}" ]]; then - echo "::error::deps-key missing .gclient-git-cache" - exit 1 - fi - if [[ ! -d "${resources}" ]]; then - echo "::error::deps-key missing src/resources (chromium-webrtc-resources)" - exit 1 - fi - dest_cache="${GITHUB_WORKSPACE}/.gclient-git-cache" - dest_res="${GITHUB_WORKSPACE}/src/resources" - rm -rf "${dest_cache}" "${dest_res}" - mv "${cache}" "${dest_cache}" - mkdir -p "${GITHUB_WORKSPACE}/src" - mv "${resources}" "${dest_res}" - test -n "$(ls -A "${dest_cache}")" - test -n "$(ls -A "${dest_res}")" - test -f "${GITHUB_WORKSPACE}/src/DEPS" - test ! -L "${GITHUB_WORKSPACE}/src" + # Tree is packed without .gclient-git-cache. Nested checkouts + # may still point alternates at a missing cache path. + for root in src/third_party src/build src/buildtools \ + src/testing src/tools src/ios; do + if [[ -d "${GITHUB_WORKSPACE}/${root}" ]]; then + find "${GITHUB_WORKSPACE}/${root}" \ + \( -path '*/.git/objects/info/alternates' -o \ + -path '*/.git/objects/info/http-alternates' \) \ + -type f -delete + fi + done - name: gclient sync and hooks - if: ${{ inputs.deps_artifact == 'deps-key' }} + if: ${{ inputs.cache_key != '' }} working-directory: src/stream_build shell: bash env: @@ -82,14 +92,14 @@ runs: # Windows is still a git-cache-only GitHub artifact. - name: Download same-run deps artifact - if: ${{ inputs.deps_artifact != 'deps-key' }} + if: ${{ inputs.deps_artifact != '' }} uses: actions/download-artifact@v8 with: name: ${{ inputs.deps_artifact }} path: ${{ runner.temp }}/deps-git-cache - name: Install same-run gclient object cache - if: ${{ inputs.deps_artifact != 'deps-key' }} + if: ${{ inputs.deps_artifact != '' }} shell: bash run: | set -euo pipefail @@ -105,7 +115,7 @@ runs: test ! -L "${GITHUB_WORKSPACE}/src" - name: gclient sync - if: ${{ inputs.deps_artifact != 'deps-key' }} + if: ${{ inputs.deps_artifact != '' }} working-directory: src/stream_build shell: bash env: diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 3478114c50..381ccf3be4 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -2,23 +2,14 @@ # Not listed in the Actions dispatch UI (workflow_call only). # Callers pass mode: build | test | package | release. # -# I/O: Linux Deps restores Hetzner deps-key.tar (skip if missing; -# skip_deps_cache skips the download for a cold make deps), shallow -# gclient sync (RUN_HOOKS=0), fetches chromium-webrtc-resources into -# src/resources, uploads GitHub artifact deps-key of .gclient-git-cache -# and src/resources (compression-level: 0; GitHub zips once). No -# gclient checkout trees (third_party/build/buildtools/…). No -# deps-key.tar on GitHub. Does not upload to Hetzner. Build/Test -# download-artifact deps-key, install git-cache + src/resources, make -# deps (gclient from cache; host GCS). Hetzner backfill (needs: deps -# only) download-artifact then -# tar cf - .gclient-git-cache src/resources | aws s3 cp - to -# artifacts//deps-key.tar in parallel with Build. -# Windows Deps stays deps-windows on GitHub. Build mode is ninja only -# (no products). Package/Release Build jobs also make package and upload -# products-* (GitHub). Package combine consumes products-* (no third -# ninja) and uploads final-*. Release attaches final-*. Tests need Deps -# only (make test, no make build). +# I/O: iOS / macOS / Android jobs HIT Hetzner build-{ios,macos,android}.tar +# (skip_deps_cache skips download), make deps, make build|test, then put +# that host's gclient tree + out/. No shared Linux Deps job. No GitHub +# deps-key. Windows Deps stays deps-windows on GitHub. Build mode is +# ninja only (no products). Package/Release Build jobs also make package +# and upload products-* (GitHub). Package combine consumes products-* +# (no third ninja) and uploads final-*. Release attaches final-*. Tests +# HIT the same per-platform key (out/ios_tests vs slice dirs). name: WebRTC make @@ -64,7 +55,7 @@ on: type: string default: "" skip_deps_cache: - description: Skip Hetzner deps-key.tar download (cold make deps) + description: Skip Hetzner build cache download (cold make deps) required: false type: boolean default: false @@ -152,103 +143,6 @@ jobs: platform_android: ${{ inputs.android }} platform_windows: ${{ inputs.windows }} - deps: - name: Deps - needs: plan - if: ${{ inputs.ios || inputs.macos || inputs.android }} - runs-on: ubuntu-latest - timeout-minutes: 360 - steps: - - uses: actions/checkout@v7 - with: - path: src - ref: ${{ inputs.webrtc_ref }} - - name: Setup WebRTC checkout - uses: ./src/.github/actions/setup-webrtc - with: - webrtc_ref: ${{ inputs.webrtc_ref }} - - name: Download cached deps-key - if: ${{ !inputs.skip_deps_cache }} - uses: ./src/.github/actions/artifact-download - with: - path: deps-key - if_missing: skip - hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - - name: gclient sync - working-directory: src/stream_build - env: - DEPS_ROOT: ${{ github.workspace }} - WEBRTC_SRC: ${{ github.workspace }}/src - GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache - TARGET_OS: ${{ needs.plan.outputs.linux_target_os }} - RUN_HOOKS: "0" - SHALLOW: "1" - JOBS: "8" - WEBRTC_REPO: https://github.com/GetStream/webrtc.git - run: make deps - # Same command as src/DEPS chromium-webrtc-resources. Do not set - # RUN_HOOKS=1 here: that runs mac_toolchain.py CIPD when mac is in - # TARGET_OS. Build restore still RUN_HOOKS=1 (rust-toolchain). - - name: Fetch chromium-webrtc-resources - working-directory: ${{ github.workspace }} - run: | - set -euo pipefail - download_from_google_storage --directory --recursive \ - --num_threads=10 --no_auth --quiet \ - --bucket chromium-webrtc-resources src/resources - # GitHub artifact is .gclient-git-cache + src/resources (one zip). - # Do not pack the gclient working tree: zip cannot store git - # hardlinks, so cache+checkout doubles objects (~33GB) and - # transplants Linux checkouts onto Mac. RUN_HOOKS=0: no hermetic - # Xcode CIPD on Linux; Build/Test restore this key then make deps - # (host GCS). Hetzner backfill tars the same members in parallel. - - name: Check deps-key members - working-directory: ${{ github.workspace }} - run: | - set -euo pipefail - test -d .gclient-git-cache - test -n "$(ls -A .gclient-git-cache)" - test -d src/resources - test -n "$(ls -A src/resources)" - - name: Upload deps-key - uses: actions/upload-artifact@v7 - with: - name: deps-key - path: | - .gclient-git-cache - src/resources - include-hidden-files: true - if-no-files-found: error - retention-days: 1 - compression-level: 0 - - hetzner_backfill: - name: Hetzner backfill - needs: [deps] - if: ${{ inputs.ios || inputs.macos || inputs.android }} - runs-on: ubuntu-latest - timeout-minutes: 360 - steps: - - uses: actions/checkout@v7 - with: - path: src - ref: ${{ inputs.webrtc_ref }} - - name: Download deps-key - uses: actions/download-artifact@v8 - with: - name: deps-key - path: ${{ github.workspace }}/deps-tree - - name: Upload deps-key to Hetzner - uses: ./src/.github/actions/artifact-put - with: - path: deps-key - source: ${{ github.workspace }}/deps-tree - hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} - hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - deps_windows: name: Deps Windows needs: plan @@ -298,7 +192,7 @@ jobs: build_ios: name: Build iOS - needs: [plan, deps] + needs: [plan] if: ${{ (inputs.mode == 'build' || inputs.mode == 'package' || inputs.mode == 'release') && inputs.ios }} runs-on: macos-26 timeout-minutes: 360 @@ -310,8 +204,12 @@ jobs: - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-key + target_os: ios + cache_key: build-ios + skip_cache: ${{ inputs.skip_deps_cache }} + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Build iOS working-directory: src/stream_build env: @@ -320,6 +218,14 @@ jobs: CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build ios + - name: Upload build-ios to Hetzner + uses: ./src/.github/actions/artifact-put + with: + path: build-ios + source: ${{ github.workspace }} + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Package iOS if: ${{ inputs.mode == 'package' || inputs.mode == 'release' }} working-directory: src/stream_build @@ -340,7 +246,7 @@ jobs: build_macos: name: Build macOS - needs: [plan, deps] + needs: [plan] if: ${{ (inputs.mode == 'build' || inputs.mode == 'package' || inputs.mode == 'release') && inputs.macos }} runs-on: macos-26 timeout-minutes: 360 @@ -352,8 +258,12 @@ jobs: - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-key + target_os: mac + cache_key: build-macos + skip_cache: ${{ inputs.skip_deps_cache }} + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Build macOS working-directory: src/stream_build env: @@ -362,6 +272,14 @@ jobs: CONFIG: ${{ inputs.config }} SKIP_DEPS: "1" run: make build macos + - name: Upload build-macos to Hetzner + uses: ./src/.github/actions/artifact-put + with: + path: build-macos + source: ${{ github.workspace }} + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Package macOS if: ${{ inputs.mode == 'package' || inputs.mode == 'release' }} working-directory: src/stream_build @@ -382,7 +300,7 @@ jobs: build_android: name: Build Android - needs: [plan, deps] + needs: [plan] if: ${{ (inputs.mode == 'build' || inputs.mode == 'package' || inputs.mode == 'release') && inputs.android }} runs-on: ubuntu-latest timeout-minutes: 360 @@ -395,8 +313,12 @@ jobs: with: webrtc_ref: ${{ inputs.webrtc_ref }} target_os: ${{ needs.plan.outputs.android_target_os }} - deps_artifact: deps-key + cache_key: build-android + skip_cache: ${{ inputs.skip_deps_cache }} install_android_packages: "true" + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Build Android working-directory: src/stream_build env: @@ -412,6 +334,14 @@ jobs: extra+=(ARCHS="${ANDROID_ARCH}") fi make build android "${extra[@]}" + - name: Upload build-android to Hetzner + uses: ./src/.github/actions/artifact-put + with: + path: build-android + source: ${{ github.workspace }} + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Package Android if: ${{ inputs.mode == 'package' || inputs.mode == 'release' }} working-directory: src/stream_build @@ -439,7 +369,7 @@ jobs: test_ios: name: Test iOS - needs: [plan, deps] + needs: [plan] if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.ios }} runs-on: macos-26 timeout-minutes: 180 @@ -451,18 +381,31 @@ jobs: - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-key + target_os: ios + cache_key: build-ios + skip_cache: ${{ inputs.skip_deps_cache }} + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Test iOS working-directory: src/stream_build env: DEPS_ROOT: ${{ github.workspace }} SKIP_DEPS: "1" run: make test ios + - name: Upload build-ios to Hetzner + if: ${{ inputs.mode == 'test' }} + uses: ./src/.github/actions/artifact-put + with: + path: build-ios + source: ${{ github.workspace }} + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} test_macos: name: Test macOS - needs: [plan, deps] + needs: [plan] if: ${{ (inputs.mode == 'test' || inputs.mode == 'release') && inputs.macos }} runs-on: macos-26 timeout-minutes: 180 @@ -474,14 +417,27 @@ jobs: - uses: ./src/.github/actions/restore-tree with: webrtc_ref: ${{ inputs.webrtc_ref }} - target_os: ${{ needs.plan.outputs.apple_target_os }} - deps_artifact: deps-key + target_os: mac + cache_key: build-macos + skip_cache: ${{ inputs.skip_deps_cache }} + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} - name: Test macOS working-directory: src/stream_build env: DEPS_ROOT: ${{ github.workspace }} SKIP_DEPS: "1" run: make test macos + - name: Upload build-macos to Hetzner + if: ${{ inputs.mode == 'test' }} + uses: ./src/.github/actions/artifact-put + with: + path: build-macos + source: ${{ github.workspace }} + hetzner_access_key: ${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_secret_access_key: ${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }} + hetzner_bucket: ${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }} test_windows: name: Test Windows diff --git a/.github/workflows/manual-platform-tests.yml b/.github/workflows/manual-platform-tests.yml index 814f87a2eb..870a4bddd7 100644 --- a/.github/workflows/manual-platform-tests.yml +++ b/.github/workflows/manual-platform-tests.yml @@ -1,8 +1,8 @@ # Manual WebRTC build. Dispatchable from a PR because this path exists on the # default branch. Implementation: .github/workflows/_make.yml -# Deps → Build (GitHub deps-key + restore-tree + make build). Hetzner -# backfill runs parallel with Build (needs: deps only). No package -# combine, no products. +# Build iOS / macOS / Android in parallel: HIT build-{ios,macos,android}, +# make deps, make build, put that host's tree + out/. No Linux Deps job. +# No package combine, no products. name: Build @@ -47,7 +47,7 @@ on: required: false default: "" skip_deps_cache: - description: Skip Hetzner deps cache (fresh gclient) + description: Skip Hetzner deps cache (fresh gclient + ninja) type: boolean default: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a999b8793b..7739f7e09f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,8 +1,8 @@ # Package WebRTC artifacts. Same path as today's Publish so a PR can dispatch it. # Does not create a GitHub release. -# _make.yml: Deps → Build → Package combine (products-* in, final-* out). -# Hetzner backfill runs parallel with Build (needs: deps only). -# Package does not rebuild from git-cache. +# _make.yml: Build (HIT + make deps + make build + make package, upload +# products-*) → Package combine (final-*). No Linux Deps job. No Test. +# No GH release. name: Package diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 11047959a1..7c17ce35b8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ # GitHub release of packaged WebRTC artifacts, then downstream wrapper publishes. # Not dispatchable until this file exists on the default branch. -# Tests run parallel with Build (needs Deps only). Hetzner backfill -# also needs Deps only. Publish waits on Test + Package. +# Tests run parallel with Build (each HIT its platform cache). Publish +# waits on Test + Package. name: Release diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46a64e4c89..1784400ab5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,6 @@ # WebRTC tests. Not dispatchable until this file exists on the default branch. -# _make.yml: Deps then make test only (no extra framework-slice make -# build). Hetzner backfill runs parallel with Test (needs: deps only). +# _make.yml: HIT build-ios / build-macos, make test only (no extra +# framework-slice make build). No Linux Deps job. name: Test diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index 9b02c4eb86..4fcf32416c 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -78,54 +78,46 @@ the tree is `webrtc/src` (real directory, not a symlink). `deps` / `$(DEPS_ROOT)/Makefile` if that file is missing (parent is outside git). CI checks out with `path: src` so `GITHUB_WORKSPACE` is the webrtc-named -folder. `DEPS_ROOT=$GITHUB_WORKSPACE`. gclient objects live at -`$DEPS_ROOT/.gclient-git-cache`. Linux Deps restores the Hetzner -`deps-key.tar` (skip if missing; Build dispatch `skip_deps_cache` -skips the download so Deps does a cold `make deps`), `gclient sync --no-history ---shallow` (`RUN_HOOKS=0`), fetches `chromium-webrtc-resources` into -`src/resources`, and uploads GitHub artifact `deps-key` of -`.gclient-git-cache` and `src/resources` (`compression-level: 0`; -GitHub zips once). Do not pack gclient working trees -(`src/third_party`, `src/build`, `src/buildtools`, …) next to the -cache: zip cannot store git hardlinks, so that dual pack doubles -objects and transplants Linux checkouts onto Mac. No `deps-key.tar` -on GitHub. Do not set `RUN_HOOKS=1` on Linux Deps (`mac` in -`TARGET_OS` would run hermetic Xcode CIPD). Do not upload to Hetzner -from Deps. Build/Test `needs: deps` only, `download-artifact` -`deps-key`, install the git-cache and `src/resources`, then -`make deps` (gclient from cache; packed resources make the -webrtc-resources hook a sha1 no-op; host GCS still fetches Apple -`rust-toolchain`; `rewrite_git_cache_alternates` retargets cache -paths) and `SKIP_DEPS=1` on build/test. `Hetzner backfill` (`needs: -deps` only) `download-artifact` then -`tar cf - .gclient-git-cache src/resources | aws s3 cp -` in -parallel with Build. -Windows Deps still uploads `deps-windows` to GitHub. Package/Release -Build jobs also `make package` and upload `products-*` (GitHub). -Package combine consumes `products-*` (no third ninja) and uploads -`final-*`. Release attaches `final-*`. Tests need Deps only and run -`make test` (no extra framework-slice build). `TARGET_OS` is only the -tokens selected this run. - -Deps packing / Hetzner: -Member list: `.gclient-git-cache` and `src/resources` only. Not -gclient checkouts (`third_party`, `build`, `buildtools`, …), not the -GetStream/webrtc `src` git worktree, `out/`, or `products/`. Same-run -handoff is `actions/upload-artifact` name `deps-key` of those paths -(`include-hidden-files: true`, `compression-level: 0`). GitHub zips -once. No `deps-key.tar` for GH. `artifact-download` is Hetzner-only -(Deps warm cache, `if_missing: skip`; skipped when `skip_deps_cache` -is true). `artifact-put` is Hetzner-only -(`tar cf - -C .gclient-git-cache src/resources | aws s3 cp -`) -to `/artifacts//deps-key.tar` with +folder. `DEPS_ROOT=$GITHUB_WORKSPACE`. `OUT` is `$DEPS_ROOT/out` +(sibling of `src`). There is no shared Linux Deps job. iOS, macOS, +and Android jobs run in parallel after Plan: + +1. `actions/checkout` `src` at `webrtc_ref` +2. HIT Hetzner `build-ios` / `build-macos` / `build-android` (`if_missing: + skip`; Build dispatch `skip_deps_cache` skips the download) +3. `make deps` (`RUN_HOOKS=1`, host GCS rust-toolchain on Apple) +4. `make build` / `make test` (`SKIP_DEPS=1`) +5. `artifact-put` that host's tree + `out/` (always, including after a + skipped HIT) + +Keys: `artifacts//build-{ios,macos,android}.tar`. +Same-OS only (Linux tree on Mac is forbidden). Members: `.gclient`, +`.gclient_entries`, `.gclient_previous_sync_commits`, `.cipd` if +present, `src/{third_party,build,buildtools,testing,tools,ios,resources}`, +`out/`. Not packed: `.gclient-git-cache`, `src/.git`, Stream-tracked +`src` files, `products/`. Restore order: checkout `src` first, extract +over gclient dirs + `out/` (not `src/.git`), `touch` those members so +ninja does not see objects older than the fresh checkout. Build +`CONFIG` is dispatch (default release); `make test` always uses debug +in `out/ios_tests` / `out/webrtc_tests`, so those subdirs do not mix +with slice dirs. Test.yml HITs the same `build-ios` / `build-macos` +keys. Windows Deps still uploads `deps-windows` to GitHub. +Package/Release Build jobs also `make package` and upload `products-*` +(GitHub). Package combine consumes `products-*` (no third ninja) and +uploads `final-*`. Release attaches `final-*`. `TARGET_OS` is the +platform of that job (`ios`, `mac`, `android,unix`). + +Hetzner: `artifact-download` / `artifact-put` stream `tar cf - … | +aws s3 cp -` to +`/artifacts//.tar` with `--endpoint-url https://hel1.your-objectstorage.com` and region `hel1`. `aws s3 cp` talks to Hetzner's S3-compatible API, not AWS. Callers pass org secrets via `with:` `${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }}`, `${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }}`, and -`${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }}`. After extract, packed -files are `touch`ed so make/ninja do not rebuild from mtime. -`products-*` / `final-*` stay on `actions/upload-artifact`. +`${{ secrets.HETZNER_BUCKET_CI_ARTIFACTS }}`. Composite actions must +not use `${{ secrets.* }}`. `products-*` / `final-*` stay on +`actions/upload-artifact`. ## Host gates diff --git a/stream_build/scripts/check.sh b/stream_build/scripts/check.sh index c1712d4cc3..01ce863e6c 100755 --- a/stream_build/scripts/check.sh +++ b/stream_build/scripts/check.sh @@ -399,35 +399,35 @@ grep -q 'object="artifacts/\${{ github.repository }}/\${OBJECT_STEM}.tar"' \ "$gha/actions/artifact-put/action.yml" grep -q 'object="artifacts/\${{ github.repository }}/\${OBJECT_STEM}.tar"' \ "$gha/actions/artifact-download/action.yml" -grep -q 'path: deps-key' "$gha/workflows/_make.yml" -! grep -q 'path: deps-key.tar' "$gha/workflows/_make.yml" -grep -q 'name: deps-key' "$gha/workflows/_make.yml" +! grep -qE 'name: Deps$' "$gha/workflows/_make.yml" +! grep -q 'name: Hetzner backfill' "$gha/workflows/_make.yml" +! grep -q 'path: deps-key' "$gha/workflows/_make.yml" +! grep -q 'name: deps-key' "$gha/workflows/_make.yml" +grep -q 'cache_key: build-ios' "$gha/workflows/_make.yml" +grep -q 'cache_key: build-macos' "$gha/workflows/_make.yml" +grep -q 'cache_key: build-android' "$gha/workflows/_make.yml" grep -q 'include-hidden-files: true' "$gha/workflows/_make.yml" grep -q 'compression-level: 0' "$gha/workflows/_make.yml" -grep -q 'name: Hetzner backfill' "$gha/workflows/_make.yml" -grep -A5 'name: Hetzner backfill' "$gha/workflows/_make.yml" | grep -q 'needs: \[deps\]' -grep -A5 'name: Build iOS' "$gha/workflows/_make.yml" | grep -q 'needs: \[plan, deps\]' -! grep -A8 'name: Build iOS' "$gha/workflows/_make.yml" | grep -q hetzner_backfill -grep -A5 'name: Test iOS' "$gha/workflows/_make.yml" | grep -q 'needs: \[plan, deps\]' -! grep -A8 'name: Test iOS' "$gha/workflows/_make.yml" | grep -q hetzner_backfill -grep -q "deps_artifact == 'deps-key'" "$gha/actions/restore-tree/action.yml" -! grep -q 'artifact-download' "$gha/actions/restore-tree/action.yml" -! grep -q 'hetzner_access_key' "$gha/actions/restore-tree/action.yml" +grep -A5 'name: Build iOS' "$gha/workflows/_make.yml" | grep -q 'needs: \[plan\]' +! grep -A8 'name: Build iOS' "$gha/workflows/_make.yml" | grep -q deps +grep -A5 'name: Test iOS' "$gha/workflows/_make.yml" | grep -q 'needs: \[plan\]' +! grep -A8 'name: Test iOS' "$gha/workflows/_make.yml" | grep -q deps +grep -q 'artifact-download' "$gha/actions/restore-tree/action.yml" +grep -q 'hetzner_access_key' "$gha/actions/restore-tree/action.yml" grep -q 'actions/download-artifact' "$gha/actions/restore-tree/action.yml" ! grep -q 'tar xf' "$gha/actions/restore-tree/action.yml" -grep -q 'if_missing: skip' "$gha/workflows/_make.yml" -grep -q 'SHALLOW: "1"' "$gha/workflows/_make.yml" -grep -q 'RUN_HOOKS: "0"' "$gha/workflows/_make.yml" -! grep -q 'RUN_HOOKS: "1"' "$gha/workflows/_make.yml" -! grep -A20 'name: Upload deps-key' "$gha/workflows/_make.yml" | grep -q 'src/third_party' -! grep -A20 'name: Upload deps-key' "$gha/workflows/_make.yml" | grep -q 'src/buildtools' -grep -A20 'name: Upload deps-key' "$gha/workflows/_make.yml" | grep -q '.gclient-git-cache' -grep -A20 'name: Upload deps-key' "$gha/workflows/_make.yml" | grep -q 'src/resources' -grep -q 'chromium-webrtc-resources' "$gha/workflows/_make.yml" -! grep -q 'src/third_party' "$gha/actions/artifact-download/action.yml" -! grep -q 'src/third_party' "$gha/actions/restore-tree/action.yml" -grep -q 'tar cf - -C "${SOURCE}" .gclient-git-cache src/resources' \ - "$gha/actions/artifact-put/action.yml" +grep -q 'if_missing: skip' "$gha/actions/restore-tree/action.yml" +grep -q 'SHALLOW: "1"' "$gha/actions/restore-tree/action.yml" +grep -q 'RUN_HOOKS: "1"' "$gha/actions/restore-tree/action.yml" +! grep -q 'RUN_HOOKS: "0"' "$gha/workflows/_make.yml" +grep -q 'src/third_party' "$gha/actions/artifact-download/action.yml" +grep -q 'src/third_party' "$gha/actions/artifact-put/action.yml" +grep -q 'src/third_party' "$gha/actions/restore-tree/action.yml" +grep -A8 'for p in .gclient' "$gha/actions/artifact-put/action.yml" | \ + grep -q 'src/third_party' +grep -A8 'for p in .gclient' "$gha/actions/artifact-put/action.yml" | grep -q ' out' +! grep -A8 'for p in .gclient' "$gha/actions/artifact-put/action.yml" | \ + grep -q 'gclient-git-cache' grep -q 'uses: ./src/.github/actions/artifact-put' "$gha/workflows/_make.yml" echo "ok" From 8a4afb7da7d94997880f10476f1cba3b27e43941 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Mon, 14 Sep 2026 19:23:27 +0300 Subject: [PATCH 19/24] Fix restoring platform tree --- .github/actions/artifact-download/action.yml | 74 ++++++++++++++++++-- .github/actions/artifact-put/action.yml | 72 +++++++++++++++++-- stream_build/AGENTS.md | 13 ++-- stream_build/scripts/check.sh | 21 +++++- 4 files changed, 159 insertions(+), 21 deletions(-) diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml index fef2acaf78..447ea55979 100644 --- a/.github/actions/artifact-download/action.yml +++ b/.github/actions/artifact-download/action.yml @@ -3,7 +3,7 @@ # hetzner_secret_access_key # hetzner_bucket name: Download WebRTC build cache from Hetzner -description: Stream a Hetzner build tarball onto $GITHUB_WORKSPACE and untar. +description: Multipart-get a Hetzner build tarball to a file, verify size, untar. inputs: path: @@ -55,13 +55,22 @@ runs: export AWS_SECRET_ACCESS_KEY="${HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS:?}" export AWS_DEFAULT_REGION=hel1 export AWS_EC2_METADATA_DISABLED=true + export AWS_MAX_ATTEMPTS=10 + export AWS_RETRY_MODE=adaptive endpoint="--endpoint-url https://hel1.your-objectstorage.com" + # File dest enables multipart GET (Range parts). A pipe to + # stdout is one streaming GET; a drop aborts the whole object. + # Peak disk is tar + extract (~2x); rm the tar after tar xf. + aws_timeouts=(--cli-connect-timeout 60 --cli-read-timeout 0) BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" [[ -n "${OBJECT_STEM}" ]] object="artifacts/${{ github.repository }}/${OBJECT_STEM}.tar" + tar_file="${RUNNER_TEMP:?}/${OBJECT_STEM}.tar" + trap 'rm -f "${tar_file}"' EXIT set +e - head_out="$(aws ${endpoint} s3api head-object \ - --bucket "${BUCKET}" --key "${object}" 2>&1)" + head_out="$(aws ${endpoint} "${aws_timeouts[@]}" s3api head-object \ + --bucket "${BUCKET}" --key "${object}" \ + --query ContentLength --output text 2>&1)" head_rc=$? set -e if [[ "${head_rc}" -ne 0 ]]; then @@ -74,16 +83,67 @@ runs: printf '%s\n' "${head_out}" exit 1 fi + expected_size="$(printf '%s' "${head_out}" | tr -d '[:space:]')" + if [[ ! "${expected_size}" =~ ^[0-9]+$ ]]; then + echo "::error::head-object missing ContentLength for ${object}" + printf '%s\n' "${head_out}" + exit 1 + fi test -d src/.git - echo "extracting ${object} -> ${GITHUB_WORKSPACE}" - # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. - # Members are gclient dirs + out/; not src/.git. - aws ${endpoint} s3 cp "s3://${BUCKET}/${object}" - | tar xf - + echo "downloading ${object} (${expected_size} bytes) -> ${tar_file}" + df -h "${GITHUB_WORKSPACE}" "${RUNNER_TEMP}" || true + file_size() { + if [[ "$(uname -s)" == Darwin ]]; then + stat -f %s "$1" + else + stat -c %s "$1" + fi + } + attempts=3 + download_ok=0 + for attempt in $(seq 1 "${attempts}"); do + rm -f "${tar_file}" + echo "download attempt ${attempt}/${attempts}" + set +e + aws ${endpoint} "${aws_timeouts[@]}" \ + s3 cp "s3://${BUCKET}/${object}" "${tar_file}" + dl_rc=$? + set -e + if [[ "${dl_rc}" -ne 0 ]]; then + echo "download attempt ${attempt} failed (exit ${dl_rc})" + elif [[ ! -f "${tar_file}" ]]; then + echo "download attempt ${attempt}: missing ${tar_file}" + else + actual_size="$(file_size "${tar_file}")" + if [[ "${actual_size}" != "${expected_size}" ]]; then + echo "size mismatch after attempt ${attempt}: local=${actual_size} expected=${expected_size}" + else + download_ok=1 + break + fi + fi + if [[ "${attempt}" -lt "${attempts}" ]]; then + sleep 10 + fi + done + if [[ "${download_ok}" -ne 1 ]]; then + actual_size="missing" + if [[ -f "${tar_file}" ]]; then + actual_size="$(file_size "${tar_file}")" + fi + echo "::error::failed to download s3://${BUCKET}/${object} after ${attempts} attempts (local=${actual_size} expected=${expected_size})" + exit 1 + fi + echo "extracting ${tar_file} -> ${GITHUB_WORKSPACE}" + tar xf "${tar_file}" + rm -f "${tar_file}" test -d src/.git test -f src/DEPS test ! -L src test -d src/third_party test -n "$(ls -A src/third_party)" + printf '%s\n' "${expected_size}" > \ + "${GITHUB_WORKSPACE}/.hetzner-hit-bytes" # Checkout set src tracked-file mtimes to now. Bump restored # tree + out/ so ninja does not see objects as stale. for p in src/third_party src/build src/buildtools \ diff --git a/.github/actions/artifact-put/action.yml b/.github/actions/artifact-put/action.yml index 9a86ebc73b..7e79ce4717 100644 --- a/.github/actions/artifact-put/action.yml +++ b/.github/actions/artifact-put/action.yml @@ -3,7 +3,7 @@ # hetzner_secret_access_key # hetzner_bucket name: Upload WebRTC build tarball to Hetzner -description: Stream an uncompressed tar of the host gclient tree and out/ to Hetzner. +description: Pack an uncompressed tar of the host tree and multipart-put it. inputs: path: @@ -39,7 +39,7 @@ runs: sudo apt-get install -y awscli fi - - name: Stream build tarball to Hetzner + - name: Upload build tarball to Hetzner shell: bash env: HETZNER_ACCESS_KEY_CI_ARTIFACTS: ${{ inputs.hetzner_access_key }} @@ -53,7 +53,10 @@ runs: export AWS_SECRET_ACCESS_KEY="${HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS:?}" export AWS_DEFAULT_REGION=hel1 export AWS_EC2_METADATA_DISABLED=true + export AWS_MAX_ATTEMPTS=10 + export AWS_RETRY_MODE=adaptive endpoint="--endpoint-url https://hel1.your-objectstorage.com" + aws_timeouts=(--cli-connect-timeout 60 --cli-read-timeout 0) [[ -n "${OBJECT_STEM}" ]] [[ -n "${SOURCE}" ]] test -d "${SOURCE}" @@ -80,7 +83,64 @@ runs: [[ ${#members[@]} -gt 0 ]] BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" object="artifacts/${{ github.repository }}/${OBJECT_STEM}.tar" - echo "streaming ${SOURCE}/{${members[*]}} -> ${object}" - # aws s3 cp talks to Hetzner's S3-compatible API, not AWS. - tar cf - -C "${SOURCE}" "${members[@]}" | \ - aws ${endpoint} s3 cp - "s3://${BUCKET}/${object}" + tar_file="${RUNNER_TEMP:?}/${OBJECT_STEM}.tar" + trap 'rm -f "${tar_file}"' EXIT + echo "packing ${SOURCE}/{${members[*]}} -> ${tar_file}" + tar cf "${tar_file}" -C "${SOURCE}" "${members[@]}" + if [[ "$(uname -s)" == Darwin ]]; then + size="$(stat -f %s "${tar_file}")" + else + size="$(stat -c %s "${tar_file}")" + fi + if [[ ! -f "${tar_file}" ]]; then + echo "::error::new tar missing ${tar_file}" + exit 1 + fi + hit_file="${GITHUB_WORKSPACE}/.hetzner-hit-bytes" + old="" + if [[ -f "${hit_file}" ]]; then + old="$(tr -d '[:space:]' < "${hit_file}")" + fi + threshold=1073741824 + skip_put=0 + if [[ "${old}" =~ ^[0-9]+$ ]]; then + if [[ "${size}" -ge "${old}" ]]; then + delta=$((size - old)) + else + delta=$((old - size)) + fi + if [[ "${delta}" -lt "${threshold}" ]]; then + echo "skip upload: old=${old} new=${size} delta=${delta} (< 1GiB)" + skip_put=1 + else + echo "upload: old=${old} new=${size} delta=${delta}" + fi + else + echo "upload: old=none new=${size} delta=n/a" + fi + if [[ "${skip_put}" -eq 0 ]]; then + df -h "${SOURCE}" "${RUNNER_TEMP}" || true + attempts=3 + upload_ok=0 + for attempt in $(seq 1 "${attempts}"); do + echo "upload attempt ${attempt}/${attempts}" + set +e + aws ${endpoint} "${aws_timeouts[@]}" \ + s3 cp "${tar_file}" "s3://${BUCKET}/${object}" + up_rc=$? + set -e + if [[ "${up_rc}" -eq 0 ]]; then + upload_ok=1 + break + fi + echo "upload attempt ${attempt} failed (exit ${up_rc})" + if [[ "${attempt}" -lt "${attempts}" ]]; then + sleep 10 + fi + done + if [[ "${upload_ok}" -ne 1 ]]; then + echo "::error::failed to upload s3://${BUCKET}/${object} after ${attempts} attempts (${size} bytes)" + exit 1 + fi + fi + rm -f "${tar_file}" diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index 4fcf32416c..ed497a3d1b 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -87,8 +87,8 @@ and Android jobs run in parallel after Plan: skip`; Build dispatch `skip_deps_cache` skips the download) 3. `make deps` (`RUN_HOOKS=1`, host GCS rust-toolchain on Apple) 4. `make build` / `make test` (`SKIP_DEPS=1`) -5. `artifact-put` that host's tree + `out/` (always, including after a - skipped HIT) +5. `artifact-put` that host's tree + `out/` (always after miss / + `skip_deps_cache`; skip PUT when HIT size delta is < 1GiB) Keys: `artifacts//build-{ios,macos,android}.tar`. Same-OS only (Linux tree on Mac is forbidden). Members: `.gclient`, @@ -107,11 +107,14 @@ Package/Release Build jobs also `make package` and upload `products-*` uploads `final-*`. Release attaches `final-*`. `TARGET_OS` is the platform of that job (`ios`, `mac`, `android,unix`). -Hetzner: `artifact-download` / `artifact-put` stream `tar cf - … | -aws s3 cp -` to +Hetzner: `artifact-download` / `artifact-put` use a local tar file +then multipart `aws s3 cp` (not a stdout/stdin pipe) to `/artifacts//.tar` with `--endpoint-url https://hel1.your-objectstorage.com` and region -`hel1`. `aws s3 cp` talks to Hetzner's S3-compatible API, not AWS. +`hel1`. A pipe GET/PUT is one HTTP body; a drop is IncompleteRead +of the whole object. File dest retries parts. Peak disk is tar + +tree (~2x); delete the tar after extract/upload. `aws s3 cp` talks +to Hetzner's S3-compatible API, not AWS. Callers pass org secrets via `with:` `${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }}`, `${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }}`, and diff --git a/stream_build/scripts/check.sh b/stream_build/scripts/check.sh index 01ce863e6c..4148cfb56f 100755 --- a/stream_build/scripts/check.sh +++ b/stream_build/scripts/check.sh @@ -392,9 +392,24 @@ gha="$ROOT/../.github" ! grep -q 'secrets\.' "$gha/actions/artifact-download/action.yml" ! grep -q 'secrets\.' "$gha/actions/artifact-put/action.yml" ! grep -q 'secrets\.' "$gha/actions/restore-tree/action.yml" -grep -q 'tar cf -' "$gha/actions/artifact-put/action.yml" -grep -q 's3 cp -' "$gha/actions/artifact-put/action.yml" -! grep -q 's3 cp "${OBJECT_STEM}.tar"' "$gha/actions/artifact-put/action.yml" +grep -q 'tar cf "${tar_file}"' "$gha/actions/artifact-put/action.yml" +! grep -q 'tar cf -' "$gha/actions/artifact-put/action.yml" +! grep -q 's3 cp -' "$gha/actions/artifact-put/action.yml" +! grep -q 's3 cp "s3://${BUCKET}/${object}" -' \ + "$gha/actions/artifact-download/action.yml" +grep -q 's3 cp "s3://${BUCKET}/${object}" "${tar_file}"' \ + "$gha/actions/artifact-download/action.yml" +grep -q 's3 cp "${tar_file}" "s3://${BUCKET}/${object}"' \ + "$gha/actions/artifact-put/action.yml" +grep -q 'AWS_MAX_ATTEMPTS' "$gha/actions/artifact-download/action.yml" +grep -q 'AWS_RETRY_MODE' "$gha/actions/artifact-download/action.yml" +grep -q 'cli-read-timeout' "$gha/actions/artifact-download/action.yml" +grep -q 'size mismatch' "$gha/actions/artifact-download/action.yml" +grep -q 'tar xf "${tar_file}"' "$gha/actions/artifact-download/action.yml" +grep -q '.hetzner-hit-bytes' "$gha/actions/artifact-download/action.yml" +grep -q '.hetzner-hit-bytes' "$gha/actions/artifact-put/action.yml" +grep -q '1073741824' "$gha/actions/artifact-put/action.yml" +grep -q 'skip upload:' "$gha/actions/artifact-put/action.yml" grep -q 'object="artifacts/\${{ github.repository }}/\${OBJECT_STEM}.tar"' \ "$gha/actions/artifact-put/action.yml" grep -q 'object="artifacts/\${{ github.repository }}/\${OBJECT_STEM}.tar"' \ From ecb64d11110165ab9a29feda3c78b8262a9b8600 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Mon, 14 Sep 2026 19:39:21 +0300 Subject: [PATCH 20/24] Retry failed cache download --- .github/actions/artifact-download/action.yml | 78 ++++++++++++++------ stream_build/AGENTS.md | 12 +-- stream_build/scripts/check.sh | 6 +- 3 files changed, 69 insertions(+), 27 deletions(-) diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml index 447ea55979..29b6f375fd 100644 --- a/.github/actions/artifact-download/action.yml +++ b/.github/actions/artifact-download/action.yml @@ -3,7 +3,7 @@ # hetzner_secret_access_key # hetzner_bucket name: Download WebRTC build cache from Hetzner -description: Multipart-get a Hetzner build tarball to a file, verify size, untar. +description: Range-get a Hetzner build tarball, resume on retry, verify size, untar. inputs: path: @@ -58,15 +58,17 @@ runs: export AWS_MAX_ATTEMPTS=10 export AWS_RETRY_MODE=adaptive endpoint="--endpoint-url https://hel1.your-objectstorage.com" - # File dest enables multipart GET (Range parts). A pipe to - # stdout is one streaming GET; a drop aborts the whole object. + # Sequential get-object leaves a valid prefix on drop. + # s3 cp multipart may not, and it truncates dest on start. # Peak disk is tar + extract (~2x); rm the tar after tar xf. aws_timeouts=(--cli-connect-timeout 60 --cli-read-timeout 0) BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" [[ -n "${OBJECT_STEM}" ]] object="artifacts/${{ github.repository }}/${OBJECT_STEM}.tar" tar_file="${RUNNER_TEMP:?}/${OBJECT_STEM}.tar" - trap 'rm -f "${tar_file}"' EXIT + partial="${tar_file}.partial" + chunk="${tar_file}.chunk" + trap 'rm -f "${tar_file}" "${partial}" "${chunk}"' EXIT set +e head_out="$(aws ${endpoint} "${aws_timeouts[@]}" s3api head-object \ --bucket "${BUCKET}" --key "${object}" \ @@ -102,33 +104,67 @@ runs: attempts=3 download_ok=0 for attempt in $(seq 1 "${attempts}"); do - rm -f "${tar_file}" - echo "download attempt ${attempt}/${attempts}" - set +e - aws ${endpoint} "${aws_timeouts[@]}" \ - s3 cp "s3://${BUCKET}/${object}" "${tar_file}" - dl_rc=$? - set -e - if [[ "${dl_rc}" -ne 0 ]]; then - echo "download attempt ${attempt} failed (exit ${dl_rc})" - elif [[ ! -f "${tar_file}" ]]; then - echo "download attempt ${attempt}: missing ${tar_file}" + if [[ -f "${partial}" ]]; then + have="$(file_size "${partial}")" else - actual_size="$(file_size "${tar_file}")" - if [[ "${actual_size}" != "${expected_size}" ]]; then - echo "size mismatch after attempt ${attempt}: local=${actual_size} expected=${expected_size}" + have=0 + fi + if [[ "${have}" -gt "${expected_size}" ]]; then + echo "partial ${have} > ${expected_size}; deleting" + rm -f "${partial}" + have=0 + fi + echo "download attempt ${attempt}/${attempts}" + if [[ "${have}" -lt "${expected_size}" ]]; then + rm -f "${chunk}" + if [[ "${have}" -eq 0 ]]; then + echo "download from 0" + set +e + aws ${endpoint} "${aws_timeouts[@]}" s3api get-object \ + --bucket "${BUCKET}" --key "${object}" \ + --range "bytes=0-" "${partial}" + dl_rc=$? + set -e else - download_ok=1 - break + echo "resume from byte ${have} / ${expected_size}" + set +e + aws ${endpoint} "${aws_timeouts[@]}" s3api get-object \ + --bucket "${BUCKET}" --key "${object}" \ + --range "bytes=${have}-" "${chunk}" + dl_rc=$? + set -e + if [[ -f "${chunk}" ]]; then + chunk_size="$(file_size "${chunk}")" + if [[ "${chunk_size}" -gt 0 ]]; then + cat "${chunk}" >> "${partial}" + fi + rm -f "${chunk}" + fi fi + if [[ "${dl_rc}" -ne 0 ]]; then + echo "download attempt ${attempt} failed (exit ${dl_rc})" + fi + fi + if [[ -f "${partial}" ]]; then + have="$(file_size "${partial}")" + else + have=0 + fi + if [[ "${have}" -eq "${expected_size}" ]]; then + mv "${partial}" "${tar_file}" + download_ok=1 + break fi + echo "size mismatch after attempt ${attempt}: local=${have} expected=${expected_size}" if [[ "${attempt}" -lt "${attempts}" ]]; then sleep 10 fi done if [[ "${download_ok}" -ne 1 ]]; then actual_size="missing" - if [[ -f "${tar_file}" ]]; then + if [[ -f "${partial}" ]]; then + actual_size="$(file_size "${partial}")" + elif [[ -f "${tar_file}" ]]; then actual_size="$(file_size "${tar_file}")" fi echo "::error::failed to download s3://${BUCKET}/${object} after ${attempts} attempts (local=${actual_size} expected=${expected_size})" diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index ed497a3d1b..3dea6970cd 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -107,14 +107,16 @@ Package/Release Build jobs also `make package` and upload `products-*` uploads `final-*`. Release attaches `final-*`. `TARGET_OS` is the platform of that job (`ios`, `mac`, `android,unix`). -Hetzner: `artifact-download` / `artifact-put` use a local tar file -then multipart `aws s3 cp` (not a stdout/stdin pipe) to +Hetzner: `artifact-download` Range-GETs with +`s3api get-object --range bytes=${have}-` into a partial file and +resumes from bytes already on disk (not `s3 cp` from 0). +`artifact-put` tars to a file then multipart `aws s3 cp` to `/artifacts//.tar` with `--endpoint-url https://hel1.your-objectstorage.com` and region `hel1`. A pipe GET/PUT is one HTTP body; a drop is IncompleteRead -of the whole object. File dest retries parts. Peak disk is tar + -tree (~2x); delete the tar after extract/upload. `aws s3 cp` talks -to Hetzner's S3-compatible API, not AWS. +of the whole object. Peak disk is tar + tree (~2x); delete the tar +after extract/upload. `aws s3 cp` / `s3api` talk to Hetzner's +S3-compatible API, not AWS. Callers pass org secrets via `with:` `${{ secrets.HETZNER_ACCESS_KEY_CI_ARTIFACTS }}`, `${{ secrets.HETZNER_SECRET_ACCESS_KEY_CI_ARTIFACTS }}`, and diff --git a/stream_build/scripts/check.sh b/stream_build/scripts/check.sh index 4148cfb56f..5bebe21f3f 100755 --- a/stream_build/scripts/check.sh +++ b/stream_build/scripts/check.sh @@ -397,8 +397,12 @@ grep -q 'tar cf "${tar_file}"' "$gha/actions/artifact-put/action.yml" ! grep -q 's3 cp -' "$gha/actions/artifact-put/action.yml" ! grep -q 's3 cp "s3://${BUCKET}/${object}" -' \ "$gha/actions/artifact-download/action.yml" -grep -q 's3 cp "s3://${BUCKET}/${object}" "${tar_file}"' \ +! grep -q 's3 cp "s3://${BUCKET}/${object}" "${tar_file}"' \ "$gha/actions/artifact-download/action.yml" +grep -q 's3api get-object' "$gha/actions/artifact-download/action.yml" +grep -q 'bytes=${have}-' "$gha/actions/artifact-download/action.yml" +grep -q 'resume from byte' "$gha/actions/artifact-download/action.yml" +grep -q 'download from 0' "$gha/actions/artifact-download/action.yml" grep -q 's3 cp "${tar_file}" "s3://${BUCKET}/${object}"' \ "$gha/actions/artifact-put/action.yml" grep -q 'AWS_MAX_ATTEMPTS' "$gha/actions/artifact-download/action.yml" From b1d86fb8b3cd6b6026d12c7c6e02710ec25dc86f Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Mon, 14 Sep 2026 19:59:52 +0300 Subject: [PATCH 21/24] Increase restore tree download attempts --- .github/actions/artifact-download/action.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml index 29b6f375fd..d4da3f8ff1 100644 --- a/.github/actions/artifact-download/action.yml +++ b/.github/actions/artifact-download/action.yml @@ -101,7 +101,10 @@ runs: stat -c %s "$1" fi } - attempts=3 + # 20.8 GiB Android tar; IncompleteRead after ~1–3 GiB + # (run 34870017073). Resume appended. 20 tries covers + # the 1.2 GiB worst chunk; remaining bytes are logged. + attempts=20 download_ok=0 for attempt in $(seq 1 "${attempts}"); do if [[ -f "${partial}" ]]; then @@ -114,11 +117,12 @@ runs: rm -f "${partial}" have=0 fi + remaining=$((expected_size - have)) echo "download attempt ${attempt}/${attempts}" if [[ "${have}" -lt "${expected_size}" ]]; then rm -f "${chunk}" if [[ "${have}" -eq 0 ]]; then - echo "download from 0" + echo "download from 0 remaining=${remaining}" set +e aws ${endpoint} "${aws_timeouts[@]}" s3api get-object \ --bucket "${BUCKET}" --key "${object}" \ @@ -126,7 +130,7 @@ runs: dl_rc=$? set -e else - echo "resume from byte ${have} / ${expected_size}" + echo "resume from byte ${have} / ${expected_size} remaining=${remaining}" set +e aws ${endpoint} "${aws_timeouts[@]}" s3api get-object \ --bucket "${BUCKET}" --key "${object}" \ @@ -150,12 +154,13 @@ runs: else have=0 fi + remaining=$((expected_size - have)) if [[ "${have}" -eq "${expected_size}" ]]; then mv "${partial}" "${tar_file}" download_ok=1 break fi - echo "size mismatch after attempt ${attempt}: local=${have} expected=${expected_size}" + echo "size mismatch after attempt ${attempt}: local=${have} expected=${expected_size} remaining=${remaining}" if [[ "${attempt}" -lt "${attempts}" ]]; then sleep 10 fi @@ -167,7 +172,11 @@ runs: elif [[ -f "${tar_file}" ]]; then actual_size="$(file_size "${tar_file}")" fi - echo "::error::failed to download s3://${BUCKET}/${object} after ${attempts} attempts (local=${actual_size} expected=${expected_size})" + remaining="n/a" + if [[ "${actual_size}" =~ ^[0-9]+$ ]]; then + remaining=$((expected_size - actual_size)) + fi + echo "::error::failed to download s3://${BUCKET}/${object} after ${attempts} attempts (local=${actual_size} expected=${expected_size} remaining=${remaining})" exit 1 fi echo "extracting ${tar_file} -> ${GITHUB_WORKSPACE}" From de1eaa12ca7c56f84e0feabe73769805fe1f1653 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Mon, 14 Sep 2026 20:25:58 +0300 Subject: [PATCH 22/24] Fix tree restoration --- .github/actions/artifact-download/action.yml | 39 +++++++++++++++++--- .github/actions/artifact-put/action.yml | 2 +- stream_build/AGENTS.md | 1 + 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml index d4da3f8ff1..7dcf9e10a7 100644 --- a/.github/actions/artifact-download/action.yml +++ b/.github/actions/artifact-download/action.yml @@ -96,7 +96,7 @@ runs: df -h "${GITHUB_WORKSPACE}" "${RUNNER_TEMP}" || true file_size() { if [[ "$(uname -s)" == Darwin ]]; then - stat -f %s "$1" + stat -f %z "$1" else stat -c %s "$1" fi @@ -118,11 +118,37 @@ runs: have=0 fi remaining=$((expected_size - have)) - echo "download attempt ${attempt}/${attempts}" + echo "download attempt ${attempt}/${attempts} resume_from=${have} remaining=${remaining}" if [[ "${have}" -lt "${expected_size}" ]]; then rm -f "${chunk}" + start="${have}" + ( + last="${start}" + stall_logged=0 + t0="$(date +%s)" + while sleep 15; do + have_now=0 + if [[ -f "${partial}" ]]; then + have_now="$(file_size "${partial}")" + fi + if [[ -f "${chunk}" ]]; then + have_now=$((have_now + $(file_size "${chunk}"))) + fi + elapsed=$(( $(date +%s) - t0 )) + pct=0 + if [[ "${expected_size}" -gt 0 ]]; then + pct=$((have_now * 100 / expected_size)) + fi + echo "download progress: ${have_now} / ${expected_size} (${pct}%) resume_from=${start} elapsed=${elapsed}s" + if [[ "${have_now}" -eq "${last}" && "${stall_logged}" -eq 0 ]]; then + echo "download progress: size not growing yet (aws may buffer until GET completes)" + stall_logged=1 + fi + last="${have_now}" + done + ) & + prog_pid=$! if [[ "${have}" -eq 0 ]]; then - echo "download from 0 remaining=${remaining}" set +e aws ${endpoint} "${aws_timeouts[@]}" s3api get-object \ --bucket "${BUCKET}" --key "${object}" \ @@ -130,7 +156,6 @@ runs: dl_rc=$? set -e else - echo "resume from byte ${have} / ${expected_size} remaining=${remaining}" set +e aws ${endpoint} "${aws_timeouts[@]}" s3api get-object \ --bucket "${BUCKET}" --key "${object}" \ @@ -145,6 +170,8 @@ runs: rm -f "${chunk}" fi fi + kill "${prog_pid}" 2>/dev/null || true + wait "${prog_pid}" 2>/dev/null || true if [[ "${dl_rc}" -ne 0 ]]; then echo "download attempt ${attempt} failed (exit ${dl_rc})" fi @@ -191,9 +218,11 @@ runs: "${GITHUB_WORKSPACE}/.hetzner-hit-bytes" # Checkout set src tracked-file mtimes to now. Bump restored # tree + out/ so ninja does not see objects as stale. + # -h: GNU/BSD no-dereference. Default GNU touch follows + # dangling NDK links (libc++.so*) and dies ENOENT (set -e). for p in src/third_party src/build src/buildtools \ src/testing src/tools src/ios src/resources out; do if [[ -d "${p}" ]]; then - find "${p}" -exec touch {} + + find "${p}" -exec touch -h {} + fi done diff --git a/.github/actions/artifact-put/action.yml b/.github/actions/artifact-put/action.yml index 7e79ce4717..b1967202ad 100644 --- a/.github/actions/artifact-put/action.yml +++ b/.github/actions/artifact-put/action.yml @@ -88,7 +88,7 @@ runs: echo "packing ${SOURCE}/{${members[*]}} -> ${tar_file}" tar cf "${tar_file}" -C "${SOURCE}" "${members[@]}" if [[ "$(uname -s)" == Darwin ]]; then - size="$(stat -f %s "${tar_file}")" + size="$(stat -f %z "${tar_file}")" else size="$(stat -c %s "${tar_file}")" fi diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index 3dea6970cd..d95f9aef24 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -110,6 +110,7 @@ platform of that job (`ios`, `mac`, `android,unix`). Hetzner: `artifact-download` Range-GETs with `s3api get-object --range bytes=${have}-` into a partial file and resumes from bytes already on disk (not `s3 cp` from 0). +While GET runs, logs `have / ContentLength (%)` every ~15s. `artifact-put` tars to a file then multipart `aws s3 cp` to `/artifacts//.tar` with `--endpoint-url https://hel1.your-objectstorage.com` and region From c48776cf723cb252b4ef22a38ffae2c90eaa3ac4 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Mon, 14 Sep 2026 22:00:55 +0300 Subject: [PATCH 23/24] Simplify caching --- .github/actions/artifact-download/action.yml | 11 ++---- .github/actions/artifact-put/action.yml | 36 ++++++++--------- .github/actions/restore-tree/action.yml | 23 ++--------- .github/workflows/_make.yml | 2 +- stream_build/AGENTS.md | 25 +++++++----- stream_build/scripts/check.sh | 41 +++++++++++++++----- stream_build/scripts/common.sh | 13 +++++-- 7 files changed, 81 insertions(+), 70 deletions(-) diff --git a/.github/actions/artifact-download/action.yml b/.github/actions/artifact-download/action.yml index 7dcf9e10a7..aa8220ca64 100644 --- a/.github/actions/artifact-download/action.yml +++ b/.github/actions/artifact-download/action.yml @@ -212,16 +212,11 @@ runs: test -d src/.git test -f src/DEPS test ! -L src - test -d src/third_party - test -n "$(ls -A src/third_party)" printf '%s\n' "${expected_size}" > \ "${GITHUB_WORKSPACE}/.hetzner-hit-bytes" - # Checkout set src tracked-file mtimes to now. Bump restored - # tree + out/ so ninja does not see objects as stale. - # -h: GNU/BSD no-dereference. Default GNU touch follows - # dangling NDK links (libc++.so*) and dies ENOENT (set -e). - for p in src/third_party src/build src/buildtools \ - src/testing src/tools src/ios src/resources out; do + # Overlay members only. Do not touch .gclient-git-cache. + # -h: GNU/BSD no-dereference if a leftover symlink exists. + for p in src/resources .cipd; do if [[ -d "${p}" ]]; then find "${p}" -exec touch -h {} + fi diff --git a/.github/actions/artifact-put/action.yml b/.github/actions/artifact-put/action.yml index b1967202ad..7757c70ba4 100644 --- a/.github/actions/artifact-put/action.yml +++ b/.github/actions/artifact-put/action.yml @@ -3,14 +3,14 @@ # hetzner_secret_access_key # hetzner_bucket name: Upload WebRTC build tarball to Hetzner -description: Pack an uncompressed tar of the host tree and multipart-put it. +description: Pack git-cache plus reusable src/resources and .cipd; multipart-put. inputs: path: description: Object stem. Uploads artifacts//.tar required: true source: - description: DEPS_ROOT with gclient checkouts, src/resources, and out/. + description: DEPS_ROOT with .gclient-git-cache; optional src/resources and .cipd. required: true hetzner_access_key: description: Hetzner object storage access key. @@ -60,27 +60,27 @@ runs: [[ -n "${OBJECT_STEM}" ]] [[ -n "${SOURCE}" ]] test -d "${SOURCE}" - test -d "${SOURCE}/src/third_party" - test -n "$(ls -A "${SOURCE}/src/third_party")" - test -d "${SOURCE}/out" - test -n "$(ls -A "${SOURCE}/out")" - if [[ ! -d "${SOURCE}/src/resources" ]] || \ - [[ -z "$(ls -A "${SOURCE}/src/resources")" ]]; then - echo "::error::build cache missing src/resources" + cache="${SOURCE}/.gclient-git-cache" + if [[ ! -d "${cache}" ]] || [[ -z "$(ls -A "${cache}")" ]]; then + echo "::error::build cache missing .gclient-git-cache" exit 1 fi - members=() - # Host tree + ninja. Do not pack .gclient-git-cache (objects - # twice) or src/.git / Stream-tracked src files. - for p in .gclient .gclient_entries \ - .gclient_previous_sync_commits .cipd \ - src/third_party src/build src/buildtools \ - src/testing src/tools src/ios src/resources out; do - if [[ -e "${SOURCE}/${p}" ]]; then + members=(.gclient-git-cache) + # Reusable only. Do not pack working trees or out/ (objects + # twice) or src/.git / Stream-tracked src files. setup-webrtc + # rewrites .gclient every job. + for p in src/resources .cipd; do + if [[ -d "${SOURCE}/${p}" ]] && \ + [[ -n "$(ls -A "${SOURCE}/${p}")" ]]; then members+=("${p}") + else + echo "skip ${p} (missing or empty)" fi done - [[ ${#members[@]} -gt 0 ]] + echo "pack members: ${members[*]}" + for p in "${members[@]}"; do + du -sk "${SOURCE}/${p}" + done BUCKET="${HETZNER_BUCKET_CI_ARTIFACTS:?}" object="artifacts/${{ github.repository }}/${OBJECT_STEM}.tar" tar_file="${RUNNER_TEMP:?}/${OBJECT_STEM}.tar" diff --git a/.github/actions/restore-tree/action.yml b/.github/actions/restore-tree/action.yml index 6962565c91..775d02490b 100644 --- a/.github/actions/restore-tree/action.yml +++ b/.github/actions/restore-tree/action.yml @@ -46,8 +46,10 @@ runs: webrtc_ref: ${{ inputs.webrtc_ref }} install_android_packages: ${{ inputs.install_android_packages }} - # Checkout src first (caller). Extract overlays gclient dirs + out/ - # without replacing src/.git. Missing object = cold make deps. + # Checkout src first (caller). Extract git-cache to GIT_CACHE_PATH + # plus src/resources and .cipd. Do not strip git alternates; + # deps.sh rewrite_git_cache_alternates retargets packed cache + # paths. Missing object = cold make deps. - name: Download host build cache if: ${{ inputs.cache_key != '' && inputs.skip_cache != 'true' }} uses: ./src/.github/actions/artifact-download @@ -58,23 +60,6 @@ runs: hetzner_secret_access_key: ${{ inputs.hetzner_secret_access_key }} hetzner_bucket: ${{ inputs.hetzner_bucket }} - - name: Drop git-cache alternates - if: ${{ inputs.cache_key != '' }} - shell: bash - run: | - set -euo pipefail - # Tree is packed without .gclient-git-cache. Nested checkouts - # may still point alternates at a missing cache path. - for root in src/third_party src/build src/buildtools \ - src/testing src/tools src/ios; do - if [[ -d "${GITHUB_WORKSPACE}/${root}" ]]; then - find "${GITHUB_WORKSPACE}/${root}" \ - \( -path '*/.git/objects/info/alternates' -o \ - -path '*/.git/objects/info/http-alternates' \) \ - -type f -delete - fi - done - - name: gclient sync and hooks if: ${{ inputs.cache_key != '' }} working-directory: src/stream_build diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 381ccf3be4..29e631167b 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -4,7 +4,7 @@ # # I/O: iOS / macOS / Android jobs HIT Hetzner build-{ios,macos,android}.tar # (skip_deps_cache skips download), make deps, make build|test, then put -# that host's gclient tree + out/. No shared Linux Deps job. No GitHub +# git-cache + src/resources + .cipd. No shared Linux Deps job. No GitHub # deps-key. Windows Deps stays deps-windows on GitHub. Build mode is # ninja only (no products). Package/Release Build jobs also make package # and upload products-* (GitHub). Package combine consumes products-* diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index d95f9aef24..fe88bf3730 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -87,18 +87,23 @@ and Android jobs run in parallel after Plan: skip`; Build dispatch `skip_deps_cache` skips the download) 3. `make deps` (`RUN_HOOKS=1`, host GCS rust-toolchain on Apple) 4. `make build` / `make test` (`SKIP_DEPS=1`) -5. `artifact-put` that host's tree + `out/` (always after miss / - `skip_deps_cache`; skip PUT when HIT size delta is < 1GiB) +5. `artifact-put` `.gclient-git-cache` plus reusable `src/resources` + and `.cipd` if non-empty (always after miss / `skip_deps_cache`; + skip PUT when HIT size delta is < 1GiB) Keys: `artifacts//build-{ios,macos,android}.tar`. -Same-OS only (Linux tree on Mac is forbidden). Members: `.gclient`, -`.gclient_entries`, `.gclient_previous_sync_commits`, `.cipd` if -present, `src/{third_party,build,buildtools,testing,tools,ios,resources}`, -`out/`. Not packed: `.gclient-git-cache`, `src/.git`, Stream-tracked -`src` files, `products/`. Restore order: checkout `src` first, extract -over gclient dirs + `out/` (not `src/.git`), `touch` those members so -ninja does not see objects older than the fresh checkout. Build -`CONFIG` is dispatch (default release); `make test` always uses debug +Same-OS only (Linux tree on Mac is forbidden). Members: required +`.gclient-git-cache`; `src/resources` and `.cipd` if non-empty. +Not packed: working trees (`src/third_party`, `src/build`, +`src/buildtools`, `src/testing`, `src/tools`, `src/ios`), `out/`, +`.gclient` / `.gclient_entries` (setup-webrtc writes `.gclient` every +job), `src/.git`, Stream-tracked `src` files, `products/`. Restore: +checkout `src`, extract git-cache to `GIT_CACHE_PATH` +(`${{ github.workspace }}/.gclient-git-cache`), `src/resources`, and +`.cipd` if present. Do not strip git alternates; `make deps` runs +`rewrite_git_cache_alternates` so Linux-packed cache paths retarget +to this runner. Always `make deps` after HIT (cheap from cache). +Build `CONFIG` is dispatch (default release); `make test` always uses debug in `out/ios_tests` / `out/webrtc_tests`, so those subdirs do not mix with slice dirs. Test.yml HITs the same `build-ios` / `build-macos` keys. Windows Deps still uploads `deps-windows` to GitHub. diff --git a/stream_build/scripts/check.sh b/stream_build/scripts/check.sh index 5bebe21f3f..cc93b6746e 100755 --- a/stream_build/scripts/check.sh +++ b/stream_build/scripts/check.sh @@ -329,6 +329,19 @@ grep -qx "$deps_tmp/webrtc/.gclient-git-cache/fake-repo/objects" \ "$deps_tmp/webrtc/src/third_party/.git/objects/info/alternates" ! grep -q '.gclient_deps/.gclient-git-cache' \ "$deps_tmp/webrtc/src/third_party/.git/objects/info/alternates" +printf '%s\n' \ + "/home/runner/work/webrtc/webrtc/.gclient-git-cache/fake-repo/objects" \ + > "$deps_tmp/webrtc/src/third_party/.git/objects/info/alternates" +PATH="$fake_bin:$PATH" \ + DEPS_ROOT="$deps_tmp/webrtc" \ + WEBRTC_SRC="$deps_tmp/webrtc/src" \ + GIT_CACHE_PATH="$deps_tmp/webrtc/.gclient-git-cache" \ + RUN_HOOKS=0 JOBS=2 \ + "$ROOT/scripts/deps.sh" sync >/dev/null +grep -qx "$deps_tmp/webrtc/.gclient-git-cache/fake-repo/objects" \ + "$deps_tmp/webrtc/src/third_party/.git/objects/info/alternates" +! grep -q '/home/runner/work' \ + "$deps_tmp/webrtc/src/third_party/.git/objects/info/alternates" PATH="$fake_bin:$PATH" \ DEPS_ROOT="$deps_tmp/webrtc" \ WEBRTC_SRC="$deps_tmp/webrtc/src" \ @@ -401,8 +414,7 @@ grep -q 'tar cf "${tar_file}"' "$gha/actions/artifact-put/action.yml" "$gha/actions/artifact-download/action.yml" grep -q 's3api get-object' "$gha/actions/artifact-download/action.yml" grep -q 'bytes=${have}-' "$gha/actions/artifact-download/action.yml" -grep -q 'resume from byte' "$gha/actions/artifact-download/action.yml" -grep -q 'download from 0' "$gha/actions/artifact-download/action.yml" +grep -q 'resume_from=' "$gha/actions/artifact-download/action.yml" grep -q 's3 cp "${tar_file}" "s3://${BUCKET}/${object}"' \ "$gha/actions/artifact-put/action.yml" grep -q 'AWS_MAX_ATTEMPTS' "$gha/actions/artifact-download/action.yml" @@ -439,14 +451,23 @@ grep -q 'if_missing: skip' "$gha/actions/restore-tree/action.yml" grep -q 'SHALLOW: "1"' "$gha/actions/restore-tree/action.yml" grep -q 'RUN_HOOKS: "1"' "$gha/actions/restore-tree/action.yml" ! grep -q 'RUN_HOOKS: "0"' "$gha/workflows/_make.yml" -grep -q 'src/third_party' "$gha/actions/artifact-download/action.yml" -grep -q 'src/third_party' "$gha/actions/artifact-put/action.yml" -grep -q 'src/third_party' "$gha/actions/restore-tree/action.yml" -grep -A8 'for p in .gclient' "$gha/actions/artifact-put/action.yml" | \ - grep -q 'src/third_party' -grep -A8 'for p in .gclient' "$gha/actions/artifact-put/action.yml" | grep -q ' out' -! grep -A8 'for p in .gclient' "$gha/actions/artifact-put/action.yml" | \ - grep -q 'gclient-git-cache' +! grep -q 'Drop git-cache alternates' "$gha/actions/restore-tree/action.yml" +grep -q 'GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache' \ + "$gha/actions/restore-tree/action.yml" +grep -q 'rewrite_git_cache_alternates' "$ROOT/scripts/deps.sh" +grep -q '.gclient-git-cache' "$gha/actions/artifact-put/action.yml" +grep -q 'src/resources' "$gha/actions/artifact-put/action.yml" +grep -q '.cipd' "$gha/actions/artifact-put/action.yml" +! grep -q 'src/third_party' "$gha/actions/artifact-put/action.yml" +! grep -q 'src/third_party' "$gha/actions/artifact-download/action.yml" +! grep -q 'src/third_party' "$gha/actions/restore-tree/action.yml" +grep -A12 'members=(.gclient-git-cache)' "$gha/actions/artifact-put/action.yml" | \ + grep -q 'src/resources' +grep -A12 'members=(.gclient-git-cache)' "$gha/actions/artifact-put/action.yml" | \ + grep -q '.cipd' +! grep -A12 'members=(.gclient-git-cache)' "$gha/actions/artifact-put/action.yml" | \ + grep -q 'third_party' +grep -q 'pack members:' "$gha/actions/artifact-put/action.yml" grep -q 'uses: ./src/.github/actions/artifact-put' "$gha/workflows/_make.yml" echo "ok" diff --git a/stream_build/scripts/common.sh b/stream_build/scripts/common.sh index c3d506214a..4e3ffaebff 100755 --- a/stream_build/scripts/common.sh +++ b/stream_build/scripts/common.sh @@ -75,8 +75,9 @@ target_os = $(quote_target_os "$target_os") EOF } -# After git-cache moves off .gclient_deps/.gclient-git-cache, nested checkouts -# still point objects/info/alternates (and origin urls) at the old path. +# After git-cache moves, nested checkouts may still point +# objects/info/alternates (and origin urls) at an old host path +# (.gclient_deps/.gclient-git-cache or /home/runner/.../.gclient-git-cache). # Rewrite those to GIT_CACHE_PATH. No-op if the cache itself is the old path. rewrite_git_cache_alternates() { local src="$1" @@ -87,8 +88,9 @@ import subprocess import sys src, new_cache = sys.argv[1], sys.argv[2].rstrip("/") -frag = ".gclient_deps/.gclient-git-cache" -if new_cache.endswith(frag): +old_layout = ".gclient_deps/.gclient-git-cache" +frag = ".gclient-git-cache" +if new_cache.endswith(old_layout): raise SystemExit(0) cmd = [ @@ -131,6 +133,9 @@ def rewrite_line(line: str) -> str: start = idx while start > 0 and line[start - 1] not in stops: start -= 1 + old_path = line[start : idx + len(frag)].rstrip("/") + if old_path == new_cache: + return line return line[:start] + new_cache + line[idx + len(frag) :] From 78a418db0051f027c95d563fa43b2e40efeae15f Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Tue, 15 Sep 2026 10:58:21 +0300 Subject: [PATCH 24/24] Align with the v1 flows --- .github/actions/prepare-android/action.yml | 256 ++++++++ .github/actions/prepare-apple/action.yml | 250 +++++++ .github/actions/prepare-common-v2/action.yml | 121 ++++ .github/actions/prepare-common/action.yml | 86 +-- .github/workflows/_make.yml | 16 +- .github/workflows/build-v2.yml | 68 ++ .github/workflows/manual-platform-tests.yml | 557 ++++++++++++++-- .github/workflows/package-v2.yml | 62 ++ .github/workflows/publish.yml | 616 ++++++++++++++++-- .../workflows/{release.yml => release-v2.yml} | 13 +- .github/workflows/{test.yml => test-v2.yml} | 11 +- stream_build/AGENTS.md | 29 +- 12 files changed, 1913 insertions(+), 172 deletions(-) create mode 100644 .github/actions/prepare-android/action.yml create mode 100644 .github/actions/prepare-apple/action.yml create mode 100644 .github/actions/prepare-common-v2/action.yml create mode 100644 .github/workflows/build-v2.yml create mode 100644 .github/workflows/package-v2.yml rename .github/workflows/{release.yml => release-v2.yml} (67%) rename .github/workflows/{test.yml => test-v2.yml} (62%) diff --git a/.github/actions/prepare-android/action.yml b/.github/actions/prepare-android/action.yml new file mode 100644 index 0000000000..7a7f932b2c --- /dev/null +++ b/.github/actions/prepare-android/action.yml @@ -0,0 +1,256 @@ +name: Prepare Android dependencies +description: Prepare a Linux-host WebRTC checkout for Android validation jobs. + +inputs: + webrtc_ref: + required: true + description: Branch, tag, or SHA for GetStream/webrtc. + target_os: + required: true + description: Comma-separated gclient target_os values. + target_os_cache_key: + required: true + description: Cache-safe target_os key suffix. + ignore_cache: + required: true + description: Whether to skip cache restore. + release_pipeline_token: + required: true + description: Token used to check out the release pipeline. + +runs: + using: composite + steps: + - name: Preserve local action metadata + shell: bash + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/local-actions" + cp -R "${GITHUB_WORKSPACE}/.github/actions/prepare-android" \ + "${RUNNER_TEMP}/local-actions/prepare-android" + + - name: Check out this repository (WebRTC tree) + uses: actions/checkout@v4 + with: + ref: ${{ inputs.webrtc_ref }} + + - name: Restore local action metadata + shell: bash + run: | + set -euo pipefail + mkdir -p "${GITHUB_WORKSPACE}/.github/actions" + rm -rf "${GITHUB_WORKSPACE}/.github/actions/prepare-android" + cp -R "${RUNNER_TEMP}/local-actions/prepare-android" \ + "${GITHUB_WORKSPACE}/.github/actions/prepare-android" + + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ inputs.release_pipeline_token }} + + - name: Install system dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y python3-pip openjdk-8-jdk lsb-release software-properties-common + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Git authentication for HTTPS fetches + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + git config --global credential.helper store + git credential approve < "${GITHUB_WORKSPACE}/.github-git-askpass" <<'EOF' + #!/usr/bin/env bash + case "$1" in + *Username*) echo "x-access-token" ;; + *Password*) echo "${GITHUB_TOKEN}" ;; + *) echo "" ;; + esac + EOF + chmod +x "${GITHUB_WORKSPACE}/.github-git-askpass" + + - name: Restore gclient source Git cache + if: ${{ inputs.ignore_cache != 'true' }} + id: gclient-cache + uses: actions/cache/restore@v4 + with: + path: | + ${{ github.workspace }}/.output/src/.git + ${{ github.workspace }}/.gclient-git-cache + key: webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}- + webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}- + webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}- + + - name: Materialize cached WebRTC source checkout + if: ${{ inputs.ignore_cache != 'true' }} + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass + GIT_TERMINAL_PROMPT: "0" + GIT_CONFIG_COUNT: "2" + GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_0: "git@github.com:" + GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_1: "ssh://git@github.com/" + run: | + set -euo pipefail + src="${GITHUB_WORKSPACE}/.output/src" + if [[ ! -d "${src}/.git" ]]; then + echo "No cached Git checkout found at ${src}/.git; gclient sync will clone it." + exit 0 + fi + + alternates="${src}/.git/objects/info/alternates" + if [[ -f "${alternates}" ]]; then + while IFS= read -r alternate; do + [[ -z "${alternate}" ]] && continue + if [[ ! -d "${alternate}" ]]; then + echo "Discarding cached Git checkout with missing alternate object store: ${alternate}" + rm -rf "${src}/.git" + exit 0 + fi + done < "${alternates}" + fi + + if ! git -C "${src}" remote set-url origin https://github.com/GetStream/webrtc.git || + ! git -C "${src}" fetch --prune origin "${{ inputs.webrtc_ref }}" || + ! git -C "${src}" checkout --force FETCH_HEAD || + ! git -C "${src}" reset --hard FETCH_HEAD || + ! git -C "${src}" clean -ffdx; then + echo "Discarding unusable cached Git checkout; gclient sync will recreate it." + rm -rf "${src}/.git" + fi + + - name: Install depot_tools + shell: bash + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: gclient sync (deps lane) + if: ${{ inputs.ignore_cache != 'true' }} + shell: bash + working-directory: release-pipeline + env: + GITHUB_TOKEN: ${{ github.token }} + GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass + GIT_TERMINAL_PROMPT: "0" + GIT_CONFIG_COUNT: "2" + GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_0: "git@github.com:" + GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_1: "ssh://git@github.com/" + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache + run: >- + bundle exec fastlane deps sync + "root:${{ github.workspace }}" + "target_os:${{ inputs.target_os }}" + number_of_jobs:2 + run_hooks:false + + - name: gclient sync without cache (deps lane) + if: ${{ inputs.ignore_cache == 'true' }} + shell: bash + working-directory: release-pipeline + env: + GITHUB_TOKEN: ${{ github.token }} + GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass + GIT_TERMINAL_PROMPT: "0" + GIT_CONFIG_COUNT: "2" + GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_0: "git@github.com:" + GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_1: "ssh://git@github.com/" + run: >- + bundle exec fastlane deps sync + "root:${{ github.workspace }}" + "target_os:${{ inputs.target_os }}" + number_of_jobs:2 + run_hooks:false + + - name: Restore gclient hook download cache + if: ${{ inputs.ignore_cache != 'true' }} + id: gclient-hook-cache + uses: actions/cache/restore@v4 + with: + path: | + ${{ github.workspace }}/.output/src/resources + ${{ github.workspace }}/.output/src/tools/clang/dsymutil/bin + key: webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}- + webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}- + + - name: gclient runhooks (deps lane) + shell: bash + working-directory: release-pipeline + run: >- + bundle exec fastlane deps runhooks + "root:${{ github.workspace }}" + + - name: Save gclient hook download cache + if: ${{ steps.gclient-hook-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: | + ${{ github.workspace }}/.output/src/resources + ${{ github.workspace }}/.output/src/tools/clang/dsymutil/bin + key: webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Save gclient source Git cache + if: ${{ steps.gclient-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: | + ${{ github.workspace }}/.output/src/.git + ${{ github.workspace }}/.gclient-git-cache + key: webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Archive prepared WebRTC tree + shell: bash + run: | + set -euo pipefail + archive="${RUNNER_TEMP}/webrtc-src-prepared-android.tar.gz" + tar -czf "${archive}" \ + -C "${GITHUB_WORKSPACE}" \ + --exclude='.output/src/.git' \ + --exclude='.output/src/.git/**' \ + .output + mv "${archive}" "${GITHUB_WORKSPACE}/webrtc-src-prepared-android.tar.gz" + + - name: Upload dependency artifact + uses: actions/upload-artifact@v4 + with: + name: webrtc-src-prepared-android + path: webrtc-src-prepared-android.tar.gz + retention-days: 2 + compression-level: 0 diff --git a/.github/actions/prepare-apple/action.yml b/.github/actions/prepare-apple/action.yml new file mode 100644 index 0000000000..e23a2cf231 --- /dev/null +++ b/.github/actions/prepare-apple/action.yml @@ -0,0 +1,250 @@ +name: Prepare Apple dependencies +description: Prepare a macOS-host WebRTC checkout for iOS and macOS validation jobs. + +inputs: + webrtc_ref: + required: true + description: Branch, tag, or SHA for GetStream/webrtc. + target_os: + required: true + description: Comma-separated gclient target_os values. + target_os_cache_key: + required: true + description: Cache-safe target_os key suffix. + ignore_cache: + required: true + description: Whether to skip cache restore. + release_pipeline_token: + required: true + description: Token used to check out the release pipeline. + +runs: + using: composite + steps: + - name: Preserve local action metadata + shell: bash + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/local-actions" + cp -R "${GITHUB_WORKSPACE}/.github/actions/prepare-apple" \ + "${RUNNER_TEMP}/local-actions/prepare-apple" + + - name: Check out this repository (WebRTC tree) + uses: actions/checkout@v4 + with: + ref: ${{ inputs.webrtc_ref }} + + - name: Restore local action metadata + shell: bash + run: | + set -euo pipefail + mkdir -p "${GITHUB_WORKSPACE}/.github/actions" + rm -rf "${GITHUB_WORKSPACE}/.github/actions/prepare-apple" + cp -R "${RUNNER_TEMP}/local-actions/prepare-apple" \ + "${GITHUB_WORKSPACE}/.github/actions/prepare-apple" + + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ inputs.release_pipeline_token }} + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Git authentication for HTTPS fetches + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + git config --global credential.helper store + git credential approve < "${GITHUB_WORKSPACE}/.github-git-askpass" <<'EOF' + #!/usr/bin/env bash + case "$1" in + *Username*) echo "x-access-token" ;; + *Password*) echo "${GITHUB_TOKEN}" ;; + *) echo "" ;; + esac + EOF + chmod +x "${GITHUB_WORKSPACE}/.github-git-askpass" + + - name: Restore gclient source Git cache + if: ${{ inputs.ignore_cache != 'true' }} + id: gclient-cache + uses: actions/cache/restore@v4 + with: + path: | + ${{ github.workspace }}/.output/src/.git + ${{ github.workspace }}/.gclient-git-cache + key: webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}- + webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}- + webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}- + + - name: Materialize cached WebRTC source checkout + if: ${{ inputs.ignore_cache != 'true' }} + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass + GIT_TERMINAL_PROMPT: "0" + GIT_CONFIG_COUNT: "2" + GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_0: "git@github.com:" + GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_1: "ssh://git@github.com/" + run: | + set -euo pipefail + src="${GITHUB_WORKSPACE}/.output/src" + if [[ ! -d "${src}/.git" ]]; then + echo "No cached Git checkout found at ${src}/.git; gclient sync will clone it." + exit 0 + fi + + alternates="${src}/.git/objects/info/alternates" + if [[ -f "${alternates}" ]]; then + while IFS= read -r alternate; do + [[ -z "${alternate}" ]] && continue + if [[ ! -d "${alternate}" ]]; then + echo "Discarding cached Git checkout with missing alternate object store: ${alternate}" + rm -rf "${src}/.git" + exit 0 + fi + done < "${alternates}" + fi + + if ! git -C "${src}" remote set-url origin https://github.com/GetStream/webrtc.git || + ! git -C "${src}" fetch --prune origin "${{ inputs.webrtc_ref }}" || + ! git -C "${src}" checkout --force FETCH_HEAD || + ! git -C "${src}" reset --hard FETCH_HEAD || + ! git -C "${src}" clean -ffdx; then + echo "Discarding unusable cached Git checkout; gclient sync will recreate it." + rm -rf "${src}/.git" + fi + + - name: Install depot_tools + shell: bash + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: gclient sync (deps lane) + if: ${{ inputs.ignore_cache != 'true' }} + shell: bash + working-directory: release-pipeline + env: + GITHUB_TOKEN: ${{ github.token }} + GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass + GIT_TERMINAL_PROMPT: "0" + GIT_CONFIG_COUNT: "2" + GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_0: "git@github.com:" + GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_1: "ssh://git@github.com/" + GIT_CACHE_PATH: ${{ github.workspace }}/.gclient-git-cache + run: >- + bundle exec fastlane deps sync + "root:${{ github.workspace }}" + "target_os:${{ inputs.target_os }}" + number_of_jobs:2 + run_hooks:false + + - name: gclient sync without cache (deps lane) + if: ${{ inputs.ignore_cache == 'true' }} + shell: bash + working-directory: release-pipeline + env: + GITHUB_TOKEN: ${{ github.token }} + GIT_ASKPASS: ${{ github.workspace }}/.github-git-askpass + GIT_TERMINAL_PROMPT: "0" + GIT_CONFIG_COUNT: "2" + GIT_CONFIG_KEY_0: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_0: "git@github.com:" + GIT_CONFIG_KEY_1: url.https://github.com/.insteadOf + GIT_CONFIG_VALUE_1: "ssh://git@github.com/" + run: >- + bundle exec fastlane deps sync + "root:${{ github.workspace }}" + "target_os:${{ inputs.target_os }}" + number_of_jobs:2 + run_hooks:false + + - name: Restore gclient hook download cache + if: ${{ inputs.ignore_cache != 'true' }} + id: gclient-hook-cache + uses: actions/cache/restore@v4 + with: + path: | + ${{ github.workspace }}/.output/src/resources + ${{ github.workspace }}/.output/src/tools/clang/dsymutil/bin + key: webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}- + webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}- + + - name: gclient runhooks (deps lane) + shell: bash + working-directory: release-pipeline + run: >- + bundle exec fastlane deps runhooks + "root:${{ github.workspace }}" + + - name: Save gclient hook download cache + if: ${{ steps.gclient-hook-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: | + ${{ github.workspace }}/.output/src/resources + ${{ github.workspace }}/.output/src/tools/clang/dsymutil/bin + key: webrtc-gclient-hook-downloads-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Save gclient source Git cache + if: ${{ steps.gclient-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: | + ${{ github.workspace }}/.output/src/.git + ${{ github.workspace }}/.gclient-git-cache + key: webrtc-gclient-src-git-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.target_os_cache_key }}-${{ hashFiles('DEPS') }}-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Archive prepared WebRTC tree + shell: bash + run: | + set -euo pipefail + archive="${RUNNER_TEMP}/webrtc-src-prepared-apple.tar.gz" + COPYFILE_DISABLE=1 tar -czf "${archive}" \ + -C "${GITHUB_WORKSPACE}" \ + --exclude='.output/src/.git' \ + --exclude='.output/src/.git/**' \ + .output + mv "${archive}" "${GITHUB_WORKSPACE}/webrtc-src-prepared-apple.tar.gz" + + - name: Upload dependency artifact + uses: actions/upload-artifact@v4 + with: + name: webrtc-src-prepared-apple + path: webrtc-src-prepared-apple.tar.gz + retention-days: 2 + compression-level: 0 diff --git a/.github/actions/prepare-common-v2/action.yml b/.github/actions/prepare-common-v2/action.yml new file mode 100644 index 0000000000..73185153f9 --- /dev/null +++ b/.github/actions/prepare-common-v2/action.yml @@ -0,0 +1,121 @@ +name: Plan platform deps +description: Map selected platforms to gclient target_os. + +inputs: + platform_ios: + required: true + description: Whether iOS is selected. + platform_macos: + required: true + description: Whether macOS is selected. + platform_android: + required: true + description: Whether Android is selected. + platform_windows: + required: false + default: "false" + description: Whether Windows is selected. + skip_maccatalyst: + required: false + default: "false" + description: Omit Mac Catalyst from the Apple deps label. + +outputs: + run_ios: + value: ${{ steps.flags.outputs.run_ios }} + run_macos: + value: ${{ steps.flags.outputs.run_macos }} + run_android: + value: ${{ steps.flags.outputs.run_android }} + run_windows: + value: ${{ steps.flags.outputs.run_windows }} + run_apple: + value: ${{ steps.flags.outputs.run_apple }} + apple_target_os: + value: ${{ steps.flags.outputs.apple_target_os }} + apple_target_os_label: + value: ${{ steps.flags.outputs.apple_target_os_label }} + android_target_os: + value: ${{ steps.flags.outputs.android_target_os }} + windows_target_os: + value: ${{ steps.flags.outputs.windows_target_os }} + linux_target_os: + value: ${{ steps.flags.outputs.linux_target_os }} + +runs: + using: composite + steps: + - id: flags + name: Resolve platform flags + shell: bash + run: | + set -euo pipefail + ios='${{ inputs.platform_ios }}' + macos='${{ inputs.platform_macos }}' + android='${{ inputs.platform_android }}' + windows='${{ inputs.platform_windows }}' + skip_maccatalyst='${{ inputs.skip_maccatalyst }}' + + # TARGET_OS is only tokens selected this run (ios, mac, android,unix, win). + apple_os_list=() + apple_label_list=() + linux_os_list=() + if [[ "${ios}" == "true" ]]; then + apple_os_list+=("ios") + apple_label_list+=("ios") + linux_os_list+=("ios") + fi + if [[ "${macos}" == "true" ]]; then + apple_os_list+=("mac") + apple_label_list+=("macos") + linux_os_list+=("mac") + fi + if [[ "${ios}" == "true" && "${skip_maccatalyst}" != "true" ]]; then + apple_label_list+=("maccatalyst") + fi + if [[ "${android}" == "true" ]]; then + linux_os_list+=("android" "unix") + fi + + join_csv() { + local IFS=, + printf '%s' "$*" + } + + run_apple=false + apple_target_os="" + apple_target_os_label="" + if [[ ${#apple_os_list[@]} -gt 0 ]]; then + run_apple=true + apple_target_os="$(join_csv "${apple_os_list[@]}")" + old_ifs="$IFS" + IFS=', ' + apple_target_os_label="${apple_label_list[*]}" + IFS="$old_ifs" + fi + + linux_target_os="" + if [[ ${#linux_os_list[@]} -gt 0 ]]; then + linux_target_os="$(join_csv "${linux_os_list[@]}")" + fi + + android_target_os="" + if [[ "${android}" == "true" ]]; then + android_target_os="android,unix" + fi + + windows_target_os="" + if [[ "${windows}" == "true" ]]; then + windows_target_os="win" + fi + + echo "run_ios=${ios}" >> "${GITHUB_OUTPUT}" + echo "run_macos=${macos}" >> "${GITHUB_OUTPUT}" + echo "run_android=${android}" >> "${GITHUB_OUTPUT}" + echo "run_windows=${windows}" >> "${GITHUB_OUTPUT}" + echo "run_apple=${run_apple}" >> "${GITHUB_OUTPUT}" + echo "apple_target_os=${apple_target_os}" >> "${GITHUB_OUTPUT}" + echo "apple_target_os_label=${apple_target_os_label}" >> "${GITHUB_OUTPUT}" + echo "android_target_os=${android_target_os}" >> "${GITHUB_OUTPUT}" + echo "windows_target_os=${windows_target_os}" >> "${GITHUB_OUTPUT}" + echo "linux_target_os=${linux_target_os}" >> "${GITHUB_OUTPUT}" diff --git a/.github/actions/prepare-common/action.yml b/.github/actions/prepare-common/action.yml index 73185153f9..a441753034 100644 --- a/.github/actions/prepare-common/action.yml +++ b/.github/actions/prepare-common/action.yml @@ -1,24 +1,16 @@ -name: Plan platform deps -description: Map selected platforms to gclient target_os. +name: Plan platform validation +description: Resolve selected platform flags into host-specific prepare plans. inputs: platform_ios: required: true - description: Whether iOS is selected. + description: Whether iOS validation is enabled. platform_macos: required: true - description: Whether macOS is selected. + description: Whether macOS validation is enabled. platform_android: required: true - description: Whether Android is selected. - platform_windows: - required: false - default: "false" - description: Whether Windows is selected. - skip_maccatalyst: - required: false - default: "false" - description: Omit Mac Catalyst from the Apple deps label. + description: Whether Android validation is enabled. outputs: run_ios: @@ -27,20 +19,16 @@ outputs: value: ${{ steps.flags.outputs.run_macos }} run_android: value: ${{ steps.flags.outputs.run_android }} - run_windows: - value: ${{ steps.flags.outputs.run_windows }} run_apple: value: ${{ steps.flags.outputs.run_apple }} apple_target_os: value: ${{ steps.flags.outputs.apple_target_os }} - apple_target_os_label: - value: ${{ steps.flags.outputs.apple_target_os_label }} + apple_target_os_cache_key: + value: ${{ steps.flags.outputs.apple_target_os_cache_key }} android_target_os: value: ${{ steps.flags.outputs.android_target_os }} - windows_target_os: - value: ${{ steps.flags.outputs.windows_target_os }} - linux_target_os: - value: ${{ steps.flags.outputs.linux_target_os }} + android_target_os_cache_key: + value: ${{ steps.flags.outputs.android_target_os_cache_key }} runs: using: composite @@ -53,50 +41,21 @@ runs: ios='${{ inputs.platform_ios }}' macos='${{ inputs.platform_macos }}' android='${{ inputs.platform_android }}' - windows='${{ inputs.platform_windows }}' - skip_maccatalyst='${{ inputs.skip_maccatalyst }}' - - # TARGET_OS is only tokens selected this run (ios, mac, android,unix, win). - apple_os_list=() - apple_label_list=() - linux_os_list=() - if [[ "${ios}" == "true" ]]; then - apple_os_list+=("ios") - apple_label_list+=("ios") - linux_os_list+=("ios") - fi - if [[ "${macos}" == "true" ]]; then - apple_os_list+=("mac") - apple_label_list+=("macos") - linux_os_list+=("mac") - fi - if [[ "${ios}" == "true" && "${skip_maccatalyst}" != "true" ]]; then - apple_label_list+=("maccatalyst") - fi - if [[ "${android}" == "true" ]]; then - linux_os_list+=("android" "unix") + if [[ "${ios}" != "true" && "${macos}" != "true" && "${android}" != "true" ]]; then + echo "Select at least one platform (enable one or more checkboxes above)." + exit 1 fi - join_csv() { - local IFS=, - printf '%s' "$*" - } + apple_os_list=() + if [[ "${ios}" == "true" ]]; then apple_os_list+=("ios"); fi + if [[ "${macos}" == "true" ]]; then apple_os_list+=("mac"); fi run_apple=false apple_target_os="" - apple_target_os_label="" if [[ ${#apple_os_list[@]} -gt 0 ]]; then run_apple=true - apple_target_os="$(join_csv "${apple_os_list[@]}")" - old_ifs="$IFS" - IFS=', ' - apple_target_os_label="${apple_label_list[*]}" - IFS="$old_ifs" - fi - - linux_target_os="" - if [[ ${#linux_os_list[@]} -gt 0 ]]; then - linux_target_os="$(join_csv "${linux_os_list[@]}")" + IFS=',' + apple_target_os="${apple_os_list[*]}" fi android_target_os="" @@ -104,18 +63,11 @@ runs: android_target_os="android,unix" fi - windows_target_os="" - if [[ "${windows}" == "true" ]]; then - windows_target_os="win" - fi - echo "run_ios=${ios}" >> "${GITHUB_OUTPUT}" echo "run_macos=${macos}" >> "${GITHUB_OUTPUT}" echo "run_android=${android}" >> "${GITHUB_OUTPUT}" - echo "run_windows=${windows}" >> "${GITHUB_OUTPUT}" echo "run_apple=${run_apple}" >> "${GITHUB_OUTPUT}" echo "apple_target_os=${apple_target_os}" >> "${GITHUB_OUTPUT}" - echo "apple_target_os_label=${apple_target_os_label}" >> "${GITHUB_OUTPUT}" + echo "apple_target_os_cache_key=${apple_target_os//,/-}" >> "${GITHUB_OUTPUT}" echo "android_target_os=${android_target_os}" >> "${GITHUB_OUTPUT}" - echo "windows_target_os=${windows_target_os}" >> "${GITHUB_OUTPUT}" - echo "linux_target_os=${linux_target_os}" >> "${GITHUB_OUTPUT}" + echo "android_target_os_cache_key=${android_target_os//,/-}" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/_make.yml b/.github/workflows/_make.yml index 29e631167b..c9fe6f309d 100644 --- a/.github/workflows/_make.yml +++ b/.github/workflows/_make.yml @@ -1,4 +1,4 @@ -# Called by Build / Test / Package / Release dispatch workflows. +# Called by Build v2 / Test v2 / Package v2 / Release v2 dispatch workflows. # Not listed in the Actions dispatch UI (workflow_call only). # Callers pass mode: build | test | package | release. # @@ -11,7 +11,7 @@ # (no third ninja) and uploads final-*. Release attaches final-*. Tests # HIT the same per-platform key (out/ios_tests vs slice dirs). -name: WebRTC make +name: WebRTC make v2 on: workflow_call: @@ -136,7 +136,7 @@ jobs: path: src - id: plan name: Plan target_os - uses: ./src/.github/actions/prepare-common + uses: ./src/.github/actions/prepare-common-v2 with: platform_ios: ${{ inputs.ios }} platform_macos: ${{ inputs.macos }} @@ -191,7 +191,7 @@ jobs: compression-level: 0 build_ios: - name: Build iOS + name: Build iOS v2 needs: [plan] if: ${{ (inputs.mode == 'build' || inputs.mode == 'package' || inputs.mode == 'release') && inputs.ios }} runs-on: macos-26 @@ -245,7 +245,7 @@ jobs: retention-days: 7 build_macos: - name: Build macOS + name: Build macOS v2 needs: [plan] if: ${{ (inputs.mode == 'build' || inputs.mode == 'package' || inputs.mode == 'release') && inputs.macos }} runs-on: macos-26 @@ -299,7 +299,7 @@ jobs: retention-days: 7 build_android: - name: Build Android + name: Build Android v2 needs: [plan] if: ${{ (inputs.mode == 'build' || inputs.mode == 'package' || inputs.mode == 'release') && inputs.android }} runs-on: ubuntu-latest @@ -510,7 +510,7 @@ jobs: echo "Selected-platform tests passed (Android tests are unwired and skipped)." build_windows: - name: Build Windows + name: Build Windows v2 needs: [plan, deps_windows] if: ${{ (inputs.mode == 'build' || inputs.mode == 'package' || inputs.mode == 'release') && inputs.windows }} runs-on: windows-latest @@ -751,7 +751,7 @@ jobs: gh release create "${release_args[@]}" trigger_downstream_releases: - name: Trigger downstream WebRTC releases + name: Trigger downstream WebRTC releases v2 needs: [plan, github_release] if: ${{ always() && !cancelled() && inputs.mode == 'release' && needs.github_release.result == 'success' }} runs-on: ubuntu-latest diff --git a/.github/workflows/build-v2.yml b/.github/workflows/build-v2.yml new file mode 100644 index 0000000000..d000a38a1f --- /dev/null +++ b/.github/workflows/build-v2.yml @@ -0,0 +1,68 @@ +# Manual WebRTC build (v2 Makefile DAG). Implementation: +# .github/workflows/_make.yml +# Build iOS / macOS / Android in parallel: HIT build-{ios,macos,android}, +# make deps, make build, put that host's tree + out/. No Linux Deps job. +# No package combine, no products. + +name: Build v2 + +run-name: >- + Build v2 config:${{ inputs.config }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && format('{0} Android{1}', (inputs.ios == 'true' || inputs.macos == 'true') && ',' || '', inputs.android_arch != '' && format(' ({0})', inputs.android_arch) || '') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + webrtc_ref: + description: Branch, tag, or SHA for this repo (GetStream/webrtc) + required: true + default: main + ios: + description: Build iOS + type: boolean + default: true + macos: + description: Build macOS + type: boolean + default: true + android: + description: Build Android + type: boolean + default: true + windows: + description: Build Windows (fails clearly if the runner is not a WebRTC host) + type: boolean + default: false + config: + description: GN configuration + type: choice + options: + - release + - debug + default: release + android_arch: + description: Optional Android ABI, for example arm64-v8a. Empty = default ABI set. + required: false + default: "" + skip_deps_cache: + description: Skip Hetzner deps cache (fresh gclient + ninja) + type: boolean + default: false + +jobs: + build: + name: Build v2 + uses: ./.github/workflows/_make.yml + secrets: inherit + with: + mode: build + webrtc_ref: ${{ inputs.webrtc_ref }} + ios: ${{ inputs.ios }} + macos: ${{ inputs.macos }} + android: ${{ inputs.android }} + windows: ${{ inputs.windows }} + config: ${{ inputs.config }} + android_arch: ${{ inputs.android_arch }} + skip_deps_cache: ${{ inputs.skip_deps_cache }} diff --git a/.github/workflows/manual-platform-tests.yml b/.github/workflows/manual-platform-tests.yml index 870a4bddd7..cbf8cae2de 100644 --- a/.github/workflows/manual-platform-tests.yml +++ b/.github/workflows/manual-platform-tests.yml @@ -1,14 +1,8 @@ -# Manual WebRTC build. Dispatchable from a PR because this path exists on the -# default branch. Implementation: .github/workflows/_make.yml -# Build iOS / macOS / Android in parallel: HIT build-{ios,macos,android}, -# make deps, make build, put that host's tree + out/. No Linux Deps job. -# No package combine, no products. +# Manual WebRTC build. Shared planning runs first, host-specific prepare jobs +# produce platform-ready trees, then selected platform builds/tests fan out. name: Build -run-name: >- - Build config:${{ inputs.config }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && format('{0} Android{1}', (inputs.ios == 'true' || inputs.macos == 'true') && ',' || '', inputs.android_arch != '' && format(' ({0})', inputs.android_arch) || '') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} - permissions: contents: read @@ -19,49 +13,530 @@ on: description: Branch, tag, or SHA for this repo (GetStream/webrtc) required: true default: main - ios: - description: Build iOS + apple: + description: Build and package Apple xcframework artifacts type: boolean default: true - macos: - description: Build macOS + android: + description: Build and package Android AAR artifact type: boolean default: true - android: - description: Build Android + debug: + description: Build WebRTC artifacts with GN is_debug=true and include Apple dSYMs when available + type: boolean + default: true + run_ios_tests: + description: Run iOS simulator tests when Apple is enabled type: boolean default: true - windows: - description: Build Windows (fails clearly if the runner is not a WebRTC host) + run_macos_tests: + description: Run macOS host tests when Apple is enabled + type: boolean + default: false + ignore_cache: + description: Skip restoring dependency caches and force a fresh sync type: boolean default: false - config: - description: GN configuration - type: choice - options: - - release - - debug - default: release android_arch: - description: Optional Android ABI, for example arm64-v8a. Empty = default ABI set. + description: Optional Android ABI to build, for example arm64-v8a. Leave empty to build the default ABI set. required: false default: "" - skip_deps_cache: - description: Skip Hetzner deps cache (fresh gclient + ninja) - type: boolean - default: false jobs: - build: - uses: ./.github/workflows/_make.yml - secrets: inherit - with: - mode: build - webrtc_ref: ${{ inputs.webrtc_ref }} - ios: ${{ inputs.ios }} - macos: ${{ inputs.macos }} - android: ${{ inputs.android }} - windows: ${{ inputs.windows }} - config: ${{ inputs.config }} - android_arch: ${{ inputs.android_arch }} - skip_deps_cache: ${{ inputs.skip_deps_cache }} + # Fail invalid manual input before any expensive WebRTC sync/build work starts. + validate_inputs: + name: Validate build inputs + runs-on: ubuntu-latest + steps: + - name: Ensure at least one platform is selected + env: + BUILD_APPLE: ${{ inputs.apple }} + BUILD_ANDROID: ${{ inputs.android }} + run: | + set -euo pipefail + if [[ "${BUILD_APPLE}" != "true" && "${BUILD_ANDROID}" != "true" ]]; then + echo "At least one of apple or android must be true." + exit 1 + fi + + - name: Validate Android build options + env: + ANDROID_ARCH: ${{ inputs.android_arch }} + run: | + set -euo pipefail + + if [[ -n "${ANDROID_ARCH}" && "${ANDROID_ARCH}" =~ [[:space:]] ]]; then + echo "::error::android_arch accepts a single ABI, for example arm64-v8a." + exit 1 + fi + + prepare_common: + name: Plan platform build + needs: validate_inputs + runs-on: ubuntu-latest + outputs: + apple_target_os: ${{ steps.plan.outputs.apple_target_os }} + apple_target_os_cache_key: ${{ steps.plan.outputs.apple_target_os_cache_key }} + android_target_os: ${{ steps.plan.outputs.android_target_os }} + android_target_os_cache_key: ${{ steps.plan.outputs.android_target_os_cache_key }} + steps: + - name: Check out workflow actions + uses: actions/checkout@v4 + + - id: plan + name: Plan platform build + uses: ./.github/actions/prepare-common + with: + # Apple artifacts include both iOS and macOS slices, so both platform flags follow apple. + platform_ios: ${{ inputs.apple }} + platform_macos: ${{ inputs.apple }} + platform_android: ${{ inputs.android }} + + prepare_apple: + name: Prepare Apple dependencies + needs: prepare_common + if: ${{ inputs.apple }} + runs-on: macos-26 + timeout-minutes: 360 + steps: + - name: Check out workflow actions + uses: actions/checkout@v4 + + - name: Prepare Apple dependencies + uses: ./.github/actions/prepare-apple + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.prepare_common.outputs.apple_target_os }} + target_os_cache_key: ${{ needs.prepare_common.outputs.apple_target_os_cache_key }} + ignore_cache: ${{ inputs.ignore_cache }} + release_pipeline_token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + prepare_android: + name: Prepare Android dependencies + needs: prepare_common + if: ${{ inputs.android }} + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - name: Check out workflow actions + uses: actions/checkout@v4 + + - name: Prepare Android dependencies + uses: ./.github/actions/prepare-android + with: + webrtc_ref: ${{ inputs.webrtc_ref }} + target_os: ${{ needs.prepare_common.outputs.android_target_os }} + target_os_cache_key: ${{ needs.prepare_common.outputs.android_target_os_cache_key }} + ignore_cache: ${{ inputs.ignore_cache }} + release_pipeline_token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + ios_test: + name: Validate iOS simulator tests + needs: prepare_apple + if: ${{ inputs.apple && inputs.run_ios_tests }} + runs-on: macos-26 + timeout-minutes: 180 + steps: + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + - name: Download prepared WebRTC tree + uses: actions/download-artifact@v4 + with: + name: webrtc-src-prepared-apple + + - name: Extract WebRTC tree + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/webrtc-tree" + tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" + + - name: CI verification (fan-out does not run gclient sync) + run: | + echo "ios test uses only the restored tree; Fastlane lane :test does not call configure_google_client." + test -f "${{ github.workspace }}/webrtc-tree/.output/src/tools/mb/mb.py" + test -f "${{ github.workspace }}/webrtc-tree/.output/src/tools_webrtc/mb/mb_config.pyl" + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install depot_tools + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: Generate local iOS mb config + run: | + python3 - <<'PY' + import ast + import pathlib + import pprint + + base_path = pathlib.Path("${{ github.workspace }}/webrtc-tree/.output/src/tools_webrtc/mb/mb_config.pyl") + output_path = pathlib.Path("${{ github.workspace }}/release-pipeline/fastlane/mb_configs/local_ios.pyl") + config = ast.literal_eval(base_path.read_text(encoding="utf-8")) + + mixins = dict(config.get("mixins", {})) + mixins["stream_local_ios_tests"] = { + "gn_args": "use_siso=false use_remoteexec=false use_reclient=false", + } + config["mixins"] = mixins + + configs = {name: list(value) for name, value in config.get("configs", {}).items()} + configs["ios_debug_local_bot_arm64"] = configs["ios_debug_bot_arm64"] + ["stream_local_ios_tests"] + config["configs"] = configs + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(f"{pprint.pformat(config, width=120)}\n", encoding="utf-8") + PY + + - name: Run iOS tests (no gclient sync in lane) + working-directory: release-pipeline + run: >- + bundle exec fastlane ios test + "root:${{ github.workspace }}/webrtc-tree/.output/src" + "builder_config:ios_debug_local_bot_arm64" + "config_file:${{ github.workspace }}/release-pipeline/fastlane/mb_configs/local_ios.pyl" + + macos_test: + name: Validate macOS host tests + needs: prepare_apple + if: ${{ inputs.apple && inputs.run_macos_tests }} + runs-on: macos-26 + timeout-minutes: 180 + steps: + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + - name: Download prepared WebRTC tree + uses: actions/download-artifact@v4 + with: + name: webrtc-src-prepared-apple + + - name: Extract WebRTC tree + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/webrtc-tree" + tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install depot_tools + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: Run macOS host tests (no gclient sync in lane) + working-directory: release-pipeline + run: >- + bundle exec fastlane macos test + "root:${{ github.workspace }}/webrtc-tree/.output/src" + + build_ios: + name: Build iOS SDK + needs: prepare_apple + if: ${{ inputs.apple }} + runs-on: macos-26 + timeout-minutes: 360 + steps: + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + - name: Download prepared Apple WebRTC tree + uses: actions/download-artifact@v4 + with: + name: webrtc-src-prepared-apple + + - name: Extract WebRTC tree + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/webrtc-tree" + tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install depot_tools + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: Build iOS SDK artifact + working-directory: release-pipeline + run: >- + bundle exec fastlane ios build + "root:${{ github.workspace }}/webrtc-tree/.output/src" + "build_root:${{ github.workspace }}/webrtc-tree/.output" + "products_root:${{ github.workspace }}/ios-products" + configure_google_client_skip:true + prepare_signing_skip:false + sign_product_skip:true + verify_signatures_skip:true + zip_product_skip:true + "build_product_arg_is_debug:${{ inputs.debug }}" + build_product_arg_treat_warnings_as_errors:false + build_product_arg_rtc_build_examples:false + skip_licenses:true + + - name: Package iOS XCFramework artifact + run: | + set -euo pipefail + ditto -c -k --sequesterRsrc --keepParent \ + "${{ github.workspace }}/ios-products/WebRTC.xcframework" \ + "${{ github.workspace }}/WebRTC-ios.xcframework.zip" + + - name: Upload iOS SDK artifact + uses: actions/upload-artifact@v4 + with: + name: apple-ios-xcframework + path: WebRTC-ios.xcframework.zip + retention-days: 2 + + build_macos: + name: Build macOS SDK + needs: prepare_apple + if: ${{ inputs.apple }} + runs-on: macos-26 + timeout-minutes: 360 + steps: + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + - name: Download prepared Apple WebRTC tree + uses: actions/download-artifact@v4 + with: + name: webrtc-src-prepared-apple + + - name: Extract WebRTC tree + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/webrtc-tree" + tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install depot_tools + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: Build macOS SDK artifact + working-directory: release-pipeline + run: >- + bundle exec fastlane macos build + "root:${{ github.workspace }}/webrtc-tree/.output/src" + "products_root:${{ github.workspace }}/macos-products" + "build_product_arg_is_debug:${{ inputs.debug }}" + build_product_arg_treat_warnings_as_errors:false + build_product_arg_rtc_build_examples:false + zip_product_skip:true + + - name: Package macOS XCFramework artifact + run: | + set -euo pipefail + ditto -c -k --sequesterRsrc --keepParent \ + "${{ github.workspace }}/macos-products/WebRTC.xcframework" \ + "${{ github.workspace }}/WebRTC-macos.xcframework.zip" + + - name: Upload macOS SDK artifact + uses: actions/upload-artifact@v4 + with: + name: apple-macos-xcframework + path: WebRTC-macos.xcframework.zip + retention-days: 2 + + package_apple: + name: Package Apple build artifact + needs: [build_ios, build_macos, ios_test, macos_test] + # Test jobs can be skipped by input. Continue when selected Apple build jobs succeeded and no selected test failed. + if: ${{ inputs.apple && !cancelled() && !failure() }} + runs-on: macos-26 + timeout-minutes: 60 + steps: + - name: Download iOS SDK artifact + uses: actions/download-artifact@v4 + with: + name: apple-ios-xcframework + path: apple-inputs/ios + + - name: Download macOS SDK artifact + uses: actions/download-artifact@v4 + with: + name: apple-macos-xcframework + path: apple-inputs/macos + + - name: Combine Apple XCFrameworks + env: + ENABLE_DEBUG: ${{ inputs.debug }} + run: | + set -euo pipefail + ditto -x -k apple-inputs/ios/WebRTC-ios.xcframework.zip apple-inputs/ios + ditto -x -k apple-inputs/macos/WebRTC-macos.xcframework.zip apple-inputs/macos + + command=(xcodebuild -create-xcframework) + while IFS= read -r framework; do + command+=(-framework "${framework}") + if [[ "${ENABLE_DEBUG}" == "true" ]]; then + mapfile -t dsyms < <(find "$(dirname "${framework}")" -name "$(basename "${framework}").dSYM" -type d | sort) + if [[ "${#dsyms[@]}" -eq 0 ]]; then + echo "::warning::Missing dSYM for ${framework}" + continue + fi + command+=(-debug-symbols "${dsyms[0]}") + fi + done < <(find apple-inputs -name '*.framework' -type d | sort) + + "${command[@]}" -output WebRTC.xcframework + ditto -c -k --sequesterRsrc --keepParent WebRTC.xcframework WebRTC.xcframework.zip + + - name: Upload Apple build artifact + uses: actions/upload-artifact@v4 + with: + name: release-apple + path: WebRTC.xcframework.zip + retention-days: 7 + + build_android: + name: Build Android AAR + needs: prepare_android + if: ${{ inputs.android }} + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + - name: Download prepared Android WebRTC tree + uses: actions/download-artifact@v4 + with: + name: webrtc-src-prepared-android + + - name: Extract WebRTC tree + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/webrtc-tree" + tar -xzf webrtc-src-prepared-android.tar.gz -C "${{ github.workspace }}/webrtc-tree" + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y python3-pip openjdk-8-jdk lsb-release software-properties-common + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install depot_tools + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: Build Android SDK artifact + working-directory: release-pipeline + env: + ANDROID_ARCH: ${{ inputs.android_arch }} + run: | + set -euo pipefail + + fastlane_args=( + android build + "root:${{ github.workspace }}/webrtc-tree/.output/src" + "products_root:${{ github.workspace }}/android-products" + "build_product_arg_is_debug:${{ inputs.debug }}" + build_product_arg_treat_warnings_as_errors:false + build_product_arg_rtc_build_examples:false + ) + + if [[ -n "${ANDROID_ARCH}" ]]; then + fastlane_args+=("archs:${ANDROID_ARCH}") + fi + + bundle exec fastlane "${fastlane_args[@]}" + + - name: Upload Android build artifact + uses: actions/upload-artifact@v4 + with: + name: release-android + path: android-products/libwebrtc.aar + retention-days: 7 diff --git a/.github/workflows/package-v2.yml b/.github/workflows/package-v2.yml new file mode 100644 index 0000000000..029b0a3a6a --- /dev/null +++ b/.github/workflows/package-v2.yml @@ -0,0 +1,62 @@ +# Package WebRTC artifacts (v2 Makefile DAG). Does not create a GitHub +# release. _make.yml: Build (HIT + make deps + make build + make package, +# upload products-*) → Package combine (final-*). No Linux Deps job. No +# Test. No GH release. + +name: Package v2 + +run-name: >- + Package v2 config:${{ inputs.config }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && format('{0} Android{1}', (inputs.ios == 'true' || inputs.macos == 'true') && ',' || '', inputs.android_arch != '' && format(' ({0})', inputs.android_arch) || '') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + webrtc_ref: + description: Branch, tag, or SHA for this repo (GetStream/webrtc) + required: true + default: main + ios: + description: Package iOS xcframework + type: boolean + default: true + macos: + description: Package macOS xcframework + type: boolean + default: true + android: + description: Package Android AAR + type: boolean + default: true + windows: + description: Package Windows libs (fails clearly if the runner is not a WebRTC host) + type: boolean + default: false + config: + description: GN configuration + type: choice + options: + - release + - debug + default: release + android_arch: + description: Optional Android ABI, for example arm64-v8a. Empty = default ABI set. + required: false + default: "" + +jobs: + package: + name: Package v2 + uses: ./.github/workflows/_make.yml + secrets: inherit + with: + mode: package + webrtc_ref: ${{ inputs.webrtc_ref }} + ios: ${{ inputs.ios }} + macos: ${{ inputs.macos }} + android: ${{ inputs.android }} + windows: ${{ inputs.windows }} + config: ${{ inputs.config }} + android_arch: ${{ inputs.android_arch }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7739f7e09f..d74465e668 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,62 +1,590 @@ -# Package WebRTC artifacts. Same path as today's Publish so a PR can dispatch it. -# Does not create a GitHub release. -# _make.yml: Build (HIT + make deps + make build + make package, upload -# products-*) → Package combine (final-*). No Linux Deps job. No Test. -# No GH release. - -name: Package - -run-name: >- - Package config:${{ inputs.config }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && format('{0} Android{1}', (inputs.ios == 'true' || inputs.macos == 'true') && ',' || '', inputs.android_arch != '' && format(' ({0})', inputs.android_arch) || '') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} +name: Publish permissions: - contents: read + contents: write on: workflow_dispatch: inputs: - webrtc_ref: - description: Branch, tag, or SHA for this repo (GetStream/webrtc) + version: + description: Release version/tag to create + required: true + branch: + description: Branch, tag, or SHA from GetStream/webrtc to publish required: true default: main - ios: - description: Package iOS xcframework + alpha: + description: Mark the GitHub release as an alpha/pre-release type: boolean - default: true - macos: - description: Package macOS xcframework + default: false + debug: + description: Build WebRTC artifacts with GN is_debug=true + type: boolean + default: false + ios: + description: Build, validate, publish Apple artifacts and trigger stream-video-swift-webrtc type: boolean default: true android: - description: Package Android AAR + description: Build, validate, publish Android artifacts and trigger stream-video-android-webrtc type: boolean default: true - windows: - description: Package Windows libs (fails clearly if the runner is not a WebRTC host) - type: boolean - default: false - config: - description: GN configuration - type: choice - options: - - release - - debug - default: release - android_arch: - description: Optional Android ABI, for example arm64-v8a. Empty = default ABI set. + changelog: + description: Optional release notes markdown. Leave empty to use generated notes. required: false default: "" + ignore_cache: + description: Skip restoring dependency caches and force a fresh sync + type: boolean + default: false jobs: - package: - uses: ./.github/workflows/_make.yml - secrets: inherit - with: - mode: package - webrtc_ref: ${{ inputs.webrtc_ref }} - ios: ${{ inputs.ios }} - macos: ${{ inputs.macos }} - android: ${{ inputs.android }} - windows: ${{ inputs.windows }} - config: ${{ inputs.config }} - android_arch: ${{ inputs.android_arch }} + # Keep this as a separate first job so invalid manual input fails before any expensive WebRTC sync/build work starts. + validate_inputs: + name: Validate release inputs + runs-on: ubuntu-latest + steps: + - name: Ensure at least one platform is selected + env: + BUILD_IOS: ${{ inputs.ios }} + BUILD_ANDROID: ${{ inputs.android }} + run: | + set -euo pipefail + if [[ "${BUILD_IOS}" != "true" && "${BUILD_ANDROID}" != "true" ]]; then + echo "At least one of ios or android must be true." + exit 1 + fi + + plan: + name: Plan release build + needs: validate_inputs + runs-on: ubuntu-latest + outputs: + apple_target_os: ${{ steps.plan.outputs.apple_target_os }} + apple_target_os_cache_key: ${{ steps.plan.outputs.apple_target_os_cache_key }} + android_target_os: ${{ steps.plan.outputs.android_target_os }} + android_target_os_cache_key: ${{ steps.plan.outputs.android_target_os_cache_key }} + steps: + - name: Check out workflow actions + uses: actions/checkout@v4 + + - id: plan + name: Plan release build + uses: ./.github/actions/prepare-common + with: + # Platform inputs control which prepare/build/test legs run. iOS releases need both iOS and macOS + # Apple artifacts, so platform_macos follows the iOS input. + platform_ios: ${{ inputs.ios }} + platform_macos: ${{ inputs.ios }} + platform_android: ${{ inputs.android }} + + prepare_apple: + name: Prepare Apple dependencies + needs: plan + if: ${{ inputs.ios }} + runs-on: macos-26 + timeout-minutes: 360 + steps: + - name: Check out workflow actions + uses: actions/checkout@v4 + + - name: Prepare Apple dependencies + uses: ./.github/actions/prepare-apple + with: + webrtc_ref: ${{ inputs.branch }} + target_os: ${{ needs.plan.outputs.apple_target_os }} + target_os_cache_key: ${{ needs.plan.outputs.apple_target_os_cache_key }} + ignore_cache: ${{ inputs.ignore_cache }} + release_pipeline_token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + prepare_android: + name: Prepare Android dependencies + needs: plan + if: ${{ inputs.android }} + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - name: Check out workflow actions + uses: actions/checkout@v4 + + - name: Prepare Android dependencies + uses: ./.github/actions/prepare-android + with: + webrtc_ref: ${{ inputs.branch }} + target_os: ${{ needs.plan.outputs.android_target_os }} + target_os_cache_key: ${{ needs.plan.outputs.android_target_os_cache_key }} + ignore_cache: ${{ inputs.ignore_cache }} + release_pipeline_token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + validate_ios: + name: Validate iOS simulator tests + needs: prepare_apple + if: ${{ inputs.ios }} + runs-on: macos-26 + timeout-minutes: 180 + steps: + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + - name: Download prepared Apple WebRTC tree + uses: actions/download-artifact@v4 + with: + name: webrtc-src-prepared-apple + + - name: Extract WebRTC tree + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/webrtc-tree" + tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install depot_tools + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: Generate local iOS mb config + run: | + python3 - <<'PY' + import ast + import pathlib + import pprint + + base_path = pathlib.Path("${{ github.workspace }}/webrtc-tree/.output/src/tools_webrtc/mb/mb_config.pyl") + output_path = pathlib.Path("${{ github.workspace }}/release-pipeline/fastlane/mb_configs/local_ios.pyl") + config = ast.literal_eval(base_path.read_text(encoding="utf-8")) + + mixins = dict(config.get("mixins", {})) + mixins["stream_local_ios_tests"] = { + "gn_args": "use_siso=false use_remoteexec=false use_reclient=false", + } + config["mixins"] = mixins + + configs = {name: list(value) for name, value in config.get("configs", {}).items()} + configs["ios_debug_local_bot_arm64"] = configs["ios_debug_bot_arm64"] + ["stream_local_ios_tests"] + config["configs"] = configs + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(f"{pprint.pformat(config, width=120)}\n", encoding="utf-8") + PY + + - name: Run iOS tests + working-directory: release-pipeline + run: >- + bundle exec fastlane ios test + "root:${{ github.workspace }}/webrtc-tree/.output/src" + "builder_config:ios_debug_local_bot_arm64" + "config_file:${{ github.workspace }}/release-pipeline/fastlane/mb_configs/local_ios.pyl" + + validate_macos: + name: Validate macOS host tests + needs: prepare_apple + if: ${{ inputs.ios }} + runs-on: macos-26 + timeout-minutes: 180 + steps: + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + - name: Download prepared Apple WebRTC tree + uses: actions/download-artifact@v4 + with: + name: webrtc-src-prepared-apple + + - name: Extract WebRTC tree + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/webrtc-tree" + tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install depot_tools + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: Run macOS host tests + working-directory: release-pipeline + run: >- + bundle exec fastlane macos test + "root:${{ github.workspace }}/webrtc-tree/.output/src" + + build_ios: + name: Build iOS SDK + needs: prepare_apple + if: ${{ inputs.ios }} + runs-on: macos-26 + timeout-minutes: 360 + steps: + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + - name: Download prepared Apple WebRTC tree + uses: actions/download-artifact@v4 + with: + name: webrtc-src-prepared-apple + + - name: Extract WebRTC tree + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/webrtc-tree" + tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install depot_tools + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: Build iOS SDK artifact + working-directory: release-pipeline + run: >- + bundle exec fastlane ios build + "root:${{ github.workspace }}/webrtc-tree/.output/src" + "build_root:${{ github.workspace }}/webrtc-tree/.output" + "products_root:${{ github.workspace }}/ios-products" + configure_google_client_skip:true + prepare_signing_skip:false + sign_product_skip:true + verify_signatures_skip:true + zip_product_skip:true + "build_product_arg_is_debug:${{ inputs.debug }}" + skip_licenses:true + + - name: Package iOS XCFramework artifact + run: | + set -euo pipefail + ditto -c -k --sequesterRsrc --keepParent \ + "${{ github.workspace }}/ios-products/WebRTC.xcframework" \ + "${{ github.workspace }}/WebRTC-ios.xcframework.zip" + + - name: Upload iOS SDK artifact + uses: actions/upload-artifact@v4 + with: + name: apple-ios-xcframework + path: WebRTC-ios.xcframework.zip + retention-days: 2 + + build_macos: + name: Build macOS SDK + needs: prepare_apple + if: ${{ inputs.ios }} + runs-on: macos-26 + timeout-minutes: 360 + steps: + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + - name: Download prepared Apple WebRTC tree + uses: actions/download-artifact@v4 + with: + name: webrtc-src-prepared-apple + + - name: Extract WebRTC tree + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/webrtc-tree" + tar -xzf webrtc-src-prepared-apple.tar.gz -C "${{ github.workspace }}/webrtc-tree" + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install depot_tools + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: Build macOS SDK artifact + working-directory: release-pipeline + run: >- + bundle exec fastlane macos build + "root:${{ github.workspace }}/webrtc-tree/.output/src" + "products_root:${{ github.workspace }}/macos-products" + "build_product_arg_is_debug:${{ inputs.debug }}" + zip_product_skip:true + + - name: Package macOS XCFramework artifact + run: | + set -euo pipefail + ditto -c -k --sequesterRsrc --keepParent \ + "${{ github.workspace }}/macos-products/WebRTC.xcframework" \ + "${{ github.workspace }}/WebRTC-macos.xcframework.zip" + + - name: Upload macOS SDK artifact + uses: actions/upload-artifact@v4 + with: + name: apple-macos-xcframework + path: WebRTC-macos.xcframework.zip + retention-days: 2 + + package_apple: + name: Package Apple release artifact + needs: [build_ios, build_macos, validate_ios, validate_macos] + if: ${{ inputs.ios }} + runs-on: macos-26 + timeout-minutes: 60 + steps: + - name: Download iOS SDK artifact + uses: actions/download-artifact@v4 + with: + name: apple-ios-xcframework + path: apple-inputs/ios + + - name: Download macOS SDK artifact + uses: actions/download-artifact@v4 + with: + name: apple-macos-xcframework + path: apple-inputs/macos + + - name: Combine Apple XCFrameworks + env: + ENABLE_DEBUG: ${{ inputs.debug }} + run: | + set -euo pipefail + ditto -x -k apple-inputs/ios/WebRTC-ios.xcframework.zip apple-inputs/ios + ditto -x -k apple-inputs/macos/WebRTC-macos.xcframework.zip apple-inputs/macos + + command=(xcodebuild -create-xcframework) + while IFS= read -r framework; do + command+=(-framework "${framework}") + if [[ "${ENABLE_DEBUG}" == "true" ]]; then + mapfile -t dsyms < <(find "$(dirname "${framework}")" -name "$(basename "${framework}").dSYM" -type d | sort) + if [[ "${#dsyms[@]}" -eq 0 ]]; then + echo "::warning::Missing dSYM for ${framework}" + continue + fi + command+=(-debug-symbols "${dsyms[0]}") + fi + done < <(find apple-inputs -name '*.framework' -type d | sort) + + "${command[@]}" -output WebRTC.xcframework + ditto -c -k --sequesterRsrc --keepParent WebRTC.xcframework WebRTC.xcframework.zip + + - name: Upload Apple release artifact + uses: actions/upload-artifact@v4 + with: + name: release-apple + path: WebRTC.xcframework.zip + retention-days: 2 + + build_android: + name: Build Android AAR + needs: prepare_android + if: ${{ inputs.android }} + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - name: Check out stream-webrtc-release-pipeline + uses: actions/checkout@v4 + with: + repository: GetStream/stream-webrtc-release-pipeline + path: release-pipeline + token: ${{ secrets.RELEASE_PIPELINE_TOKEN || github.token }} + + - name: Download prepared Android WebRTC tree + uses: actions/download-artifact@v4 + with: + name: webrtc-src-prepared-android + + - name: Extract WebRTC tree + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/webrtc-tree" + tar -xzf webrtc-src-prepared-android.tar.gz -C "${{ github.workspace }}/webrtc-tree" + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y python3-pip openjdk-8-jdk lsb-release software-properties-common + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + working-directory: release-pipeline + bundler-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install depot_tools + run: | + if [[ ! -d "${HOME}/depot_tools" ]]; then + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "${HOME}/depot_tools" + fi + echo "PATH=${HOME}/depot_tools:${PATH}" >> "${GITHUB_ENV}" + echo "DEPOT_TOOLS_UPDATE=0" >> "${GITHUB_ENV}" + + - name: Build Android SDK artifact + working-directory: release-pipeline + run: >- + bundle exec fastlane android build + "root:${{ github.workspace }}/webrtc-tree/.output/src" + "products_root:${{ github.workspace }}/android-products" + "build_product_arg_is_debug:${{ inputs.debug }}" + + - name: Upload Android SDK artifact + uses: actions/upload-artifact@v4 + with: + name: release-android + path: android-products/libwebrtc.aar + retention-days: 2 + + publish: + name: Publish GitHub release + # GitHub Actions needs are static. Jobs for disabled platforms are skipped quickly, and this condition + # lets publish continue as long as no selected platform failed or was cancelled. + needs: [validate_ios, validate_macos, package_apple, build_android] + if: ${{ !cancelled() && !failure() }} + runs-on: ubuntu-latest + steps: + - name: Check out release branch + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: Download release artifacts + uses: actions/download-artifact@v4 + with: + pattern: release-* + merge-multiple: true + path: release-assets + + - name: Create GitHub release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_BRANCH: ${{ inputs.branch }} + IS_ALPHA: ${{ inputs.alpha }} + CHANGELOG: ${{ inputs.changelog }} + run: | + set -euo pipefail + notes_file="$(mktemp)" + if [[ -n "${CHANGELOG}" ]]; then + printf '%s\n' "${CHANGELOG}" > "${notes_file}" + else + printf 'Automated WebRTC SDK release %s.\n' "${RELEASE_VERSION}" > "${notes_file}" + fi + + release_args=( + "${RELEASE_VERSION}" + release-assets/* + --repo "${{ github.repository }}" + --target "${RELEASE_BRANCH}" + --title "${RELEASE_VERSION}" + --notes-file "${notes_file}" + ) + if [[ "${IS_ALPHA}" == "true" ]]; then + release_args+=(--prerelease) + else + release_args+=(--latest) + fi + gh release create "${release_args[@]}" + + trigger_downstream_releases: + name: Trigger downstream WebRTC releases + needs: publish + if: ${{ !cancelled() && !failure() }} + runs-on: ubuntu-latest + steps: + - name: Trigger stream-video-swift-webrtc release + # Only trigger the Swift wrapper when Apple artifacts were part of this WebRTC release. + if: ${{ inputs.ios }} + env: + GH_TOKEN: ${{ secrets.CROSS_REPO_TRIGGER_RELEASE_TOKEN }} + RELEASE_VERSION: ${{ inputs.version }} + IS_ALPHA: ${{ inputs.alpha }} + run: | + set -euo pipefail + webrtc_release_url="https://github.com/${{ github.repository }}/releases/tag/${RELEASE_VERSION}" + gh workflow run publish-from-webrtc.yml \ + --repo GetStream/stream-video-swift-webrtc \ + --field "webrtc_release_url=${webrtc_release_url}" \ + --field "pre_release=${IS_ALPHA}" + + - name: Trigger stream-video-android-webrtc release + # Only trigger the Android wrapper when Android artifacts were part of this WebRTC release. + if: ${{ inputs.android }} + env: + GH_TOKEN: ${{ secrets.CROSS_REPO_TRIGGER_RELEASE_TOKEN }} + RELEASE_VERSION: ${{ inputs.version }} + IS_ALPHA: ${{ inputs.alpha }} + run: | + set -euo pipefail + webrtc_release_url="https://github.com/${{ github.repository }}/releases/tag/${RELEASE_VERSION}" + gh workflow run publish-from-webrtc.yml \ + --repo GetStream/stream-video-android-webrtc \ + --field "webrtc_release_url=${webrtc_release_url}" \ + --field "pre_release=${IS_ALPHA}" diff --git a/.github/workflows/release.yml b/.github/workflows/release-v2.yml similarity index 67% rename from .github/workflows/release.yml rename to .github/workflows/release-v2.yml index 7c17ce35b8..6976414d04 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release-v2.yml @@ -1,12 +1,12 @@ -# GitHub release of packaged WebRTC artifacts, then downstream wrapper publishes. -# Not dispatchable until this file exists on the default branch. -# Tests run parallel with Build (each HIT its platform cache). Publish -# waits on Test + Package. +# GitHub release of packaged WebRTC artifacts (v2 Makefile DAG), then +# downstream wrapper publishes. Not dispatchable until this file exists +# on the default branch. Tests run parallel with Build (each HIT its +# platform cache). Publish waits on Test + Package. -name: Release +name: Release v2 run-name: >- - Release config:${{ inputs.config }}${{ inputs.prerelease == 'true' && ' Pre-Release' || '' }} ${{ inputs.version }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && format('{0} Android{1}', (inputs.ios == 'true' || inputs.macos == 'true') && ',' || '', inputs.android_arch != '' && format(' ({0})', inputs.android_arch) || '') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} + Release v2 config:${{ inputs.config }}${{ inputs.prerelease == 'true' && ' Pre-Release' || '' }} ${{ inputs.version }}${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.android == 'true' && format('{0} Android{1}', (inputs.ios == 'true' || inputs.macos == 'true') && ',' || '', inputs.android_arch != '' && format(' ({0})', inputs.android_arch) || '') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true' || inputs.android == 'true') && ', Windows' || ' Windows') || '' }} permissions: contents: write @@ -59,6 +59,7 @@ on: jobs: release: + name: Release v2 uses: ./.github/workflows/_make.yml secrets: inherit with: diff --git a/.github/workflows/test.yml b/.github/workflows/test-v2.yml similarity index 62% rename from .github/workflows/test.yml rename to .github/workflows/test-v2.yml index 1784400ab5..80a7a1f5d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test-v2.yml @@ -1,11 +1,11 @@ -# WebRTC tests. Not dispatchable until this file exists on the default branch. -# _make.yml: HIT build-ios / build-macos, make test only (no extra -# framework-slice make build). No Linux Deps job. +# WebRTC tests (v2 Makefile DAG). Not dispatchable until this file exists +# on the default branch. _make.yml: HIT build-ios / build-macos, make test +# only (no extra framework-slice make build). No Linux Deps job. -name: Test +name: Test v2 run-name: >- - Test config:debug${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true') && ', Windows' || ' Windows') || '' }} + Test v2 config:debug${{ inputs.ios == 'true' && ' iOS' || '' }}${{ inputs.macos == 'true' && (inputs.ios == 'true' && ', macOS' || ' macOS') || '' }}${{ inputs.windows == 'true' && ((inputs.ios == 'true' || inputs.macos == 'true') && ', Windows' || ' Windows') || '' }} permissions: contents: read @@ -32,6 +32,7 @@ on: jobs: test: + name: Test v2 uses: ./.github/workflows/_make.yml secrets: inherit with: diff --git a/stream_build/AGENTS.md b/stream_build/AGENTS.md index fe88bf3730..f2d12d7488 100644 --- a/stream_build/AGENTS.md +++ b/stream_build/AGENTS.md @@ -105,7 +105,7 @@ checkout `src`, extract git-cache to `GIT_CACHE_PATH` to this runner. Always `make deps` after HIT (cheap from cache). Build `CONFIG` is dispatch (default release); `make test` always uses debug in `out/ios_tests` / `out/webrtc_tests`, so those subdirs do not mix -with slice dirs. Test.yml HITs the same `build-ios` / `build-macos` +with slice dirs. test-v2.yml HITs the same `build-ios` / `build-macos` keys. Windows Deps still uploads `deps-windows` to GitHub. Package/Release Build jobs also `make package` and upload `products-*` (GitHub). Package combine consumes `products-*` (no third ninja) and @@ -171,3 +171,30 @@ make combine SKIP_LICENSES=1 make rename apple make rename android ``` + +## GitHub Actions DAGs + +Two independent dispatch DAGs. Either can `workflow_dispatch` without +the other. + +**v2 (Makefile / this tree)** + +| UI name | File | +|---|---| +| Build v2 | `.github/workflows/build-v2.yml` | +| Test v2 | `.github/workflows/test-v2.yml` | +| Package v2 | `.github/workflows/package-v2.yml` | +| Release v2 | `.github/workflows/release-v2.yml` | + +Reusable: `.github/workflows/_make.yml` (`name: WebRTC make`). +Actions: `restore-tree`, `artifact-put`, `artifact-download`, +`setup-webrtc`, `prepare-common-v2`. + +**main (legacy Fastlane / stream-webrtc-release-pipeline)** + +| UI name | File | +|---|---| +| Build | `.github/workflows/manual-platform-tests.yml` | +| Publish | `.github/workflows/publish.yml` | + +Actions: `prepare-common`, `prepare-apple`, `prepare-android`.