diff --git a/.github/actions/setup-build/action.yaml b/.github/actions/setup-build/action.yaml index f277b4d21f9..89e3dac3c8a 100644 --- a/.github/actions/setup-build/action.yaml +++ b/.github/actions/setup-build/action.yaml @@ -5,6 +5,10 @@ description: >- repository checked out with its submodules and VCPKG_ROOT set. inputs: + msvc-environment: + description: "true to give the job MSVC's environment on Windows, for a Ninja build" + required: false + default: "false" r2-cache-mode: description: "read or readwrite to use the R2 vcpkg binary cache; empty to go without it" required: false @@ -62,20 +66,59 @@ runs: shell: bash run: brew install autoconf autoconf-archive automake libtool nasm unzip zip + # The image carries the Xcodes besides the one the build uses and an + # Android SDK, none of which the build reads. They go in the background, + # while the ports build, rather than holding the job up. + - name: macOS disk cleanup + if: runner.os == 'macOS' + shell: bash + run: | + df -h / + keep="$(cd "${DEVELOPER_DIR:-$(xcode-select -p)}/../.." && pwd -P)" + if [[ "$keep" != /Applications/*.app ]]; then + echo "::warning::The build's Xcode is not an application under /Applications ($keep); leaving the image as it is" + exit 0 + fi + doomed=() + shopt -s nullglob + for xcode in /Applications/Xcode_*.app; do + if [[ "$(cd "$xcode" && pwd -P)" != "$keep" ]]; then + doomed+=("$xcode") + fi + done + if [[ -n "${ANDROID_HOME:-}" && -d "$ANDROID_HOME" ]]; then + doomed+=("$ANDROID_HOME") + fi + echo "Keeping $keep; removing ${doomed[*]}" + if (( ${#doomed[@]} )); then + nohup sudo rm -rf "${doomed[@]}" > /dev/null 2>&1 & + fi + - name: Setup python uses: actions/setup-python@v7 with: python-version: "3.14" - # Linux builds with the CMake floor, so the floor is a floor. + # Linux builds with the CMake floor, so the floor is a floor; the Visual + # Studio 2026 generator needs a newer one. Both are pinned: vcpkg builds + # the ports with this CMake and hashes its version into every package + # ABI, so a CMake release on PyPI would otherwise miss the whole binary + # cache until a protected build filled it again. - name: Install python dependencies shell: bash run: | if [[ "$RUNNER_OS" == "Linux" ]] - then pip3 install llsd cmake==4.0.0 ninja - else pip3 install llsd cmake ninja + then pip3 install llsd cmake==4.0.0 ninja==1.13.2 + else pip3 install llsd cmake==4.4.3 ninja==1.13.2 fi + # What a Developer Command Prompt has, for the runner's architecture, + # but Visual Studio's CMake, Ninja and vcpkg. + - name: MSVC environment + if: runner.os == 'Windows' && inputs.msvc-environment == 'true' + shell: pwsh + run: python scripts/ci/msvc_environment.py + - name: Bootstrap vcpkg Windows if: runner.os == 'Windows' shell: pwsh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3b47d277d72..941ce312283 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -75,17 +75,40 @@ jobs: RELEASE_RUN: ${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'Alchemy') && 'Y' || '' }} FROM_FORK: ${{ github.event.pull_request.head.repo.fork }} steps: - # which-branch installs PyGithub for tag builds. - - name: Setup python - uses: actions/setup-python@v7 - with: - python-version: "3.14" - + # The branch the build is of. A pull request or push names its own; a + # tag build is of the branch whose tip the tagged commit is, and takes + # its release notes from that branch's open pull request: whatever the + # description says after a line reading only "relnotes:". - name: Determine source branch id: which-branch - uses: AlchemyViewer/viewer-build-util/which-branch@v3 - with: - token: ${{ github.token }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + if [[ "$GITHUB_REF_TYPE" != "tag" ]]; then + echo "branch=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" + exit 0 + fi + + branch="$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/branches-where-head" \ + --jq '[.[].name] | sort | first // empty')" + echo "branch=$branch" >> "$GITHUB_OUTPUT" + if [[ -z "$branch" ]]; then + echo "::warning::No branch has $GITHUB_SHA at its tip" + exit 0 + fi + + # head= wants owner:branch; a bare branch name is ignored, and every + # open pull request comes back. + body="$(gh api -X GET "repos/$GITHUB_REPOSITORY/pulls" \ + -f state=open -f "head=$GITHUB_REPOSITORY_OWNER:$branch" \ + --jq 'first.body // empty' | tr -d '\r')" + relnotes="$(sed -n '/^[[:space:]]*relnotes:[[:space:]]*$/,$p' <<< "$body" | tail -n +2)" + if [[ -n "$relnotes" ]]; then + # A delimiter the notes cannot contain. + delimiter="relnotes_$(openssl rand -hex 16)" + printf 'relnotes<<%s\n%s\n%s\n' "$delimiter" "$relnotes" "$delimiter" >> "$GITHUB_OUTPUT" + fi - name: Set Variables id: setvar @@ -234,7 +257,6 @@ jobs: continue-on-error: ${{ startsWith(matrix.build_variant, 'Tests') }} outputs: viewer_version: ${{ steps.build.outputs.viewer_version }} - imagename: ${{ steps.build.outputs.imagename }} env: # Sets Xcode version used for build DEVELOPER_DIR: "/Applications/Xcode.app/Contents/Developer" @@ -267,9 +289,11 @@ jobs: # the individual test fails. LOGFAIL: DEBUG master_message_template_checkout: ${{ github.workspace }}/.master-message-template - # vcpkg Setup + # vcpkg Setup. No local binary cache: nothing reads it again on a + # runner that lives for one job, and it would hold a zip of every port + # built. The setup adds the R2 cache where the job may use it. VCPKG_ROOT: ${{ github.workspace }}/vcpkg - VCPKG_BINARY_SOURCES: "clear;default,readwrite" + VCPKG_BINARY_SOURCES: "clear" steps: - name: Checkout code uses: actions/checkout@v7 @@ -291,6 +315,7 @@ jobs: - name: Set up the build uses: ./.github/actions/setup-build with: + msvc-environment: true r2-cache-mode: ${{ needs.setup.outputs.r2_cache_mode }} r2-endpoint-url: ${{ vars.VCPKG_R2_ENDPOINT_URL }} r2-bucket: ${{ vars.VCPKG_R2_BUCKET }} @@ -315,11 +340,15 @@ jobs: cmake_preset_postfix="-os" fi - SIGNING=() + compiler_args=() case "$RUNNER_OS" in Windows) - cmake_preset_name="vs2026${cmake_preset_postfix}-$BUILD_ARCH" + # Ninja with the MSVC environment the setup gave the job, for + # the runner's architecture; named, so no other compiler on the + # image's path is taken for it. + cmake_preset_name="ninja${cmake_preset_postfix}" build_directory="build-Windows-$cmake_preset_name" + compiler_args=(-DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl) if [[ "$BUILD_ARCH" == "arm64" ]] then vcpkg_triplet="arm64-windows-alchemy-release" @@ -346,7 +375,12 @@ jobs: echo "build_directory=$build_directory" >> "$GITHUB_OUTPUT" + # Each port's sources, build tree and staged package go once it is + # installed: a cache miss builds every port, and their leftovers + # would take the disk the viewer's own build needs. cmake -S indra --preset $cmake_preset_name \ + "${compiler_args[@]}" \ + -DVCPKG_INSTALL_OPTIONS=--clean-after-build \ -DVCPKG_TARGET_TRIPLET="$vcpkg_triplet" \ -DAL_BUILD_VIEWER:BOOL="$AL_BUILD_VIEWER" \ -DAL_BUILD_APPEARANCE_UTILITY:BOOL="$AL_BUILD_APPEARANCE_UTILITY" \ @@ -362,8 +396,7 @@ jobs: -DAL_SENTRY_DSN:STRING="$AL_SENTRY_DSN" \ -DAL_CHANNEL:STRING="${AL_CHANNEL}" \ -DAL_GRID:STRING="$VIEWER_GRID" \ - $template_verifier_master_url \ - "${SIGNING[@]}" + $template_verifier_master_url - name: Build id: build @@ -418,9 +451,10 @@ jobs: echo "viewer_version=$viewer_version" >> "$GITHUB_OUTPUT" fi - # The package: the install tree on Windows, which the signing job - # packages with Velopack from it and its package.env metadata; the - # CPack archive elsewhere. + # The package. On Windows and macOS, what the packaging jobs sign and + # package with Velopack, guided by package.env: the install tree on + # Windows; on macOS the archive of the stripped bundle, which is + # signed ad-hoc and signed again there. On Linux, the release itself. if $AL_BUILD_VIEWER then case "$RUNNER_OS" in @@ -429,10 +463,12 @@ jobs: echo "viewer_app=$RUNNER_TEMP/app" >> "$GITHUB_OUTPUT" ;; *) - cpack --config "$BUILD_DIRECTORY/CPackConfig.cmake" -C "$cmake_config" -B "$RUNNER_TEMP/package" - viewer_app="$(ls "$RUNNER_TEMP"/package/*.tar.xz "$RUNNER_TEMP"/package/*.dmg 2>/dev/null | head -n 1 || true)" + cpack --config "$BUILD_DIRECTORY/CPackConfig.cmake" -C "$cmake_config" -G TXZ -B "$RUNNER_TEMP/package" + # The copy of the tree CPack archived. + rm -rf "$RUNNER_TEMP/package/_CPack_Packages" + viewer_app="$(ls "$RUNNER_TEMP"/package/*.tar.xz 2>/dev/null | head -n 1 || true)" if [[ -z "$viewer_app" ]]; then - echo "::error::No .tar.xz or .dmg package under $RUNNER_TEMP/package" + echo "::error::No .tar.xz package under $RUNNER_TEMP/package" exit 1 fi echo "viewer_app=$viewer_app" >> "$GITHUB_OUTPUT" @@ -440,9 +476,9 @@ jobs: esac fi - - name: Upload Windows package metadata + - name: Upload package metadata uses: actions/upload-artifact@v7 - if: runner.os == 'Windows' && matrix.configuration == 'release' && matrix.build_variant == 'Viewer' + if: runner.os != 'Linux' && matrix.configuration == 'release' && matrix.build_variant == 'Viewer' with: name: "${{ steps.build.outputs.artifact }}-metadata" path: ${{ steps.configure.outputs.build_directory }}/newview/package.env @@ -454,44 +490,15 @@ jobs: with: name: "${{ steps.build.outputs.artifact }}-app" if-no-files-found: error - # The Windows install tree is uncompressed; the .tar.xz and .dmg - # already are. + # The Windows install tree is uncompressed; the .tar.xz already is. compression-level: ${{ runner.os == 'Windows' && 9 || 0 }} path: | ${{ steps.build.outputs.viewer_app }} - # The other upload of nontrivial size is the symbol file. Use a distinct - # artifact for that too. - - name: Upload Windows symbol file - uses: actions/upload-artifact@v7 - if: runner.os == 'Windows' && matrix.configuration == 'release' && matrix.build_variant == 'Viewer' && needs.setup.outputs.sentry_dsn != '' - with: - name: "${{ steps.build.outputs.artifact }}-symbols" - if-no-files-found: error - path: | - ${{ steps.configure.outputs.build_directory }}/symbols/Release/${{ needs.setup.outputs.viewer_channel }}.sym.tar.xz - - - name: Upload macOS symbol file - uses: actions/upload-artifact@v7 - if: runner.os == 'macOS' && matrix.configuration == 'release' && matrix.build_variant == 'Viewer' && needs.setup.outputs.sentry_dsn != '' - with: - name: "${{ steps.build.outputs.artifact }}-symbols" - if-no-files-found: error - path: | - ${{ steps.configure.outputs.build_directory }}/symbols/Release/${{ needs.setup.outputs.viewer_channel }}.xcarchive.zip - - - name: Upload Linux symbol file - uses: actions/upload-artifact@v7 - if: runner.os == 'Linux' && matrix.configuration == 'release' && matrix.build_variant == 'Viewer' && needs.setup.outputs.sentry_dsn != '' - with: - name: "${{ steps.build.outputs.artifact }}-symbols" - if-no-files-found: error - path: | - ${{ steps.configure.outputs.build_directory }}/symbols/Release/${{ needs.setup.outputs.viewer_channel }}.sym.tar.xz - - # Sentry symbolicates a crash report with the debug files of the - # binaries that shipped, in pairs: PE and PDB, Mach-O and dSYM, ELF and - # its split .debug. The build tree still holds both halves side by side. + # Sentry symbolicates a crash report with the symbol store: every binary + # that shipped with its debug information beside it, PE and PDB, Mach-O + # and dSYM, ELF and its split .debug filed by build ID. The store is + # made from the staged tree, after the package is. - name: Upload debug files to Sentry if: matrix.configuration == 'release' && matrix.build_variant == 'Viewer' && needs.setup.outputs.sentry_dsn != '' shell: bash @@ -510,37 +517,14 @@ jobs: pip3 install sentry-cli==3.7.0 - case "$RUNNER_OS" in - Windows) - candidates=( - "$BUILD_DIRECTORY/newview/Release" - "$BUILD_DIRECTORY/symbols/Release" - "$BUILD_DIRECTORY/dullahan/Release" - ) - ;; - macOS) - candidates=( - "$BUILD_DIRECTORY"/newview/Release/*.app/Contents/MacOS - "$BUILD_DIRECTORY"/newview/Release/*.app/Contents/Frameworks - "$BUILD_DIRECTORY"/newview/Release/*.app/Contents/Resources/llplugin - "$BUILD_DIRECTORY/symbols/Release" - ) - ;; - Linux) - candidates=( - "$BUILD_DIRECTORY/newview/Release/bin" - "$BUILD_DIRECTORY/newview/Release/llplugin" - "$BUILD_DIRECTORY/symbols/Release" - ) - ;; - esac + cmake --build "$BUILD_DIRECTORY" --config Release --target symbols + sentry-cli debug-files upload --include-sources --wait "$BUILD_DIRECTORY/symbols/Release" - paths=() - for candidate in "${candidates[@]}"; do - [[ -e "$candidate" ]] && paths+=("$candidate") - done - - sentry-cli debug-files upload --include-sources --wait "${paths[@]}" + # The headroom the row finished with, failed or not. + - name: Report disk space + if: always() + shell: bash + run: df -h "$GITHUB_WORKSPACE" # - name: Upload appearance utility package # uses: actions/upload-artifact@v7 @@ -551,6 +535,8 @@ jobs: # ${{ steps.configure.outputs.build_directory }}/llappearanceutility/Release/appearance-utility-bin # ${{ steps.configure.outputs.build_directory }}/llappearanceutility/Release/appearance-utility-headless-bin + # Pull requests are packaged unsigned: the code they build has not been + # reviewed, and the certificates are not lent to it. sign-and-package-windows: needs: [setup, build] timeout-minutes: 30 @@ -558,9 +544,11 @@ jobs: matrix: arch: [x64, arm64] runs-on: windows-2025-vs2026 + env: + SIGN_PACKAGES: ${{ github.event_name != 'pull_request' }} steps: # The app and its metadata come from the build's artifacts; the tree - # contributes only the Velopack tool manifest. + # contributes only the .NET tool manifest. - name: Checkout code uses: actions/checkout@v7 with: @@ -574,7 +562,7 @@ jobs: with: dotnet-version: '10.x' - - name: Install Velopack CLI + - name: Install Velopack CLI and AzureSignTool run: dotnet tool restore - name: Fetch Windows app @@ -589,17 +577,24 @@ jobs: name: Windows${{ needs.setup.outputs.build_suffix }}-${{ matrix.arch }}-metadata path: .metadata - - name: Load Velopack metadata + - name: Load package metadata shell: bash run: | + # Every key, upper-cased, into the environment of the steps below. while IFS='=' read -r key value; do - if [[ "$key" == velopack_* ]]; then + if [[ -n "$key" ]]; then printf '%s=%s\n' "${key^^}" "$value" >> "$GITHUB_ENV" fi done < .metadata/package.env - - name: Build and sign Velopack package + - name: Package with Velopack shell: bash + env: + AZURE_KEY_VAULT_URI: ${{ secrets.AZURE_KEY_VAULT_URI }} + AZURE_KEY_VAULT_CERTIFICATE: ${{ secrets.AZURE_KEY_VAULT_CERTIFICATE }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} run: | set -x @@ -627,95 +622,310 @@ jobs: vpk_args+=(--splashImage ".app/$VELOPACK_SPLASH") fi + # Every binary vpk packs, and the Setup.exe and Update.exe it adds, + # signed by AzureSignTool with the certificate in Azure Key Vault. + # vpk runs the template through cmd.exe, which expands the + # %VARIABLES%, so the credentials stay off the command line it logs. + if [[ "$SIGN_PACKAGES" == "true" && -n "$AZURE_KEY_VAULT_URI" && -n "$AZURE_KEY_VAULT_CERTIFICATE" \ + && -n "$AZURE_CLIENT_ID" && -n "$AZURE_CLIENT_SECRET" && -n "$AZURE_TENANT_ID" ]]; then + vpk_args+=(--signTemplate "dotnet azuresigntool sign -kvu %AZURE_KEY_VAULT_URI% -kvc %AZURE_KEY_VAULT_CERTIFICATE% -kvi %AZURE_CLIENT_ID% -kvs %AZURE_CLIENT_SECRET% -kvt %AZURE_TENANT_ID% -tr http://timestamp.digicert.com -td sha256 {{file...}}") + else + echo "::notice::No Azure Key Vault signing certificate: the Windows package is not signed" + fi + "${vpk_args[@]}" - name: Rename Velopack outputs shell: bash run: | - # Move Setup.exe into .app for the installer upload step - setup="Releases/${VELOPACK_PACK_ID}-${VELOPACK_CHANNEL}-Setup.exe" - if [[ -f "$setup" ]]; then - mv "$setup" ".app/${VELOPACK_INSTALLER_BASE}_Setup.exe" - echo "Moved $setup to .app/${VELOPACK_INSTALLER_BASE}_Setup.exe" - fi + # The installer and the portable archive take the package's name. + mkdir .installer + mv "Releases/${VELOPACK_PACK_ID}-${VELOPACK_CHANNEL}-Setup.exe" ".installer/${IMAGENAME}_Setup.exe" + mv "Releases/${VELOPACK_PACK_ID}-${VELOPACK_CHANNEL}-Portable.zip" "Releases/${IMAGENAME}_Portable.zip" + + - name: Post the installer + uses: actions/upload-artifact@v7 + with: + name: "Windows${{ needs.setup.outputs.build_suffix }}-${{ matrix.arch }}-installer" + path: .installer/ + if-no-files-found: error + + - name: Upload Velopack releases + uses: actions/upload-artifact@v7 + with: + name: "Windows${{ needs.setup.outputs.build_suffix }}-${{ matrix.arch }}-releases" + path: Releases/ + if-no-files-found: error + + # Pull requests are packaged signed ad-hoc and not notarized, for the reason + # above. + sign-and-package-mac: + needs: [setup, build] + timeout-minutes: 60 + strategy: + matrix: + arch: [arm64] + runs-on: xcode-27 + env: + DEVELOPER_DIR: "/Applications/Xcode.app/Contents/Developer" + SIGN_PACKAGES: ${{ github.event_name != 'pull_request' }} + steps: + # The app and its metadata come from the build's artifacts; the tree + # contributes the .NET tool manifest, the signing script, the + # entitlements and the disk image's layout. + - name: Checkout code + uses: actions/checkout@v7 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha || github.sha }} + sparse-checkout: | + /dotnet-tools.json + /indra/cmake/ViewerCodeSign.cmake + /indra/newview/slplugin.entitlements + /indra/newview/installers/darwin/ + /indra/dullahan + sparse-checkout-cone-mode: false - # Rename Portable.zip to include version - portable="Releases/${VELOPACK_PACK_ID}-${VELOPACK_CHANNEL}-Portable.zip" - if [[ -f "$portable" ]]; then - mv "$portable" "Releases/${VELOPACK_INSTALLER_BASE}_Portable.zip" - echo "Moved $portable to Releases/${VELOPACK_INSTALLER_BASE}_Portable.zip" + # The CEF helpers' entitlements are dullahan's. + - name: Checkout dullahan + run: git submodule update --init --depth 1 -- indra/dullahan + + - name: Setup .NET for Velopack + uses: actions/setup-dotnet@v6 + with: + dotnet-version: '10.x' + + - name: Install Velopack CLI + run: dotnet tool restore + + - name: Setup python + uses: actions/setup-python@v7 + with: + python-version: "3.14" + + # The badge_icons extra badges the volume icon through Quartz. + - name: Install CMake and dmgbuild + run: pip install cmake "dmgbuild[badge_icons]==1.6.7" + + - name: Fetch macOS app + uses: actions/download-artifact@v8 + with: + name: macOS${{ needs.setup.outputs.build_suffix }}-${{ matrix.arch }}-app + path: .package + + - name: Fetch macOS package metadata + uses: actions/download-artifact@v8 + with: + name: macOS${{ needs.setup.outputs.build_suffix }}-${{ matrix.arch }}-metadata + path: .metadata + + - name: Load package metadata + shell: bash + run: | + # Every key, upper-cased, into the environment of the steps below. + while IFS='=' read -r key value; do + if [[ -n "$key" ]]; then + printf '%s=%s\n' "$(tr '[:lower:]' '[:upper:]' <<< "$key")" "$value" >> "$GITHUB_ENV" + fi + done < .metadata/package.env + + - name: Unpack the app + id: unpack + shell: bash + run: | + mkdir .unpacked + tar -xJf .package/*.tar.xz -C .unpacked + app="$(find .unpacked -maxdepth 2 -name '*.app' -print -quit)" + if [[ -z "$app" ]]; then + echo "::error::No .app in the macOS package" + exit 1 fi + # Extended attributes codesign refuses to seal over. + xattr -cr "$app" + echo "app=$PWD/$app" >> "$GITHUB_OUTPUT" - - name: Find Velopack installer - id: find-installer + - name: Set up the signing keychain + id: keychain + if: env.SIGN_PACKAGES == 'true' shell: bash + env: + MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} + MACOS_CERTIFICATE_PASSWORD: ${{ secrets.MACOS_CERTIFICATE_PASSWORD }} + MACOS_NOTARY_KEY: ${{ secrets.MACOS_NOTARY_KEY }} + MACOS_NOTARY_KEY_ID: ${{ secrets.MACOS_NOTARY_KEY_ID }} + MACOS_NOTARY_ISSUER_ID: ${{ secrets.MACOS_NOTARY_ISSUER_ID }} run: | - installer="$(ls -t .app/*_Setup.exe 2>/dev/null | head -n 1 || true)" - if [[ -z "$installer" ]]; then - echo "::error::No Velopack installer found under .app" + if [[ -z "$MACOS_CERTIFICATE" || -z "$MACOS_CERTIFICATE_PASSWORD" ]]; then + echo "::notice::No Developer ID certificate: the macOS package is signed ad-hoc and not notarized" + exit 0 + fi + + certificate="$RUNNER_TEMP/certificate.p12" + notary_key="$RUNNER_TEMP/notary.p8" + trap 'rm -f "$certificate" "$notary_key"' EXIT + + # A keychain of the job's own, unlocked for its length, and on the + # search list so codesign finds the certificate's chain. + keychain="$RUNNER_TEMP/signing.keychain-db" + password="$(openssl rand -base64 24)" + echo "::add-mask::$password" + security create-keychain -p "$password" "$keychain" + echo "keychain=$keychain" >> "$GITHUB_OUTPUT" + security set-keychain-settings -lut 21600 "$keychain" + security unlock-keychain -p "$password" "$keychain" + searchlist=() + while IFS= read -r path; do + searchlist+=("$path") + done < <(security list-keychains -d user | sed -e 's/^[[:space:]]*"//' -e 's/"$//') + security list-keychains -d user -s "$keychain" "${searchlist[@]}" + + base64 --decode > "$certificate" <<< "$MACOS_CERTIFICATE" + security import "$certificate" -k "$keychain" -f pkcs12 -P "$MACOS_CERTIFICATE_PASSWORD" -T /usr/bin/codesign + security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$password" "$keychain" > /dev/null + + # The identity by its hash, which no other certificate can share. + identity="$(security find-identity -v -p codesigning "$keychain" \ + | awk '/"Developer ID Application: / { print $2; exit }')" + if [[ -z "$identity" ]]; then + echo "::error::MACOS_CERTIFICATE holds no Developer ID Application identity" exit 1 fi + echo "identity=$identity" >> "$GITHUB_OUTPUT" + + if [[ -z "$MACOS_NOTARY_KEY" || -z "$MACOS_NOTARY_KEY_ID" ]]; then + echo "::notice::No App Store Connect API key: the macOS package is signed but not notarized" + exit 0 + fi + # A team key has an issuer; an individual key must be given none. + issuer=() + if [[ -n "$MACOS_NOTARY_ISSUER_ID" ]]; then + issuer=(--issuer "$MACOS_NOTARY_ISSUER_ID") + fi + # notarytool keeps the API key in the job's keychain, as a profile. + printf '%s\n' "$MACOS_NOTARY_KEY" > "$notary_key" + xcrun notarytool store-credentials alchemy-notary --key "$notary_key" \ + --key-id "$MACOS_NOTARY_KEY_ID" "${issuer[@]}" --keychain "$keychain" + echo "notary_profile=alchemy-notary" >> "$GITHUB_OUTPUT" + + # Always, ad-hoc without an identity: unpacking the archive decomposes + # some file names, which breaks the seal the build made. + - name: Sign the app inside out + shell: bash + env: + APP: ${{ steps.unpack.outputs.app }} + IDENTITY: ${{ steps.keychain.outputs.identity }} + KEYCHAIN: ${{ steps.keychain.outputs.keychain }} + run: | + cmake \ + -D "AL_SIGN_BUNDLE=$APP" \ + -D "AL_SIGN_IDENTITY=$IDENTITY" \ + -D "AL_SIGN_KEYCHAIN=$KEYCHAIN" \ + -D "AL_SIGN_PLUGIN_ENTITLEMENTS=$PWD/indra/newview/slplugin.entitlements" \ + -D "AL_SIGN_HELPER_ENTITLEMENTS=$PWD/indra/dullahan/src/dullahan.entitlements" \ + -P indra/cmake/ViewerCodeSign.cmake + + # vpk signs the updater it adds and seals the bundle again with the + # viewer's entitlements; given a profile, it notarizes and staples it. + - name: Package with Velopack + shell: bash + env: + APP: ${{ steps.unpack.outputs.app }} + IDENTITY: ${{ steps.keychain.outputs.identity }} + KEYCHAIN: ${{ steps.keychain.outputs.keychain }} + NOTARY_PROFILE: ${{ steps.keychain.outputs.notary_profile }} + run: | + vpk_args=( + dotnet vpk pack + --packId "$VELOPACK_PACK_ID" + --packVersion "$VELOPACK_PACK_VERSION" + --packAuthors "$VELOPACK_PACK_AUTHORS" + --packTitle "$VELOPACK_PACK_TITLE" + --packDir "$APP" + --mainExe "$VELOPACK_MAIN_EXE" + --channel "$VELOPACK_CHANNEL" + --runtime "$VELOPACK_RUNTIME" + --noInst + --signAppIdentity "${IDENTITY:--}" + --signDisableDeep + --signEntitlements "$PWD/indra/newview/slplugin.entitlements" + ) + if [[ -n "$KEYCHAIN" ]]; then + vpk_args+=(--keychain "$KEYCHAIN") + fi + if [[ -n "$NOTARY_PROFILE" ]]; then + vpk_args+=(--notaryProfile "$NOTARY_PROFILE") + fi + "${vpk_args[@]}" + + # From the bundle vpk sealed, updater and all, so a viewer installed + # from the image updates itself. + - name: Build the disk image + id: dmg + shell: bash + run: | + mkdir .dmg-app .installer + ditto -x -k "Releases/${VELOPACK_PACK_ID}-${VELOPACK_CHANNEL}-Portable.zip" .dmg-app + app="$(find .dmg-app -maxdepth 1 -name '*.app' -print -quit)" + installer="$PWD/.installer/$IMAGENAME.dmg" + dmgbuild -s indra/newview/installers/darwin/dmg_settings.py -D "app=$app" "$VELOPACK_PACK_TITLE" "$installer" echo "installer=$installer" >> "$GITHUB_OUTPUT" + - name: Sign and notarize the disk image + if: steps.keychain.outputs.identity != '' + shell: bash + env: + INSTALLER: ${{ steps.dmg.outputs.installer }} + IDENTITY: ${{ steps.keychain.outputs.identity }} + KEYCHAIN: ${{ steps.keychain.outputs.keychain }} + NOTARY_PROFILE: ${{ steps.keychain.outputs.notary_profile }} + run: | + codesign --force --sign "$IDENTITY" --keychain "$KEYCHAIN" --timestamp "$INSTALLER" + if [[ -z "$NOTARY_PROFILE" ]]; then + exit 0 + fi + + # notarytool can exit 0 on a rejection, so the verdict is read from + # its report, and the log fetched when it is not Accepted. + notary=(--keychain-profile "$NOTARY_PROFILE" --keychain "$KEYCHAIN") + result="$(xcrun notarytool submit "$INSTALLER" "${notary[@]}" --wait --output-format json)" || true + echo "$result" + id="$(plutil -extract id raw -o - - <<< "$result" 2>/dev/null || true)" + status="$(plutil -extract status raw -o - - <<< "$result" 2>/dev/null || true)" + if [[ "$status" != "Accepted" ]]; then + if [[ -n "$id" ]]; then + xcrun notarytool log "$id" "${notary[@]}" + fi + echo "::error::Notarizing the disk image ended ${status:-without a verdict}" + exit 1 + fi + xcrun stapler staple "$INSTALLER" + spctl --assess --type open --context context:primary-signature --verbose "$INSTALLER" + - name: Post the installer uses: actions/upload-artifact@v7 with: - name: "Windows${{ needs.setup.outputs.build_suffix }}-${{ matrix.arch }}-installer" - path: ${{ steps.find-installer.outputs.installer }} + name: "macOS${{ needs.setup.outputs.build_suffix }}-${{ matrix.arch }}-installer" + path: ${{ steps.dmg.outputs.installer }} if-no-files-found: error + compression-level: 0 - name: Upload Velopack releases uses: actions/upload-artifact@v7 with: - name: "Windows${{ needs.setup.outputs.build_suffix }}-${{ matrix.arch }}-releases" + name: "macOS${{ needs.setup.outputs.build_suffix }}-${{ matrix.arch }}-releases" path: Releases/ if-no-files-found: error + compression-level: 0 - # sign-and-package-mac: - # env: - # NOTARIZE_CREDS_MACOS: ${{ secrets.NOTARIZE_CREDS_MACOS }} - # SIGNING_CERT_MACOS: ${{ secrets.SIGNING_CERT_MACOS }} - # SIGNING_CERT_MACOS_IDENTITY: ${{ secrets.SIGNING_CERT_MACOS_IDENTITY }} - # SIGNING_CERT_MACOS_PASSWORD: ${{ secrets.SIGNING_CERT_MACOS_PASSWORD }} - # needs: [setup, build] - # runs-on: macos-latest - # if: needs.setup.outputs.build_type == 'proprietary' - # steps: - # - name: Unpack Mac notarization credentials - # if: env.NOTARIZE_CREDS_MACOS - # id: note-creds - # shell: bash - # run: | - # # In NOTARIZE_CREDS_MACOS we expect to find: - # # USERNAME="..." - # # PASSWORD="..." - # # TEAM_ID="..." - # eval "${{ env.NOTARIZE_CREDS_MACOS }}" - # echo "::add-mask::$USERNAME" - # echo "::add-mask::$PASSWORD" - # echo "::add-mask::$TEAM_ID" - # echo "note_user=$USERNAME" >> "$GITHUB_OUTPUT" - # echo "note_pass=$PASSWORD" >> "$GITHUB_OUTPUT" - # echo "note_team=$TEAM_ID" >> "$GITHUB_OUTPUT" - # # If we didn't manage to retrieve all of these credentials, better - # # find out sooner than later. - # [[ -n "$USERNAME" && -n "$PASSWORD" && -n "$TEAM_ID" ]] - - # - name: Sign and package Mac viewer - # if: env.SIGNING_CERT_MACOS && env.SIGNING_CERT_MACOS_IDENTITY && env.SIGNING_CERT_MACOS_PASSWORD && steps.note-creds.outputs.note_user && steps.note-creds.outputs.note_pass && steps.note-creds.outputs.note_team - # uses: secondlife/viewer-build-util/sign-pkg-mac@v2 - # with: - # channel: ${{ needs.setup.outputs.viewer_channel }} - # imagename: ${{ needs.build.outputs.imagename }} - # cert_base64: ${{ env.SIGNING_CERT_MACOS }} - # cert_name: ${{ env.SIGNING_CERT_MACOS_IDENTITY }} - # cert_pass: ${{ env.SIGNING_CERT_MACOS_PASSWORD }} - # note_user: ${{ steps.note-creds.outputs.note_user }} - # note_pass: ${{ steps.note-creds.outputs.note_pass }} - # note_team: ${{ steps.note-creds.outputs.note_team }} + - name: Remove the signing keychain + if: always() && steps.keychain.outputs.keychain != '' + shell: bash + env: + KEYCHAIN: ${{ steps.keychain.outputs.keychain }} + run: security delete-keychain "$KEYCHAIN" release: - needs: [setup, build, sign-and-package-windows] + needs: [setup, build, sign-and-package-windows, sign-and-package-mac] runs-on: ubuntu-latest timeout-minutes: 30 if: needs.setup.outputs.release_run @@ -755,9 +965,9 @@ jobs: append_body: true fail_on_unmatched_files: true files: | - Windows*-installer/*.exe + *-installer/* + *-releases/* *.tar.xz - Windows*-releases/* - name: post release URL env: diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 71894f99404..2ce4dc3e87c 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -72,7 +72,8 @@ jobs: category: /language:c-cpp/os:windows env: VCPKG_ROOT: ${{ github.workspace }}/vcpkg - VCPKG_BINARY_SOURCES: "clear;default,readwrite" + # No local binary cache, as in the build: the runner lives for one job. + VCPKG_BINARY_SOURCES: "clear" # MSBuild keeps its worker nodes alive between builds. One started by # the configure, before tracing began, would compile untraced. MSBUILDDISABLENODEREUSE: 1 diff --git a/doc/BUILD.md b/doc/BUILD.md index d3f973fdc6b..73150ae6702 100644 --- a/doc/BUILD.md +++ b/doc/BUILD.md @@ -370,6 +370,8 @@ Options are defined in [`indra/CMakeLists.txt`](../indra/CMakeLists.txt). The mo |:-----------------|:------------|:-------------------------------------------------------| | `AL_USE_OPENXR` | OFF | OpenXR VR support (experimental) | | `AL_USE_SDL_WINDOW` | ON on Linux | SDL-based window management (Linux only; GL through EGL on Wayland and X11 alike) | +| `AL_SIGNING_IDENTITY` | empty | macOS Developer ID the bundle is signed with; empty signs ad-hoc | +| `AL_NOTARY_PROFILE` | empty | macOS notarytool keychain profile the `velopack` target notarizes with; empty skips notarization | ### Crash reporting @@ -461,11 +463,24 @@ cpack --config build--/CPackSourceConfig.cmake (or the `package_source` target under Ninja). Uncommitted changes are not in it, and cpack says so. -The Windows installer and the update packages come from [Velopack](https://velopack.io): configure with `-DAL_USE_VELOPACK=ON`, run `dotnet tool restore` once so the `vpk` tool is available, and build the `velopack` target. It installs into `newview/velopack//app` and writes the installer and the update feed to `newview/velopack//Releases`. Each Windows architecture has its own Velopack channel, named for its runtime, since an installed viewer updates from its channel's feed: `win-x64` and `win-arm64`. The channel also names the feed, `releases.win-x64.json` or `releases.win-arm64.json`, and the files vpk writes. +The Windows installer and the update packages for Windows and macOS come from [Velopack](https://velopack.io): configure with `-DAL_USE_VELOPACK=ON`, run `dotnet tool restore` once so the `vpk` tool is available, and build the `velopack` target. It installs into `newview/velopack//app` and writes the update feed, and on Windows the installer, to `newview/velopack//Releases`. Each platform and architecture has its own Velopack channel, named for its runtime, since an installed viewer updates from its channel's feed: `win-x64`, `win-arm64`, `osx-arm64` and `osx-x64`. The channel also names the feed, `releases..json`, and the files vpk writes. On macOS vpk adds its updater to the bundle and seals it again, with `AL_SIGNING_IDENTITY` or ad-hoc, and notarizes it when `AL_NOTARY_PROFILE` names a profile stored with `xcrun notarytool store-credentials`. The third-party attribution is generated, not kept by hand: `cmake/Attribution.cmake` reads every installed port's `vcpkg.spdx.json` and `copyright` and writes `app_settings/packages-info.txt` (what the About floater's Licences tab shows) and `licenses.txt` (every licence text). What vcpkg cannot know — the pieces under `indra/externals/`, the SDKs from outside vcpkg, and a holder or licence a port's files do not state — is in `cmake/attribution.json`, as is the list of installed ports that ship nothing and are skipped: build tools, empty ports that stand for a system library, and what is built only for those. A newly added port whose `vcpkg.json` declares no `license` stops the build with its name; fix the port, add an override to the table, or, if the viewer ships none of it, skip it with the reason (and the platform, when the port is empty only on some). -On macOS the install step signs the bundle inside out — ad-hoc, or with `-DAL_ENABLE_SIGNING=ON -DAL_SIGNING_IDENTITY=` — so the CEF helpers keep their sandbox entitlements. On Linux the binaries carry an `$ORIGIN`-relative RPATH and find the data one directory above the executable, so the tree runs from wherever it is unpacked. +On macOS the install step signs the bundle inside out — ad-hoc, or with `-DAL_SIGNING_IDENTITY=` — so the CEF helpers keep their sandbox entitlements, and the package step seals it again after stripping the executable. The disk image is APFS: HFS+ decomposes file names, which breaks the seal over the font stand-ins with Japanese names. On Linux the binaries carry an `$ORIGIN`-relative RPATH and find the data one directory above the executable, so the tree runs from wherever it is unpacked. + +The hosted build (`.github/workflows/build.yaml`) packages Windows and macOS in jobs of their own, after the build, from the build's install tree or stripped bundle and its `newview/package.env`. Pull requests are packaged unsigned; other builds are signed when the repository has the secrets, and without them are packaged unsigned with a notice: + +| Secret | What | +|:--|:--| +| `AZURE_KEY_VAULT_URI`, `AZURE_KEY_VAULT_CERTIFICATE` | The Azure Key Vault and the name of the Windows code-signing certificate in it | +| `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID` | The Entra application AzureSignTool signs as | +| `MACOS_CERTIFICATE`, `MACOS_CERTIFICATE_PASSWORD` | The Developer ID Application certificate and key, as a base64 `.p12`, and its password | +| `MACOS_NOTARY_KEY`, `MACOS_NOTARY_KEY_ID`, `MACOS_NOTARY_ISSUER_ID` | An App Store Connect API key (the `.p8`'s text) and its key ID, to notarize with, and the issuer ID of a team key; an individual key has none | + +`scripts/signing/macos_signing_secrets.py` makes the macOS secrets from the exported `.p12` and the API key's `.p8`, checking each as the build will use it, and stores them with `gh secret set` (`--repo`) or writes them to files (`--output`); its header says where each comes from. + +vpk signs every Windows binary, its `Setup.exe` and `Update.exe` through AzureSignTool. On macOS the job signs the bundle inside out with `ViewerCodeSign.cmake`, vpk adds its updater, seals, notarizes and staples the bundle, and the job builds the disk image from that bundle with [dmgbuild](https://dmgbuild.readthedocs.io) (`indra/newview/installers/darwin/dmg_settings.py`), then signs, notarizes and staples the image. ## Troubleshooting diff --git a/dotnet-tools.json b/dotnet-tools.json index 1fb3120d89e..bb069f49a9e 100644 --- a/dotnet-tools.json +++ b/dotnet-tools.json @@ -3,11 +3,18 @@ "isRoot": true, "tools": { "vpk": { - "version": "1.2.0", + "version": "1.2.161", "commands": [ "vpk" ], "rollForward": false + }, + "azuresigntool": { + "version": "7.0.1", + "commands": [ + "azuresigntool" + ], + "rollForward": false } } -} \ No newline at end of file +} diff --git a/indra/CMakeLists.txt b/indra/CMakeLists.txt index 1654ead62a8..0b9056352f5 100644 --- a/indra/CMakeLists.txt +++ b/indra/CMakeLists.txt @@ -200,9 +200,20 @@ cmake_dependent_option( # WebRTC voice. Off in sanitized builds, which cannot link libwebrtc. option(AL_USE_WEBRTC "Build with WebRTC voice" ON) -# Signing -option(AL_ENABLE_SIGNING "Enable signing the viewer" OFF) -set(AL_SIGNING_IDENTITY "" CACHE STRING "Specifies the signing identity to use, if necessary.") +# macOS signing. The install step always signs the bundle: with this +# identity, or ad-hoc when it is empty. +set( + AL_SIGNING_IDENTITY + "" + CACHE STRING + "The macOS Developer ID to sign the bundle with; empty signs ad-hoc" +) +set( + AL_NOTARY_PROFILE + "" + CACHE STRING + "The notarytool keychain profile the macOS velopack target notarizes with; empty skips notarization" +) # Warn about any option name from before the AL_ scheme. include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/RetiredOptions.cmake) diff --git a/indra/cmake/00-Common.cmake b/indra/cmake/00-Common.cmake index d1adc89384d..b49963dc489 100644 --- a/indra/cmake/00-Common.cmake +++ b/indra/cmake/00-Common.cmake @@ -438,7 +438,7 @@ if(DARWIN) set(CMAKE_XCODE_ATTRIBUTE_COMPILATION_CACHE_ENABLE_CACHING YES) set(CMAKE_XCODE_ATTRIBUTE_GCC_GENERATE_DEBUGGING_SYMBOLS YES) - set(CMAKE_XCODE_ATTRIBUTE_DEBUG_INFORMATION_FORMAT "dwarf") # dSYMs only where a target asks + set(CMAKE_XCODE_ATTRIBUTE_DEBUG_INFORMATION_FORMAT "dwarf") # the symbols target makes the dSYMs set(CMAKE_XCODE_ATTRIBUTE_GCC_FAST_MATH NO) # Xcode does not read -march; this is the Darwin row of AlchemyTarget.cmake # in the spelling it does read. diff --git a/indra/cmake/ConfigurationReport.cmake b/indra/cmake/ConfigurationReport.cmake index 188f0ed0026..1057717e0b1 100644 --- a/indra/cmake/ConfigurationReport.cmake +++ b/indra/cmake/ConfigurationReport.cmake @@ -51,7 +51,6 @@ set( AL_ENABLE_TRACY_LOCAL_ONLY AL_ENABLE_TRACY_GPU AL_BUILD_TRACY_GUI - AL_ENABLE_SIGNING AL_USE_VELOPACK AL_ENABLE_RELEASE_DEBUG_LOGGING AL_BUILD_PACKAGE @@ -326,10 +325,16 @@ function(al_configuration_report) endif() _al_report_row("Package" "${package_kind}") endif() - if(AL_ENABLE_SIGNING) - _al_report_row("Signing" "identity '${AL_SIGNING_IDENTITY}'") - else() - _al_report_row("Signing" "off") + if(DARWIN) + if(AL_SIGNING_IDENTITY) + set(signing "identity '${AL_SIGNING_IDENTITY}'") + else() + set(signing "ad-hoc") + endif() + if(AL_USE_VELOPACK AND AL_NOTARY_PROFILE) + string(APPEND signing ", Velopack notarized with profile '${AL_NOTARY_PROFILE}'") + endif() + _al_report_row("Signing" "${signing}") endif() if(AL_BUILD_TESTS) diff --git a/indra/cmake/Linking.cmake b/indra/cmake/Linking.cmake index 1be5d1b693d..66f7e01359c 100644 --- a/indra/cmake/Linking.cmake +++ b/indra/cmake/Linking.cmake @@ -3,10 +3,8 @@ include_guard() include(Variables) -set( - SYMBOLS_STAGING_DIR - ${INDRA_BINARY_DIR}/symbols/$,$/,>${AL_CHANNEL} -) +# The symbol store; see ViewerSymbols.cmake. +set(SYMBOLS_DIR ${INDRA_BINARY_DIR}/symbols$<$:/$>) if(WINDOWS OR DARWIN) set( diff --git a/indra/cmake/RetiredOptions.cmake b/indra/cmake/RetiredOptions.cmake index 68ed79f2380..7deef912d21 100644 --- a/indra/cmake/RetiredOptions.cmake +++ b/indra/cmake/RetiredOptions.cmake @@ -53,7 +53,8 @@ set( GCC_DISABLE_FATAL_WARNINGS=AL_ENABLE_WARNINGS_AS_ERRORS CLANG_DISABLE_FATAL_WARNINGS=AL_ENABLE_WARNINGS_AS_ERRORS AL_WARNINGS_AS_ERRORS=AL_ENABLE_WARNINGS_AS_ERRORS - ENABLE_SIGNING=AL_ENABLE_SIGNING + ENABLE_SIGNING=AL_SIGNING_IDENTITY + AL_ENABLE_SIGNING=AL_SIGNING_IDENTITY SIGNING_IDENTITY=AL_SIGNING_IDENTITY INSTALL_PROPRIETARY=AL_ENABLE_PROPRIETARY RELEASE_CRASH_REPORTING=AL_ENABLE_CRASH_REPORTING diff --git a/indra/cmake/ViewerCodeSign.cmake b/indra/cmake/ViewerCodeSign.cmake index 8f4c5e850a1..dd5966585c5 100644 --- a/indra/cmake/ViewerCodeSign.cmake +++ b/indra/cmake/ViewerCodeSign.cmake @@ -1,8 +1,16 @@ # -*- cmake -*- # -# Signs the installed macOS bundle inside out, run from the install rules -# with AL_SIGN_BUNDLE, AL_SIGN_IDENTITY, AL_SIGN_PLUGIN_ENTITLEMENTS and -# AL_SIGN_HELPER_ENTITLEMENTS set. +# Signs a macOS bundle inside out. Run from the install rules, from the +# package step after it strips the executable, and by the hosted build's +# macOS packaging job as `cmake -D... -P ViewerCodeSign.cmake`, with: +# +# AL_SIGN_BUNDLE the .app +# AL_SIGN_IDENTITY a Developer ID, or empty for ad-hoc +# AL_SIGN_PLUGIN_ENTITLEMENTS the viewer's and the plugin hosts' +# AL_SIGN_HELPER_ENTITLEMENTS the CEF helpers' +# AL_SIGN_KEYCHAIN optional, the keychain holding the identity +# AL_SIGN_BUNDLE_ONLY optional, re-seal the outer bundle alone: for +# when only the main executable has changed # # Every nested piece is signed explicitly, deepest first: --deep would re-sign # the helpers with the outer bundle's identity and entitlements and strip the @@ -10,6 +18,11 @@ # Developer ID when one is configured, otherwise ad-hoc so a local build runs, # sandbox helpers included, without a certificate. +foreach(var AL_SIGN_BUNDLE AL_SIGN_PLUGIN_ENTITLEMENTS AL_SIGN_HELPER_ENTITLEMENTS) + if("${${var}}" STREQUAL "") + message(FATAL_ERROR "ViewerCodeSign.cmake needs ${var}") + endif() +endforeach() if(NOT IS_DIRECTORY "${AL_SIGN_BUNDLE}") message(FATAL_ERROR "No bundle to sign at ${AL_SIGN_BUNDLE}") endif() @@ -21,16 +34,24 @@ else() set(identity "-") set(timestamp "") endif() +set(keychain "") +if(AL_SIGN_KEYCHAIN) + set(keychain --keychain "${AL_SIGN_KEYCHAIN}") +endif() message(STATUS "Signing ${AL_SIGN_BUNDLE} (${identity})") function(al_codesign path) set(entitlements "") - if(ARGC GREATER 1 AND EXISTS "${ARGV1}") + if(ARGC GREATER 1) + if(NOT EXISTS "${ARGV1}") + message(FATAL_ERROR "No entitlements at ${ARGV1}") + endif() set(entitlements --entitlements "${ARGV1}") endif() execute_process( COMMAND - codesign --force --sign "${identity}" --options runtime ${timestamp} ${entitlements} "${path}" + codesign --force --sign "${identity}" ${keychain} --options runtime ${timestamp} + ${entitlements} "${path}" RESULT_VARIABLE result ) if(result) @@ -58,41 +79,40 @@ endfunction() set(contents "${AL_SIGN_BUNDLE}/Contents") -# 1. Loose Mach-O files anywhere in the bundle, so every enclosing bundle -# seals over already-signed code. -file( - GLOB_RECURSE loose - LIST_DIRECTORIES false - "${contents}/*.dylib" - "${contents}/*.so" - "${contents}/*.bin" - "${contents}/*.dat" -) -foreach(path IN LISTS loose) - al_codesign("${path}") -endforeach() +if(NOT AL_SIGN_BUNDLE_ONLY) + # 1. Loose Mach-O libraries anywhere in the bundle, so every enclosing + # bundle seals over already-signed code. Data files are sealed by their + # bundle's resource rules; a signature of their own would live in + # extended attributes, which archives and update packages drop. + file(GLOB_RECURSE loose LIST_DIRECTORIES false "${contents}/*.dylib" "${contents}/*.so") + foreach(path IN LISTS loose) + if(NOT IS_SYMLINK "${path}") + al_codesign("${path}") + endif() + endforeach() -# 2. Frameworks, deepest first. -file(GLOB_RECURSE frameworks LIST_DIRECTORIES true "${contents}/*.framework") -list(FILTER frameworks INCLUDE REGEX "\\.framework$") -al_deepest_first(frameworks) -foreach(path IN LISTS frameworks) - al_codesign("${path}") -endforeach() + # 2. Frameworks, deepest first. + file(GLOB_RECURSE frameworks LIST_DIRECTORIES true "${contents}/*.framework") + list(FILTER frameworks INCLUDE REGEX "\\.framework$") + al_deepest_first(frameworks) + foreach(path IN LISTS frameworks) + al_codesign("${path}") + endforeach() -# 3. Nested applications: the CEF helpers with their own entitlements, then -# the plugin hosts, deepest first. -file(GLOB_RECURSE apps LIST_DIRECTORIES true "${contents}/*.app") -list(FILTER apps INCLUDE REGEX "\\.app$") -al_deepest_first(apps) -foreach(path IN LISTS apps) - get_filename_component(name "${path}" NAME) - if(name MATCHES "^DullahanHelper") - al_codesign("${path}" "${AL_SIGN_HELPER_ENTITLEMENTS}") - else() - al_codesign("${path}" "${AL_SIGN_PLUGIN_ENTITLEMENTS}") - endif() -endforeach() + # 3. Nested applications: the CEF helpers with their own entitlements, then + # the plugin hosts, deepest first. + file(GLOB_RECURSE apps LIST_DIRECTORIES true "${contents}/*.app") + list(FILTER apps INCLUDE REGEX "\\.app$") + al_deepest_first(apps) + foreach(path IN LISTS apps) + get_filename_component(name "${path}" NAME) + if(name MATCHES "^DullahanHelper") + al_codesign("${path}" "${AL_SIGN_HELPER_ENTITLEMENTS}") + else() + al_codesign("${path}" "${AL_SIGN_PLUGIN_ENTITLEMENTS}") + endif() + endforeach() +endif() # 4. The viewer itself, sealing everything above. The plugin entitlements # carry disable-library-validation, which the hardened runtime needs to diff --git a/indra/cmake/ViewerInstall.cmake b/indra/cmake/ViewerInstall.cmake index 975cb6afbc6..21e14fede04 100644 --- a/indra/cmake/ViewerInstall.cmake +++ b/indra/cmake/ViewerInstall.cmake @@ -60,6 +60,9 @@ if(DARWIN) set(AL_INSTALL_LIBDIR "${AL_INSTALL_BUNDLE}/Contents/Frameworks") set(AL_INSTALL_PLUGINDIR "${AL_INSTALL_DATADIR}") set(al_ca_bundle_dir "${AL_INSTALL_DATADIR}") + # What ViewerCodeSign.cmake signs with, here and in the package steps. + set(AL_SIGN_PLUGIN_ENTITLEMENTS "${CMAKE_CURRENT_SOURCE_DIR}/slplugin.entitlements") + set(AL_SIGN_HELPER_ENTITLEMENTS "${INDRA_SOURCE_DIR}/dullahan/src/dullahan.entitlements") elseif(LINUX) set(AL_INSTALL_DATADIR ".") set(AL_INSTALL_BINDIR "bin") @@ -424,13 +427,16 @@ if(TARGET media_plugin_libvlc) set(vlc_frameworks_dir "${AL_INSTALL_PLUGINDIR}/media_plugin_libvlc.app/Contents/Frameworks") file(GLOB vlc_libraries "${al_vcpkg_dir}/lib/libvlc*.dylib*") install(FILES ${vlc_libraries} DESTINATION "${vlc_frameworks_dir}" COMPONENT plugins) + # Not plugins.dat: VLC trusts its cache only while every plugin keeps the + # size and time it was cached with, and signing changes both. A data file + # under Frameworks would also need a signature of its own, which lives in + # extended attributes that archives and update packages drop. install( DIRECTORY "${VLC_PLUGINS_DIR}/" DESTINATION "${vlc_frameworks_dir}/plugins" COMPONENT plugins FILES_MATCHING PATTERN "*.dylib" - PATTERN "plugins.dat" ) endif() endif() @@ -632,19 +638,17 @@ if(DARWIN) ) endforeach() - # Ad-hoc, or with AL_SIGNING_IDENTITY, inside out; the hosted build signs - # in its own step. - if(NOT DEFINED ENV{GITHUB_ACTIONS}) - install( - CODE - "set(AL_SIGN_BUNDLE \"\${CMAKE_INSTALL_PREFIX}/${AL_INSTALL_BUNDLE}\") + # Ad-hoc, or with AL_SIGNING_IDENTITY, inside out. The hosted build signs + # ad-hoc here and again with its Developer ID in the packaging job. + install( + CODE + "set(AL_SIGN_BUNDLE \"\${CMAKE_INSTALL_PREFIX}/${AL_INSTALL_BUNDLE}\") set(AL_SIGN_IDENTITY \"${AL_SIGNING_IDENTITY}\") -set(AL_SIGN_PLUGIN_ENTITLEMENTS \"${al_newview_dir}/slplugin.entitlements\") -set(AL_SIGN_HELPER_ENTITLEMENTS \"${INDRA_SOURCE_DIR}/dullahan/src/dullahan.entitlements\")" - COMPONENT viewer - ) - install(SCRIPT "${CMAKE_CURRENT_LIST_DIR}/ViewerCodeSign.cmake" COMPONENT viewer) - endif() +set(AL_SIGN_PLUGIN_ENTITLEMENTS \"${AL_SIGN_PLUGIN_ENTITLEMENTS}\") +set(AL_SIGN_HELPER_ENTITLEMENTS \"${AL_SIGN_HELPER_ENTITLEMENTS}\")" + COMPONENT viewer + ) + install(SCRIPT "${CMAKE_CURRENT_LIST_DIR}/ViewerCodeSign.cmake" COMPONENT viewer) endif() if(LINUX) diff --git a/indra/cmake/ViewerPackage.cmake b/indra/cmake/ViewerPackage.cmake index ef3ca9f482f..1027dc2a10d 100644 --- a/indra/cmake/ViewerPackage.cmake +++ b/indra/cmake/ViewerPackage.cmake @@ -15,37 +15,29 @@ include_guard() set(al_velopack_authors "Alchemy Viewer Project") set(al_velopack_splash_color "#00a5dc") set(al_velopack_version "${VIEWER_SHORT_VERSION}-${VIEWER_VERSION_REVISION}") +# An installed viewer updates from releases..json, so each platform +# and architecture has a channel of its own, named for its runtime. The +# channel also names the files vpk writes. +set(al_velopack_os "") if(DARWIN) set(al_velopack_title "${AL_APP_NAME}") set(al_velopack_main_exe "${product}") -else() + set(al_velopack_os osx) +elseif(WINDOWS) set(al_velopack_title "${AL_APP_NAME_ONEWORD}") set(al_velopack_main_exe "${OUTPUT_BINARY_NAME}.exe") - # An installed viewer updates from releases..json, so each - # architecture has a channel of its own, named for its runtime. The - # channel also names the installer, the portable archive and the packages - # vpk writes. + set(al_velopack_os win) +endif() + +# What the hosted build's packaging jobs need to know, one key=value per +# line. The jobs export every key to the environment in upper case. +if(al_velopack_os) if(BUILD_TARGET_IS_ARM64) - set(al_velopack_runtime win-arm64) + set(al_velopack_runtime ${al_velopack_os}-arm64) else() - set(al_velopack_runtime win-x64) + set(al_velopack_runtime ${al_velopack_os}-x64) endif() set(al_velopack_channel ${al_velopack_runtime}) -endif() - -# What the hosted build's packaging and signing steps need to know, one -# key=value per line for GITHUB_OUTPUT. -if(DARWIN) - set( - al_package_env - "velopack_mac_pack_id=${AL_APP_NAME_ONEWORD} -velopack_mac_pack_version=${al_velopack_version} -velopack_mac_pack_title=${al_velopack_title} -velopack_mac_main_exe=${al_velopack_main_exe} -velopack_mac_bundle_id=${MACOSX_BUNDLE_GUI_IDENTIFIER} -" - ) -elseif(WINDOWS) set( al_package_env "velopack_pack_id=${AL_APP_NAME_ONEWORD} @@ -53,14 +45,19 @@ velopack_pack_version=${al_velopack_version} velopack_pack_title=${al_velopack_title} velopack_pack_authors=${al_velopack_authors} velopack_main_exe=${al_velopack_main_exe} -velopack_icon=install_icon.ico -velopack_splash=install_splash.gif -velopack_splash_color=${al_velopack_splash_color} -velopack_installer_base=${AL_PACKAGE_NAME} velopack_channel=${al_velopack_channel} velopack_runtime=${al_velopack_runtime} " ) + if(WINDOWS) + string( + APPEND al_package_env + "velopack_icon=install_icon.ico +velopack_splash=install_splash.gif +velopack_splash_color=${al_velopack_splash_color} +" + ) + endif() else() set(al_package_env "") endif() @@ -108,22 +105,29 @@ set(CPACK_AL_REPOSITORY "${al_repository_dir}") if(WINDOWS) set(CPACK_GENERATOR ZIP) elseif(DARWIN) + # A plain disk image with the Applications link beside the bundle. The + # hosted build makes its own, laid out, from the notarized bundle. APFS, + # not HFS+: HFS+ decomposes file names, and the bundle's seal holds the + # names of the font stand-ins with Japanese names as they were composed. set(CPACK_GENERATOR DragNDrop) set(CPACK_DMG_VOLUME_NAME "${AL_APP_NAME}") - set( - CPACK_DMG_BACKGROUND_IMAGE - "${CMAKE_CURRENT_SOURCE_DIR}/installers/darwin/release-dmg/background.jpg" - ) - set(CPACK_DMG_DS_STORE "${CMAKE_CURRENT_SOURCE_DIR}/installers/darwin/release-dmg/_DS_Store") + set(CPACK_DMG_FILESYSTEM APFS) + set(CPACK_DMG_FORMAT ULMO) else() set(CPACK_GENERATOR TXZ) endif() # Release archives on Linux and macOS are stripped of debug information -# after the install into the package staging area. +# after the install into the package staging area; on macOS the bundle is +# then sealed again as the install signed it. if(NOT WINDOWS) set(CPACK_PRE_BUILD_SCRIPTS "${CMAKE_CURRENT_LIST_DIR}/ViewerStrip.cmake") endif() +if(DARWIN) + set(CPACK_AL_SIGN_IDENTITY "${AL_SIGNING_IDENTITY}") + set(CPACK_AL_SIGN_PLUGIN_ENTITLEMENTS "${AL_SIGN_PLUGIN_ENTITLEMENTS}") + set(CPACK_AL_SIGN_HELPER_ENTITLEMENTS "${AL_SIGN_HELPER_ENTITLEMENTS}") +endif() include(CPack) @@ -131,19 +135,34 @@ if(AL_USE_VELOPACK) set(velopack_dir "${CMAKE_CURRENT_BINARY_DIR}/velopack/$") set(velopack_releases "${velopack_dir}/Releases") if(DARWIN) + # The install signed every nested piece; vpk signs the updater it adds, + # seals the bundle again with the viewer's entitlements, and notarizes it + # when given a notarytool profile. Ad-hoc without an identity. + if(AL_SIGNING_IDENTITY) + set(velopack_identity "${AL_SIGNING_IDENTITY}") + else() + set(velopack_identity "-") + endif() set( velopack_args --packDir "${velopack_dir}/app/${AL_INSTALL_BUNDLE}" --mainExe "${al_velopack_main_exe}" - --bundleId - "${MACOSX_BUNDLE_GUI_IDENTIFIER}" - --icon - "${BRANDING_SOURCE_DIR}/viewer/icons/${ICON_PATH}/alchemy.icns" + --channel + "${al_velopack_channel}" + --runtime + "${al_velopack_runtime}" --noInst + --signAppIdentity + "${velopack_identity}" + --signDisableDeep + --signEntitlements + "${AL_SIGN_PLUGIN_ENTITLEMENTS}" ) - set(velopack_rename "") + if(AL_SIGNING_IDENTITY AND AL_NOTARY_PROFILE) + list(APPEND velopack_args --notaryProfile "${AL_NOTARY_PROFILE}") + endif() else() set( velopack_args @@ -164,6 +183,12 @@ if(AL_USE_VELOPACK) --runtime "${al_velopack_runtime}" ) + endif() + # The Windows installer and portable archive take the package's name; vpk's + # names for the macOS outputs carry the channel, which keeps them apart + # from the Windows arm64 ones on a release page. + set(velopack_rename "") + if(WINDOWS) set( velopack_rename COMMAND diff --git a/indra/cmake/ViewerStrip.cmake b/indra/cmake/ViewerStrip.cmake index 507f8d04ca9..db965929c41 100644 --- a/indra/cmake/ViewerStrip.cmake +++ b/indra/cmake/ViewerStrip.cmake @@ -4,7 +4,9 @@ # archive is written: strips debug information from the Release binaries. # `strip -S` keeps the symbol table, so a crash log still names functions. # On Linux that is every ELF file under bin/ and lib/; on macOS the viewer -# executable, whose dSYM is generated separately. +# executable. The symbols target keeps what is stripped, from the build +# tree's copies (ViewerSymbols.cmake). The install signed the +# bundle and stripping breaks the seal, so the bundle is sealed again. if(NOT CPACK_BUILD_CONFIG STREQUAL "Release") return() @@ -24,9 +26,18 @@ function(al_strip_file path) endfunction() if(APPLE) - file(GLOB executables "${CPACK_TEMPORARY_INSTALL_DIRECTORY}/*.app/Contents/MacOS/*") - foreach(path IN LISTS executables) - al_strip_file("${path}") + file(GLOB bundles LIST_DIRECTORIES true "${CPACK_TEMPORARY_INSTALL_DIRECTORY}/*.app") + foreach(bundle IN LISTS bundles) + file(GLOB executables "${bundle}/Contents/MacOS/*") + foreach(path IN LISTS executables) + al_strip_file("${path}") + endforeach() + set(AL_SIGN_BUNDLE "${bundle}") + set(AL_SIGN_IDENTITY "${CPACK_AL_SIGN_IDENTITY}") + set(AL_SIGN_PLUGIN_ENTITLEMENTS "${CPACK_AL_SIGN_PLUGIN_ENTITLEMENTS}") + set(AL_SIGN_HELPER_ENTITLEMENTS "${CPACK_AL_SIGN_HELPER_ENTITLEMENTS}") + set(AL_SIGN_BUNDLE_ONLY ON) + include("${CMAKE_CURRENT_LIST_DIR}/ViewerCodeSign.cmake") endforeach() else() file( diff --git a/indra/cmake/ViewerSymbols.cmake b/indra/cmake/ViewerSymbols.cmake new file mode 100644 index 00000000000..7009f123627 --- /dev/null +++ b/indra/cmake/ViewerSymbols.cmake @@ -0,0 +1,251 @@ +# -*- cmake -*- +# +# The symbol store: every binary the viewer ships with its debug information +# beside it, which is what Sentry symbolicates crash reports with and what a +# debugger needs to read a shipped build. Run by the `symbols` target as +# `cmake -D... -P ViewerSymbols.cmake`, over the tree the viewer's staging +# step installed, with: +# +# AL_SYMBOLS_FORMAT elf, macho or pe +# AL_SYMBOLS_MANIFEST the staging install's manifest: the files that ship +# AL_SYMBOLS_VIEWER the viewer executable, which the manifest leaves out: +# the viewer links into the staged tree, and the +# install records nothing it finds already in place +# AL_SYMBOLS_PREFIX the prefix the staging installed to +# AL_SYMBOLS_DIR the store, made again from nothing on every run +# AL_SYMBOLS_OBJCOPY elf: objcopy, to split the debug information off +# AL_SYMBOLS_READELF elf: readelf, for build IDs and section lists +# AL_SYMBOLS_DSYMUTIL macho: dsymutil +# AL_SYMBOLS_DUMPBIN pe: dumpbin, for the PDB each binary names +# AL_SYMBOLS_VCPKG pe: the vcpkg installed tree, whose PDBs are found +# by name when the path the binary names is gone +# +# The store per format: +# elf by build ID, as gdb, debuginfod and distribution debug packages +# lay one out: .build-id/xx/yyyy is the binary without its debug +# information, .build-id/xx/yyyy.debug the debug information alone +# with its sections compressed with zstd. +# macho the shipped tree's Mach-O files at their paths in it, each with +# a dSYM beside it where it has a debug map to make one from. +# pe the shipped tree's executables and DLLs at their paths in it, +# each with the PDB it names beside it. +# +# Mach-O and PE binaries are hard links into the staged tree, so they cost +# the store nothing. Binaries without debug information still go in: their +# symbol tables and unwind information let a crash report walk through them. + +foreach( + var + AL_SYMBOLS_FORMAT + AL_SYMBOLS_MANIFEST + AL_SYMBOLS_VIEWER + AL_SYMBOLS_PREFIX + AL_SYMBOLS_DIR +) + if("${${var}}" STREQUAL "") + message(FATAL_ERROR "ViewerSymbols.cmake needs ${var}") + endif() +endforeach() +if(AL_SYMBOLS_FORMAT STREQUAL "elf") + set(tools AL_SYMBOLS_OBJCOPY AL_SYMBOLS_READELF) +elseif(AL_SYMBOLS_FORMAT STREQUAL "macho") + set(tools AL_SYMBOLS_DSYMUTIL) +elseif(AL_SYMBOLS_FORMAT STREQUAL "pe") + set(tools AL_SYMBOLS_DUMPBIN) +else() + message(FATAL_ERROR "AL_SYMBOLS_FORMAT is elf, macho or pe, not ${AL_SYMBOLS_FORMAT}") +endif() +foreach(var IN LISTS tools) + if(NOT EXISTS "${${var}}") + message(FATAL_ERROR "ViewerSymbols.cmake needs ${var} for ${AL_SYMBOLS_FORMAT}: '${${var}}'") + endif() +endforeach() +if(NOT EXISTS "${AL_SYMBOLS_MANIFEST}") + message(FATAL_ERROR "No staging manifest at ${AL_SYMBOLS_MANIFEST}: build the viewer first") +endif() + +file(REMOVE_RECURSE "${AL_SYMBOLS_DIR}") +file(MAKE_DIRECTORY "${AL_SYMBOLS_DIR}") +file(STRINGS "${AL_SYMBOLS_MANIFEST}" files) +list(APPEND files "${AL_SYMBOLS_VIEWER}") +list(REMOVE_DUPLICATES files) + +# Runs a tool and fails with its output when the tool fails. +function(al_symbols_run) + execute_process( + COMMAND ${ARGN} + RESULT_VARIABLE result + OUTPUT_VARIABLE output + ERROR_VARIABLE output + ) + if(result) + list(JOIN ARGN " " command) + message(FATAL_ERROR "${command} failed (${result}):\n${output}") + endif() +endfunction() + +# Links a file into the store, or copies it where a link cannot be made. +function(al_symbols_link path destination) + cmake_path(GET destination PARENT_PATH directory) + file(MAKE_DIRECTORY "${directory}") + file(CREATE_LINK "${path}" "${destination}" COPY_ON_ERROR) +endfunction() + +# The PDBs of the prebuilt libraries, by lower-case file name. Not debug/: +# its PDBs have the same names as the release ones. +if(AL_SYMBOLS_FORMAT STREQUAL "pe" AND AL_SYMBOLS_VCPKG) + file(GLOB vcpkg_pdbs "${AL_SYMBOLS_VCPKG}/bin/*.pdb") + file(GLOB_RECURSE vcpkg_tool_pdbs "${AL_SYMBOLS_VCPKG}/tools/*.pdb") + foreach(pdb IN LISTS vcpkg_pdbs vcpkg_tool_pdbs) + cmake_path(GET pdb FILENAME name) + string(TOLOWER "${name}" name) + string(MAKE_C_IDENTIFIER "${name}" key) + set("vcpkg_pdb_${key}" "${pdb}") + endforeach() +endif() + +set(compression zstd) +set(binary_count 0) +set(debug_count 0) + +foreach(path IN LISTS files) + if(IS_SYMLINK "${path}" OR NOT EXISTS "${path}" OR IS_DIRECTORY "${path}") + continue() + endif() + file(RELATIVE_PATH relative "${AL_SYMBOLS_PREFIX}" "${path}") + + if(AL_SYMBOLS_FORMAT STREQUAL "elf") + file(READ "${path}" magic LIMIT 4 HEX) + if(NOT magic STREQUAL "7f454c46") + continue() + endif() + execute_process( + COMMAND "${AL_SYMBOLS_READELF}" --notes --section-headers --wide "${path}" + RESULT_VARIABLE result + OUTPUT_VARIABLE headers + ERROR_VARIABLE headers + ) + if(result) + message(FATAL_ERROR "${AL_SYMBOLS_READELF} failed (${result}) on ${path}:\n${headers}") + endif() + if(NOT headers MATCHES "Build ID: ([0-9a-f]+)") + message(WARNING "${relative} has no build ID to file it under; it is not in the store") + continue() + endif() + string(SUBSTRING "${CMAKE_MATCH_1}" 0 2 head) + string(SUBSTRING "${CMAKE_MATCH_1}" 2 -1 tail) + set(destination "${AL_SYMBOLS_DIR}/.build-id/${head}/${tail}") + # The same binary installed under two names is filed once. + if(EXISTS "${destination}") + continue() + endif() + file(MAKE_DIRECTORY "${AL_SYMBOLS_DIR}/.build-id/${head}") + math(EXPR binary_count "${binary_count} + 1") + if(NOT headers MATCHES "[ \t]\\.z?debug_info[ \t]") + al_symbols_link("${path}" "${destination}") + continue() + endif() + # zstd where this objcopy was built with it, zlib where it was not. + execute_process( + COMMAND + "${AL_SYMBOLS_OBJCOPY}" --only-keep-debug --compress-debug-sections=${compression} "${path}" + "${destination}.debug" + RESULT_VARIABLE result + OUTPUT_VARIABLE output + ERROR_VARIABLE output + ) + if(result AND compression STREQUAL "zstd") + message( + WARNING + "${AL_SYMBOLS_OBJCOPY} cannot compress with zstd; the store uses zlib:\n${output}" + ) + set(compression zlib) + al_symbols_run( + "${AL_SYMBOLS_OBJCOPY}" + --only-keep-debug + --compress-debug-sections=zlib + "${path}" + "${destination}.debug" + ) + elseif(result) + message(FATAL_ERROR "${AL_SYMBOLS_OBJCOPY} failed (${result}) on ${path}:\n${output}") + endif() + al_symbols_run("${AL_SYMBOLS_OBJCOPY}" --strip-debug "${path}" "${destination}") + math(EXPR debug_count "${debug_count} + 1") + elseif(AL_SYMBOLS_FORMAT STREQUAL "macho") + # Thin 64-bit Mach-O, or universal. + file(READ "${path}" magic LIMIT 4 HEX) + if(NOT magic MATCHES "^(cffaedfe|cafebabe)$") + continue() + endif() + set(destination "${AL_SYMBOLS_DIR}/${relative}") + al_symbols_link("${path}" "${destination}") + math(EXPR binary_count "${binary_count} + 1") + # dsymutil reads the debug map the linker left, which points at the + # object files. A binary built elsewhere has none, and dsymutil says so. + execute_process( + COMMAND "${AL_SYMBOLS_DSYMUTIL}" -o "${destination}.dSYM" "${path}" + RESULT_VARIABLE result + OUTPUT_VARIABLE output + ERROR_VARIABLE output + ) + if(output MATCHES "no debug symbols in executable") + file(REMOVE_RECURSE "${destination}.dSYM") + elseif(result) + message(FATAL_ERROR "${AL_SYMBOLS_DSYMUTIL} failed (${result}) on ${path}:\n${output}") + else() + string(REGEX MATCHALL "unable to open object file" missing "${output}") + list(LENGTH missing missing) + if(missing) + message( + WARNING + "The dSYM of ${relative} lacks ${missing} object files dsymutil could not open" + ) + endif() + math(EXPR debug_count "${debug_count} + 1") + endif() + else() + string(TOLOWER "${path}" lower) + if(NOT lower MATCHES "\\.(exe|dll)$") + continue() + endif() + set(destination "${AL_SYMBOLS_DIR}/${relative}") + al_symbols_link("${path}" "${destination}") + math(EXPR binary_count "${binary_count} + 1") + # The CodeView record names the PDB the linker wrote. A prebuilt + # library's names the tree it was built in, so it is found by name. + execute_process( + COMMAND "${AL_SYMBOLS_DUMPBIN}" /nologo /headers "${path}" + RESULT_VARIABLE result + OUTPUT_VARIABLE headers + ERROR_VARIABLE headers + ) + if(result) + message(FATAL_ERROR "${AL_SYMBOLS_DUMPBIN} failed (${result}) on ${path}:\n${headers}") + endif() + if(NOT headers MATCHES "Format: RSDS, {[^}]*}, [0-9]+, ([^\r\n]+)") + continue() + endif() + string(STRIP "${CMAKE_MATCH_1}" pdb) + file(TO_CMAKE_PATH "${pdb}" pdb) + cmake_path(GET pdb FILENAME pdb_name) + if(NOT EXISTS "${pdb}") + string(TOLOWER "${pdb_name}" key) + string(MAKE_C_IDENTIFIER "${key}" key) + if(NOT DEFINED "vcpkg_pdb_${key}") + continue() + endif() + set(pdb "${vcpkg_pdb_${key}}") + endif() + cmake_path(GET destination PARENT_PATH directory) + if(NOT EXISTS "${directory}/${pdb_name}") + al_symbols_link("${pdb}" "${directory}/${pdb_name}") + math(EXPR debug_count "${debug_count} + 1") + endif() + endif() +endforeach() + +message( + STATUS + "Symbol store ${AL_SYMBOLS_DIR}: ${binary_count} binaries, ${debug_count} with debug files" +) diff --git a/indra/newview/CMakeLists.txt b/indra/newview/CMakeLists.txt index 438a70ab1a9..e9905f235dc 100644 --- a/indra/newview/CMakeLists.txt +++ b/indra/newview/CMakeLists.txt @@ -2163,21 +2163,6 @@ if(AL_BUILD_VIEWER) XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER "${MACOSX_BUNDLE_GUI_IDENTIFIER}" ) - # Support generating dsym for non-xcode generators - if(NOT XCODE) - if(AL_USE_SENTRY) - set(VIEWER_DEBUG_BUNDLE "${SYMBOLS_STAGING_DIR}/dSYMs/${product}.dSYM") - find_program(DSYMUTIL_PROGRAM dsymutil REQUIRED) - add_custom_command( - TARGET ${AL_VIEWER_BINARY_NAME} - POST_BUILD - COMMAND - ${DSYMUTIL_PROGRAM} -o ${VIEWER_DEBUG_BUNDLE} $ - COMMENT "Generating Alchemy.dSYM" - ) - endif() - endif() - target_link_libraries(${AL_VIEWER_BINARY_NAME} PRIVATE llphysicsextensionsos) # The bundle's localised strings carry the version. @@ -2344,92 +2329,56 @@ ${settings_install} include(ViewerPackage) # After every link, stage the tree the viewer runs from: the install rules - # applied to the build directory the viewer already sits in. + # applied to the build directory the viewer already sits in. The install's + # manifest, which the next install anywhere overwrites, is kept per + # configuration as the list of what ships. + set(staged_manifest "${CMAKE_CURRENT_BINARY_DIR}/install_manifest_staged_$.txt") add_custom_command( TARGET ${AL_VIEWER_BINARY_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} --install ${CMAKE_BINARY_DIR} --config $ --prefix ${VIEWER_STAGING_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/install_manifest.txt ${staged_manifest} COMMENT "Staging the viewer into ${VIEWER_STAGING_DIR}" VERBATIM ) - if(AL_BUILD_PACKAGE AND AL_ENABLE_CRASH_REPORTING) - if(AL_USE_SENTRY) - # The debug information Sentry symbolicates crash reports with, staged - # per platform as .sym.tar.xz or the dSYM archive. It is part - # of the default build: CPack's package target is created at generate - # time, so nothing can be made to depend on it. - if(WINDOWS) - # Redirect symbols to staging directory for upload. - set_target_properties( - ${AL_VIEWER_BINARY_NAME} - PROPERTIES PDB_OUTPUT_DIRECTORY "${SYMBOLS_STAGING_DIR}" - ) - - set(VIEWER_APP_SYMBOLS_ARCHIVE "${SYMBOLS_STAGING_DIR}.sym.tar.xz") - - # Just pack up a tarball containing only the .pdb files for the - # executables. - add_custom_command( - OUTPUT "${VIEWER_APP_SYMBOLS_ARCHIVE}" - COMMAND tar cJf "${AL_CHANNEL}.sym.tar.xz" "${AL_CHANNEL}" - DEPENDS "${AL_VIEWER_BINARY_NAME}" - WORKING_DIRECTORY "${SYMBOLS_STAGING_DIR}/.." - COMMENT "Packing viewer PDBs into ${VIEWER_APP_SYMBOLS_ARCHIVE}" - VERBATIM - ) - add_custom_target(generate_symbols ALL DEPENDS "${VIEWER_APP_SYMBOLS_ARCHIVE}") - add_dependencies(generate_symbols ${AL_VIEWER_BINARY_NAME}) - endif() - if(DARWIN) - set(VIEWER_APP_XCARCHIVE "${SYMBOLS_STAGING_DIR}.xcarchive.zip") - - # we only need an xcarchive with dSYMs (including the application) - set_target_properties( - ${AL_VIEWER_BINARY_NAME} - PROPERTIES - XCODE_ATTRIBUTE_DEBUG_INFORMATION_FORMAT "dwarf-with-dsym" - XCODE_ATTRIBUTE_DWARF_DSYM_FOLDER_PATH "${SYMBOLS_STAGING_DIR}/dSYMs" - ) - - add_custom_command( - OUTPUT "${VIEWER_APP_XCARCHIVE}" - COMMAND zip -r "${VIEWER_APP_XCARCHIVE}" "${AL_CHANNEL}" - WORKING_DIRECTORY "${SYMBOLS_STAGING_DIR}/.." - DEPENDS "${AL_VIEWER_BINARY_NAME}" - COMMENT "Generating ${VIEWER_APP_XCARCHIVE} for upload" - VERBATIM - ) - add_custom_target(generate_symbols ALL DEPENDS "${VIEWER_APP_XCARCHIVE}") - endif() - if(LINUX) - # The debug information, split from the build-tree binary before the - # package strips its own copy; the two share a GNU build ID, which is - # how they are paired again. - find_program(OBJCOPY_PROGRAM objcopy REQUIRED) - set(VIEWER_APP_SYMBOLS_ARCHIVE "${SYMBOLS_STAGING_DIR}.sym.tar.xz") - set( - VIEWER_DEBUG_FILE - "${SYMBOLS_STAGING_DIR}/$.debug" - ) - add_custom_command( - OUTPUT "${VIEWER_APP_SYMBOLS_ARCHIVE}" - COMMAND ${CMAKE_COMMAND} -E make_directory "${SYMBOLS_STAGING_DIR}" - COMMAND - ${OBJCOPY_PROGRAM} --only-keep-debug --compress-debug-sections=zlib - $ "${VIEWER_DEBUG_FILE}" - COMMAND tar cJf "${AL_CHANNEL}.sym.tar.xz" "${AL_CHANNEL}" - DEPENDS "${AL_VIEWER_BINARY_NAME}" - WORKING_DIRECTORY "${SYMBOLS_STAGING_DIR}/.." - COMMENT "Packing the viewer's debug information into ${VIEWER_APP_SYMBOLS_ARCHIVE}" - VERBATIM - ) - add_custom_target(generate_symbols ALL DEPENDS "${VIEWER_APP_SYMBOLS_ARCHIVE}") - endif() - endif() + # The symbol store (ViewerSymbols.cmake): every binary in the staged tree + # with its debug information beside it, which Sentry symbolicates crash + # reports with and a debugger reads a shipped build with. Made on request + # rather than by every build. + if(WINDOWS) + cmake_path(GET CMAKE_LINKER PARENT_PATH linker_dir) + find_program(DUMPBIN_PROGRAM dumpbin HINTS "${linker_dir}") + set( + symbols_options + -DAL_SYMBOLS_FORMAT=pe + "-DAL_SYMBOLS_DUMPBIN=${DUMPBIN_PROGRAM}" + "-DAL_SYMBOLS_VCPKG=${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}" + ) + elseif(DARWIN) + find_program(DSYMUTIL_PROGRAM dsymutil) + set(symbols_options -DAL_SYMBOLS_FORMAT=macho "-DAL_SYMBOLS_DSYMUTIL=${DSYMUTIL_PROGRAM}") + else() + set( + symbols_options + -DAL_SYMBOLS_FORMAT=elf + "-DAL_SYMBOLS_OBJCOPY=${CMAKE_OBJCOPY}" + "-DAL_SYMBOLS_READELF=${CMAKE_READELF}" + ) endif() + add_custom_target( + symbols + COMMAND + ${CMAKE_COMMAND} ${symbols_options} "-DAL_SYMBOLS_MANIFEST=${staged_manifest}" + "-DAL_SYMBOLS_VIEWER=$" + "-DAL_SYMBOLS_PREFIX=${VIEWER_STAGING_DIR}" "-DAL_SYMBOLS_DIR=${SYMBOLS_DIR}" -P + "${INDRA_SOURCE_DIR}/cmake/ViewerSymbols.cmake" + COMMENT "Making the symbol store in ${SYMBOLS_DIR}" + VERBATIM + ) + add_dependencies(symbols ${AL_VIEWER_BINARY_NAME}) elseif(AL_BUILD_TESTS) diff --git a/indra/newview/installers/darwin/apple-notarize.sh b/indra/newview/installers/darwin/apple-notarize.sh deleted file mode 100755 index cc4435e614b..00000000000 --- a/indra/newview/installers/darwin/apple-notarize.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh -if [[ $SKIP_NOTARIZATION == "true" ]]; then - echo "Skipping notarization" - exit 0 -fi - -CONFIG_FILE="$build_secrets_checkout/code-signing-osx/notarize_creds.sh" -if [[ -f "$CONFIG_FILE" ]]; then - source "$CONFIG_FILE" - app_file="$1" - zip_file=${app_file/app/zip} - ditto -c -k --keepParent "$app_file" "$zip_file" - if [[ -f "$zip_file" ]]; then - res=$(xcrun altool --notarize-app --primary-bundle-id "com.secondlife.viewer" \ - --username $USERNAME \ - --password $PASSWORD \ - --asc-provider $ASC_PROVIDER \ - --file "$zip_file" 2>&1) - echo $res - - requestUUID=$(echo $res | awk '/RequestUUID/ { print $NF; }') - if [[ -n $requestUUID ]]; then - in_progress=1 - while [[ $in_progress -eq 1 ]]; do - sleep 30 - res=$(xcrun altool --notarization-info "$requestUUID" \ - --username $USERNAME \ - --password $PASSWORD 2>&1) - if [[ $res != *"in progress"* ]]; then - in_progress=0 - fi - echo "." - done - # log results - echo $res - - #remove temporary file - rm "$zip_file" - - if [[ $res == *"success"* ]]; then - xcrun stapler staple "$app_file" - exit 0 - elif [[ $res == *"invalid"* ]]; then - echo "Notarization error: failed to process the app file" - exit 1 - else - echo "Notarization error: unknown response status" - fi - else - echo "Notarization error: couldn't get request UUID" - exit 1 - fi - else - echo "Notarization error: ditto failed" - exit 1 - fi -fi diff --git a/indra/newview/installers/darwin/dmg-cleanup.applescript b/indra/newview/installers/darwin/dmg-cleanup.applescript deleted file mode 100644 index 8a71b392f92..00000000000 --- a/indra/newview/installers/darwin/dmg-cleanup.applescript +++ /dev/null @@ -1,28 +0,0 @@ --- First, convert the disk image to "read-write" format with Disk Utility or hdiutil --- Mount the image, open the disk image window in the Finder and make it frontmost, then run this script from inside Script Editor --- After running the script, unmount the disk image, re-mount it, and copy the .DS_Store file off from the command line. - -tell application "Finder" - - set foo to every item in front window - repeat with i in foo - if the name of i is "Applications" then - set the position of i to {391, 165} - else if the name of i ends with ".app" then - set the position of i to {121, 166} - end if - end repeat - - -- There doesn't seem to be a way to set the background picture with applescript, but all the saved .DS_Store files should already have that set correctly. - - set foo to front window - set current view of foo to icon view - set toolbar visible of foo to false - set statusbar visible of foo to false - set the bounds of foo to {100, 100, 600, 449} - - -- set the position of front window to {100, 100} - -- get {name, position} of every item of front window - - get properties of front window -end tell diff --git a/indra/newview/installers/darwin/dmg_settings.py b/indra/newview/installers/darwin/dmg_settings.py new file mode 100644 index 00000000000..a3e17514416 --- /dev/null +++ b/indra/newview/installers/darwin/dmg_settings.py @@ -0,0 +1,45 @@ +# dmgbuild settings for the disk image the hosted build makes from the +# notarized bundle: +# +# dmgbuild -s dmg_settings.py -D app=/.app "" .dmg +# +# The window shows the bundle and a link to /Applications either side of +# dmgbuild's own arrow, and the volume's icon is badged with the viewer's. +# Badging needs `dmgbuild[badge_icons]`. + +import os.path +import plistlib + +application = defines["app"] # noqa: F821 -- dmgbuild provides defines +appname = os.path.basename(application) + + +def app_icon(app): + with open(os.path.join(app, "Contents", "Info.plist"), "rb") as f: + icon = plistlib.load(f)["CFBundleIconFile"] + if not os.path.splitext(icon)[1]: + icon += ".icns" + return os.path.join(app, "Contents", "Resources", icon) + + +# APFS, not HFS+: HFS+ decomposes file names, and the bundle's seal holds the +# names of the font stand-ins with Japanese names as they were composed. +filesystem = "APFS" +# LZMA, the smallest of hdiutil's formats. LZFSE (ULFO) builds in half the +# time but is a third larger, and a higher lzma level gains nothing. +format = "ULMO" +files = [application] +symlinks = {"Applications": "/Applications"} +badge_icon = app_icon(application) + +background = "builtin-arrow" +window_rect = ((100, 100), (640, 280)) +default_view = "icon-view" +show_status_bar = False +show_tab_view = False +show_toolbar = False +show_pathbar = False +show_sidebar = False +icon_size = 128 +text_size = 14 +icon_locations = {appname: (140, 120), "Applications": (500, 120)} diff --git a/indra/newview/installers/darwin/fix_application_icon_position.sh b/indra/newview/installers/darwin/fix_application_icon_position.sh deleted file mode 100755 index 841defe96cd..00000000000 --- a/indra/newview/installers/darwin/fix_application_icon_position.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# just run this script each time after you change the installer's name to fix the icon misalignment -mydir="$(dirname "$0")" -# If there's more than one DMG in more than one build directory, pick the most -# recent one. -dmgfile="$(ls -t "$mydir/../../../../build-darwin-*/newview/*.dmg" | head -n 1)" -dmgwork="$HOME/Desktop/TempBuild.dmg" -mounted="/Volumes/Second Life Installer" -cp -r "$dmgfile" "$dmgwork" -hdid "$dmgwork" -open -a finder "$mounted" -osascript "$mydir/dmg-cleanup.applescript" -umount "$mounted"/ -hdid "$dmgwork" -open -a finder "$mounted" -#cp "$mounted"/.DS_Store ~/Desktop/_DS_Store -#chflags nohidden ~/Desktop/_DS_Store -#cp ~/Desktop/_DS_Store "$mydir/firstlook-dmg/_DS_Store" -#cp ~/Desktop/_DS_Store "$mydir/publicnightly-dmg/_DS_Store" -#cp ~/Desktop/_DS_Store "$mydir/release-dmg/_DS_Store" -#cp ~/Desktop/_DS_Store "$mydir/releasecandidate-dmg/_DS_Store" -#umount "$mounted"/ -#rm ~/Desktop/_DS_Store "$dmgwork" diff --git a/indra/newview/installers/darwin/release-dmg/Applications-alias.r b/indra/newview/installers/darwin/release-dmg/Applications-alias.r deleted file mode 100644 index f9be0536c6c..00000000000 --- a/indra/newview/installers/darwin/release-dmg/Applications-alias.r +++ /dev/null @@ -1,21 +0,0 @@ -data 'alis' (0) { - $"0000 0000 0112 0002 0001 036D 6277 0000" /* ...........mbw.. */ - $"0000 0000 0000 0000 0000 0000 0000 0000" /* ................ */ - $"0000 0000 0000 C135 A717 482B 0000 0000" /* ......�5�.H+.... */ - $"0002 0C41 7070 6C69 6361 7469 6F6E 7300" /* ...Applications. */ - $"0000 0000 0000 0000 0000 0000 0000 0000" /* ................ */ - $"0000 0000 0000 0000 0000 0000 0000 0000" /* ................ */ - $"0000 0000 0000 0000 0000 0000 0000 0000" /* ................ */ - $"0000 00E2 4F18 C2E8 9FB0 0000 0000 0000" /* ...�O.�蟰...... */ - $"0000 FFFF FFFF 0000 0920 0000 0000 0000" /* ..����..� ...... */ - $"0000 0000 0000 0000 0003 6D62 7700 0010" /* ..........mbw... */ - $"0008 0000 C136 0987 0000 0011 0008 0000" /* ....�6Ƈ........ */ - $"C2E9 0220 0000 0001 0000 0002 0010 6D62" /* ��. ..........mb */ - $"773A 4170 706C 6963 6174 696F 6E73 000E" /* w:Applications.. */ - $"001A 000C 0041 0070 0070 006C 0069 0063" /* .....A.p.p.l.i.c */ - $"0061 0074 0069 006F 006E 0073 000F 0008" /* .a.t.i.o.n.s.... */ - $"0003 006D 0062 0077 0012 000C 4170 706C" /* ...m.b.w....Appl */ - $"6963 6174 696F 6E73 0013 0001 2F00 FFFF" /* ications..../.�� */ - $"0000" /* .. */ -}; - diff --git a/indra/newview/installers/darwin/release-dmg/_DS_Store b/indra/newview/installers/darwin/release-dmg/_DS_Store deleted file mode 100644 index 747ca961d82..00000000000 Binary files a/indra/newview/installers/darwin/release-dmg/_DS_Store and /dev/null differ diff --git a/indra/newview/installers/darwin/release-dmg/_VolumeIcon.icns b/indra/newview/installers/darwin/release-dmg/_VolumeIcon.icns deleted file mode 100644 index 272b496e7d6..00000000000 Binary files a/indra/newview/installers/darwin/release-dmg/_VolumeIcon.icns and /dev/null differ diff --git a/indra/newview/installers/darwin/release-dmg/background.jpg b/indra/newview/installers/darwin/release-dmg/background.jpg deleted file mode 100644 index e7064d95452..00000000000 Binary files a/indra/newview/installers/darwin/release-dmg/background.jpg and /dev/null differ diff --git a/indra/vcpkg-configuration.json b/indra/vcpkg-configuration.json index 821a4e73398..f4144755621 100644 --- a/indra/vcpkg-configuration.json +++ b/indra/vcpkg-configuration.json @@ -2,7 +2,7 @@ "registries": [ { "kind": "git", - "baseline": "ef285cdf2812f05d3763a7275f849cfac8245e9d", + "baseline": "87d8ff664aac1a654ddc27b28e991888dc0fe123", "reference": "main", "repository": "https://github.com/AlchemyViewer/alchemy-registry", "packages": [ diff --git a/scripts/ci/msvc_environment.py b/scripts/ci/msvc_environment.py new file mode 100644 index 00000000000..78220343d0a --- /dev/null +++ b/scripts/ci/msvc_environment.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Give a GitHub Actions job the MSVC build environment, for Ninja. + +vcvarsall.bat puts cl, link, rc, mt and the Windows SDK on the path, with +the INCLUDE and LIB they read, for one target architecture. This runs it +for the runner's own architecture under the Visual Studio the Visual +Studio generator would pick, and hands the steps after it what it changed: +the variables through GITHUB_ENV and the new PATH entries through +GITHUB_PATH. Two of its changes stay out: Visual Studio's own CMake and +Ninja, which would shadow the pinned ones every vcpkg package ABI hashes, +and VCPKG_ROOT, which names Visual Studio's vcpkg rather than the +submodule. +""" + +import os +from pathlib import Path +import subprocess + +# Visual Studio 2026, which the Visual Studio 18 2026 generator uses. +VERSION_RANGE = "[18.0,19.0)" +ARCHITECTURES = {"X64": "x64", "ARM64": "arm64"} +# Set by the developer prompt but not wanted in the job. +EXCLUDED_VARIABLES = {"PATH", "VCPKG_ROOT", "PROMPT"} +# Path entries of the tools the job pins or brings itself. +EXCLUDED_PATH_PARTS = ("\\commonextensions\\microsoft\\cmake\\", "\\vc\\vcpkg") + + +def architecture(runner_arch): + try: + return ARCHITECTURES[runner_arch] + except KeyError: + raise ValueError(f"No MSVC architecture for a {runner_arch or 'nameless'} runner") from None + + +def parse_set(output): + """The variables `set` printed, keyed by upper-case name as Windows + compares them.""" + variables = {} + for line in output.splitlines(): + name, sep, value = line.partition("=") + if sep and name: + variables[name.upper()] = value + return variables + + +def _path_key(entry): + return entry.rstrip("\\/").lower() + + +def job_environment(before, after): + """What vcvarsall changed: the variables to set, and the PATH entries + to add, in the order it put them.""" + before = {name.upper(): value for name, value in before.items()} + after = {name.upper(): value for name, value in after.items()} + variables = {} + for name, value in after.items(): + if name in EXCLUDED_VARIABLES or before.get(name) == value: + continue + if any(c in name + value for c in "\r\n\0"): + raise ValueError(f"{name} does not fit on one line of GITHUB_ENV") + variables[name] = value + + known = {_path_key(entry) for entry in before.get("PATH", "").split(";") if entry} + entries = [] + for entry in after.get("PATH", "").split(";"): + key = _path_key(entry) + if not entry or key in known or any(part in key + "\\" for part in EXCLUDED_PATH_PARTS): + continue + known.add(key) + entries.append(entry) + return variables, entries + + +def visual_studio(env): + vswhere = (Path(env.get("ProgramFiles(x86)", r"C:\Program Files (x86)")) + / "Microsoft Visual Studio" / "Installer" / "vswhere.exe") + result = subprocess.run( + [str(vswhere), "-version", VERSION_RANGE, "-products", "*", "-latest", + "-property", "installationPath"], + check=True, text=True, stdout=subprocess.PIPE, + ) + installation = result.stdout.strip() + if not installation: + raise ValueError(f"No Visual Studio {VERSION_RANGE} on this runner") + return Path(installation) + + +def vcvars(installation, arch): + vcvarsall = installation / "VC" / "Auxiliary" / "Build" / "vcvarsall.bat" + # cmd /s keeps the quotes inside the outer pair as written. + result = subprocess.run( + f'cmd /d /s /c ""{vcvarsall}" {arch} >nul && set"', + check=True, text=True, stdout=subprocess.PIPE, + ) + return parse_set(result.stdout) + + +def main(): + env = dict(os.environ) + arch = architecture(env.get("RUNNER_ARCH", "")) + installation = visual_studio(env) + variables, entries = job_environment(env, vcvars(installation, arch)) + with open(env["GITHUB_ENV"], "a", encoding="utf-8", newline="\n") as stream: + for name, value in variables.items(): + stream.write(f"{name}={value}\n") + # The runner puts each line of GITHUB_PATH ahead of the ones before it. + with open(env["GITHUB_PATH"], "a", encoding="utf-8", newline="\n") as stream: + for entry in reversed(entries): + stream.write(f"{entry}\n") + print(f"MSVC {arch} from {installation}: {len(variables)} variables, {len(entries)} path entries") + for name in ("VCTOOLSVERSION", "WINDOWSSDKVERSION"): + if name in variables: + print(f" {name}={variables[name]}") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/test_msvc_environment.py b/scripts/ci/test_msvc_environment.py new file mode 100644 index 00000000000..467b8054843 --- /dev/null +++ b/scripts/ci/test_msvc_environment.py @@ -0,0 +1,66 @@ +import unittest + +import msvc_environment as msvc + + +VS = r"C:\Program Files\Microsoft Visual Studio\18\Enterprise" +TOOLS = VS + r"\VC\Tools\MSVC\14.50.35717\bin\HostX64\x64" +SDK = r"C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64" +CMAKE = VS + r"\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin" +NINJA = VS + r"\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja" +PYTHON = r"C:\hostedtoolcache\windows\Python\3.14.7\x64\Scripts" + + +class MsvcEnvironmentTests(unittest.TestCase): + def setUp(self): + self.before = { + "Path": PYTHON + r";C:\Windows\system32", + "VCPKG_ROOT": r"D:\a\Alchemy\Alchemy\vcpkg", + "RUNNER_ARCH": "X64", + } + self.after = { + "PATH": ";".join([TOOLS, CMAKE, NINJA, VS + r"\VC\vcpkg", SDK, PYTHON, r"C:\Windows\System32\\"]), + "VCPKG_ROOT": VS + r"\VC\vcpkg", + "RUNNER_ARCH": "X64", + "INCLUDE": VS + r"\VC\Tools\MSVC\14.50.35717\include", + "VSCMD_ARG_TGT_ARCH": "x64", + "PROMPT": "$P$G", + } + + def test_architecture_follows_the_runner(self): + self.assertEqual(msvc.architecture("X64"), "x64") + self.assertEqual(msvc.architecture("ARM64"), "arm64") + for runner_arch in ("", "X86", "x64"): + with self.assertRaises(ValueError): + msvc.architecture(runner_arch) + + def test_parse_set_keys_names_as_windows_compares_them(self): + output = "Path=C:\\a;C:\\b\r\nINCLUDE=C:\\x=y\r\nnot a variable\r\n" + self.assertEqual(msvc.parse_set(output), {"PATH": "C:\\a;C:\\b", "INCLUDE": "C:\\x=y"}) + + def test_hands_on_only_what_vcvars_changed(self): + variables, _ = msvc.job_environment(self.before, self.after) + self.assertEqual(variables, { + "INCLUDE": self.after["INCLUDE"], + "VSCMD_ARG_TGT_ARCH": "x64", + }) + + def test_keeps_the_submodule_vcpkg_and_the_pinned_cmake_and_ninja(self): + variables, entries = msvc.job_environment(self.before, self.after) + self.assertNotIn("VCPKG_ROOT", variables) + self.assertNotIn("PATH", variables) + self.assertEqual(entries, [TOOLS, SDK]) + + def test_path_entries_compare_without_case_or_trailing_separator(self): + self.after["PATH"] = ";".join([TOOLS, TOOLS.upper() + "\\", PYTHON.lower()]) + _, entries = msvc.job_environment(self.before, self.after) + self.assertEqual(entries, [TOOLS]) + + def test_a_variable_cannot_add_lines_to_the_job_environment(self): + self.after["INCLUDE"] = "C:\\include\nINJECTED=yes" + with self.assertRaises(ValueError): + msvc.job_environment(self.before, self.after) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/signing/macos_signing_secrets.py b/scripts/signing/macos_signing_secrets.py new file mode 100755 index 00000000000..173a9cbddc6 --- /dev/null +++ b/scripts/signing/macos_signing_secrets.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Turn the macOS signing certificate and notarization key into the build's secrets. + +The hosted build (.github/workflows/build.yaml) signs with a Developer ID +Application certificate and notarizes with an App Store Connect API key, +read from these repository secrets: + + MACOS_CERTIFICATE the certificate and its private key, a .p12, in base64 + MACOS_CERTIFICATE_PASSWORD the .p12's password + MACOS_NOTARY_KEY the API key, the .p8's text + MACOS_NOTARY_KEY_ID the API key's ID + MACOS_NOTARY_ISSUER_ID the issuer ID of a team key; unset for an individual key + +Each is checked as the build will use it. The .p12 must hold one private +key and one Developer ID Application identity that macOS trusts, imported +into a keychain of the script's own as the build imports it; it is stored +encrypted again under a password the script makes, in the format the +build's `security import` reads. The API key is tried against the notary +service unless --no-verify says not to. Then the secrets are stored with +`gh secret set`, or written to files, a file a secret, to paste into the +repository's settings. + +Nothing secret is printed or put on a command line: passwords reach +openssl through a pipe, and gh takes each value on its standard input. +What the script writes along the way stays in a private directory it +removes. + +The .p12: in Keychain Access, under My Certificates, select the +"Developer ID Application: ()" certificate, which carries its +private key, and use File > Export Items to save it as a .p12 with a +password. The API key: in App Store Connect, under Users and Access > +Integrations, a team key with the Developer role, or an individual key. +Its .p8 downloads once, as AuthKey_.p8; a team key's issuer ID +heads the page. + + scripts/signing/macos_signing_secrets.py --certificate DeveloperID.p12 \\ + --notary-key AuthKey_ABCDE12345.p8 --issuer-id --repo AlchemyViewer/Alchemy +""" + +import argparse +import base64 +import datetime +import getpass +import os +from pathlib import Path +import re +import secrets +import subprocess +import sys +import tempfile +import uuid + +# The system's LibreSSL: what it writes, macOS's security tool reads. +OPENSSL = "/usr/bin/openssl" +EXPIRY_WARNING_DAYS = 30 + + +class Error(Exception): + pass + + +def run(command, **kwargs): + kwargs.setdefault("stdout", subprocess.PIPE) + kwargs.setdefault("stderr", subprocess.PIPE) + return subprocess.run([str(part) for part in command], check=False, **kwargs) + + +def write_private(path, data): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as f: + f.write(data) + + +def openssl(arguments, stdin=None, **pipes): + """Runs openssl, feeding it each of `pipes` through a pipe of its own. + + In the arguments, @fd:NAME@ becomes fd:N, for -passin and -passout, and + @path:NAME@ becomes /dev/fd/N, for a file. + """ + fds = {} + try: + for name, data in pipes.items(): + read_end, write_end = os.pipe() + fds[name] = read_end + # Small enough to sit in the pipe until openssl reads it. + os.write(write_end, data) + os.close(write_end) + command = [OPENSSL] + for argument in map(str, arguments): + for name, fd in fds.items(): + argument = argument.replace(f"@fd:{name}@", f"fd:{fd}").replace(f"@path:{name}@", f"/dev/fd/{fd}") + command.append(argument) + return run(command, input=stdin, pass_fds=tuple(fds.values())) + finally: + for fd in fds.values(): + os.close(fd) + + +def search_list(): + result = run(["security", "list-keychains", "-d", "user"], text=True) + return [line.strip().strip('"') for line in result.stdout.splitlines() if line.strip()] + + +def identities(keychain, *options): + """The identities find-identity lists, by hash, as (name, problem).""" + result = run(["security", "find-identity", *options, keychain], text=True) + found = {} + for sha1, name, problem in re.findall( + r'^\s*\d+\)\s+([0-9A-F]{40})\s+"([^"]*)"(?:\s+\(([^)]*)\))?\s*$', result.stdout, re.MULTILINE + ): + found.setdefault(sha1, (name, problem)) + return found + + +def certificate_expiry(keychain, sha1): + result = run(["security", "find-certificate", "-a", "-Z", "-p", keychain], text=True) + match = re.search( + rf"SHA-1 hash: {sha1}\n(-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----)", + result.stdout, re.DOTALL, + ) + if not match: + raise Error("could not find the identity's certificate") + result = run([OPENSSL, "x509", "-noout", "-enddate"], input=match.group(1), text=True) + end = result.stdout.strip().removeprefix("notAfter=") + return datetime.datetime.strptime(end, "%b %d %H:%M:%S %Y %Z").replace(tzinfo=datetime.timezone.utc) + + +def check_identity(p12, password, workdir): + """Imports a .p12 into a keychain of its own, as the build does, and checks what it holds.""" + keychain = workdir / "check.keychain-db" + keychain_password = secrets.token_urlsafe(24) + before = search_list() + try: + result = run(["security", "create-keychain", "-p", keychain_password, keychain]) + if result.returncode: + raise Error(f"could not create a keychain to check the certificate in: {result.stderr.decode().strip()}") + result = run(["security", "import", p12, "-k", keychain, "-f", "pkcs12", "-P", password]) + if result.returncode: + raise Error(f"macOS's security tool cannot import the certificate: {result.stderr.decode().strip()}") + + dump = run(["security", "dump-keychain", keychain], text=True).stdout + keys = len(re.findall(r"^class: 0x00000010\b", dump, re.MULTILINE)) + if keys != 1: + raise Error(f"the .p12 holds {keys} private keys; export only the Developer ID Application certificate") + + found = identities(keychain) + if len(found) != 1: + raise Error(f"the .p12 holds {len(found)} identities; export only the Developer ID Application certificate") + sha1, (name, _) = next(iter(found.items())) + if not name.startswith("Developer ID Application: "): + raise Error(f'"{name}" is not a Developer ID Application certificate') + + if sha1 not in identities(keychain, "-v", "-p", "codesigning"): + _, problem = identities(keychain, "-p", "codesigning").get(sha1, (name, "")) + raise Error(f'macOS does not accept "{name}" for code signing ({problem or "no reason given"})') + + expiry = certificate_expiry(keychain, sha1) + return name, expiry + finally: + run(["security", "delete-keychain", keychain]) + if search_list() != before: + run(["security", "list-keychains", "-d", "user", "-s", *before]) + + +def prepare_certificate(path, workdir): + password = getpass.getpass(f"Password for {path.name}: ") + + # The key and certificates, unencrypted, through memory only. + result = openssl( + ["pkcs12", "-in", path, "-passin", "@fd:password@", "-nodes"], password=password.encode() + b"\n" + ) + if result.returncode: + raise Error(f"could not open {path}: wrong password, or not a .p12") + keys = re.findall( + rb"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----\n", result.stdout, re.DOTALL + ) + certificates = b"".join( + re.findall(rb"-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----\n", result.stdout, re.DOTALL) + ) + if len(keys) != 1: + raise Error(f"the .p12 holds {len(keys)} private keys; export only the Developer ID Application certificate") + + # Encrypted again under a password made here, and, to check it with the + # security tool without putting that password on a command line, a twin + # made the same way under a throwaway one. openssl reads the key and the + # certificates in two passes, so the key comes through a pipe of its own. + def export(new_password): + result = openssl( + ["pkcs12", "-export", "-inkey", "@path:key@", "-passout", "@fd:password@"], + stdin=certificates, key=keys[0], password=new_password.encode() + b"\n", + ) + if result.returncode: + raise Error(f"could not encrypt the certificate again: {result.stderr.decode().strip()}") + return result.stdout + + new_password = secrets.token_urlsafe(32) + p12 = export(new_password) + throwaway = secrets.token_urlsafe(24) + twin = workdir / "twin.p12" + write_private(twin, export(throwaway)) + name, expiry = check_identity(twin, throwaway, workdir) + + # And the one stored opens with its password. + result = openssl( + ["pkcs12", "-noout", "-passin", "@fd:password@"], stdin=p12, password=new_password.encode() + b"\n" + ) + if result.returncode: + raise Error("the certificate encrypted again does not open with its password") + + days = (expiry - datetime.datetime.now(datetime.timezone.utc)).days + print(f"Certificate: {name}, expires {expiry:%Y-%m-%d}", flush=True) + if days < EXPIRY_WARNING_DAYS: + print(f"warning: the certificate expires in {days} days", file=sys.stderr) + return { + "MACOS_CERTIFICATE": base64.b64encode(p12).decode(), + "MACOS_CERTIFICATE_PASSWORD": new_password, + } + + +def prepare_notary_key(path, key_id, issuer_id, verify): + if not key_id: + match = re.fullmatch(r"AuthKey_([A-Z0-9]+)\.p8", path.name) + if not match: + raise Error("pass --key-id, or keep the .p8's name, AuthKey_.p8") + key_id = match.group(1) + if not re.fullmatch(r"[A-Z0-9]{10,}", key_id): + raise Error(f"{key_id} is not an App Store Connect key ID") + if issuer_id: + try: + issuer_id = str(uuid.UUID(issuer_id)) + except ValueError: + raise Error(f"{issuer_id} is not an issuer ID, which is a UUID") from None + + text = path.read_text().replace("\r\n", "\n").strip() + if not re.fullmatch(r"-----BEGIN PRIVATE KEY-----\n[A-Za-z0-9+/=\n]+\n-----END PRIVATE KEY-----", text) or run( + [OPENSSL, "pkey", "-noout"], input=text.encode() + ).returncode: + raise Error(f"{path} is not an App Store Connect API key") + + if verify: + command = ["xcrun", "notarytool", "history", "--key", path, "--key-id", key_id, "--output-format", "json"] + if issuer_id: + command += ["--issuer", issuer_id] + result = run(command, text=True, timeout=120) + if result.returncode: + kind = "a team key with that issuer" if issuer_id else "an individual key (a team key needs --issuer-id)" + raise Error(f"the notary service refused the key as {kind}: {result.stderr.strip()}") + print(f"Notary key: {key_id}, accepted by the notary service") + else: + print(f"Notary key: {key_id}, not tried against the notary service") + + values = {"MACOS_NOTARY_KEY": text, "MACOS_NOTARY_KEY_ID": key_id} + if issuer_id: + values["MACOS_NOTARY_ISSUER_ID"] = issuer_id + return values + + +def store_in_github(repo, values, remove, assume_yes): + print(f"\nIn {repo}:") + for name in values: + print(f" set {name}") + for name in remove: + print(f" remove {name}") + if not assume_yes and input("Go ahead? [y/N] ").strip().lower() not in ("y", "yes"): + raise Error("nothing stored") + + for name, value in values.items(): + result = run(["gh", "secret", "set", name, "--repo", repo], input=value.encode()) + if result.returncode: + raise Error( + f"gh could not set {name}, though any listed above it are set: {result.stderr.decode().strip()}" + ) + if remove: + result = run(["gh", "secret", "list", "--repo", repo, "--json", "name", "--jq", ".[].name"], text=True) + if result.returncode: + raise Error(f"gh could not list the secrets: {result.stderr.strip()}") + for name in set(remove) & set(result.stdout.split()): + result = run(["gh", "secret", "delete", name, "--repo", repo]) + if result.returncode: + raise Error(f"gh could not remove {name}: {result.stderr.decode().strip()}") + print("Stored.") + + +def store_in_files(directory, values, remove): + directory.mkdir(mode=0o700) + for name, value in values.items(): + write_private(directory / name, value.encode()) + print(f"\nWrote {', '.join(values)} to {directory}, readable by you alone.") + for name in remove: + print(f"Leave {name} unset: an individual key has no issuer.") + print("Paste each into the repository's Actions secrets, then delete the directory.") + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__.split("\n\n")[0], + epilog="See the top of this script for what each secret is and where to get it.", + ) + parser.add_argument("--certificate", type=Path, help="the Developer ID Application .p12") + parser.add_argument("--notary-key", type=Path, help="the App Store Connect API key, AuthKey_.p8") + parser.add_argument("--key-id", help="the API key's ID, when the .p8's name does not carry it") + parser.add_argument("--issuer-id", help="the issuer ID, for a team key; leave it out for an individual key") + parser.add_argument("--no-verify", action="store_true", help="do not try the API key against the notary service") + where = parser.add_mutually_exclusive_group(required=True) + where.add_argument("--repo", help="store the secrets in this repository, OWNER/NAME, with gh") + where.add_argument("--output", type=Path, help="write the secrets to files in this new directory") + parser.add_argument("--yes", action="store_true", help="store in the repository without asking") + args = parser.parse_args() + + if sys.platform != "darwin": + parser.error("this needs macOS's security tool") + if not args.certificate and not args.notary_key: + parser.error("give --certificate, --notary-key, or both") + if args.notary_key is None and (args.key_id or args.issuer_id): + parser.error("--key-id and --issuer-id go with --notary-key") + if args.output and args.output.exists(): + parser.error(f"{args.output} exists; name a new directory") + + try: + values = {} + remove = [] + with tempfile.TemporaryDirectory(prefix="macos-signing-") as workdir: + if args.certificate: + values.update(prepare_certificate(args.certificate, Path(workdir))) + if args.notary_key: + values.update(prepare_notary_key(args.notary_key, args.key_id, args.issuer_id, not args.no_verify)) + if not args.issuer_id: + remove.append("MACOS_NOTARY_ISSUER_ID") + + if args.repo: + store_in_github(args.repo, values, remove, args.yes) + else: + store_in_files(args.output, values, remove) + except (Error, OSError, subprocess.TimeoutExpired) as error: + sys.exit(f"error: {error}") + except KeyboardInterrupt: + sys.exit("interrupted") + + +if __name__ == "__main__": + main()