test_workflow.yml

name: Shared Test Workflow

on:
  workflow_call:
    inputs:
      configuration:
        required: true
        type: string
      skip_gtest:
        description: "Skip C++ gtest execution (use when gtests run in a separate job)"
        required: false
        type: boolean
        default: false
      slow_tests:
        description: "Run the slow/e2e test suite (testSlow<Config>) instead of the regular suite"
        required: false
        type: boolean
        default: false

permissions:
  contents: read
  actions: read

jobs:
  cache-jdks:
    # This job is used to cache the JDKs for the test jobs
    uses: ./.github/workflows/cache_java.yml
  filter-musl-configs:
    # Sanitizers (asan/tsan) are not supported on musl - filter them out
    runs-on: ubuntu-latest
    outputs:
      configs: ${{ steps.filter.outputs.configs }}
      has_configs: ${{ steps.filter.outputs.has_configs }}
    steps:
      - id: filter
        run: |
          configs=$(echo '${{ inputs.configuration }}' | jq -c '[.[] | select(. != "asan" and . != "tsan")]')
          if [ "$configs" = "[]" ]; then
            echo "has_configs=false" >> $GITHUB_OUTPUT
          else
            echo "has_configs=true" >> $GITHUB_OUTPUT
          fi
          echo "configs=$configs" >> $GITHUB_OUTPUT
  test-linux-glibc-amd64:
    needs: cache-jdks
    # The default job name has no room for slow_tests -- it is a workflow
    # input, not a matrix axis -- so two calls to this workflow in the same
    # run (one regular, one slow) would otherwise show identical job names
    # and be indistinguishable to generate-test-summary.sh.
    name: test-linux-glibc-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
    strategy:
      fail-fast: false
      matrix:
        # java_version: [ "8", "8-orcl", "8-j9", "8-zing", "8-ibm", "11", "11-j9", "11-zing", "17", "17-j9", "17-zing", "17-graal", "21", "21-zing", "21-graal", "25", "25-graal" ]
        # FIXME: Zing disabled - Azul pulled public CDN access
        java_version: [ "8", "8-orcl", "8-j9", "8-ibm", "11", "11-j9", "17", "17-j9", "17-graal", "21", "21-graal", "25", "25-graal" ]
        config: ${{ fromJson(inputs.configuration) }}
    runs-on: ubuntu-latest
    timeout-minutes: 180
    steps:
      - name: Set enabled flag
        id: set_enabled
        run: |
          echo "enabled=true" >> $GITHUB_OUTPUT
          if [[ "${{ matrix.java_version }}" =~ -zing ]]; then
            if [[ "${{ matrix.config }}" != "release" ]] && [[ "${{ matrix.config }}" != "debug" ]]; then
              echo "enabled=false" >> $GITHUB_OUTPUT
            fi
          fi
          # IBM SDK 8 (OpenJ9 2.9) deadlocks inside JVMTI during the slow/e2e suite
          # (class-unload + System.gc() + profiler.dump/stop); regular testDebug passes.
          # Eclipse OpenJ9 8 (8-j9) and 17-j9 slow suites pass, so this is
          # IBM-SDK-8-specific. Recurring nightly + release slow-test hangs since ~2026-08-19.
          if [[ "${{ matrix.java_version }}" == "8-ibm" && "${{ inputs.slow_tests }}" == "true" ]]; then
            echo "enabled=false" >> $GITHUB_OUTPUT
          fi
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: steps.set_enabled.outputs.enabled == 'true'
        with:
          fetch-depth: 0
      - name: Cache Gradle Wrapper Binaries
        if: steps.set_enabled.outputs.enabled == 'true'
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.gradle/wrapper/dists
          key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
          restore-keys: |
            gradle-wrapper-${{ runner.os }}-
      - name: Cache Gradle User Home
        if: steps.set_enabled.outputs.enabled == 'true'
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.gradle/caches
          key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
          restore-keys: |
            gradle-caches-${{ runner.os }}-
      - name: Setup cached JDK
        id: cache-jdk
        if: steps.set_enabled.outputs.enabled == 'true'
        uses: ./.github/actions/setup_cached_java
        with:
          version: ${{ matrix.java_version }}
          arch: 'amd64'
      - name: Setup OS
        if: steps.set_enabled.outputs.enabled == 'true'
        run: |
          sudo apt-get update
          sudo apt-get install -y curl zip unzip libgtest-dev libgmock-dev binutils gdb
          # Install debug symbols for system libraries
          sudo apt-get install -y libc6-dbg
          if [[ ${{ matrix.java_version }} =~ "-zing" ]]; then
            sudo apt-get install -y g++-9 gcc-9
            sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9
            sudo update-alternatives --set gcc /usr/bin/gcc-9
          fi
      - name: Extract Versions
        if: steps.set_enabled.outputs.enabled == 'true'
        uses: ./.github/actions/extract_versions
      - name: Test
        if: steps.set_enabled.outputs.enabled == 'true'
        run: |
          set +e
          export KEEP_JFRS=true
          export TEST_COMMIT=${{ github.sha }}
          export TEST_CONFIGURATION=glibc/${{ matrix.java_version }}-${{ matrix.config }}-amd64
          export LIBC=glibc
          export SANITIZER=${{ matrix.config }}

          GRADLEW_PREFIX=""
          if [[ "${{ matrix.config }}" == "asan" ]]; then
            # Reduce ASLR entropy so the JVM doesn't land in ASan's shadow range
            # [0x00007fff7000-0x10007fff7fff]; 28 (the default) causes random aborts.
            if ! sudo sysctl -w vm.mmap_rnd_bits=8 2>/dev/null; then
              echo "::warning::Cannot set vm.mmap_rnd_bits; falling back to setarch --addr-no-randomize"
              GRADLEW_PREFIX="setarch $(uname -m) --addr-no-randomize"
            fi
          fi

          # GDB watchdog for debug config: dumps all native threads if the JVM hangs
          # inside JVMTI (e.g. the IBM SDK 8 slow-suite deadlock) before the 180-min
          # job timeout fires, so a thread dump lands in the test-reports artifact.
          # prepare_reports.sh copies build/logs/gdb-watchdog.log on failure.
          if [[ "${{ matrix.config }}" == "debug" ]]; then
            mkdir -p build/logs
            (
              sleep 1200  # 20 minutes — well past the normal ~6 min runtime
              echo "::warning::GDB watchdog triggered after 20 minutes"
              for pid in $(pgrep -f 'java.*ddprof-test'); do
                echo "=== Thread dump for PID $pid ===" >> build/logs/gdb-watchdog.log
                gdb -batch -ex 'thread apply all bt full' -p "$pid" >> build/logs/gdb-watchdog.log 2>&1 || true
              done
            ) &
            GDB_WATCHDOG_PID=$!
          fi

          # GraalVM pre-allocates fixed memory that conflicts with ASan's shadow range
          # [0x00007fff7000-0x10007fff7fff]; vm.mmap_rnd_bits tuning does not help (google/sanitizers#856).
          if [[ "${{ matrix.config }}" == "asan" && "${{ matrix.java_version }}" =~ graal ]]; then
            echo "::notice::Skipping ASan for GraalVM (incompatible shadow memory ranges — google/sanitizers#856)"
            exit 0
          fi

          # Default: a failure naming no test is a build error, not worth a
          # retry.
          export RETRY_ON_NO_TEST_FAILURES=0
          if [[ "${{ matrix.config }}" == "asan" ]]; then
            # ASan init can nondeterministically collide with the JVM's
            # ASLR-influenced mmap layout (google/sanitizers#856). The JVM
            # aborts before any test runs, so that failure names no test and
            # would otherwise be classed as a build error and left unretried.
            export RETRY_ON_NO_TEST_FAILURES=1
          fi

          # The slow/e2e suite already runs the best part of an hour, so a
          # retry would risk the 180-minute job timeout, and it records
          # failures without re-running them. ASan keeps its second attempt
          # even when slow, but only for the init abort above (0 named
          # failures): MAX_FAILURES_TO_RETRY=0 stops a slow ASan run that
          # actually failed named tests from being retried too, which would
          # re-run the full slow suite a second time and risk that same
          # timeout.
          export MAX_ATTEMPTS=2
          if [[ "${{ inputs.slow_tests }}" == "true" ]]; then
            if [[ "${{ matrix.config }}" == "asan" ]]; then
              export MAX_FAILURES_TO_RETRY=0
            else
              export MAX_ATTEMPTS=1
            fi
          fi

          .github/scripts/run_tests_with_retry.sh \
            "glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64${{ inputs.slow_tests && '-slow' || '' }}" -- \
            ${GRADLEW_PREFIX} ./gradlew -PCI -PkeepJFRs ${{ inputs.skip_gtest == true && '-Pskip-gtest' || '' }} :ddprof-test:test${{ inputs.slow_tests && 'Slow' || '' }}${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs
          EXIT_CODE=$?

          # Kill the watchdog if tests finished before it fired
          if [[ -n "${GDB_WATCHDOG_PID:-}" ]]; then
            kill "$GDB_WATCHDOG_PID" 2>/dev/null || true
          fi

          if [ $EXIT_CODE -ne 0 ]; then
            echo "glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64" >> failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt
            exit 1
          fi
      - name: Generate Unwinding Report
        if: success() && matrix.config == 'debug' && steps.set_enabled.outputs.enabled == 'true'
        run: |
          ./gradlew -PCI :ddprof-test:unwindingReport --no-daemon
      - name: Add Unwinding Report to Job Summary
        if: success() && matrix.config == 'debug' && steps.set_enabled.outputs.enabled == 'true' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != ''
        run: |
          echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (amd64)" >> $GITHUB_STEP_SUMMARY
          cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY
      - name: Upload build artifacts
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: success() && steps.set_enabled.outputs.enabled == 'true'
        with:
          name: (build) test-linux-glibc-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: build/
      - name: Upload failures
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: failures-glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64-${{ inputs.slow_tests && 'slow' || 'regular' }}
          path: failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt
      - name: Prepare reports
        if: always() && steps.set_enabled.outputs.enabled == 'true'
        run: |
          .github/scripts/prepare_reports.sh
      - name: Upload unwinding reports
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: success() && matrix.config == 'debug'
        with:
          name: (unwinding-reports) unwinding-linux-glibc-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: unwinding-reports
      - name: Upload test reports
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: (test-reports) test-linux-glibc-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: test-reports
      # Always uploaded, unlike the reports above: a cell that went green only
      # because of a retry produces no failure artifact, and that is exactly the
      # run whose evidence the summary needs. The file is a few hundred bytes.
      - name: Upload CI outcome
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: always()
        with:
          name: (ci-outcome) test-linux-glibc-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: ci-outcome
          if-no-files-found: ignore
      - name: Upload signal-safety violation log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: signal-safety-violation-glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64-${{ inputs.slow_tests && 'slow' || 'regular' }}
          path: /tmp/signal-safety-violation.txt
          if-no-files-found: ignore
      - name: Upload ASan logs
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure() && matrix.config == 'asan'
        with:
          name: asan-logs-glibc-${{ matrix.java_version }}-amd64-${{ inputs.slow_tests && 'slow' || 'regular' }}
          path: /tmp/asan.log.*
          if-no-files-found: ignore

  test-linux-musl-amd64:
    needs: [cache-jdks, filter-musl-configs]
    if: needs.filter-musl-configs.outputs.has_configs == 'true'
    # See the comment on test-linux-glibc-amd64's name above.
    name: test-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
    strategy:
     fail-fast: false
     matrix:
       java_version: [ "8-librca", "11-librca", "17-librca", "21-librca", "25-librca" ]
       config: ${{ fromJson(needs.filter-musl-configs.outputs.configs) }}
    runs-on: ubuntu-latest
    container:
      image: "alpine:3.23"
      options: --cpus 4 --workdir /github/workspace -v /home/runner/work/_temp:/home/runner/work/_temp
    timeout-minutes: 180
    steps:
      - name: Setup OS
        run: |
          # dl-cdn.alpinelinux.org intermittently fails the index fetch with a TLS error;
          # retry so a transient mirror blip does not fail the job. musl-dbg provides
          # debug symbols for musl libc.
          for attempt in 1 2 3 4 5; do
            if apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake gtest-dev gmock tar binutils python3 musl-dbg >/dev/null; then
              break
            fi
            if [ "$attempt" -eq 5 ]; then
              exit 1
            fi
            sleep $((attempt * 10))
          done
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
      - name: Cache Gradle Wrapper Binaries
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.gradle/wrapper/dists
          key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
          restore-keys: |
            gradle-wrapper-${{ runner.os }}-
      - name: Cache Gradle User Home
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.gradle/caches
          key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
          restore-keys: |
            gradle-caches-${{ runner.os }}-
      - name: Setup cached JDK
        id: cache-jdk
        uses: ./.github/actions/setup_cached_java
        with:
          version: ${{ matrix.java_version }}
          arch: 'amd64-musl'
      - name: Extract Versions
        uses: ./.github/actions/extract_versions
      - name: Test
        shell: bash
        run: |
          set +e

          export KEEP_JFRS=true
          export TEST_COMMIT=${{ github.sha }}
          export TEST_CONFIGURATION=musl/${{ matrix.java_version }}-${{ matrix.config }}-amd64
          # make sure the job knows it is running on musl
          export LIBC=musl
          export SANITIZER=${{ matrix.config }}

          # due to env hell in GHA containers, we need to re-do the logic from Extract Versions here
          JAVA_VERSION=$(${{ env.JAVA_TEST_HOME }}/bin/java -version 2>&1 | awk -F '"' '/version/ {
            split($2, v, "[._]");
            if (v[2] == "") {
              # Version is like "24": assume it is major only and add .0.0
              printf "%s.0.0\n", v[1]
            } else if (v[1] == "1") {
              # Java 8 or older: Format is "1.major.minor_update"
              printf "%s.%s.%s\n", v[2], v[3], v[4]
            } else {
              # Java 9 or newer: Format is "major.minor.patch"
              printf "%s.%s.%s\n", v[1], v[2], v[3]
            }
          }')
          export JAVA_VERSION
          echo "JAVA_VERSION=${JAVA_VERSION}"

          export MAX_ATTEMPTS=${{ inputs.slow_tests && 1 || 2 }}

          .github/scripts/run_tests_with_retry.sh \
            "musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64${{ inputs.slow_tests && '-slow' || '' }}" -- \
            ./gradlew -PCI -PkeepJFRs :ddprof-test:test${{ inputs.slow_tests && 'Slow' || '' }}${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs
          EXIT_CODE=$?

          if [ $EXIT_CODE -ne 0 ]; then
            echo "musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64" >> failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt
            exit 1
          fi
      - name: Generate Unwinding Report
        if: success() && matrix.config == 'debug'
        run: |
          ./gradlew -PCI :ddprof-test:unwindingReport --no-daemon
      - name: Add Unwinding Report to Job Summary
        if: success() && matrix.config == 'debug' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != ''
        run: |
          echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (amd64-musl)" >> $GITHUB_STEP_SUMMARY
          cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY
      - name: Upload build artifacts
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: success()
        with:
          name: (build) test-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: build/
      - name: Upload failures
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: failures-musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64-${{ inputs.slow_tests && 'slow' || 'regular' }}
          path: failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt
      - name: Prepare reports
        if: always()
        run: |
          .github/scripts/prepare_reports.sh
      - name: Upload unwinding reports
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: success() && matrix.config == 'debug'
        with:
          name: (unwinding-reports) unwinding-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: unwinding-reports
      - name: Upload test reports
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: (test-reports) test-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: test-reports
      # Always uploaded, unlike the reports above: a cell that went green only
      # because of a retry produces no failure artifact, and that is exactly the
      # run whose evidence the summary needs. The file is a few hundred bytes.
      - name: Upload CI outcome
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: always()
        with:
          name: (ci-outcome) test-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: ci-outcome
          if-no-files-found: ignore
      - name: Upload signal-safety violation log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: signal-safety-violation-musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64-${{ inputs.slow_tests && 'slow' || 'regular' }}
          path: /tmp/signal-safety-violation.txt
          if-no-files-found: ignore
      - name: Upload ASan logs
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure() && matrix.config == 'asan'
        with:
          name: asan-logs-musl-${{ matrix.java_version }}-amd64-${{ inputs.slow_tests && 'slow' || 'regular' }}
          path: /tmp/asan.log.*
          if-no-files-found: ignore

  test-linux-glibc-aarch64:
    needs: cache-jdks
    # See the comment on test-linux-glibc-amd64's name above.
    name: test-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
    strategy:
      fail-fast: false
      matrix:
        # java_version: [ "8", "8-j9", "8-zing", "11", "11-j9", "11-zing", "17", "17-j9", "17-zing", "17-graal", "21", "21-zing", "21-graal", "23", "23-graal" ]
        # FIXME: Hotspot 8 and 11 versions are rather crashy in ASGCT on aarch64, so we are skipping them for now
        # java_version: [ "8-j9", "8-zing", "11-j9", "11-zing", "17", "17-j9", "17-zing", "17-graal", "21", "21-zing", "21-graal", "25", "25-graal" ]
        # FIXME: Zing disabled - Azul pulled public CDN access
        java_version: [ "8-j9", "11-j9", "17", "17-j9", "17-graal", "21", "21-graal", "25", "25-graal" ]
        config: ${{ fromJson(inputs.configuration) }}
    runs-on:
      group: ARM LINUX SHARED
      labels: arm-4core-linux
    timeout-minutes: 180
    steps:
      - name: Set enabled flag
        id: set_enabled
        run: |
          echo "enabled=true" >> $GITHUB_OUTPUT
          if [[ "${{ matrix.java_version }}" =~ -zing ]]; then
            if [[ "${{ matrix.config }}" != "release" ]] && [[ "${{ matrix.config }}" != "debug" ]]; then
              echo "enabled=false" >> $GITHUB_OUTPUT
            fi
          fi
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: steps.set_enabled.outputs.enabled == 'true'
        with:
          fetch-depth: 0
      - name: Cache Gradle Wrapper Binaries
        if: steps.set_enabled.outputs.enabled == 'true'
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.gradle/wrapper/dists
          key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
          restore-keys: |
            gradle-wrapper-${{ runner.os }}-
      - name: Cache Gradle User Home
        if: steps.set_enabled.outputs.enabled == 'true'
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.gradle/caches
          key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
          restore-keys: |
            gradle-caches-${{ runner.os }}-
      - name: Setup cached JDK
        id: cache-jdk
        if: steps.set_enabled.outputs.enabled == 'true'
        uses: ./.github/actions/setup_cached_java
        with:
          version: ${{ matrix.java_version }}
          arch: 'aarch64'
      - name: Setup OS
        if: steps.set_enabled.outputs.enabled == 'true'
        run: |
          # ports.ubuntu.com's IPv6 route is intermittently unreachable from GH-hosted
          # aarch64 runners; force IPv4 so apt doesn't stall/fail on a dead IPv6 path,
          # and retry on top in case of other transient connection blips.
          printf 'Acquire::ForceIPv4 "true";\nAcquire::Retries "5";\n' | sudo tee /etc/apt/apt.conf.d/99force-ipv4 >/dev/null
          sudo apt update -y
          sudo apt remove -y g++
          sudo apt autoremove -y
          sudo apt install -y curl zip unzip clang make build-essential binutils gdb libgtest-dev libgmock-dev
          # Install debug symbols for system libraries
          sudo apt install -y libc6-dbg
          if [[ ${{ matrix.java_version }} =~ "-zing" ]]; then
            sudo apt -y install g++-9 gcc-9
            sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9
            sudo update-alternatives --set gcc /usr/bin/gcc-9
          fi
      - name: Extract Versions
        if: steps.set_enabled.outputs.enabled == 'true'
        uses: ./.github/actions/extract_versions
      - name: Test
        if: steps.set_enabled.outputs.enabled == 'true'
        run: |
          set +e
          export KEEP_JFRS=true
          export TEST_COMMIT=${{ github.sha }}
          export TEST_CONFIGURATION=glibc/${{ matrix.java_version }}-${{ matrix.config }}-aarch64
          export LIBC=glibc
          export SANITIZER=${{ matrix.config }}

          GRADLEW_PREFIX=""
          # For ASAN: launch a gdb watchdog that dumps all native threads before the
          # 30-minute Gradle timeout kills the JVM, so we can diagnose hangs in native code.
          if [[ "${{ matrix.config }}" == "asan" ]]; then
            # Reduce ASLR entropy so the JVM doesn't land in ASan's shadow range; 28 causes random aborts.
            if ! sudo sysctl -w vm.mmap_rnd_bits=8 2>/dev/null; then
              echo "::warning::Cannot set vm.mmap_rnd_bits; falling back to setarch --addr-no-randomize"
              GRADLEW_PREFIX="setarch $(uname -m) --addr-no-randomize"
            fi
            mkdir -p build/logs
            (
              sleep 1500  # 25 minutes — fires before the 30-min Gradle timeout
              echo "::warning::GDB watchdog triggered after 25 minutes"
              for pid in $(pgrep -f 'java.*ddprof-test'); do
                echo "=== Thread dump for PID $pid ===" >> build/logs/gdb-watchdog.log
                gdb -batch -ex 'thread apply all bt full' -p "$pid" >> build/logs/gdb-watchdog.log 2>&1 || true
              done
            ) &
            GDB_WATCHDOG_PID=$!
          fi

          # GraalVM pre-allocates fixed memory that conflicts with ASan's shadow range
          # [0x00007fff7000-0x10007fff7fff]; vm.mmap_rnd_bits tuning does not help (google/sanitizers#856).
          if [[ "${{ matrix.config }}" == "asan" && "${{ matrix.java_version }}" =~ graal ]]; then
            echo "::notice::Skipping ASan for GraalVM (incompatible shadow memory ranges — google/sanitizers#856)"
            exit 0
          fi

          # Default: a failure naming no test is a build error, not worth a
          # retry.
          export RETRY_ON_NO_TEST_FAILURES=0
          if [[ "${{ matrix.config }}" == "asan" ]]; then
            # ASan init can nondeterministically collide with the JVM's
            # ASLR-influenced mmap layout (google/sanitizers#856). The JVM
            # aborts before any test runs, so that failure names no test and
            # would otherwise be classed as a build error and left unretried.
            export RETRY_ON_NO_TEST_FAILURES=1
          fi

          # The slow/e2e suite already runs the best part of an hour, so a
          # retry would risk the 180-minute job timeout, and it records
          # failures without re-running them. ASan keeps its second attempt
          # even when slow, but only for the init abort above (0 named
          # failures): MAX_FAILURES_TO_RETRY=0 stops a slow ASan run that
          # actually failed named tests from being retried too, which would
          # re-run the full slow suite a second time and risk that same
          # timeout.
          export MAX_ATTEMPTS=2
          if [[ "${{ inputs.slow_tests }}" == "true" ]]; then
            if [[ "${{ matrix.config }}" == "asan" ]]; then
              export MAX_FAILURES_TO_RETRY=0
            else
              export MAX_ATTEMPTS=1
            fi
          fi

          .github/scripts/run_tests_with_retry.sh \
            "glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64${{ inputs.slow_tests && '-slow' || '' }}" -- \
            ${GRADLEW_PREFIX} ./gradlew -PCI -PkeepJFRs ${{ inputs.skip_gtest == true && '-Pskip-gtest' || '' }} :ddprof-test:test${{ inputs.slow_tests && 'Slow' || '' }}${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs
          EXIT_CODE=$?

          # Kill the watchdog if tests finished before it fired
          if [[ -n "${GDB_WATCHDOG_PID:-}" ]]; then
            kill "$GDB_WATCHDOG_PID" 2>/dev/null || true
          fi

          if [ $EXIT_CODE -ne 0 ]; then
            echo "glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64" >> failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt
            exit 1
          fi
      - name: Generate Unwinding Report
        if: success() && matrix.config == 'debug'
        run: |
          ./gradlew -PCI :ddprof-test:unwindingReport --no-daemon
      - name: Add Unwinding Report to Job Summary
        if: success() && matrix.config == 'debug' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != ''
        run: |
          echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (aarch64)" >> $GITHUB_STEP_SUMMARY
          cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY
      - name: Upload build artifacts
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: success()
        with:
          name: (build) test-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: build/
      - name: Upload failures
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: failures-glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64-${{ inputs.slow_tests && 'slow' || 'regular' }}
          path: failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt
      - name: Prepare reports
        if: always() && steps.set_enabled.outputs.enabled == 'true'
        run: |
          .github/scripts/prepare_reports.sh
      - name: Upload unwinding reports
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: success() && matrix.config == 'debug'
        with:
          name: (unwinding-reports) unwinding-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: unwinding-reports
      - name: Upload test reports
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: (test-reports) test-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: test-reports
      # Always uploaded, unlike the reports above: a cell that went green only
      # because of a retry produces no failure artifact, and that is exactly the
      # run whose evidence the summary needs. The file is a few hundred bytes.
      - name: Upload CI outcome
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: always()
        with:
          name: (ci-outcome) test-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
          path: ci-outcome
          if-no-files-found: ignore
      - name: Upload signal-safety violation log
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: signal-safety-violation-glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64-${{ inputs.slow_tests && 'slow' || 'regular' }}
          path: /tmp/signal-safety-violation.txt
          if-no-files-found: ignore
      - name: Upload ASan logs
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure() && matrix.config == 'asan'
        with:
          name: asan-logs-glibc-${{ matrix.java_version }}-aarch64-${{ inputs.slow_tests && 'slow' || 'regular' }}
          path: /tmp/asan.log.*
          if-no-files-found: ignore

  test-linux-musl-aarch64:
     needs: [cache-jdks, filter-musl-configs]
     if: needs.filter-musl-configs.outputs.has_configs == 'true'
     # See the comment on test-linux-glibc-amd64's name above.
     name: test-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
     strategy:
       fail-fast: false
       matrix:
         java_version: [ "8-librca", "11-librca", "17-librca", "21-librca", "25-librca" ]
         config: ${{ fromJson(needs.filter-musl-configs.outputs.configs) }}
     runs-on:
       group: ARM LINUX SHARED
       labels: arm-4core-linux-ubuntu24.04
     timeout-minutes: 180
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
          fetch-depth: 0
       - name: Cache Gradle Wrapper Binaries
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ~/.gradle/wrapper/dists
           key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
           restore-keys: |
             gradle-wrapper-${{ runner.os }}-
       - name: Cache Gradle User Home
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ~/.gradle/caches
           key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
           restore-keys: |
             gradle-caches-${{ runner.os }}-
       - name: Setup cached JDK
         id: cache-jdk
         uses: ./.github/actions/setup_cached_java
         with:
           version: ${{ matrix.java_version }}
           arch: 'aarch64-musl'
       - name: Extract Versions
         uses: ./.github/actions/extract_versions
       - name: Test
         run: |
           set +e
           # the effective JAVA_VERSION is computed in the test_alpine_aarch64.sh script
           mkdir -p build/logs
           export MAX_ATTEMPTS=${{ inputs.slow_tests && 1 || 2 }}

           .github/scripts/run_tests_with_retry.sh \
             "musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64${{ inputs.slow_tests && '-slow' || '' }}" -- \
             docker run --cpus 4 --rm -v /tmp:/tmp -v "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}" -w "${GITHUB_WORKSPACE}" alpine:3.21 /bin/sh -c "
               \"$GITHUB_WORKSPACE/.github/scripts/test_alpine_aarch64.sh\" \
               \"${{ github.sha }}\" \"musl/${{ matrix.java_version }}-${{ matrix.config }}-aarch64\" \
               \"${{ matrix.config }}\" \"${{ env.JAVA_HOME }}\" \"${{ env.JAVA_TEST_HOME }}\" \
               \"${{ inputs.slow_tests }}\"
             "

           EXIT_CODE=$?

           if [ $EXIT_CODE -ne 0 ]; then
             echo "musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64" >> failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt
             exit 1
           fi
       - name: Fix permissions for artifact upload
         if: always()
         run: |
           # Files created by Docker container may have restrictive permissions
           # that prevent GitHub Actions from reading them during artifact creation.
           # Use a+rX (umask-independent) where X adds execute only to directories.
           sudo chmod -R a+rX build/ ddprof-test/build/ ddprof-lib/build/ || true
       - name: Generate Unwinding Report
         if: success() && matrix.config == 'debug'
         run: |
           docker run --cpus 4 --rm -v /tmp:/tmp -v "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}" -w "${GITHUB_WORKSPACE}" alpine:3.21 /bin/sh -c "
             \"$GITHUB_WORKSPACE/.github/scripts/unwinding_report_alpine_aarch64.sh\" \
             \"${{ github.sha }}\" \"musl/${{ matrix.java_version }}-${{ matrix.config }}-aarch64\" \
             \"${{ matrix.config }}\" \"${{ env.JAVA_HOME }}\" \"${{ env.JAVA_TEST_HOME }}\"
           "
           # Fix permissions after Docker container execution (both top-level and subproject builds).
           # Use a+rX (umask-independent) where X adds execute only to directories.
           sudo chmod -R a+rX build/ ddprof-test/build/ ddprof-lib/build/ || true
       - name: Add Unwinding Report to Job Summary
         if: success() && matrix.config == 'debug' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != ''
         run: |
           echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (aarch64-musl)" >> $GITHUB_STEP_SUMMARY
           cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY
       - name: Upload build artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: success()
         with:
           name: (build) test-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
           path: build/
       - name: Upload failures
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: failure()
         with:
           name: failures-musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64-${{ inputs.slow_tests && 'slow' || 'regular' }}
           path: failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt
       - name: Prepare reports
         if: always()
         run: |
           .github/scripts/prepare_reports.sh
       - name: Upload unwinding reports
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: success() && matrix.config == 'debug'
         with:
           name: (unwinding-reports) unwinding-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
           path: unwinding-reports
       - name: Upload test reports
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: failure()
         with:
           name: (test-reports) test-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
           path: test-reports
       # Always uploaded, unlike the reports above: a cell that went green only
       # because of a retry produces no failure artifact, and that is exactly the
       # run whose evidence the summary needs. The file is a few hundred bytes.
       - name: Upload CI outcome
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: always()
         with:
           name: (ci-outcome) test-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}, ${{ inputs.slow_tests && 'slow' || 'regular' }})
           path: ci-outcome
           if-no-files-found: ignore
       - name: Upload signal-safety violation log
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: failure()
         with:
           name: signal-safety-violation-musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64-${{ inputs.slow_tests && 'slow' || 'regular' }}
           path: /tmp/signal-safety-violation.txt
           if-no-files-found: ignore
       - name: Upload ASan logs
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: failure() && matrix.config == 'asan'
         with:
           name: asan-logs-musl-${{ matrix.java_version }}-aarch64-${{ inputs.slow_tests && 'slow' || 'regular' }}
           path: /tmp/asan.log.*
           if-no-files-found: ignore