_test.yml

# Reusable: unit + integration tests on all platforms
name: Test

on:
  workflow_call:
    inputs:
      skip_perf:
        description: 'Skip incremental perf tests (phases 2-7)'
        type: boolean
        default: true
      broad_platforms:
        description: 'Test the broad platform matrix (older glibc + extra OS versions) instead of the core set'
        type: boolean
        default: false
      shard_suites:
        description: 'Split each C-suite leg across parallel shard jobs (ubuntu x3, windows x2); off = one job per leg, byte-identical to the pre-shard topology'
        type: boolean
        default: false
      platforms:
        description: "Platform set: 'all' (default) or 'linux' (skip macOS and Windows legs)"
        type: string
        default: 'all'
      shard_profile:
        description: 'Shard counts when shard_suites is on: standard = ubuntu x3, windows x2 (dry run, release); pr = ubuntu x86 x1, ubuntu arm64 x2, windows x1 (fewer jobs re-paying setup)'
        type: string
        default: standard
      lanes:
        description: 'PR lane selection (scripts/ci/select-lanes.sh JSON list): only those legs and jobs run; "all" = every one (dry run, release)'
        type: string
        default: all
      test_selection:
        description: 'PR test selection (smart CI): the artifact the select-tests job uploaded; the Linux x86_64 test-unix leg runs only its tests, the arm64, macOS and Windows legs run every test. Empty = every test (dry run, release, and any PR the selection does not narrow)'
        type: string
        default: ''

permissions:
  contents: read

jobs:
  # Emit the platform matrices as JSON. The CORE set is the default (fast,
  # unchanged); the BROAD set adds extra free runners (older glibc /
  # additional OS versions) for a wider "does it build everywhere" picture.
  setup-matrix:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      unix: ${{ steps.set.outputs.unix }}
      windows: ${{ steps.set.outputs.windows }}
      tsan: ${{ steps.set.outputs.tsan }}
      has_unix: ${{ steps.set.outputs.has_unix }}
      has_windows: ${{ steps.set.outputs.has_windows }}
      has_tsan: ${{ steps.set.outputs.has_tsan }}
    steps:
      - name: Compute matrices
        id: set
        env:
          BROAD: ${{ inputs.broad_platforms }}
          SHARDS: ${{ inputs.shard_suites }}
          PLATFORMS: ${{ inputs.platforms }}
          SHARD_PROFILE: ${{ inputs.shard_profile }}
          LANES: ${{ inputs.lanes }}
        run: |
          # Every CORE leg is tagged with the lane (scripts/ci/select-lanes.sh)
          # it serves; the tag is what lane selection filters on below.
          # The lane names unix-macos14 and tsan-mac are kept as lane keys
          # (select-lanes.sh, ci-ok and the lane contracts key on them); both
          # run on macos-15 because GitHub retires the macos-14 image on 2026-11-02.
          CORE_UNIX='[
            {"os":"ubuntu-latest","cc":"gcc","cxx":"g++","lane":"unix-x86"},
            {"os":"ubuntu-24.04-arm","cc":"gcc","cxx":"g++","lane":"unix-arm64"},
            {"os":"macos-15","cc":"cc","cxx":"c++","lane":"unix-macos14"},
            {"os":"macos-15-intel","cc":"cc","cxx":"c++","lane":"unix-macos-intel"}
          ]'
          # Broad matrix legs are REQUIRED gates (no optional/continue-on-error
          # escape hatches): the release dry run must be all-green, every platform.
          # macos-15 is the CORE macOS arm64 leg now, so it is not repeated here.
          BROAD_UNIX='[
            {"os":"ubuntu-22.04","cc":"gcc","cxx":"g++"},
            {"os":"ubuntu-22.04-arm","cc":"gcc","cxx":"g++"}
          ]'
          # Each Windows leg pins the msys2 environment + package arch to the
          # RUNNER architecture so the build is native, never emulated:
          #   x86-64 runners -> CLANG64 (mingw-w64-clang-x86_64-*)
          #   ARM64  runner  -> CLANGARM64 (mingw-w64-clang-aarch64-*)
          # windows-11-arm previously used the x86-64 CLANG64 toolchain, so its
          # binary ran under Windows-on-ARM x86-64 emulation and ASan's function
          # interception crashed (interception_win: unhandled instruction). With
          # the native ARM64 toolchain ASan instruments native ARM64 code, so it
          # is a real (non-optional) gate, not a tolerated emulated-flake.
          CORE_WIN='[{"os":"windows-latest","msystem":"CLANG64","pkg":"x86_64","lane":"windows"}]'
          BROAD_WIN='[{"os":"windows-2025","msystem":"CLANG64","pkg":"x86_64"},{"os":"windows-11-arm","msystem":"CLANGARM64","pkg":"aarch64"}]'
          if [ "$BROAD" = "true" ]; then
            UNIX=$(jq -cn --argjson a "$CORE_UNIX" --argjson b "$BROAD_UNIX" '$a + $b')
            WIN=$(jq -cn --argjson a "$CORE_WIN" --argjson b "$BROAD_WIN" '$a + $b')
          else
            UNIX=$(jq -cn --argjson a "$CORE_UNIX" '$a')
            WIN=$(jq -cn --argjson a "$CORE_WIN" '$a')
          fi
          # The ThreadSanitizer legs (test-tsan), tagged the same way.
          TSAN='[
            {"os":"ubuntu-latest","cc":"clang","cxx":"clang++","lane":"tsan-x86"},
            {"os":"ubuntu-24.04-arm","cc":"clang","cxx":"clang++","lane":"tsan-arm"},
            {"os":"macos-15","cc":"cc","cxx":"c++","lane":"tsan-mac"}
          ]'
          # Lane selection. "all" (dry run, release) keeps every leg and drops
          # the tags, so those matrices -- and their job names -- stay exactly
          # as before lane selection. A PR keeps only its selected legs, and
          # the tag stays in the job name, where ci-ok reads it back. An
          # untagged leg could never be selected: that is a configuration
          # error here, never a leg silently dropped.
          if [ "$LANES" = "all" ]; then
            UNIX=$(jq -c 'map(del(.lane))' <<< "$UNIX")
            WIN=$(jq -c 'map(del(.lane))' <<< "$WIN")
            TSAN=$(jq -c 'map(del(.lane))' <<< "$TSAN")
          else
            for M in "$UNIX" "$WIN" "$TSAN"; do
              jq -e 'all(.[]; has("lane"))' <<< "$M" > /dev/null || { echo "::error::untagged test leg in $M"; exit 1; }
            done
            UNIX=$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$UNIX")
            WIN=$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$WIN")
            TSAN=$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$TSAN")
          fi
          # Linux-only release: drop macOS legs. The Windows job is skipped by
          # its job-level `if`, so its matrix is computed but never expanded.
          case "$PLATFORMS" in
            all) ;;
            linux)
              UNIX=$(jq -c 'map(select(.os | startswith("ubuntu")))' <<< "$UNIX")
              TSAN=$(jq -c 'map(select(.os | startswith("ubuntu")))' <<< "$TSAN")
              ;;
            *) echo "::error::unknown platforms input: $PLATFORMS"; exit 1 ;;
          esac
          # Suite sharding (opt-in via the shard_suites input): each platform's
          # C-suite run is split across N runner jobs. CBM_TEST_SHARD="i/N"
          # slices deterministically inside run-tests-parallel.sh; each
          # shard's union guard proves it ran exactly its slice; shards are
          # minted HERE from one expansion so a missing shard cannot be
          # configured silently; and the shard-completeness job re-proves the
          # cross-shard union at runtime. Ubuntu runners are plentiful (3
          # shards); Windows gets 2 (every extra shard re-pays ~5 min of
          # MSYS2 setup); macOS stays at 1 — mac legs are not the critical
          # path and mac runner concurrency ceilings are the tightest.
          # The pr profile shards less: a PR runs few legs and every shard
          # re-pays its setup, so x86 and Windows run whole and only arm64,
          # the slowest leg (~72 min unsharded), splits in two.
          case "$SHARD_PROFILE" in
            standard) N_X86=3; N_ARM=3; N_WIN=2 ;;
            pr) N_X86=1; N_ARM=2; N_WIN=1 ;;
            *) echo "::error::unknown shard_profile '$SHARD_PROFILE'"; exit 1 ;;
          esac
          if [ "$SHARDS" = "true" ]; then
            UNIX=$(jq -c --argjson x86 "$N_X86" --argjson arm "$N_ARM" '[.[] | . as $e
              | (if ($e.os | startswith("ubuntu")) then
                   (if ($e.os | endswith("-arm")) then $arm else $x86 end)
                 else 1 end) as $n
              | range(1; $n + 1) as $i
              | $e + {shard: "\($i)/\($n)"}]' <<< "$UNIX")
            WIN=$(jq -c --argjson n "$N_WIN" '[.[] | . as $e
              | range(1; $n + 1) as $i
              | $e + {shard: "\($i)/\($n)"}]' <<< "$WIN")
          fi
          echo "unix={\"include\":$UNIX}" >> "$GITHUB_OUTPUT"
          echo "windows={\"include\":$WIN}" >> "$GITHUB_OUTPUT"
          echo "tsan={\"include\":$TSAN}" >> "$GITHUB_OUTPUT"
          # GitHub rejects an empty matrix, so a job whose legs were all
          # deselected is skipped through these flags instead.
          echo "has_unix=$(jq 'length > 0' <<< "$UNIX")" >> "$GITHUB_OUTPUT"
          echo "has_windows=$(jq 'length > 0' <<< "$WIN")" >> "$GITHUB_OUTPUT"
          echo "has_tsan=$(jq 'length > 0' <<< "$TSAN")" >> "$GITHUB_OUTPUT"

  test-unix:
    needs: setup-matrix
    if: ${{ needs.setup-matrix.outputs.has_unix == 'true' }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.setup-matrix.outputs.unix) }}
    runs-on: ${{ matrix.os }}
    # Broad-only legs (extra OS versions) are informational: visible but
    # non-blocking, so a flaky/less-common runner can't block a release.
    continue-on-error: ${{ matrix.optional == true }}
    timeout-minutes: 240
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install deps (Ubuntu)
        if: startsWith(matrix.os, 'ubuntu')
        timeout-minutes: 10
        run: APT_RETRIES=3; [ -f /etc/apt/apt-mirrors.txt ] && APT_RETRIES=0; sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 --error-on=any update && sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y zlib1g-dev ccache

      # Ubuntu 23.10+ ships kernel.apparmor_restrict_unprivileged_userns=1, which
      # blocks unshare(CLONE_NEWUSER) for unprivileged users. Without this, the
      # #1830 single-uid-userns smoke test SKIPS on every 24.04 leg, and its only
      # real-namespace coverage lived on the two ubuntu-22.04 legs -- which are
      # broad-matrix only (never PR CI) and which GitHub retires on 2027-04-17.
      # A deliberate relaxation of a security check was therefore guarded by a
      # test that ran in no PR and was on a clock. Hosted runners have
      # passwordless sudo, so one sysctl puts it on the CORE matrix instead.
      # Best-effort: a runner that refuses simply returns to skipping.
      - name: Permit unprivileged user namespaces (Ubuntu)
        if: startsWith(matrix.os, 'ubuntu')
        run: |
          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || \
            echo "could not lift the userns restriction; namespace tests will skip"
          # Then SAY whether it worked. The runner prints per-shard aggregates
          # ("N passed, M skipped") and never individual test names, so a
          # namespace test that silently returns to skipping is invisible in a
          # green log -- the same false assurance that let #1830 ship with a
          # test that never executed. This line is the evidence. Probed exactly
          # as the test does it: unprivileged, no sudo.
          if unshare -U -r true 2>/dev/null; then
            echo "CBM_USERNS_AVAILABLE=yes"
          else
            echo "CBM_USERNS_AVAILABLE=no (namespace-dependent tests will skip)"
          fi

      - name: Install ccache (macOS)
        if: startsWith(matrix.os, 'macos')
        run: command -v ccache >/dev/null 2>&1 || brew install ccache

      # Verified compiler cache: CCACHE_COMPILERCHECK=content keys every entry
      # on the compiler-binary CONTENT plus the fully preprocessed input, so a
      # hit is provably the identical compilation — a stale or foreign cache
      # can only miss, never return wrong output (see scripts/env.sh).
      # Keys embed github.ref; the final ref-less restore key falls back to
      # the newest cache GitHub's scoping permits (a PR can only ever see its
      # own ref and the base branch, and base-branch caches are written only
      # by base-branch runs), so a fresh PR starts from main's warm cache
      # instead of building cold. Content verification is what makes that
      # safe; the fallback buys ~8-12 minutes on every first-of-ref build.
      - name: Compiler cache (content-verified, restore)
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ github.workspace }}/.ccache
          key: ccache-test-${{ matrix.os }}-${{ matrix.cc }}-${{ github.ref }}-${{ github.sha }}
          # The cross-ref fallback is what keeps fresh PRs warm: without it a
          # new PR always started from a cold cache (observed 0% hit legs).
          # ccache verifies content hashes itself, so a stale cache can only
          # miss, never mis-compile.
          restore-keys: |
            ccache-test-${{ matrix.os }}-${{ matrix.cc }}-${{ github.ref }}-
            ccache-test-${{ matrix.os }}-${{ matrix.cc }}-

      - name: Fetch the test selection
        # Only the Linux x86_64 leg narrows (user decision 2026-10-16): the
        # coverage that justifies an omission comes from the Linux x86_64
        # producer, so arm64, macOS and Windows code under #ifdef has static
        # evidence only and those legs run every test.
        if: ${{ inputs.test_selection != '' && runner.os == 'Linux' && runner.arch == 'X64' }}
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ${{ inputs.test_selection }}
          path: ${{ runner.temp }}/test-selection

      - name: Test
        run: scripts/test.sh CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}
        env:
          CBM_TEST_SHARD: ${{ matrix.shard }}
          CBM_TEST_LEG: ${{ matrix.os }}-${{ matrix.cc }}
          CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }}
          CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && runner.os == 'Linux' && runner.arch == 'X64' && format('{0}/test-selection', runner.temp) || '' }}
          CCACHE_DIR: ${{ github.workspace }}/.ccache
          CCACHE_MAXSIZE: 1500M

      # Every shard of a leg builds the identical runner, so one shard's
      # cache carries the complete object set: shard 1 (or the unsharded
      # job) saves, the rest only restore — same save volume as before
      # sharding, no quota thrash, no same-key save races. Saving on failure
      # too keeps red iterations warm.
      - name: Shard manifest (completeness proof)
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: shard-manifest-${{ matrix.os }}-${{ matrix.cc }}-${{ strategy.job-index }}
          path: build/c/test-logs/shard-manifest.txt
          # The harness writes the manifest BEFORE any suite runs, so a
          # missing file here means the job died before the harness started
          # (contract step / build) — that failure is already the job's red;
          # warn instead of stacking a second error on top of it. The
          # shard-completeness job still gates the cross-shard union.
          if-no-files-found: warn

      # The job log carries a red suite's SUMMARY; these are the complete logs
      # behind it, so a failure can be read instead of re-run. The harness
      # copies the log of every suite that did not end green, plus results.txt,
      # into test-logs/failed/ (scripts/suite-failure-report.sh) — typically a
      # few kilobytes for a red suite, not the ~25 MB of a whole run's logs.
      # Runs only on a failed job. warn, not error: a job that failed OUTSIDE
      # the suite wave (contract step, build, prod-binary guards) has no such
      # directory, and a second error would only bury the first.
      - name: Failing suite logs
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: suite-logs-${{ matrix.os }}-${{ matrix.cc }}-${{ strategy.job-index }}
          path: build/c/test-logs/failed/
          retention-days: 7
          if-no-files-found: warn

      - name: Compiler cache (save)
        if: always() && (matrix.shard == null || startsWith(matrix.shard, '1/'))
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ github.workspace }}/.ccache
          key: ccache-test-${{ matrix.os }}-${{ matrix.cc }}-${{ github.ref }}-${{ github.sha }}

      - name: ccache stats
        if: always()
        run: ccache -s || true
        env:
          CCACHE_DIR: ${{ github.workspace }}/.ccache

  # The package wrappers own their own runtime-set publication and lease-lock
  # implementations. Run the same suites against both host families so Unix
  # inode/flock behavior and Windows handle/link behavior are each gating,
  # without adding Node/Go provisioning to every sanitizer/container leg.
  test-package-wrappers:
    if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'pkg-wrappers') }}
    strategy:
      fail-fast: false
      matrix:
        os: ${{ fromJSON(inputs.platforms == 'linux' && '["ubuntu-latest"]' || '["ubuntu-latest","windows-latest"]') }}
    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
        with:
          go-version-file: pkg/go/go.mod
          cache: false

      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: "22"

      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Test Go, npm, and PyPI runtime-set wrappers
        shell: bash
        run: scripts/ci/test-package-wrappers.sh

  # CodeAtlasWeb (graph-ui) leg: the frontend's own vitest suite (2000+
  # tests; the render tests run under jsdom), its hardcoded-string and
  # promise scans, its portable frozen acceptance checks, and a production
  # build with the same npm ci the product build runs. One canonical script
  # (scripts/ci/test-ui.sh) is the whole leg, here and locally. No browser
  # is downloaded: the browser proofs (graph-ui/tools/smoke-*.mjs) need a
  # running server and Playwright browsers, and they stay a local, manually
  # dispatched venue documented in graph-ui/README.md rather than a hosted
  # gate.
  test-ui:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: "22"

      - name: CodeAtlasWeb leg (install, unit, style, promises, acceptance, build)
        run: scripts/ci/test-ui.sh

  # ThreadSanitizer data-race gate. The threaded production surfaces (watcher,
  # subprocess, httpd, pipeline, worker pool, diagnostics, the runnable
  # daemon-coordination paths) plus allocator concurrency all run under TSan
  # here — see TEST_TSAN_SUITES in Makefile.cbm. Runs on both Linux
  # architectures AND native ARM64 macOS: the threading code is shared, so a
  # race is usually caught on all three, but scheduler differences mean each
  # platform can surface a race the others miss. Windows has no TSan runtime on
  # any toolchain (documented irreducible gap); this shared-code coverage is
  # its substitute.
  # Memory-diagnostics lane (user decision 2026-08-03: runner cost accepted).
  # Newest pinned LLVM sanitizer runtime + straighter stacks over the full
  # parallel suite — catches what an older compiler-rt can miss. Same
  # canonical scripts/test.sh wave as every other leg.
  test-diag:
    if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'diag') }}
    # Pinned for the same reason as the lint jobs: the pinned diagnostic
    # toolchain below installs from llvm-toolchain-noble-21.
    runs-on: ubuntu-24.04
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install deps
        timeout-minutes: 10
        run: APT_RETRIES=3; [ -f /etc/apt/apt-mirrors.txt ] && APT_RETRIES=0; sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 --error-on=any update && sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y zlib1g-dev ccache

      - name: Install LLVM 21 (pinned diagnostic toolchain)
        timeout-minutes: 10
        run: |
          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
          echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-21 main" | sudo tee /etc/apt/sources.list.d/llvm-21.list
          APT_RETRIES=3; [ -f /etc/apt/apt-mirrors.txt ] && APT_RETRIES=0; sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 --error-on=any update
          sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y clang-21

      - name: Compiler cache (content-verified, restore)
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ github.workspace }}/.ccache
          key: ccache-diag-${{ github.ref }}-${{ github.sha }}
          restore-keys: |
            ccache-diag-${{ github.ref }}-
            ccache-diag-

      # Also enables the ASan checks that are off by default in every other
      # lane: stack-use-after-return, stack-use-after-scope, and strict string
      # checks. Mirrors `make -f Makefile.cbm diag`, including its reasoning
      # for leaving detect_invalid_pointer_pairs out.
      - name: Test (clang-21, ASan+UBSan + off-by-default ASan checks)
        env:
          ASAN_OPTIONS: "detect_stack_use_after_return=1:strict_string_checks=1:detect_stack_use_after_scope=1"
        run: |
          scripts/test.sh CC=clang-21 CXX=clang++-21 \
            SANITIZE="-fsanitize=address,undefined -fno-omit-frame-pointer -fno-optimize-sibling-calls"

      # Same step as in test-unix: the complete logs of the suites that did
      # not end green, only when the job failed.
      - name: Failing suite logs
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: suite-logs-diag
          path: build/c/test-logs/failed/
          retention-days: 7
          if-no-files-found: warn

      - name: Compiler cache (save)
        if: always()
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ github.workspace }}/.ccache
          key: ccache-diag-${{ github.ref }}-${{ github.sha }}

  # MemorySanitizer lane (stage 2). Uninitialized READS are the one
  # memory-error class no other lane covers dynamically. The image carries
  # MSan-instrumented libc++/libc++abi/libunwind + zlib (vendored C deps
  # instrument in-tree); scripts/msan.sh drives the build and the suite.
  #
  # THIS JOB IS THE AUTHORITATIVE VENUE for the lane. It previously ran WITHOUT
  # the script's default exclusions to settle whether those were an aarch64
  # artifact — the local ladder cannot answer that, having no faithful x86-64
  # emulation. It has now answered it, and the answer SPLIT the list:
  #
  #   - five suites aborted with "stack-overflow ... nested bug" on both
  #     architectures. That was not deep recursion: it was MSan crashing while
  #     it reported real uninitialized reads (reporting with the slow unwinder
  #     exhausted the main thread's stack). scripts/msan.sh now unwinds with
  #     frame pointers, the
  #     reports print, the reads are fixed, and the five suites run again;
  #   - `cli` does not overflow here at all — it fails in the install path for
  #     an unrelated, still-undiagnosed reason;
  #   - `incremental` was a shadow-memory RSS artifact, now fixed in the test
  #     itself rather than skipped, so it stays in the lane.
  #
  # Per-cause detail lives in scripts/msan.sh, which is the single authoritative
  # list both venues now share.
  test-msan:
    if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'msan') }}
    runs-on: ubuntu-latest
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Buildx cache
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: /tmp/.buildx-msan
          key: buildx-msan-${{ hashFiles('test-infrastructure/Dockerfile.msan') }}

      - name: Build MSan image (cached layers)
        run: scripts/ci/msan-lane.sh build

      - name: MSan suite (exclusions per scripts/msan.sh)
        run: scripts/ci/msan-lane.sh run

  # macOS leak coverage. LeakSanitizer runs by default under ASan on Linux, so
  # test-unix's ubuntu legs have always had it; on macOS it is off by default
  # and Apple's clang refuses to turn it on, which left an entire platform with
  # zero leak detection. Upstream (Homebrew) LLVM supports LSan on darwin/arm64,
  # so this leg is the ordinary ASan suite built with that toolchain and run
  # with detect_leaks=1. Mirrors the local `make -f Makefile.cbm test-lsan`.
  test-lsan-macos:
    if: ${{ inputs.platforms != 'linux' && (inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'lsan-macos')) }}
    runs-on: macos-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      # Pinned: the unversioned `llvm` formula tracks Homebrew stable, so the
      # leg's compiler changed under us when stable moved 22 -> 23 (a new
      # -Wunused-but-set-global under -Werror broke the build on whichever
      # runner drew the 23 bottle, and passed on the ones that drew 22). A
      # test verdict must be a function of the code, not of the image snapshot;
      # moving to a newer major is a deliberate one-line bump here.
      - name: Install LSan-capable toolchain
        run: brew install llvm@22 ccache

      - name: Compiler cache (content-verified, restore)
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ github.workspace }}/.ccache
          key: ccache-lsan-${{ github.ref }}-${{ github.sha }}
          restore-keys: |
            ccache-lsan-${{ github.ref }}-
            ccache-lsan-

      - name: Test (Homebrew clang, ASan + LeakSanitizer)
        env:
          ASAN_OPTIONS: "detect_leaks=1:halt_on_error=1"
        run: |
          LLVM_PREFIX="$(brew --prefix llvm@22)"
          scripts/test.sh CC="$LLVM_PREFIX/bin/clang" CXX="$LLVM_PREFIX/bin/clang++"

      # Same step as in test-unix: the complete logs of the suites that did
      # not end green, only when the job failed.
      - name: Failing suite logs
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: suite-logs-lsan-macos
          path: build/c/test-logs/failed/
          retention-days: 7
          if-no-files-found: warn

      - name: Compiler cache (save)
        if: always()
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ github.workspace }}/.ccache
          key: ccache-lsan-${{ github.ref }}-${{ github.sha }}

  test-tsan:
    needs: setup-matrix
    if: ${{ needs.setup-matrix.outputs.has_tsan == 'true' }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.setup-matrix.outputs.tsan) }}
    runs-on: ${{ matrix.os }}
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install deps (Ubuntu)
        if: runner.os == 'Linux'
        timeout-minutes: 10
        run: APT_RETRIES=3; [ -f /etc/apt/apt-mirrors.txt ] && APT_RETRIES=0; sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 --error-on=any update && sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y clang zlib1g-dev ccache

      - name: Install deps (macOS)
        if: runner.os == 'macOS'
        run: brew install ccache

      - name: Compiler cache (content-verified)
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ github.workspace }}/.ccache
          key: ccache-tsan-${{ matrix.os }}-${{ github.ref }}-${{ github.sha }}
          restore-keys: |
            ccache-tsan-${{ matrix.os }}-${{ github.ref }}-
            ccache-tsan-${{ matrix.os }}-

      - name: ThreadSanitizer tests
        # Calls make directly (no env.sh), so route the compilers through
        # ccache's masquerade dir: /usr/lib/ccache on Debian, libexec under the
        # Homebrew prefix on macOS. On Linux, reduce mmap ASLR entropy first:
        # modern (esp. aarch64) kernels randomize high enough that TSan's
        # shadow mapping aborts with "unexpected memory mapping" before any
        # test runs. macOS TSan needs no such adjustment.
        run: |
          if [ "$RUNNER_OS" = "macOS" ]; then
            export PATH="$(brew --prefix ccache)/libexec:$PATH"
          else
            export PATH=/usr/lib/ccache:$PATH
            sudo sysctl -w vm.mmap_rnd_bits=28 || true
          fi
          # The TSan leg lives in the canonical test entry (same file the
          # compose test-tsan service runs); env.sh supplies the verified
          # ccache masquerade.
          scripts/test.sh --tsan CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}
        env:
          CCACHE_DIR: ${{ github.workspace }}/.ccache
          CCACHE_MAXSIZE: 1500M

      - name: ccache stats
        if: always()
        run: ccache -s || true
        env:
          CCACHE_DIR: ${{ github.workspace }}/.ccache

  test-windows:
    needs: setup-matrix
    if: ${{ inputs.platforms != 'linux' && needs.setup-matrix.outputs.has_windows == 'true' }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.setup-matrix.outputs.windows) }}
    runs-on: ${{ matrix.os }}
    continue-on-error: ${{ matrix.optional == true }}
    timeout-minutes: 240
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      # Defender posture: real-time protection is POLICY-LOCKED OFF on hosted
      # Windows images (verified 2026-07-27: WinDefend starts and
      # Set-MpPreference accepts, but RTP stays off — untamperable from the
      # job). The local VM leg enforces Defender-ON via the canonical Defender
      # preflight script, so AV-interaction coverage lives there; runners run
      # without RTP by platform constraint.

      - uses: msys2/setup-msys2@ec48f7c5447b3140e2b088413ae3a55687bccb6e # v2
        with:
          msystem: ${{ matrix.msystem }}
          path-type: inherit
          install: >-
            mingw-w64-clang-${{ matrix.pkg }}-clang
            mingw-w64-clang-${{ matrix.pkg }}-compiler-rt
            mingw-w64-clang-${{ matrix.pkg }}-zlib
            mingw-w64-clang-${{ matrix.pkg }}-python3
            mingw-w64-clang-${{ matrix.pkg }}-ccache
            make
            git
            zip

      - name: Create protected per-user temp root
        # One implementation, shared with _soak.yml and the VM leg
        # (test-infrastructure/vm/vm-run-tests.sh) — see the script's header for
        # why the harness cannot run under a shared or ACL-inherited temp.
        shell: pwsh
        run: |
          $root = & scripts/ci/new-protected-temp-root.ps1 `
            -Prefix 'cbm-ci-tmp-' `
            -ProtectDir (Join-Path $env:GITHUB_WORKSPACE "build\c")
          "CBM_CI_TEMP_ROOT=$root" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8

      - name: Compiler cache (content-verified)
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ github.workspace }}/.ccache
          key: ccache-test-${{ matrix.os }}-${{ matrix.msystem }}-${{ github.ref }}-${{ github.sha }}
          restore-keys: |
            ccache-test-${{ matrix.os }}-${{ matrix.msystem }}-${{ github.ref }}-
            ccache-test-${{ matrix.os }}-${{ matrix.msystem }}-

      - name: Test
        shell: msys2 {0}
        # AddressSanitizer is unavailable on native ARM64 Windows (LLVM ships no
        # libclang_rt.asan for aarch64-w64-windows-gnu) and cannot intercept the
        # system DLLs under x86-64 emulation either. Instead of running the
        # ARM64 leg unsanitized, it runs UBSan in TRAP mode: -fsanitize-trap
        # needs no runtime library (the very thing aarch64-w64-windows-gnu
        # lacks), so it instruments natively and turns any undefined behavior
        # into an illegal-instruction trap; -fstack-protector-strong adds
        # stack-smash coverage the heap tools miss. To see WHICH check fired,
        # reproduce under the emulated x86_64 UBSan (win.sh ubsan-build), which
        # carries the full message. x86-64 Windows keeps full ASan+UBSan; the
        # other native-ARM legs (Linux/macOS) carry ASan+LSan+TSan.
        run: |
          # Native tests resolve temp via TEMP/TMP (cbm_tmpdir); shell tools use
          # TMPDIR. Route both through the protected root created above so no
          # fixture lands under a shared or ACL-inherited ancestry.
          export TEMP="$(cygpath -m "$CBM_CI_TEMP_ROOT")"
          export TMP="$TEMP"
          export TMPDIR="$(cygpath -u "$CBM_CI_TEMP_ROOT")"
          # No per-leg flag logic here: test.sh itself applies the CLANGARM64
          # trap-UBSan default, so local and CI build identical test binaries.
          scripts/test.sh CC=clang CXX=clang++
        env:
          CBM_TEST_SHARD: ${{ matrix.shard }}
          CBM_TEST_LEG: ${{ matrix.os }}-${{ matrix.msystem }}
          CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }}
          # Windows legs run every test: see the Linux x86_64-only fetch in test-unix.
          CBM_TEST_SELECTION_DIR: ''
          CCACHE_DIR: ${{ github.workspace }}/.ccache
          CCACHE_MAXSIZE: 1500M

      - name: ccache stats
        if: always()
        shell: msys2 {0}
        run: ccache -s || true
        env:
          CCACHE_DIR: ${{ github.workspace }}/.ccache

      # The harness prints only each failing suite's last 15 lines, which cuts
      # off sanitizer reports. Keep the full per-suite logs of a failed shard.
      - name: Upload test logs (on failure)
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: test-logs-${{ matrix.os }}-${{ matrix.msystem }}-${{ strategy.job-index }}
          path: build/c/test-logs/*.log
          retention-days: 7
          if-no-files-found: ignore

      - name: Shard manifest (completeness proof)
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: shard-manifest-${{ matrix.os }}-${{ matrix.msystem }}-${{ strategy.job-index }}
          path: build/c/test-logs/shard-manifest.txt
          # The harness writes the manifest BEFORE any suite runs, so a
          # missing file here means the job died before the harness started
          # (contract step / build) — that failure is already the job's red;
          # warn instead of stacking a second error on top of it. The
          # shard-completeness job still gates the cross-shard union.
          if-no-files-found: warn

      # Same step as in test-unix: the complete logs of the suites that did
      # not end green, only when the job failed.
      - name: Failing suite logs
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: suite-logs-${{ matrix.os }}-${{ matrix.msystem }}-${{ strategy.job-index }}
          path: build/c/test-logs/failed/
          retention-days: 7
          if-no-files-found: warn

      # Shard 1 (or the unsharded job) saves the compiler cache; every shard
      # builds the identical runner, so one cache carries the full object set.
      - name: Compiler cache (save)
        if: always() && (matrix.shard == null || startsWith(matrix.shard, '1/'))
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ github.workspace }}/.ccache
          key: ccache-test-${{ matrix.os }}-${{ matrix.msystem }}-${{ github.ref }}-${{ github.sha }}

  # Cross-shard completeness: re-proves at runtime that each leg's shards
  # agreed on the full suite list and that the union of their slices IS that
  # list. This is the guard the per-shard union checks cannot provide: a
  # mis-plumbed CBM_TEST_SHARD (two jobs running the same slice) passes every
  # per-shard guard while a slice runs nowhere — only a cross-shard view
  # catches it. Runs and gates on unsharded topologies too (every leg then
  # contributes one 1/1 manifest whose slice must equal the full list).
  shard-completeness:
    needs: [test-unix, test-windows]
    if: always()
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      # The union proof lives in the canonical scripts/ci entry, not inline
      # YAML (venue-parity contract) — checkout (into repo/, so the manifest
      # download below stays untouched) provides the script.
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          path: repo

      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: shard-manifest-*
          path: manifests
      # A PR with no test leg selected uploads no manifest; the script
      # accepts that only when the selection really holds no leg.
      - name: Verify cross-shard union per leg
        env:
          LANES: ${{ inputs.lanes }}
        run: repo/scripts/ci/verify-shard-union.sh manifests --lanes "$LANES"

  # Windows product-surface regression guards. Distinct from test-windows above
  # (the sanitizer C suite): these drive a real product binary + external UI pack
  # over stdio / CLI / HTTP and fail if a Windows bug already fixed on main comes
  # back -- non-ASCII repo paths dropping definitions (#636/#357, fixed by #700),
  # the PreToolUse hook augmenter no-op on drive-letter cwd (#618, fixed by #619),
  # the UI directory picker not enumerating drives (#548, roots field), and non-ASCII
  # CLI arguments being mangled by the narrow-argv main() (#423/#20, fixed by the
  # wide-argv entrypoint that reads GetCommandLineW).
  test-windows-guards:
    if: ${{ inputs.platforms != 'linux' && (inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'windows-guards')) }}
    runs-on: windows-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - uses: msys2/setup-msys2@ec48f7c5447b3140e2b088413ae3a55687bccb6e # v2
        with:
          msystem: CLANG64
          path-type: inherit
          install: >-
            mingw-w64-clang-x86_64-clang
            mingw-w64-clang-x86_64-zlib
            mingw-w64-clang-x86_64-ccache
            make
            git

      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: "22"

      - name: Compiler cache (content-verified)
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ github.workspace }}/.ccache
          key: ccache-guards-${{ github.ref }}-${{ github.sha }}
          restore-keys: |
            ccache-guards-${{ github.ref }}-
            ccache-guards-

      - name: Build product binary with external UI assets and guard seams
        shell: msys2 {0}
        # --with-ui builds the frontend and its content-addressed asset pack, so
        # the drive-picker guard's HTTP UI is available. Functional gate only
        # (no sanitizers).
        run: scripts/build.sh --with-ui CC=clang CXX=clang++ TEST_SEAMS=1
        env:
          CCACHE_DIR: ${{ github.workspace }}/.ccache
          CCACHE_MAXSIZE: 1500M

      - name: Windows regression guards (#636/#357, #618, #548, #423/#20)
        shell: pwsh
        # -GuardsOnly runs the four green guards and gates on them. All guards run
        # against the real supervisor->worker spawn path: under the mandatory
        # coordination daemon, CBM_INDEX_SUPERVISOR=0 is a fail-closed refusal, so
        # the driver no longer sets it anywhere.
        run: ./scripts/test-windows.ps1 -GuardsOnly -Binary build/c/codebase-memory-cli.exe