diff --git a/.github/workflows/_ci-suite.yaml b/.github/workflows/_ci-suite.yaml index 58b5b5fd..788b2c8f 100644 --- a/.github/workflows/_ci-suite.yaml +++ b/.github/workflows/_ci-suite.yaml @@ -60,7 +60,7 @@ jobs: toolchain_components: "clippy, rustfmt, rust-src" enable_magic_cache: true enable_rust_cache: true - run_script: just ci-check + operation: quality cargo-graph: if: ${{ inputs.include_cargo_graph }} @@ -74,12 +74,7 @@ jobs: toolchain_components: "rust-src" enable_magic_cache: true enable_rust_cache: true - run_script: | - mkdir -p target/cargo-rail - cargo rail config validate --strict - cargo rail config migrate --check - cargo rail unify --check --explain --format json \ - --output target/cargo-rail/unify-result.json + operation: cargo-graph artifact_name: cargo-graph-${{ inputs.cache_key_prefix }} artifact_path: target/cargo-rail/unify-result.json artifact_always: true @@ -95,9 +90,7 @@ jobs: toolchain_components: "rust-src" enable_magic_cache: true enable_rust_cache: true - run_script: | - just check-feature-matrix - just test-feature-matrix + operation: feature-contracts ci: name: CI (${{ matrix.target.name }}) @@ -115,25 +108,9 @@ jobs: toolchain_components: ${{ contains(matrix.target.name, 'windows-msvc') && 'clippy' || 'clippy, rustfmt, rust-src' }} enable_magic_cache: ${{ matrix.target.type == 'runson' }} enable_rust_cache: true - run_script: | - export RSCRYPTO_TEST_MODE="${{ inputs.test_mode }}" - case "${{ matrix.target.name }}" in - x86_64-pc-windows-msvc) - echo "Windows x86_64 lane: compile smoke plus BLAKE3 ASM runtime vectors" - cargo clippy --workspace --lib --all-features -- -D warnings - cargo test --workspace --all-features --no-run - cargo test --workspace --features blake3 --test blake3_official_vectors --test blake3_differential - ;; - aarch64-pc-windows-msvc) - echo "Windows AArch64 lane: compile-only smoke" - cargo clippy --workspace --lib --all-features -- -D warnings - cargo test --workspace --all-features --no-run - ;; - *) - bash scripts/ci/native-check.sh --all-targets - bash scripts/test/test.sh --all - ;; - esac + operation: native + target: ${{ matrix.target.name }} + test_mode: ${{ inputs.test_mode }} ci-ibm-s390x: name: CI (IBM Z s390x) @@ -146,14 +123,8 @@ jobs: toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true - pre_script: | - uname -a - lscpu - sed -n '1,50p' /proc/cpuinfo - run_script: | - export RSCRYPTO_TEST_MODE="${{ inputs.test_mode }}" - bash scripts/ci/native-check.sh - bash scripts/test/test.sh --all + operation: native-ibm + test_mode: ${{ inputs.test_mode }} ci-ibm-power10: name: CI (IBM POWER10 ppc64le) @@ -166,14 +137,8 @@ jobs: toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true - pre_script: | - uname -a - lscpu - sed -n '1,50p' /proc/cpuinfo - run_script: | - export RSCRYPTO_TEST_MODE="${{ inputs.test_mode }}" - bash scripts/ci/native-check.sh - bash scripts/test/test.sh --all + operation: native-ibm + test_mode: ${{ inputs.test_mode }} cross-targets: name: Cross Targets @@ -186,7 +151,7 @@ jobs: toolchain_components: "clippy, rust-src" enable_magic_cache: true enable_rust_cache: true - run_script: bash scripts/ci/cross-targets.sh deep + operation: cross-targets supply-chain: name: Supply Chain (${{ inputs.supply_chain_mode }}) @@ -198,14 +163,5 @@ jobs: tools_mode: supply-chain enable_magic_cache: true enable_rust_cache: true - run_script: | - if [[ "${{ inputs.supply_chain_mode }}" == "full" ]]; then - cargo deny check all - # RustCrypto `rsa` is used only as a dev/test/bench oracle. Production - # RSA verification is implemented in `src/auth/rsa.rs`; keep this - # scoped to the known Marvin advisory until the oracle dependency is - # removed or fixed. - cargo audit --ignore RUSTSEC-2023-0071 - else - cargo deny check advisories - fi + operation: supply-chain + supply_chain_mode: ${{ inputs.supply_chain_mode }} diff --git a/.github/workflows/_rust-job.yaml b/.github/workflows/_rust-job.yaml index 976caacf..bf1381d7 100644 --- a/.github/workflows/_rust-job.yaml +++ b/.github/workflows/_rust-job.yaml @@ -3,6 +3,9 @@ name: _rust-job on: workflow_call: inputs: + operation: + required: true + type: string runner: required: true type: string @@ -30,13 +33,62 @@ on: required: false type: boolean default: true - pre_script: + test_mode: required: false type: string default: "" - run_script: - required: true + target: + required: false + type: string + default: "" + platform: + required: false + type: string + default: "" + supply_chain_mode: + required: false + type: string + default: "" + base_sha: + required: false + type: string + default: "" + miri_tree_borrows: + required: false + type: boolean + default: false + bench_targets: + required: false + type: string + default: all + bench_filter: + required: false + type: string + default: "" + bench_quick: + required: false + type: boolean + default: false + dudect_timeout: + required: false + type: number + default: 1800 + dudect_filter: + required: false + type: string + default: "" + dudect_gate: + required: false type: string + default: required + binsec_timeout: + required: false + type: number + default: 900 + upload_raw_artifacts: + required: false + type: boolean + default: false artifact_name: required: false type: string @@ -73,18 +125,26 @@ jobs: enable-magic-cache: ${{ inputs.enable_magic_cache }} enable-rust-cache: ${{ inputs.enable_rust_cache }} - - name: Pre Run - if: inputs.pre_script != '' - shell: bash - env: - PRE_SCRIPT: ${{ inputs.pre_script }} - run: bash -euo pipefail <<<"$PRE_SCRIPT" - - name: Run shell: bash env: - RUN_SCRIPT: ${{ inputs.run_script }} - run: bash -euo pipefail <<<"$RUN_SCRIPT" + RSCRYPTO_CI_OPERATION: ${{ inputs.operation }} + RSCRYPTO_CI_RUNNER: ${{ inputs.runner }} + RSCRYPTO_CI_TEST_MODE: ${{ inputs.test_mode }} + RSCRYPTO_CI_TARGET: ${{ inputs.target }} + RSCRYPTO_CI_PLATFORM: ${{ inputs.platform }} + RSCRYPTO_CI_SUPPLY_CHAIN_MODE: ${{ inputs.supply_chain_mode }} + RSCRYPTO_CI_BASE_SHA: ${{ inputs.base_sha }} + RSCRYPTO_CI_MIRI_TREE_BORROWS: ${{ inputs.miri_tree_borrows && 'true' || 'false' }} + RSCRYPTO_CI_BENCH_TARGETS: ${{ inputs.bench_targets }} + RSCRYPTO_CI_BENCH_FILTER: ${{ inputs.bench_filter }} + RSCRYPTO_CI_BENCH_QUICK: ${{ inputs.bench_quick && 'true' || 'false' }} + RSCRYPTO_CI_DUDECT_TIMEOUT: ${{ inputs.dudect_timeout }} + RSCRYPTO_CI_DUDECT_FILTER: ${{ inputs.dudect_filter }} + RSCRYPTO_CI_DUDECT_GATE: ${{ inputs.dudect_gate }} + RSCRYPTO_CI_BINSEC_TIMEOUT: ${{ inputs.binsec_timeout }} + RSCRYPTO_CI_UPLOAD_RAW_ARTIFACTS: ${{ inputs.upload_raw_artifacts && 'true' || 'false' }} + run: scripts/ci/run-rust-job.sh - name: Upload Artifact (always) if: always() && inputs.artifact_name != '' && inputs.artifact_path != '' && inputs.artifact_always diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 8e1a74d6..26417e10 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -74,28 +74,10 @@ jobs: toolchain_components: ${{ matrix.toolchain_components }} enable_magic_cache: false enable_rust_cache: true - pre_script: | - echo "Bench host: ${{ matrix.platform }} (${{ matrix.os }})" - uname -a - lscpu - sed -n '1,50p' /proc/cpuinfo - run_script: | - RUN_DATE="$(date -u +"%Y-%m-%d")" - RUN_TIME="$(date -u +"%H_%M_%S")" - RUN_COMMIT="$(git rev-parse HEAD 2>/dev/null || echo unknown)" - BENCH_OUTPUT_DIR=benchmark-results \ - BENCH_RESULTS_DIR=benchmark-results \ - BENCH_RUN_DATE="$RUN_DATE" \ - BENCH_RUN_TIME="$RUN_TIME" \ - BENCH_RUN_OS="${{ matrix.os }}" \ - BENCH_RUN_ARCH="${{ matrix.platform }}" \ - BENCH_RUN_COMMIT="$RUN_COMMIT" \ - BENCH_RUN_MODE=ci \ - BENCH_ONLY="${{ inputs.targets || 'all' }}" \ - BENCH_FILTER="${{ inputs.filter }}" \ - BENCH_QUICK=${{ inputs.quick && 'true' || 'false' }} \ - BENCH_ALLOW_FULL_HASHES_COMP=true \ - BENCH_PLATFORM="${{ matrix.platform }}" \ - scripts/ci/run-bench.sh + operation: benchmark + platform: ${{ matrix.platform }} + bench_targets: ${{ inputs.targets || 'all' }} + bench_filter: ${{ inputs.filter }} + bench_quick: ${{ inputs.quick }} artifact_name: benchmark-${{ matrix.artifact_suffix }} artifact_path: benchmark-results/ diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7ea2f73f..74cf62c6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -72,9 +72,8 @@ jobs: toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true - run_script: | - set -euo pipefail - GITHUB_BASE_SHA="${{ github.event.pull_request.base.sha }}" bash scripts/ci/dependabot-smoke.sh + operation: dependabot-smoke + base_sha: ${{ github.event.pull_request.base.sha }} suite: name: CI Suite diff --git a/.github/workflows/ct.yaml b/.github/workflows/ct.yaml index c0fbe10c..df8a3748 100644 --- a/.github/workflows/ct.yaml +++ b/.github/workflows/ct.yaml @@ -11,8 +11,8 @@ on: dudect_timeout: description: "Timeout in seconds for each per-case dudect command." required: false - type: string - default: "1800" + type: number + default: 1800 dudect_filter: description: "Optional comma-separated DudeCT case/name/filter substrings. Empty = all cases in selected gate." required: false @@ -26,8 +26,8 @@ on: binsec_timeout: description: "Timeout in seconds for each BINSEC kernel batch on Linux targets." required: false - type: string - default: "900" + type: number + default: 900 upload_raw_artifacts: description: "Upload raw CT build/proof artifacts for debugging. Default uploads compact reports only." required: false @@ -43,8 +43,8 @@ on: dudect_timeout: description: "Timeout in seconds for each per-case dudect command." required: false - type: string - default: "1800" + type: number + default: 1800 dudect_filter: description: "Optional comma-separated DudeCT case/name/filter substrings. Empty = all cases in selected gate." required: false @@ -62,8 +62,8 @@ on: binsec_timeout: description: "Timeout in seconds for each BINSEC kernel batch on Linux targets." required: false - type: string - default: "900" + type: number + default: 900 upload_raw_artifacts: description: "Upload raw CT build/proof artifacts for debugging. Default uploads compact reports only." required: false @@ -118,61 +118,14 @@ jobs: toolchain_components: ${{ matrix.toolchain_components }} enable_magic_cache: ${{ matrix.enable_magic_cache }} enable_rust_cache: ${{ matrix.enable_rust_cache }} - pre_script: | - set -euo pipefail - mkdir -p target/ct-evidence-package - { - echo "CT platform: ${{ matrix.platform }}" - echo "CT target: ${{ matrix.target }}" - echo "CT runner: ${{ matrix.runner }}" - uname -a || true - rustc -vV - cargo -V - lscpu || true - sed -n '1,80p' /proc/cpuinfo || true - } 2>&1 | tee "target/ct-evidence-package/host-${{ matrix.artifact_suffix }}.log" - run_script: | - set -euo pipefail - evidence_dir="target/ct-evidence-package" - mkdir -p "$evidence_dir" - status=0 - raw_artifacts="${{ inputs.upload_raw_artifacts && 'true' || 'false' }}" - raw_dudect_filter="${{ inputs.dudect_filter || '' }}" - raw_dudect_gate="${{ inputs.dudect_gate || 'required' }}" - dudect_filter="$(printf '%s' "$raw_dudect_filter" | tr -d '[:space:]')" - dudect_gate="$(printf '%s' "$raw_dudect_gate" | tr -d '[:space:]')" - if [[ "$raw_dudect_filter" != "$dudect_filter" ]]; then - echo "Normalized whitespace in DudeCT filter input." - fi - case "$dudect_gate" in - required|diagnostic|all) ;; - *) - echo "Invalid DudeCT gate: $dudect_gate" >&2 - exit 2 - ;; - esac - args=( - --target "${{ matrix.target }}" - --dudect-timeout "${{ inputs.dudect_timeout || '1800' }}" - --binsec-timeout "${{ inputs.binsec_timeout || '900' }}" - --dudect-gate "$dudect_gate" - ) - if [[ -n "$dudect_filter" ]]; then - args+=(--dudect-filter "$dudect_filter") - fi - package_args=(--target "${{ matrix.target }}" --suffix "${{ matrix.artifact_suffix }}" --out-dir "$evidence_dir") - - { - echo "Running: scripts/ct/full.py ${args[*]}" - scripts/ct/full.py "${args[@]}" - } 2>&1 | tee "$evidence_dir/ct-full-${{ matrix.artifact_suffix }}.log" || status=$? - - if [[ "$raw_artifacts" == "true" ]]; then - package_args+=(--raw) - fi - scripts/ct/python.sh scripts/ct/package_evidence.py "${package_args[@]}" - - exit "$status" + operation: constant-time + platform: ${{ matrix.platform }} + target: ${{ matrix.target }} + dudect_timeout: ${{ inputs.dudect_timeout }} + dudect_filter: ${{ inputs.dudect_filter }} + dudect_gate: ${{ inputs.dudect_gate }} + binsec_timeout: ${{ inputs.binsec_timeout }} + upload_raw_artifacts: ${{ inputs.upload_raw_artifacts }} artifact_name: ct-${{ matrix.artifact_suffix }} artifact_path: target/ct-evidence-package/ artifact_always: true diff --git a/.github/workflows/riscv.yaml b/.github/workflows/riscv.yaml index dc2b7e9e..ac86d0c0 100644 --- a/.github/workflows/riscv.yaml +++ b/.github/workflows/riscv.yaml @@ -55,16 +55,7 @@ jobs: toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true - pre_script: | - uname -a - lscpu - sed -n '1,50p' /proc/cpuinfo - run_script: | - export RSCRYPTO_TEST_MODE=weekly - export RSCRYPTO_CI_RESOURCE_PROFILE=constrained - export RSCRYPTO_SKIP_DOCTESTS=1 - bash scripts/ci/native-check.sh - bash scripts/test/test.sh --all + operation: native-riscv ct: if: ${{ github.event_name == 'schedule' || inputs.mode == 'evidence' || inputs.mode == 'constant-time' || inputs.mode == 'all' }} @@ -75,8 +66,8 @@ jobs: uses: ./.github/workflows/ct.yaml with: platforms: rise-riscv - dudect_timeout: "1800" - binsec_timeout: "900" + dudect_timeout: 1800 + binsec_timeout: 900 upload_raw_artifacts: true benchmark: @@ -94,29 +85,11 @@ jobs: toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true - pre_script: | - echo "Bench host: rise-riscv (linux)" - uname -a - lscpu - sed -n '1,50p' /proc/cpuinfo - run_script: | - RUN_DATE="$(date -u +"%Y-%m-%d")" - RUN_TIME="$(date -u +"%H_%M_%S")" - RUN_COMMIT="$(git rev-parse HEAD 2>/dev/null || echo unknown)" - BENCH_OUTPUT_DIR=benchmark-results \ - BENCH_RESULTS_DIR=benchmark-results \ - BENCH_RUN_DATE="$RUN_DATE" \ - BENCH_RUN_TIME="$RUN_TIME" \ - BENCH_RUN_OS=linux \ - BENCH_RUN_ARCH=rise-riscv \ - BENCH_RUN_COMMIT="$RUN_COMMIT" \ - BENCH_RUN_MODE=ci \ - BENCH_ONLY="${{ inputs.targets || 'all' }}" \ - BENCH_FILTER="${{ inputs.filter }}" \ - BENCH_QUICK=${{ inputs.quick && 'true' || 'false' }} \ - BENCH_ALLOW_FULL_HASHES_COMP=true \ - BENCH_PLATFORM=rise-riscv \ - scripts/ci/run-bench.sh + operation: benchmark + platform: rise-riscv + bench_targets: ${{ inputs.targets || 'all' }} + bench_filter: ${{ inputs.filter }} + bench_quick: ${{ inputs.quick }} artifact_name: benchmark-rise-riscv artifact_path: benchmark-results/ diff --git a/.github/workflows/rsa.yaml b/.github/workflows/rsa.yaml index cb7537aa..393e1b0c 100644 --- a/.github/workflows/rsa.yaml +++ b/.github/workflows/rsa.yaml @@ -29,14 +29,7 @@ jobs: toolchain_components: "miri, rust-src" enable_magic_cache: true enable_rust_cache: true - run_script: | - set -euo pipefail - mkdir -p ci-evidence - { - uname -a - lscpu - just test-miri --rsa - } 2>&1 | tee ci-evidence/rsa-miri-linux-x64.log + operation: rsa-miri artifact_name: rsa-miri-linux-x64 artifact_path: ci-evidence/rsa-miri-linux-x64.log artifact_always: true @@ -59,16 +52,8 @@ jobs: tools_mode: minimal enable_magic_cache: true enable_rust_cache: true - run_script: | - set -euo pipefail - mkdir -p ci-evidence - { - uname -a - lscpu - RSCRYPTO_RSA_LEAKAGE_SAMPLES="${RSCRYPTO_RSA_LEAKAGE_SAMPLES:-4000}" \ - RSCRYPTO_RSA_LEAKAGE_T_THRESHOLD="${RSCRYPTO_RSA_LEAKAGE_T_THRESHOLD:-8.0}" \ - just test-rsa-leakage - } 2>&1 | tee "ci-evidence/rsa-leakage-${{ matrix.target.name }}.log" + operation: rsa-leakage + target: ${{ matrix.target.name }} artifact_name: rsa-leakage-${{ matrix.target.name }} artifact_path: ci-evidence/rsa-leakage-${{ matrix.target.name }}.log artifact_always: true diff --git a/.github/workflows/weekly.yaml b/.github/workflows/weekly.yaml index f3a1e7c8..a26f365c 100644 --- a/.github/workflows/weekly.yaml +++ b/.github/workflows/weekly.yaml @@ -49,7 +49,7 @@ jobs: toolchain_components: "miri, rust-src" enable_magic_cache: true enable_rust_cache: true - run_script: just test-miri + operation: miri # ─── Miri memory-safety (tree borrows). ── miri-tree-borrows: @@ -63,9 +63,8 @@ jobs: toolchain_components: "miri, rust-src" enable_magic_cache: true enable_rust_cache: true - run_script: | - export MIRIFLAGS="-Zmiri-tree-borrows" - just test-miri + operation: miri + miri_tree_borrows: true # ─── Fuzzing (libFuzzer full + scoped packages). ── fuzzing: @@ -86,23 +85,7 @@ jobs: tools_mode: fuzz enable_magic_cache: true enable_rust_cache: true - pre_script: | - echo "RSCRYPTO_FUZZ_DURATION_SECS=60" >> "$GITHUB_ENV" - run_script: | - just test-fuzz --all - rm -rf fuzz-output - mkdir -p fuzz-output - mapfile -t corpus_dirs < <( - { - [ -d fuzz/corpus ] && printf '%s\n' fuzz/corpus - find fuzz-packages -mindepth 2 -maxdepth 2 -type d -name corpus - } | sort - ) - if [ "${#corpus_dirs[@]}" -eq 0 ]; then - tar -czf fuzz-output/corpus.tar.gz --files-from /dev/null - else - tar -czf fuzz-output/corpus.tar.gz "${corpus_dirs[@]}" - fi + operation: fuzz artifact_name: fuzz-output-weekly-${{ matrix.target.name }} artifact_path: | fuzz-output/corpus.tar.gz @@ -129,7 +112,7 @@ jobs: toolchain_components: "rust-src" enable_magic_cache: true enable_rust_cache: true - run_script: just test-fuzz-asan --all + operation: fuzz-asan # --- Native ML-KEM promotion gate on the physical AWS targets. --- mlkem-graviton: @@ -151,14 +134,8 @@ jobs: toolchain_components: "clippy, rustfmt, rust-src" enable_magic_cache: false enable_rust_cache: true - pre_script: | - echo "ML-KEM gate host: ${{ matrix.display_name }} (${{ matrix.platform }})" - uname -a - lscpu - sed -n '1,80p' /proc/cpuinfo - run_script: | - MLKEM_AARCH64_GATE_PLATFORM="${{ matrix.platform }}" \ - bash scripts/ci/mlkem-aarch64-gate.sh + operation: mlkem-aarch64 + platform: ${{ matrix.platform }} artifact_name: mlkem-aarch64-gate-${{ matrix.platform }} artifact_path: mlkem-aarch64-gate/ artifact_always: true @@ -169,8 +146,8 @@ jobs: uses: ./.github/workflows/ct.yaml with: platforms: all - dudect_timeout: "1800" - binsec_timeout: "900" + dudect_timeout: 1800 + binsec_timeout: 900 upload_raw_artifacts: true rsa: diff --git a/justfile b/justfile index 0a78a373..0dc8a39c 100644 --- a/justfile +++ b/justfile @@ -243,6 +243,7 @@ check-actions: @scripts/ci/dependabot-smoke-test.sh @scripts/ci/check-ci-ownership.sh @scripts/ci/check-ci-ownership-test.sh + @scripts/ci/run-rust-job-test.sh @scripts/ci/emit-manual-matrix-test.sh @scripts/ci/pre-push-test.sh @scripts/ci/release-evidence-check-test.sh diff --git a/scripts/README.md b/scripts/README.md index 10ad987f..1c78310b 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -9,17 +9,17 @@ its caller. |--------|---------| | `check/check.sh` | `just check` | | `check/check-all.sh` | `just check-all` | -| `check/check-feature-matrix.sh`| `just check-feature-matrix`, `scripts/check/check.sh`, `_ci-suite.yaml` | +| `check/check-feature-matrix.sh`| `just check-feature-matrix`, `scripts/check/check.sh`, `ci/run-rust-job.sh` | | `check/asm-ledger.sh` | `scripts/check/check.sh` | | `check/zeroize-evidence.sh` | `just check-zeroize-evidence`, `scripts/check/check-all.sh` | -| `ci/ci-check.sh` | `just ci-check`, `_ci-suite.yaml` (quality lane) | -| `ci/native-check.sh` | `_ci-suite.yaml` (native Linux/IBM lanes), `riscv.yaml` | -| `test/test.sh` | `just test`, `just test-all`, `_ci-suite.yaml`, `riscv.yaml` | -| `test/test-feature-matrix.sh` | `just test-feature-matrix`, `scripts/check/check.sh`, `_ci-suite.yaml` | -| `test/test-miri.sh` | `just test-miri`, `weekly.yaml` | -| `test/test-fuzz.sh` | `just test-fuzz`, `weekly.yaml` | -| `test/test-fuzz-asan.sh` | `just test-fuzz-asan` | -| `test/test-rsa-leakage.sh` | `just test-rsa-leakage`, `rsa.yaml` | +| `ci/ci-check.sh` | `just ci-check`, `ci/run-rust-job.sh` | +| `ci/native-check.sh` | `ci/run-rust-job.sh` | +| `test/test.sh` | `just test`, `just test-all`, `ci/run-rust-job.sh` | +| `test/test-feature-matrix.sh` | `just test-feature-matrix`, `scripts/check/check.sh`, `ci/run-rust-job.sh` | +| `test/test-miri.sh` | `just test-miri`, `ci/run-rust-job.sh` | +| `test/test-fuzz.sh` | `just test-fuzz`, `ci/run-rust-job.sh` | +| `test/test-fuzz-asan.sh` | `just test-fuzz-asan`, `ci/run-rust-job.sh` | +| `test/test-rsa-leakage.sh` | `just test-rsa-leakage`, `ci/run-rust-job.sh` | | `test/test-coverage.sh` | `just test-coverage`, `just test-fuzz-coverage`, `weekly.yaml` | | `bench/bench.sh` | `just bench`, `just bench-quick` | | `ci/check-action-pins.sh` | `just check-actions`, `ci/ci-check.sh`, `ci/dependabot-smoke.sh` | @@ -27,6 +27,7 @@ its caller. | `ci/dependabot-smoke-test.sh` | `just check-actions` | | `ci/check-ci-ownership.sh` | `just check-actions`, `ci/check-ci-ownership-test.sh` | | `ci/check-ci-ownership-test.sh`| `just check-actions` | +| `ci/run-rust-job-test.sh` | `just check-actions` | | `ci/emit-manual-matrix-test.sh`| `just check-actions` | | `ci/pre-push-test.sh` | `just check-actions` | | `ci/release-plan-check.sh` | `just release-check` | @@ -46,7 +47,7 @@ its caller. | `ci/pre-push.sh` | `just push`, `just push-full` | | `ct/artifacts.sh` | `just ct`, `just ct-artifacts`, `scripts/ct/full.py` | | `ct/dudect.sh` | `just ct-dudect`, `scripts/ct/full.py` | -| `ct/python.sh` | CT recipes and Python-backed CT, check, and release scripts | +| `ct/python.sh` | CT recipes, `ci/run-rust-job.sh`, and Python-backed CT, check, and release scripts | | `update/update-all.sh` | `just update`, `just update-check` | The optimized secret-lifecycle inspection performed by @@ -66,7 +67,7 @@ claim in [`docs/secret-lifecycle.md`](../docs/secret-lifecycle.md). | Script | Callers | |--------|---------| -| `ci/run-bench.sh` | `scripts/bench/bench.sh`, `bench.yaml`, `riscv.yaml` | +| `ci/run-bench.sh` | `scripts/bench/bench.sh`, `ci/mlkem-aarch64-gate.sh`, `ci/run-rust-job.sh` | | `bench/blake3-gap-gate.sh` | `scripts/ci/run-bench.sh` | ## CI-only (not surfaced via `just`) @@ -74,11 +75,12 @@ claim in [`docs/secret-lifecycle.md`](../docs/secret-lifecycle.md). | Script | Callers | |--------|---------| | `ci/install-tools.sh` | `.github/actions/setup/action.yaml` | -| `ci/dependabot-smoke.sh` | `ci.yaml` (Dependabot PR lane) | +| `ci/run-rust-job.sh` | `.github/workflows/_rust-job.yaml` | +| `ci/dependabot-smoke.sh` | `ci/run-rust-job.sh` | | `ci/emit-manual-matrix.sh` | `bench.yaml`, `ct.yaml` | -| `ci/mlkem-aarch64-gate.sh` | `weekly.yaml` | +| `ci/mlkem-aarch64-gate.sh` | `ci/run-rust-job.sh` | | `ci/nostd-wasm-suite.sh` | `ci/cross-targets.sh` | -| `ci/cross-targets.sh` | `_ci-suite.yaml` (consolidated cross-target lane) | +| `ci/cross-targets.sh` | `ci/run-rust-job.sh` | ## Shared Libraries (sourced, not invoked) diff --git a/scripts/ci/check-ci-ownership-test.sh b/scripts/ci/check-ci-ownership-test.sh index 12ae03e0..367ff909 100755 --- a/scripts/ci/check-ci-ownership-test.sh +++ b/scripts/ci/check-ci-ownership-test.sh @@ -55,6 +55,32 @@ make_fixture "$draft_runs_suite" yq eval '.jobs.suite.if = "always()"' -i "$draft_runs_suite/.github/workflows/ci.yaml" expect_failure "$draft_runs_suite" "draft PR can run the expensive suite" +shell_fragment_input="$TMP_ROOT/shell-fragment-input" +make_fixture "$shell_fragment_input" +printf '\n run_script: echo caller-controlled\n' \ + >>"$shell_fragment_input/.github/workflows/_ci-suite.yaml" +expect_failure "$shell_fragment_input" "reusable workflow accepts executable shell fragments" + +missing_typed_operation="$TMP_ROOT/missing-typed-operation" +make_fixture "$missing_typed_operation" +sed -i.bak '/operation: quality/d' "$missing_typed_operation/.github/workflows/_ci-suite.yaml" +rm -f "$missing_typed_operation/.github/workflows/_ci-suite.yaml.bak" +expect_failure "$missing_typed_operation" "reusable Rust job caller omits its operation" + +unsupported_typed_operation="$TMP_ROOT/unsupported-typed-operation" +make_fixture "$unsupported_typed_operation" +sed -i.bak 's/operation: quality/operation: arbitrary-shell/' \ + "$unsupported_typed_operation/.github/workflows/_ci-suite.yaml" +rm -f "$unsupported_typed_operation/.github/workflows/_ci-suite.yaml.bak" +expect_failure "$unsupported_typed_operation" "reusable Rust job caller selects an unsupported operation" + +evaluated_workflow_input="$TMP_ROOT/evaluated-workflow-input" +make_fixture "$evaluated_workflow_input" +sed -i.bak 's#run: scripts/ci/run-rust-job.sh#run: echo "${{ inputs.operation }}"#' \ + "$evaluated_workflow_input/.github/workflows/_rust-job.yaml" +rm -f "$evaluated_workflow_input/.github/workflows/_rust-job.yaml.bak" +expect_failure "$evaluated_workflow_input" "workflow input is evaluated as shell code" + duplicate_matrix="$TMP_ROOT/duplicate-matrix" make_fixture "$duplicate_matrix" printf '\n# duplicate owner\n run: just test-feature-matrix\n' >>"$duplicate_matrix/.github/workflows/weekly.yaml" @@ -74,13 +100,13 @@ expect_failure "$fake_musl" "MUSL label without a MUSL target invocation" missing_cross_owner="$TMP_ROOT/missing-cross-owner" make_fixture "$missing_cross_owner" -sed -i.bak '/scripts\/ci\/cross-targets\.sh/d' "$missing_cross_owner/.github/workflows/_ci-suite.yaml" +sed -i.bak '/operation: cross-targets/d' "$missing_cross_owner/.github/workflows/_ci-suite.yaml" rm -f "$missing_cross_owner/.github/workflows/_ci-suite.yaml.bak" expect_failure "$missing_cross_owner" "missing cross-target owner" missing_graph_owner="$TMP_ROOT/missing-graph-owner" make_fixture "$missing_graph_owner" -sed -i.bak '/cargo rail unify --check/d' "$missing_graph_owner/.github/workflows/_ci-suite.yaml" +sed -i.bak '/operation: cargo-graph/d' "$missing_graph_owner/.github/workflows/_ci-suite.yaml" rm -f "$missing_graph_owner/.github/workflows/_ci-suite.yaml.bak" expect_failure "$missing_graph_owner" "missing Cargo graph assurance owner" diff --git a/scripts/ci/check-ci-ownership.sh b/scripts/ci/check-ci-ownership.sh index 3001aaff..1afe9eeb 100755 --- a/scripts/ci/check-ci-ownership.sh +++ b/scripts/ci/check-ci-ownership.sh @@ -19,6 +19,7 @@ fi WORKFLOWS="$ROOT/.github/workflows" CI="$WORKFLOWS/ci.yaml" SUITE="$WORKFLOWS/_ci-suite.yaml" +RUST_JOB="$WORKFLOWS/_rust-job.yaml" WEEKLY="$WORKFLOWS/weekly.yaml" RISCV="$WORKFLOWS/riscv.yaml" RELEASE="$WORKFLOWS/release.yaml" @@ -29,6 +30,7 @@ COMPILE_MATRIX="$ROOT/scripts/check/check-feature-matrix.sh" EXECUTABLE_MATRIX="$ROOT/scripts/test/test-feature-matrix.sh" CHECK_ALL="$ROOT/scripts/check/check-all.sh" CI_CHECK="$ROOT/scripts/ci/ci-check.sh" +RUN_RUST_JOB="$ROOT/scripts/ci/run-rust-job.sh" RELEASE_PREFLIGHT="$ROOT/scripts/ci/release-preflight.sh" RELEASE_EVIDENCE="$ROOT/scripts/ci/release-evidence-check.sh" RELEASE_SOURCE="$ROOT/scripts/ci/package-release-source.sh" @@ -86,6 +88,7 @@ require_unique_feature_sets() { require_file "$CI" require_file "$SUITE" +require_file "$RUST_JOB" require_file "$WEEKLY" require_file "$RISCV" require_file "$RELEASE" @@ -96,6 +99,7 @@ require_file "$COMPILE_MATRIX" require_file "$EXECUTABLE_MATRIX" require_file "$CHECK_ALL" require_file "$CI_CHECK" +require_file "$RUN_RUST_JOB" require_file "$RELEASE_PREFLIGHT" require_file "$RELEASE_EVIDENCE" require_file "$RELEASE_SOURCE" @@ -137,6 +141,41 @@ fi == $(yq eval '.jobs | length' "$CI") ]] \ || fail "every CI job must defer draft pull requests" +if grep -ERn '^[[:space:]]+(pre_script|run_script):' "$WORKFLOWS" >/dev/null; then + fail "reusable workflows must not accept executable shell fragments" +fi +rust_job_calls=$(count_matches 'uses:[[:space:]]+\./\.github/workflows/_rust-job\.yaml' "$WORKFLOWS") +rust_job_operations=$(count_matches '^[[:space:]]+operation:[[:space:]]+[-[:alnum:]]+[[:space:]]*$' "$WORKFLOWS") +[[ "$rust_job_calls" -eq "$rust_job_operations" ]] \ + || fail "every reusable Rust job caller must select exactly one typed operation" +while IFS= read -r operation; do + [[ -n "$operation" ]] || continue + grep -Eq "^[[:space:]]+$operation\\)" "$RUN_RUST_JOB" \ + || fail "reusable Rust job caller selects unsupported operation: $operation" +done < <( + awk '/^[[:space:]]+operation:[[:space:]]+[-[:alnum:]]+[[:space:]]*$/ { print $2 }' \ + "$WORKFLOWS"/*.yaml | sort -u +) +[[ $(yq eval '.on.workflow_call.inputs.operation.required' "$RUST_JOB") == "true" ]] \ + || fail "the reusable Rust job operation must be required" +[[ $(yq eval '.on.workflow_call.inputs.operation.type' "$RUST_JOB") == "string" ]] \ + || fail "the reusable Rust job operation must be typed as a string" +[[ $(yq eval '[.jobs.run.steps[] | select(has("run"))] | length' "$RUST_JOB") -eq 1 ]] \ + || fail "the reusable Rust job must expose one fixed command step" +[[ $(yq eval '.jobs.run.steps[] | select(has("run")) | .run' "$RUST_JOB") == "scripts/ci/run-rust-job.sh" ]] \ + || fail "the reusable Rust job must invoke the repository-owned dispatcher" +# shellcheck disable=SC2016 # GitHub expression is an intentional literal workflow contract. +grep -Fq 'RSCRYPTO_CI_OPERATION: ${{ inputs.operation }}' "$RUST_JOB" \ + || fail "the reusable Rust job must pass its operation as environment data" +if yq eval '.. | select(tag == "!!map" and has("run") and (.run | tag == "!!str")) | .run' \ + "$WORKFLOWS"/*.yaml | grep -Eq '\$\{\{[[:space:]]*inputs\.'; then + fail "workflow inputs must not be interpolated into shell programs" +fi +if grep -En '(^|[[:space:]])eval[[:space:]]|(^|[[:space:]])(bash|sh)[[:space:]]+-c|<<<' \ + "$RUN_RUST_JOB" >/dev/null; then + fail "the Rust job dispatcher must not invoke a dynamic shell interpreter" +fi + [[ $(count_feature_sets "$COMPILE_MATRIX") -eq 29 ]] \ || fail "compile feature matrix must retain all 29 profiles" [[ $(count_feature_sets "$EXECUTABLE_MATRIX") -eq 38 ]] \ @@ -147,9 +186,9 @@ require_unique_feature_sets "$EXECUTABLE_MATRIX" grep -Eq 'HOST_ARGS\+=\(--feature-matrix\)' "$CHECK_ALL" \ || fail "local check-all must retain one explicit feature-matrix execution" -[[ $(count_matches 'just test-feature-matrix' "$WORKFLOWS") -eq 1 ]] \ +[[ $(count_matches 'just test-feature-matrix' "$WORKFLOWS" "$RUN_RUST_JOB") -eq 1 ]] \ || fail "ordinary workflows must have exactly one executable feature-matrix owner" -[[ $(count_matches 'just check-feature-matrix' "$WORKFLOWS") -eq 1 ]] \ +[[ $(count_matches 'just check-feature-matrix' "$WORKFLOWS" "$RUN_RUST_JOB") -eq 1 ]] \ || fail "ordinary workflows must have exactly one compile feature-matrix owner" # shellcheck disable=SC2016 # `$crate` is an intentional literal in the release-preflight contract regex. [[ $(count_matches 'cargo semver-checks --package "\$crate" --all-features' "$RELEASE_PREFLIGHT") -eq 1 ]] \ @@ -158,7 +197,7 @@ if grep -ERn 'cargo semver-checks' "$WORKFLOWS" >/dev/null; then fail "ordinary workflows must leave version-aware SemVer analysis to cargo-rail release planning" fi -if grep -ERn 'just check --all|check-all\.sh' "$WORKFLOWS" >/dev/null; then +if grep -ERn 'just check --all|check-all\.sh' "$WORKFLOWS" "$RUN_RUST_JOB" >/dev/null; then fail "native workflows must not invoke comprehensive cross-target checks" fi @@ -166,12 +205,18 @@ if grep -En 'test-feature-matrix|check-feature-matrix' "$WEEKLY" >/dev/null; the fail "weekly must inherit feature contracts from the reusable suite" fi -[[ $(count_matches 'scripts/ci/cross-targets\.sh' "$SUITE") -eq 1 ]] \ +[[ $(count_matches 'operation:[[:space:]]+cross-targets' "$SUITE") -eq 1 ]] \ || fail "the reusable suite must have exactly one cross-target owner" -[[ $(count_matches 'scripts/ci/native-check\.sh' "$SUITE") -eq 3 ]] \ +[[ $(count_matches 'scripts/ci/cross-targets\.sh' "$RUN_RUST_JOB") -eq 1 ]] \ + || fail "the Rust job dispatcher must define exactly one cross-target operation" +[[ $(count_matches 'operation:[[:space:]]+native$' "$SUITE") -eq 1 ]] \ + || fail "the reusable suite must own exactly one target-matrix native operation" +[[ $(count_matches 'operation:[[:space:]]+native-ibm' "$SUITE") -eq 2 ]] \ || fail "the reusable suite must own only Linux, IBM Z, and POWER10 native validation" -[[ $(count_matches 'scripts/ci/native-check\.sh' "$RISCV") -eq 1 ]] \ +[[ $(count_matches 'operation:[[:space:]]+native-riscv' "$RISCV") -eq 1 ]] \ || fail "the RISC-V workflow must own exactly one native validation lane" +[[ $(count_matches 'scripts/ci/native-check\.sh' "$RUN_RUST_JOB") -eq 3 ]] \ + || fail "the Rust job dispatcher must retain Linux, IBM, and RISC-V native operations" if grep -Ein 'riscv' "$SUITE" >/dev/null; then fail "the reusable CI suite must not own physical RISC-V work" fi @@ -187,8 +232,10 @@ fi || fail "the RISC-V workflow must select only the RISE CT lane" [[ $(yq eval '.jobs.ct.with.upload_raw_artifacts' "$RISCV") == "true" ]] \ || fail "the RISC-V workflow must retain raw CT artifacts for release promotion" -[[ $(count_matches 'cargo rail unify --check' "$SUITE") -eq 1 ]] \ +[[ $(count_matches 'operation:[[:space:]]+cargo-graph' "$SUITE") -eq 1 ]] \ || fail "the reusable suite must have exactly one Cargo graph assurance owner" +[[ $(count_matches 'cargo rail unify --check' "$RUN_RUST_JOB") -eq 1 ]] \ + || fail "the Rust job dispatcher must define exactly one Cargo graph assurance operation" if grep -En 'cargo rail unify --check' "$RELEASE_PREFLIGHT" >/dev/null; then fail "tag preflight must consume exact-commit Weekly graph assurance instead of repeating it" fi diff --git a/scripts/ci/emit-manual-matrix.sh b/scripts/ci/emit-manual-matrix.sh index 166e6662..e0e32a3e 100755 --- a/scripts/ci/emit-manual-matrix.sh +++ b/scripts/ci/emit-manual-matrix.sh @@ -56,34 +56,34 @@ normalize_platform() { # Bench matrix rows. Shape matches the inputs to _rust-job.yaml: # runner, timeout_minutes, tools_mode, toolchain_components -# Plus bench-specific plumbing (platform, os, display_name, artifact_suffix). +# Plus bench-specific plumbing (platform, display_name, artifact_suffix). append_row_for_platform() { local platform="${1:-}" local runner_uarch="runs-on=${GH_RUN_ID_VAL}/runner=" case "$platform" in amd-zen4) - ROWS+=("{\"platform\":\"amd-zen4\",\"os\":\"linux\",\"display_name\":\"AMD Zen4\",\"artifact_suffix\":\"amd-zen4\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}amd-zen4\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") + ROWS+=("{\"platform\":\"amd-zen4\",\"display_name\":\"AMD Zen4\",\"artifact_suffix\":\"amd-zen4\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}amd-zen4\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") ;; intel-spr) - ROWS+=("{\"platform\":\"intel-spr\",\"os\":\"linux\",\"display_name\":\"Intel Sapphire Rapids\",\"artifact_suffix\":\"intel-spr\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}intel-spr\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") + ROWS+=("{\"platform\":\"intel-spr\",\"display_name\":\"Intel Sapphire Rapids\",\"artifact_suffix\":\"intel-spr\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}intel-spr\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") ;; intel-icl) - ROWS+=("{\"platform\":\"intel-icl\",\"os\":\"linux\",\"display_name\":\"Intel Ice Lake\",\"artifact_suffix\":\"intel-icl\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}intel-icl\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") + ROWS+=("{\"platform\":\"intel-icl\",\"display_name\":\"Intel Ice Lake\",\"artifact_suffix\":\"intel-icl\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}intel-icl\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") ;; amd-zen5) - ROWS+=("{\"platform\":\"amd-zen5\",\"os\":\"linux\",\"display_name\":\"AMD Zen5\",\"artifact_suffix\":\"amd-zen5\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}amd-zen5\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") + ROWS+=("{\"platform\":\"amd-zen5\",\"display_name\":\"AMD Zen5\",\"artifact_suffix\":\"amd-zen5\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}amd-zen5\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") ;; graviton3) - ROWS+=("{\"platform\":\"graviton3\",\"os\":\"linux\",\"display_name\":\"AWS Graviton3\",\"artifact_suffix\":\"graviton3\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}graviton3\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") + ROWS+=("{\"platform\":\"graviton3\",\"display_name\":\"AWS Graviton3\",\"artifact_suffix\":\"graviton3\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}graviton3\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") ;; graviton4) - ROWS+=("{\"platform\":\"graviton4\",\"os\":\"linux\",\"display_name\":\"AWS Graviton4\",\"artifact_suffix\":\"graviton4\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}graviton4\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") + ROWS+=("{\"platform\":\"graviton4\",\"display_name\":\"AWS Graviton4\",\"artifact_suffix\":\"graviton4\",\"timeout_minutes\":${RUNSON_TIMEOUT_MINUTES},\"runner\":\"${runner_uarch}graviton4\",\"tools_mode\":\"minimal\",\"toolchain_components\":\"${COMPONENTS_STD}\"}") ;; ibm-s390x) - ROWS+=("{\"platform\":\"ibm-s390x\",\"os\":\"linux\",\"display_name\":\"IBM Z s390x\",\"artifact_suffix\":\"ibm-s390x\",\"timeout_minutes\":${IBM_TIMEOUT_MINUTES},\"runner\":\"ubuntu-24.04-s390x\",\"tools_mode\":\"ibm\",\"toolchain_components\":\"clippy, rustfmt\"}") + ROWS+=("{\"platform\":\"ibm-s390x\",\"display_name\":\"IBM Z s390x\",\"artifact_suffix\":\"ibm-s390x\",\"timeout_minutes\":${IBM_TIMEOUT_MINUTES},\"runner\":\"ubuntu-24.04-s390x\",\"tools_mode\":\"ibm\",\"toolchain_components\":\"clippy, rustfmt\"}") ;; ibm-power10) - ROWS+=("{\"platform\":\"ibm-power10\",\"os\":\"linux\",\"display_name\":\"IBM POWER10 ppc64le\",\"artifact_suffix\":\"ibm-power10\",\"timeout_minutes\":${IBM_TIMEOUT_MINUTES},\"runner\":\"ubuntu-24.04-ppc64le-p10\",\"tools_mode\":\"ibm\",\"toolchain_components\":\"clippy, rustfmt\"}") + ROWS+=("{\"platform\":\"ibm-power10\",\"display_name\":\"IBM POWER10 ppc64le\",\"artifact_suffix\":\"ibm-power10\",\"timeout_minutes\":${IBM_TIMEOUT_MINUTES},\"runner\":\"ubuntu-24.04-ppc64le-p10\",\"tools_mode\":\"ibm\",\"toolchain_components\":\"clippy, rustfmt\"}") ;; *) echo "error: unsupported bench platform '$platform'" >&2 diff --git a/scripts/ci/run-rust-job-test.sh b/scripts/ci/run-rust-job-test.sh new file mode 100755 index 00000000..f0f6d233 --- /dev/null +++ b/scripts/ci/run-rust-job-test.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DISPATCHER="$SCRIPT_DIR/run-rust-job.sh" +TMP_ROOT="$(mktemp -d)" +trap 'rm -rf "$TMP_ROOT"' EXIT + +fail() { + echo "rust job regression failure: $*" >&2 + exit 1 +} + +expect_failure() { + if "$@" >/dev/null 2>&1; then + fail "command unexpectedly succeeded: $*" + fi +} + +FIXTURE="$TMP_ROOT/repo" +CAPTURE="$TMP_ROOT/capture" +BIN="$TMP_ROOT/bin" +mkdir -p "$FIXTURE/scripts/ci" "$FIXTURE/scripts/ct" "$CAPTURE" "$BIN" +cp "$DISPATCHER" "$FIXTURE/scripts/ci/run-rust-job.sh" + +cat >"$BIN/just" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$@" >"$RSCRYPTO_CI_CAPTURE_DIR/just.args" +EOF + +for command in uname lscpu sed rustc cargo; do + cat >"$BIN/$command" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +done + +cat >"$FIXTURE/scripts/ci/run-bench.sh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s' "$BENCH_ONLY" >"$RSCRYPTO_CI_CAPTURE_DIR/bench-targets" +printf '%s' "$BENCH_FILTER" >"$RSCRYPTO_CI_CAPTURE_DIR/bench-filter" +printf '%s' "$BENCH_QUICK" >"$RSCRYPTO_CI_CAPTURE_DIR/bench-quick" +EOF + +cat >"$FIXTURE/scripts/ct/full.py" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$@" >"$RSCRYPTO_CI_CAPTURE_DIR/ct.args" +EOF + +cat >"$FIXTURE/scripts/ct/python.sh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$@" >"$RSCRYPTO_CI_CAPTURE_DIR/ct-package.args" +EOF + +chmod +x \ + "$BIN/just" \ + "$BIN/uname" \ + "$BIN/lscpu" \ + "$BIN/sed" \ + "$BIN/rustc" \ + "$BIN/cargo" \ + "$FIXTURE/scripts/ci/run-bench.sh" \ + "$FIXTURE/scripts/ct/full.py" \ + "$FIXTURE/scripts/ct/python.sh" + +TEST_PATH="$BIN:$PATH" +RUNNER=(env PATH="$TEST_PATH" RSCRYPTO_CI_CAPTURE_DIR="$CAPTURE" bash "$FIXTURE/scripts/ci/run-rust-job.sh") + +RSCRYPTO_CI_OPERATION=quality "${RUNNER[@]}" +[[ $(<"$CAPTURE/just.args") == "ci-check" ]] || fail "quality selected the wrong command" + +sentinel="$TMP_ROOT/injected" +# shellcheck disable=SC2016 # Command substitution is an intentional literal injection payload. +shell_payload='$(touch "'"$sentinel"'"); echo injected; #' +expect_failure env \ + PATH="$TEST_PATH" \ + RSCRYPTO_CI_CAPTURE_DIR="$CAPTURE" \ + RSCRYPTO_CI_OPERATION="quality; $shell_payload" \ + bash "$FIXTURE/scripts/ci/run-rust-job.sh" +[[ ! -e "$sentinel" ]] || fail "operation input was evaluated as shell code" + +expect_failure env \ + PATH="$TEST_PATH" \ + RSCRYPTO_CI_CAPTURE_DIR="$CAPTURE" \ + RSCRYPTO_CI_OPERATION=dependabot-smoke \ + RSCRYPTO_CI_BASE_SHA="$shell_payload" \ + bash "$FIXTURE/scripts/ci/run-rust-job.sh" +[[ ! -e "$sentinel" ]] || fail "commit input was evaluated as shell code" + +multiline_payload="$shell_payload"$'\n'"second line" +env \ + PATH="$TEST_PATH" \ + RSCRYPTO_CI_CAPTURE_DIR="$CAPTURE" \ + RSCRYPTO_CI_OPERATION=benchmark \ + RSCRYPTO_CI_PLATFORM=amd-zen4 \ + RSCRYPTO_CI_BENCH_TARGETS="$multiline_payload" \ + RSCRYPTO_CI_BENCH_FILTER="$multiline_payload" \ + RSCRYPTO_CI_BENCH_QUICK=true \ + bash "$FIXTURE/scripts/ci/run-rust-job.sh" >/dev/null +[[ $(<"$CAPTURE/bench-targets") == "$multiline_payload" ]] || fail "benchmark targets were not passed literally" +[[ $(<"$CAPTURE/bench-filter") == "$multiline_payload" ]] || fail "benchmark filter was not passed literally" +[[ $(<"$CAPTURE/bench-quick") == "true" ]] || fail "benchmark boolean was not preserved" +[[ ! -e "$sentinel" ]] || fail "benchmark input was evaluated as shell code" + +# shellcheck disable=SC2016 # Command substitution is an intentional literal injection payload. +ct_payload='$(touch${IFS}'"$sentinel"')' +env \ + PATH="$TEST_PATH" \ + RSCRYPTO_CI_CAPTURE_DIR="$CAPTURE" \ + RSCRYPTO_CI_OPERATION=constant-time \ + RSCRYPTO_CI_RUNNER=test-runner \ + RSCRYPTO_CI_PLATFORM=amd-zen4 \ + RSCRYPTO_CI_TARGET=x86_64-unknown-linux-gnu \ + RSCRYPTO_CI_DUDECT_TIMEOUT=1800 \ + RSCRYPTO_CI_DUDECT_FILTER="$ct_payload" \ + RSCRYPTO_CI_DUDECT_GATE=required \ + RSCRYPTO_CI_BINSEC_TIMEOUT=900 \ + RSCRYPTO_CI_UPLOAD_RAW_ARTIFACTS=false \ + bash "$FIXTURE/scripts/ci/run-rust-job.sh" >/dev/null +grep -Fxq -- "$ct_payload" "$CAPTURE/ct.args" || fail "DudeCT filter was not passed as one literal argument" +[[ ! -e "$sentinel" ]] || fail "DudeCT filter was evaluated as shell code" + +expect_failure env \ + PATH="$TEST_PATH" \ + RSCRYPTO_CI_CAPTURE_DIR="$CAPTURE" \ + RSCRYPTO_CI_OPERATION=constant-time \ + RSCRYPTO_CI_PLATFORM=amd-zen4 \ + RSCRYPTO_CI_TARGET=x86_64-unknown-linux-gnu \ + RSCRYPTO_CI_DUDECT_TIMEOUT="1800; $shell_payload" \ + RSCRYPTO_CI_BINSEC_TIMEOUT=900 \ + RSCRYPTO_CI_UPLOAD_RAW_ARTIFACTS=false \ + bash "$FIXTURE/scripts/ci/run-rust-job.sh" +[[ ! -e "$sentinel" ]] || fail "numeric input was evaluated as shell code" + +if grep -En '(^|[[:space:]])eval[[:space:]]|(^|[[:space:]])(bash|sh)[[:space:]]+-c|<<<' "$DISPATCHER" >/dev/null; then + fail "dispatcher contains a dynamic shell interpreter" +fi + +echo "Rust job dispatcher regression tests passed" diff --git a/scripts/ci/run-rust-job.sh b/scripts/ci/run-rust-job.sh new file mode 100755 index 00000000..595a0c03 --- /dev/null +++ b/scripts/ci/run-rust-job.sh @@ -0,0 +1,376 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$REPO_ROOT" + +die() { + echo "rust job error: $*" >&2 + exit 2 +} + +require_nonempty() { + local name=$1 + local value=$2 + [[ -n "$value" ]] || die "$name is required" +} + +require_one_of() { + local name=$1 + local value=$2 + shift 2 + + local allowed + for allowed in "$@"; do + if [[ "$value" == "$allowed" ]]; then + return 0 + fi + done + + die "invalid $name: $value" +} + +require_bool() { + require_one_of "$1" "$2" true false +} + +require_positive_integer() { + local name=$1 + local value=$2 + [[ "$value" =~ ^[1-9][0-9]*$ ]] || die "$name must be a positive integer" +} + +require_commit_sha() { + local value=$1 + [[ "$value" =~ ^([[:xdigit:]]{40}|[[:xdigit:]]{64})$ ]] \ + || die "base_sha must be a full commit ID" +} + +host_diagnostics() { + local cpuinfo_lines=$1 + uname -a + lscpu + sed -n "1,${cpuinfo_lines}p" /proc/cpuinfo +} + +run_quality() { + just ci-check +} + +run_cargo_graph() { + mkdir -p target/cargo-rail + cargo rail config validate --strict + cargo rail config migrate --check + cargo rail unify --check --explain --format json \ + --output target/cargo-rail/unify-result.json +} + +run_feature_contracts() { + just check-feature-matrix + just test-feature-matrix +} + +run_native() { + local target=${RSCRYPTO_CI_TARGET:-} + local test_mode=${RSCRYPTO_CI_TEST_MODE:-} + require_one_of target "$target" \ + x86_64-unknown-linux-gnu \ + aarch64-unknown-linux-gnu \ + x86_64-pc-windows-msvc \ + aarch64-pc-windows-msvc + require_one_of test_mode "$test_mode" local commit weekly + export RSCRYPTO_TEST_MODE="$test_mode" + + case "$target" in + x86_64-pc-windows-msvc) + echo "Windows x86_64 lane: compile smoke plus BLAKE3 ASM runtime vectors" + cargo clippy --workspace --lib --all-features -- -D warnings + cargo test --workspace --all-features --no-run + cargo test --workspace --features blake3 \ + --test blake3_official_vectors \ + --test blake3_differential + ;; + aarch64-pc-windows-msvc) + echo "Windows AArch64 lane: compile-only smoke" + cargo clippy --workspace --lib --all-features -- -D warnings + cargo test --workspace --all-features --no-run + ;; + x86_64-unknown-linux-gnu | aarch64-unknown-linux-gnu) + bash scripts/ci/native-check.sh --all-targets + bash scripts/test/test.sh --all + ;; + esac +} + +run_native_ibm() { + local test_mode=${RSCRYPTO_CI_TEST_MODE:-} + require_one_of test_mode "$test_mode" local commit weekly + export RSCRYPTO_TEST_MODE="$test_mode" + + host_diagnostics 50 + bash scripts/ci/native-check.sh + bash scripts/test/test.sh --all +} + +run_native_riscv() { + host_diagnostics 50 + export RSCRYPTO_TEST_MODE=weekly + export RSCRYPTO_CI_RESOURCE_PROFILE=constrained + export RSCRYPTO_SKIP_DOCTESTS=1 + bash scripts/ci/native-check.sh + bash scripts/test/test.sh --all +} + +run_cross_targets() { + bash scripts/ci/cross-targets.sh deep +} + +run_supply_chain() { + local mode=${RSCRYPTO_CI_SUPPLY_CHAIN_MODE:-} + require_one_of supply_chain_mode "$mode" light full + + if [[ "$mode" == "full" ]]; then + cargo deny check all + # RustCrypto `rsa` is used only as a dev/test/bench oracle. Production RSA + # verification is implemented in `src/auth/rsa.rs`; keep this scoped to the + # known Marvin advisory until the oracle dependency is removed or fixed. + cargo audit --ignore RUSTSEC-2023-0071 + else + cargo deny check advisories + fi +} + +run_dependabot_smoke() { + local base_sha=${RSCRYPTO_CI_BASE_SHA:-} + require_commit_sha "$base_sha" + export GITHUB_BASE_SHA="$base_sha" + bash scripts/ci/dependabot-smoke.sh +} + +run_miri() { + local tree_borrows=${RSCRYPTO_CI_MIRI_TREE_BORROWS:-false} + require_bool miri_tree_borrows "$tree_borrows" + if [[ "$tree_borrows" == "true" ]]; then + export MIRIFLAGS=-Zmiri-tree-borrows + fi + just test-miri +} + +run_fuzz() { + export RSCRYPTO_FUZZ_DURATION_SECS=60 + just test-fuzz --all + + rm -rf -- fuzz-output + mkdir -p fuzz-output + mapfile -t corpus_dirs < <( + { + [[ -d fuzz/corpus ]] && printf '%s\n' fuzz/corpus + find fuzz-packages -mindepth 2 -maxdepth 2 -type d -name corpus + } | sort + ) + if [[ "${#corpus_dirs[@]}" -eq 0 ]]; then + tar -czf fuzz-output/corpus.tar.gz --files-from /dev/null + else + tar -czf fuzz-output/corpus.tar.gz "${corpus_dirs[@]}" + fi +} + +run_fuzz_asan() { + just test-fuzz-asan --all +} + +run_mlkem_aarch64() { + local platform=${RSCRYPTO_CI_PLATFORM:-} + local display_name + require_one_of platform "$platform" graviton3 graviton4 + case "$platform" in + graviton3) display_name="AWS Graviton3" ;; + graviton4) display_name="AWS Graviton4" ;; + esac + + echo "ML-KEM gate host: $display_name ($platform)" + host_diagnostics 80 + export MLKEM_AARCH64_GATE_PLATFORM="$platform" + bash scripts/ci/mlkem-aarch64-gate.sh +} + +run_benchmark() { + local platform=${RSCRYPTO_CI_PLATFORM:-} + local targets=${RSCRYPTO_CI_BENCH_TARGETS:-all} + local filter=${RSCRYPTO_CI_BENCH_FILTER:-} + local quick=${RSCRYPTO_CI_BENCH_QUICK:-false} + local run_date + local run_time + local run_commit + require_one_of platform "$platform" \ + amd-zen4 intel-spr intel-icl amd-zen5 \ + graviton3 graviton4 ibm-s390x ibm-power10 rise-riscv + require_bool bench_quick "$quick" + + echo "Bench host: $platform (linux)" + host_diagnostics 50 + + run_date="$(date -u +"%Y-%m-%d")" + run_time="$(date -u +"%H_%M_%S")" + run_commit="$(git rev-parse HEAD 2>/dev/null || echo unknown)" + + export BENCH_OUTPUT_DIR=benchmark-results + export BENCH_RESULTS_DIR=benchmark-results + export BENCH_RUN_DATE="$run_date" + export BENCH_RUN_TIME="$run_time" + export BENCH_RUN_COMMIT="$run_commit" + export BENCH_RUN_OS=linux + export BENCH_RUN_ARCH="$platform" + export BENCH_RUN_MODE=ci + export BENCH_ONLY="$targets" + export BENCH_FILTER="$filter" + export BENCH_QUICK="$quick" + export BENCH_ALLOW_FULL_HASHES_COMP=true + export BENCH_PLATFORM="$platform" + scripts/ci/run-bench.sh +} + +ct_target_for_platform() { + case "$1" in + amd-zen4 | intel-spr | intel-icl | amd-zen5) + echo x86_64-unknown-linux-gnu + ;; + graviton3 | graviton4) + echo aarch64-unknown-linux-gnu + ;; + ibm-s390x) + echo s390x-unknown-linux-gnu + ;; + ibm-power10) + echo powerpc64le-unknown-linux-gnu + ;; + rise-riscv) + echo riscv64gc-unknown-linux-gnu + ;; + *) + die "invalid CT platform: $1" + ;; + esac +} + +run_constant_time() { + local platform=${RSCRYPTO_CI_PLATFORM:-} + local runner=${RSCRYPTO_CI_RUNNER:-} + local dudect_timeout=${RSCRYPTO_CI_DUDECT_TIMEOUT:-1800} + local binsec_timeout=${RSCRYPTO_CI_BINSEC_TIMEOUT:-900} + local raw_dudect_filter=${RSCRYPTO_CI_DUDECT_FILTER:-} + local raw_dudect_gate=${RSCRYPTO_CI_DUDECT_GATE:-required} + local raw_artifacts=${RSCRYPTO_CI_UPLOAD_RAW_ARTIFACTS:-false} + local target=${RSCRYPTO_CI_TARGET:-} + local expected_target + expected_target="$(ct_target_for_platform "$platform")" + [[ "$target" == "$expected_target" ]] || die "CT target does not match platform" + require_positive_integer dudect_timeout "$dudect_timeout" + require_positive_integer binsec_timeout "$binsec_timeout" + require_bool upload_raw_artifacts "$raw_artifacts" + + local evidence_dir=target/ct-evidence-package + mkdir -p "$evidence_dir" + { + echo "CT platform: $platform" + echo "CT target: $target" + echo "CT runner: $runner" + uname -a || true + rustc -vV + cargo -V + lscpu || true + sed -n '1,80p' /proc/cpuinfo || true + } 2>&1 | tee "$evidence_dir/host-$platform.log" + + local dudect_filter + local dudect_gate + dudect_filter="$(printf '%s' "$raw_dudect_filter" | tr -d '[:space:]')" + dudect_gate="$(printf '%s' "$raw_dudect_gate" | tr -d '[:space:]')" + if [[ "$raw_dudect_filter" != "$dudect_filter" ]]; then + echo "Normalized whitespace in DudeCT filter input." + fi + require_one_of dudect_gate "$dudect_gate" required diagnostic all + + local -a args=( + --target "$target" + --dudect-timeout "$dudect_timeout" + --binsec-timeout "$binsec_timeout" + --dudect-gate "$dudect_gate" + ) + if [[ -n "$dudect_filter" ]]; then + args+=(--dudect-filter "$dudect_filter") + fi + + local -a package_args=( + --target "$target" + --suffix "$platform" + --out-dir "$evidence_dir" + ) + local status=0 + { + printf 'Running:' + printf ' %q' scripts/ct/full.py "${args[@]}" + printf '\n' + scripts/ct/full.py "${args[@]}" + } 2>&1 | tee "$evidence_dir/ct-full-$platform.log" || status=$? + + if [[ "$raw_artifacts" == "true" ]]; then + package_args+=(--raw) + fi + scripts/ct/python.sh scripts/ct/package_evidence.py "${package_args[@]}" + return "$status" +} + +run_rsa_miri() { + mkdir -p ci-evidence + { + uname -a + lscpu + just test-miri --rsa + } 2>&1 | tee ci-evidence/rsa-miri-linux-x64.log +} + +run_rsa_leakage() { + local target=${RSCRYPTO_CI_TARGET:-} + require_one_of target "$target" linux-x64 linux-arm64 + mkdir -p ci-evidence + { + uname -a + lscpu + RSCRYPTO_RSA_LEAKAGE_SAMPLES="${RSCRYPTO_RSA_LEAKAGE_SAMPLES:-4000}" \ + RSCRYPTO_RSA_LEAKAGE_T_THRESHOLD="${RSCRYPTO_RSA_LEAKAGE_T_THRESHOLD:-8.0}" \ + just test-rsa-leakage + } 2>&1 | tee "ci-evidence/rsa-leakage-$target.log" +} + +main() { + if [[ $# -ne 0 ]]; then + die "usage: scripts/ci/run-rust-job.sh" + fi + + local operation=${RSCRYPTO_CI_OPERATION:-} + require_nonempty operation "$operation" + case "$operation" in + quality) run_quality ;; + cargo-graph) run_cargo_graph ;; + feature-contracts) run_feature_contracts ;; + native) run_native ;; + native-ibm) run_native_ibm ;; + native-riscv) run_native_riscv ;; + cross-targets) run_cross_targets ;; + supply-chain) run_supply_chain ;; + dependabot-smoke) run_dependabot_smoke ;; + miri) run_miri ;; + fuzz) run_fuzz ;; + fuzz-asan) run_fuzz_asan ;; + mlkem-aarch64) run_mlkem_aarch64 ;; + benchmark) run_benchmark ;; + constant-time) run_constant_time ;; + rsa-miri) run_rsa_miri ;; + rsa-leakage) run_rsa_leakage ;; + *) die "unsupported operation: $operation" ;; + esac +} + +main "$@"