diff --git a/OneBranchPipelines/conda-build-pipeline.yml b/OneBranchPipelines/conda-build-pipeline.yml index fc5598478..50fa08ee2 100644 --- a/OneBranchPipelines/conda-build-pipeline.yml +++ b/OneBranchPipelines/conda-build-pipeline.yml @@ -21,6 +21,7 @@ # (drop_ConsolidateConda_ConsolidateArtifacts). Publishing is the companion # conda-release-pipeline.yml. Validated locally that the recipe builds + imports; the # per-OS legs + SDL settings need an actual ADO run to confirm. +# Successful main wheel builds trigger Official Conda builds using that exact producer run. # ========================================================================================= name: $(Year:YY)$(DayOfYear)$(Rev:.r)-CondaBuild @@ -52,7 +53,7 @@ parameters: variables: - name: effectiveOneBranchType - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + ${{ if or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.Reason'], 'ResourceTrigger')) }}: value: 'Official' ${{ else }}: value: '${{ parameters.oneBranchType }}' @@ -68,9 +69,12 @@ resources: # The wheel build pipeline whose consolidated wheel artifacts this pipeline repackages. pipelines: - pipeline: buildPipeline - source: 'Build-Release-Package-Pipeline' + source: '\OneBranch\Build-Release-Package-Pipeline' branch: main - trigger: none + trigger: + branches: + include: + - refs/heads/main extends: template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' @@ -356,14 +360,24 @@ extends: - bash: | set -euo pipefail sudo apt-get update - sudo apt-get install -y qemu-user-static binfmt-support libc6-arm64-cross - # qemu-user-static + binfmt-support register the aarch64 binfmt_misc handler - # on install (update-binfmts), so host-level qemu-user emulation works WITHOUT - # a `docker run --privileged multiarch/qemu-user-static` (an unpinned public - # image SDL would flag). Verify the handler is present and fail loudly if not, - # rather than silently emulating nothing. - test -e /proc/sys/fs/binfmt_misc/qemu-aarch64 || { echo "ERROR: aarch64 binfmt handler not registered after apt install" >&2; exit 1; } + sudo apt-get install -y qemu-user qemu-user-binfmt libc6-arm64-cross + # Ubuntu 26.04 provides static emulators in qemu-user; qemu-user-binfmt + # supplies systemd registration files, not update-binfmts entries. + sudo systemctl restart systemd-binfmt.service + qemu-aarch64 --version + test -x /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 || { echo "ERROR: ARM64 glibc loader is missing after apt install" >&2; exit 1; } + handler="/proc/sys/fs/binfmt_misc/qemu-aarch64" + grep -qx enabled "$handler" || { echo "ERROR: aarch64 binfmt handler is missing or disabled after systemd-binfmt restart" >&2; exit 1; } + interpreter="$(sed -n 's/^interpreter //p' "$handler")" + # Ubuntu 24.04 can register a binfmt-P symlink to the same emulator. + if [[ -z "$interpreter" ]] || ! installed="$(readlink -e "/usr/bin/qemu-aarch64")" \ + || ! registered="$(readlink -e "$interpreter")" || [[ "$registered" != "$installed" ]]; then + echo "ERROR: aarch64 binfmt handler does not resolve to the installed qemu-user emulator" >&2 + exit 1 + fi + echo "##vso[task.setvariable variable=ready;isOutput=true]true" displayName: 'Install libc6-arm64-cross + register QEMU binfmt (aarch64)' + name: setupArm64Qemu # continueOnError so a transient apt/binfmt flake fails ONLY the best-effort # aarch64 leg (which is itself continueOnError), never the shared Linux job -- a # blocking setup step here would discard the already-validated linux-64 packages @@ -374,6 +388,7 @@ extends: condaSubdir: 'linux-aarch64' condaTargetSubdir: 'linux-aarch64' continueOnError: true + condition: and(succeeded(), eq(variables['setupArm64Qemu.ready'], 'true')) odbcWheelDir: '$(Pipeline.Workspace)/odbc_wheels' odbcWheelFilter: 'mssql_python_odbc-*manylinux_2_28_aarch64.whl' pythonVersions: '${{ parameters.pythonVersions }}' diff --git a/OneBranchPipelines/steps/conda-build-validate-step-posix.yml b/OneBranchPipelines/steps/conda-build-validate-step-posix.yml index a440cae91..7adfb3db0 100644 --- a/OneBranchPipelines/steps/conda-build-validate-step-posix.yml +++ b/OneBranchPipelines/steps/conda-build-validate-step-posix.yml @@ -67,6 +67,10 @@ parameters: - name: continueOnError type: boolean default: false + # Emulated callers can require successful provisioning without blocking native artifacts. + - name: condition + type: string + default: succeeded() steps: - bash: | @@ -102,4 +106,5 @@ steps: MSSQL_CONDA_PYTHON_VERSIONS: ${{ parameters.pythonVersions }} CONDA_TARGET_SUBDIR: ${{ parameters.condaTargetSubdir }} displayName: 'Conda build + validate (${{ parameters.condaSubdir }})' + condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} diff --git a/conda/README.md b/conda/README.md index 432fc7125..894973867 100644 --- a/conda/README.md +++ b/conda/README.md @@ -78,6 +78,19 @@ version before rendering/building. The shared orchestrator derives and supplies automatically; omitted input fails recipe rendering instead of choosing a release. Both native installation and cross extraction require the bulk-copy initializer and a compatible extension filename. The separate native audits still validate binary headers. +Linux ARM64 builds on the x64 Ubuntu agent require working QEMU emulation. The pipeline +installs `qemu-user`, `qemu-user-binfmt` and `libc6-arm64-cross`, reloads the systemd binfmt +registrations, and checks the enabled aarch64 handler, its emulator and the ARM64 loader. +The handler's interpreter must resolve to the installed emulator; valid Ubuntu 24.04 +`binfmt-P` symlinks are accepted, but missing, dangling or different interpreters are not. +On Ubuntu 26.04, `qemu-user-static` is only a virtual package, not an install candidate. +The recipe sets `QEMU_LD_PREFIX` to the cross libc directory when available; failed +Linux interpreter execution stops with an emulation diagnostic, not a missing-wheel +error. Interpreter-free wheel extraction remains restricted to `osx-*` targets. +The ARM64 build runs only after every setup check passes; failed setup skips that leg +without blocking native Linux artifact publication. ARM64 failures still preserve native +Linux artifacts, but a missing ARM64 matrix blocks release. + For RS-dependent bindings, the orchestrator also supplies the derived `MSSQL_RS_VERSION` and a validated `rs-wheel-cp.txt` selection in `WHEELS_DIR`. Recipes install or extract that entire wheel, including private libraries and distribution metadata, before @@ -259,6 +272,24 @@ path is passed to the publishing task through a read-only task output. Dependenc use the publisher's target-specific hash lock and explicit package-feed-proxy `--index-url` with pip's `--isolated` mode. Both production and rehearsal use this same code. +On `refs/heads/main`, successful completion of `Build-Release-Package-Pipeline` +automatically starts an Official `Conda-Build-Pipeline` run. This resource trigger +waits for the entire upstream pipeline, not just an individual stage, and retains +the exact triggering wheel producer run through `buildPipeline.runID`. +An Official Conda build does not by itself make its input wheels production-eligible. +The release provenance gate checks the recorded wheel run's effective Official mode +(including scheduled Official runs), enabled SDL and Windows signing settings, and +completed/succeeded timeline evidence for every recorded platform build job: wheel +ESRP malware scans and Component Governance, plus Windows native malware scanning, +signing, Authenticode verification and signed-wheel verification. The ODBC wheel scan +is also required. Missing, skipped, failed or mismatched evidence blocks production; +non-publishing validation reports the ineligibility without granting publication. +Automation stops after the Conda build: `Conda-Release-Pipeline` remains manually +queued, with no build-completion trigger for release, rehearsal or publication. +Direct CI and PR triggers remain disabled on both Conda pipelines. The build-trigger YAML +must be present in the ADO repository's configured default branch, with the existing +pipeline-resource authorizations in place, before automatic Conda builds can run. + With `publishToConda=false` (the default), the release pipeline now rehearses the publisher instead of only printing an upload plan. Both modes use the Official template, Windows 2022 managed container, network isolation, independent producer diff --git a/conda/mssql-python/build.sh b/conda/mssql-python/build.sh index beecf5037..8f341a885 100644 --- a/conda/mssql-python/build.sh +++ b/conda/mssql-python/build.sh @@ -29,7 +29,7 @@ if "$PYTHON" -c "import sys" >/dev/null 2>&1; then "$PYTHON" -m pip install --no-deps --no-index "$WHEELS_DIR/$rs_whl" -vv fi core_suffix="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX"))')" -else +elif [[ "${target_platform:-${CONDA_SUBDIR:-}}" == osx-* ]]; then core_suffix=".cpython-${CONDA_PY}-darwin.so" echo "Host Python '$PYTHON' is not executable on this agent (non-emulated cross-build);" echo "extracting both wheels into \$SP_DIR without running Python." @@ -67,6 +67,10 @@ else if [ -n "$rs_ver" ]; then unzip -oq "$WHEELS_DIR/$rs_whl" -d "$SP_DIR" fi +else + echo "ERROR: target Python '$PYTHON' cannot execute for '${target_platform:-${CONDA_SUBDIR:-unknown}}'; only osx-* targets support wheel extraction without Python." >&2 + echo "For linux-aarch64, check QEMU binfmt registration and the ARM64 glibc loader/QEMU_LD_PREFIX before rebuilding." >&2 + exit 1 fi if [ -n "$rs_ver" ]; then diff --git a/eng/conda_tools/provenance.py b/eng/conda_tools/provenance.py index af6d93035..2d0a2096e 100644 --- a/eng/conda_tools/provenance.py +++ b/eng/conda_tools/provenance.py @@ -92,6 +92,96 @@ def _verify_run(build: dict, run: dict, pipeline_id: int, run_id: int) -> dict: return {"pipeline": pipeline_id, "run": run_id, "branch": branch, "commit": commit} +def _wheel_security_reasons(get_json: Callable[[str], dict], build: dict, run: dict) -> list[str]: + parameters = run.get("templateParameters") + if not isinstance(parameters, dict): + return ["wheel producer security parameters are missing"] + reasons = [] + # Scheduled producers select Official even when the parameter defaults to NonOfficial. + if build.get("reason") != "schedule" and parameters.get("oneBranchType") != "Official": + reasons.append("wheel producer did not use Official mode") + for name in ("runSdlTasks", "signWindowsBinaries"): + if parameters.get(name) is not True: + reasons.append(f"wheel producer {name} was not enabled") + if reasons: + return reasons + + expected_jobs = {"ODBC_BuildAll.BuildWheel.__default"} + for parameter, prefix, fields, job in ( + ("windowsConfigs", "Win_py", ("pyVer", "arch"), "BuildWheel"), + ("macosConfigs", "MacOS_py", ("pyVer",), "BuildWheel"), + ("linuxConfigs", "Linux_", ("tag", "arch"), "BuildWheels"), + ): + configs = parameters.get(parameter) + if not isinstance(configs, list) or not configs: + return [f"wheel producer {parameter} evidence is missing"] + for config in configs: + if not isinstance(config, dict) or any( + not isinstance(config.get(field), str) + or not re.fullmatch(r"[a-zA-Z0-9_]+", config[field]) + for field in fields + ): + return [f"wheel producer {parameter} evidence is invalid"] + identifier = prefix + "_".join(config[field] for field in fields) + f".{job}.__default" + if identifier in expected_jobs: + return [f"wheel producer {parameter} contains duplicate jobs"] + expected_jobs.add(identifier) + + timeline = get_json(f"build/builds/{build['id']}/timeline?api-version=7.1").get("records") + if not isinstance(timeline, list) or not all(isinstance(record, dict) for record in timeline): + return ["wheel producer security timeline is missing or invalid"] + for identifier in sorted(expected_jobs): + jobs = [ + record + for record in timeline + if record.get("type") == "Job" and record.get("identifier") == identifier + ] + if ( + len(jobs) != 1 + or not isinstance(jobs[0].get("id"), str) + or not jobs[0]["id"] + or jobs[0].get("state") != "completed" + or jobs[0].get("result") != "succeeded" + or type(jobs[0].get("attempt")) is not int + or jobs[0]["attempt"] < 1 + ): + reasons.append(f"wheel producer job {identifier} is missing or did not succeed") + continue + job_record = jobs[0] + wheel_kind = "ODBC" if identifier.startswith("ODBC_") else "Python" + required_tasks = [ + (f"ESRP MalwareScanning - {wheel_kind} Wheels (Official)", "EsrpMalwareScanning"), + ("Component Governance Detection", "ComponentGovernanceComponentDetection"), + ] + if identifier.startswith("Win_"): + required_tasks.extend( + [ + ("ESRP MalwareScanning - Native extension", "EsrpMalwareScanning"), + ("ESRP CodeSigning - Native extension (.pyd)", "EsrpCodeSigning"), + ("Verify Authenticode signature (native extension)", "PowerShell"), + ("Verify signed .pyd inside built wheel", "PowerShell"), + ] + ) + for name, task_type in required_tasks: + tasks = [ + record + for record in timeline + if record.get("type") == "Task" + and record.get("parentId") == job_record["id"] + and record.get("name") == name + and record.get("attempt") == job_record["attempt"] + ] + if ( + len(tasks) != 1 + or tasks[0].get("state") != "completed" + or tasks[0].get("result") != "succeeded" + or not isinstance(tasks[0].get("task"), dict) + or tasks[0]["task"].get("name") != task_type + ): + reasons.append(f"wheel producer {identifier} lacks completed/succeeded {name}") + return reasons + + def verify_provenance( get_json: Callable[[str], dict], *, @@ -122,12 +212,8 @@ def verify_provenance( "buildNumber" ): raise ValueError("Recorded upstream wheel version differs from its Build API build number.") - wheel = _verify_run( - wheel_build, - get_json(f"pipelines/{_WHEEL_PIPELINE_ID}/runs/{wheel_run_id}?api-version=7.1"), - _WHEEL_PIPELINE_ID, - wheel_run_id, - ) + wheel_run = get_json(f"pipelines/{_WHEEL_PIPELINE_ID}/runs/{wheel_run_id}?api-version=7.1") + wheel = _verify_run(wheel_build, wheel_run, _WHEEL_PIPELINE_ID, wheel_run_id) reasons = [ f"{name} source is {branch}, not {_MAIN}" for name, branch in ( @@ -137,6 +223,7 @@ def verify_provenance( ) if branch != _MAIN ] + reasons.extend(_wheel_security_reasons(get_json, wheel_build, wheel_run)) if publish and reasons: raise ValueError("Production publication is ineligible: " + "; ".join(reasons)) return { diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py index 6af251129..b90d6067c 100644 --- a/tests/test_027_conda_release_metadata.py +++ b/tests/test_027_conda_release_metadata.py @@ -54,6 +54,166 @@ def _promoter_cli(argv): _MP_VER = "1.13.0" +@pytest.mark.skipif(sys.platform != "linux", reason="QEMU provisioning runs on Linux") +@pytest.mark.parametrize( + "state", + [ + "ready", + "symlinked-interpreter", + "broken-symlink", + "missing-handler", + "disabled", + "wrong-interpreter", + "missing-loader", + ], +) +def test_linux_qemu_setup_requires_supported_packages_and_enabled_handler(tmp_path, state): + import os + import shutil + import subprocess + import textwrap + + bash = shutil.which("bash") + if not bash: + pytest.skip("QEMU setup regression requires bash") + if subprocess.run( + [bash, "-c", 'readlink -e "$1" >/dev/null 2>&1', "bash", tmp_path.as_posix()], + capture_output=True, + timeout=15, + ).returncode: + pytest.skip("QEMU setup regression requires readlink -e") + source = (_ROOT / "OneBranchPipelines/conda-build-pipeline.yml").read_text() + setup = source.split( + " displayName: 'Install libc6-arm64-cross + register QEMU binfmt (aarch64)'", + 1, + )[0].rsplit(" - bash: |\n", 1)[1] + handler = tmp_path / "handler" + emulator = tmp_path / "qemu-aarch64" + emulator.touch() + interpreter = emulator + if state in {"symlinked-interpreter", "broken-symlink"}: + interpreter = tmp_path / "aarch64-binfmt-P" + try: + interpreter.symlink_to( + emulator if state == "symlinked-interpreter" else tmp_path / "missing-emulator" + ) + except OSError as error: + pytest.skip(f"Symlink regression requires symlink support: {error}") + elif state == "wrong-interpreter": + interpreter = tmp_path / "other-emulator" + interpreter.touch() + if state != "missing-handler": + handler.write_text( + ("disabled" if state == "disabled" else "enabled") + + "\ninterpreter " + + interpreter.as_posix() + + "\n", + newline="\n", + ) + setup = setup.replace("/proc/sys/fs/binfmt_misc/qemu-aarch64", handler.as_posix()) + setup = setup.replace("/usr/bin/qemu-aarch64", emulator.as_posix()) + mocks = r""" +sudo() { + case "$*" in + "apt-get update"|"apt-get install -y qemu-user qemu-user-binfmt libc6-arm64-cross"|\ + "systemctl restart systemd-binfmt.service") printf '%s\n' "$*" ;; + *) echo "Unexpected privileged command: $*" >&2; return 1 ;; + esac +} +qemu-aarch64() { echo "QEMU_VERSION_CHECK"; } +test() { + if [[ "$*" == "-x /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1" ]]; then + [[ "$TEST_STATE" != missing-loader ]] + else + builtin test "$@" + fi +} +""" + result = subprocess.run( + [bash, "-c", mocks + textwrap.dedent(setup)], + env=dict(os.environ, TEST_STATE=state), + capture_output=True, + text=True, + timeout=15, + ) + output = result.stdout + result.stderr + if state in {"ready", "symlinked-interpreter"}: + assert result.returncode == 0, output + assert ( + output.splitlines().count( + "apt-get install -y qemu-user qemu-user-binfmt libc6-arm64-cross" + ) + == 1 + ), output + assert "systemctl restart systemd-binfmt.service" in output + assert "QEMU_VERSION_CHECK" in output + assert "##vso[task.setvariable variable=ready;isOutput=true]true" in output + else: + assert result.returncode != 0, output + assert "ERROR:" in output + assert ("ARM64 glibc loader" if state == "missing-loader" else "aarch64 binfmt") in output + assert "##vso[task.setvariable variable=ready;isOutput=true]true" not in output + + +def test_linux_qemu_setup_gates_arm64_build_only(): + source = (_ROOT / "OneBranchPipelines/conda-build-pipeline.yml").read_text() + linux = source.split("- stage: CondaLinux\n", 1)[1].split("- stage: ConsolidateConda\n", 1)[0] + steps = re.split(r"^ - ", linux, flags=re.MULTILINE)[1:] + setup = next(step for step in steps if "register QEMU binfmt (aarch64)" in step) + arm64 = next(step for step in steps if "condaSubdir: 'linux-aarch64'" in step) + native = next(step for step in steps if "condaSubdir: 'linux-64'" in step) + publish = next(step for step in steps if "displayName: 'Publish Linux conda artifact'" in step) + assert " name: setupArm64Qemu\n" in setup + assert " continueOnError: true\n" in setup + assert " continueOnError: true\n" in arm64 + assert re.findall(r"^ condition: (.+)$", arm64, re.MULTILINE) == [ + "and(succeeded(), eq(variables['setupArm64Qemu.ready'], 'true'))" + ] + for step in (native, publish): + assert re.findall(r"^\s+condition: (.+)$", step, re.MULTILINE) in ([], ["succeeded()"]) + + template = (_ROOT / "OneBranchPipelines/steps/conda-build-validate-step-posix.yml").read_text() + parameters, build = template.split("\nsteps:\n", 1) + assert re.search(r"- name: condition\n type: string\n default: succeeded\(\)", parameters) + assert re.search(r"^ condition: \$\{\{ parameters\.condition \}\}$", build, re.MULTILINE) + + +def test_conda_build_completion_trigger_only_main(): + source = (_ROOT / "OneBranchPipelines/conda-build-pipeline.yml").read_text() + assert "\ntrigger: none\npr: none\n" in source + resource = source.split("\n pipelines:\n", 1)[1].split("\nextends:\n", 1)[0] + assert resource.count("- pipeline: buildPipeline") == 1 + assert " source: '\\OneBranch\\Build-Release-Package-Pipeline'\n" in resource + assert " branch: main\n" in resource + assert resource.split(" trigger:\n", 1)[1].strip() == ( + "branches:\n include:\n - refs/heads/main" + ) + assert "$(resources.pipeline.buildPipeline.runID)" in source + + +def test_automatic_conda_builds_use_official_template(): + source = (_ROOT / "OneBranchPipelines/conda-build-pipeline.yml").read_text() + assert ( + " - name: effectiveOneBranchType\n" + " ${{ if or(eq(variables['Build.Reason'], 'Schedule'), " + "eq(variables['Build.Reason'], 'ResourceTrigger')) }}:\n" + " value: 'Official'\n" + " ${{ else }}:\n" + " value: '${{ parameters.oneBranchType }}'\n" + ) in source + assert "template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml" in source + + +def test_conda_release_remains_manual(): + source = (_ROOT / "OneBranchPipelines/conda-release-pipeline.yml").read_text() + assert "\ntrigger: none\npr: none\n" in source + resource = source.split("\n pipelines:\n", 1)[1].split("\nextends:\n", 1)[0] + assert re.findall(r"^ trigger: (.+)$", resource, re.MULTILINE) == ["none"] + assert " trigger:\n" not in resource + assert "PUBLISH_TO_CONDA: '${{ parameters.publishToConda }}'" in source + assert (" ${{ if eq(parameters.publishToConda, true) }}:\n" " binskim:\n") in source + + @pytest.mark.parametrize( "source_present", [False, True], ids=["wheel-tests-only", "source-checkout"] ) diff --git a/tests/test_035_conda_macho_assert.py b/tests/test_035_conda_macho_assert.py index d851a9510..194244052 100644 --- a/tests/test_035_conda_macho_assert.py +++ b/tests/test_035_conda_macho_assert.py @@ -190,6 +190,47 @@ def _realistic_payload(binding, arm64=None, x86_64=None): return payload +@pytest.mark.parametrize( + ("target", "subdir"), + [("linux-64", ""), ("linux-aarch64", ""), ("", "linux-aarch64")], +) +def test_unix_recipe_rejects_failed_linux_python_before_wheel_extraction(tmp_path, target, subdir): + bash = shutil.which("bash") + if not bash: + pytest.skip("Direct Unix recipe execution requires bash") + wheels = tmp_path / "wheels" + wheels.mkdir() + arch = "aarch64" if "aarch64" in (target or subdir) else "x86_64" + (wheels / f"mssql_python-1.18.0-cp312-cp312-manylinux_2_28_{arch}.whl").touch() + site_packages = tmp_path / "site-packages" + result = subprocess.run( + [bash, (_ROOT / "conda/mssql-python/build.sh").as_posix()], + env=dict( + os.environ, + PYTHON=(tmp_path / "nonexecutable-python").as_posix(), + PKG_NAME="mssql-python", + PKG_VERSION="1.18.0", + CONDA_PY="312", + WHEELS_DIR=wheels.as_posix(), + SP_DIR=site_packages.as_posix(), + MSSQL_ODBC_VERSION="18.6.2.2", + MSSQL_RS_VERSION="", + target_platform=target, + CONDA_SUBDIR=subdir, + ), + capture_output=True, + text=True, + timeout=15, + ) + output = result.stdout + result.stderr + assert result.returncode != 0 + assert f"cannot execute for '{target or subdir}'" in output + assert "QEMU binfmt" in output and "QEMU_LD_PREFIX" in output + assert "no mssql-python==" not in output + assert "extracting both wheels" not in output + assert not site_packages.exists() + + @pytest.mark.parametrize("cross_build", [False, True]) @pytest.mark.parametrize( ("rs_owned", "state"), diff --git a/tests/test_036_conda_provenance.py b/tests/test_036_conda_provenance.py index 5bd2a2ca7..3aa655bff 100644 --- a/tests/test_036_conda_provenance.py +++ b/tests/test_036_conda_provenance.py @@ -701,11 +701,66 @@ def chain(release_sources): producer_run["resources"]["pipelines"] = { "buildPipeline": {"pipeline": {"id": 173176}, "version": "26250.2"} } + wheel["reason"] = "manual" + wheel_run["templateParameters"] = { + "oneBranchType": "Official", + "runSdlTasks": True, + "signWindowsBinaries": True, + "windowsConfigs": [{"pyVer": "312", "arch": "x64"}, {"pyVer": "312", "arch": "arm64"}], + "macosConfigs": [{"pyVer": "312"}], + "linuxConfigs": [{"tag": "manylinux_2_28", "arch": "aarch64"}], + } + timeline = [] + for identifier in ( + "Win_py312_x64.BuildWheel.__default", + "Win_py312_arm64.BuildWheel.__default", + "MacOS_py312.BuildWheel.__default", + "Linux_manylinux_2_28_aarch64.BuildWheels.__default", + "ODBC_BuildAll.BuildWheel.__default", + ): + timeline.append( + { + "id": identifier, + "identifier": identifier, + "type": "Job", + "state": "completed", + "result": "succeeded", + "attempt": 1, + } + ) + label = "ODBC" if identifier.startswith("ODBC_") else "Python" + tasks = [ + (f"ESRP MalwareScanning - {label} Wheels (Official)", "EsrpMalwareScanning"), + ("Component Governance Detection", "ComponentGovernanceComponentDetection"), + ] + if identifier.startswith("Win_"): + tasks.extend( + [ + ("ESRP MalwareScanning - Native extension", "EsrpMalwareScanning"), + ("ESRP CodeSigning - Native extension (.pyd)", "EsrpCodeSigning"), + ("Verify Authenticode signature (native extension)", "PowerShell"), + ("Verify signed .pyd inside built wheel", "PowerShell"), + ] + ) + timeline.extend( + { + "id": f"{identifier}-{index}", + "parentId": identifier, + "type": "Task", + "name": name, + "task": {"name": task}, + "state": "completed", + "result": "succeeded", + "attempt": 1, + } + for index, (name, task) in enumerate(tasks) + ) records = { "build/builds/174195?api-version=7.1": producer, "pipelines/2318/runs/174195?api-version=7.1": producer_run, "build/builds/173176?api-version=7.1": wheel, "pipelines/2199/runs/173176?api-version=7.1": wheel_run, + "build/builds/173176/timeline?api-version=7.1": {"records": timeline}, } records.update(_source_record(path, content) for path, content in release_sources.items()) return records @@ -737,6 +792,86 @@ def test_exact_recorded_wheel_run_is_verified_not_packaging_mode_or_build_number assert result["versions"] == {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} +@pytest.mark.parametrize( + "problem", + [ + "nonofficial", + "missing-parameters", + "sdl-disabled", + "signing-disabled", + "empty-matrix", + "missing-timeline", + "missing-job", + "missing-wheel-scan", + "missing-odbc-scan", + "missing-signing", + "missing-signature-verification", + "failed-task", + "skipped-task", + "wrong-task-kind", + "wrong-job", + "previous-attempt", + ], +) +def test_wheel_security_evidence_required_for_production(chain, problem): + run = chain["pipelines/2199/runs/173176?api-version=7.1"] + timeline = chain["build/builds/173176/timeline?api-version=7.1"]["records"] + params = run["templateParameters"] + scan = next( + record + for record in timeline + if record.get("name") == "ESRP MalwareScanning - Python Wheels (Official)" + and record["parentId"].startswith("Win_py312_arm64.") + ) + if problem == "nonofficial": + params["oneBranchType"] = "NonOfficial" + elif problem == "missing-parameters": + del run["templateParameters"] + elif problem == "sdl-disabled": + params["runSdlTasks"] = False + elif problem == "signing-disabled": + params["signWindowsBinaries"] = False + elif problem == "empty-matrix": + params["windowsConfigs"] = [] + elif problem == "missing-timeline": + timeline.clear() + elif problem == "missing-job": + timeline[:] = [record for record in timeline if record.get("id") != scan["parentId"]] + elif problem == "missing-wheel-scan": + timeline.remove(scan) + elif problem in {"missing-odbc-scan", "missing-signing", "missing-signature-verification"}: + label = { + "missing-odbc-scan": "ESRP MalwareScanning - ODBC Wheels (Official)", + "missing-signing": "ESRP CodeSigning - Native extension (.pyd)", + "missing-signature-verification": "Verify signed .pyd inside built wheel", + }[problem] + timeline.remove(next(record for record in timeline if record.get("name") == label)) + elif problem in {"failed-task", "skipped-task"}: + scan["result"] = problem.removesuffix("-task") + elif problem == "wrong-task-kind": + scan["task"]["name"] = "CmdLine" + elif problem == "wrong-job": + scan["parentId"] = "unrelated-job" + elif problem == "previous-attempt": + next(record for record in timeline if record["id"] == scan["parentId"])["attempt"] = 2 + with pytest.raises(ValueError, match="Production publication is ineligible"): + _verify(chain) + result = _verify(chain, publish=False) + assert result["productionEligible"] is False + assert result["productionIneligibilityReasons"] + + +def test_scheduled_official_wheel_run_still_requires_completed_security_checks(chain): + chain["build/builds/173176?api-version=7.1"]["reason"] = "schedule" + chain["pipelines/2199/runs/173176?api-version=7.1"]["templateParameters"][ + "oneBranchType" + ] = "NonOfficial" + assert _verify(chain)["productionEligible"] is True + chain["build/builds/173176/timeline?api-version=7.1"]["records"].clear() + with pytest.raises(ValueError, match="Production publication is ineligible"): + _verify(chain) + + @pytest.mark.parametrize( "content", [ @@ -1002,6 +1137,31 @@ def test_provenance_cli_executes_same_chain_in_both_modes(monkeypatch, chain, ca ) +@pytest.mark.parametrize("publish", ["true", "false"]) +def test_provenance_cli_cannot_promote_unsigned_wheels( + monkeypatch, provenance_cli_env, chain, capsys, publish +): + monkeypatch.setenv("PUBLISH_TO_CONDA", publish) + monkeypatch.setattr(provenance, "ado_get_json", chain.__getitem__) + chain["pipelines/2199/runs/173176?api-version=7.1"]["templateParameters"][ + "signWindowsBinaries" + ] = False + status = provenance.cli() + output = capsys.readouterr() + if publish == "true": + assert status == 1 + assert "Production publication is ineligible" in output.err + assert "##vso[task.setvariable" not in output.out + else: + assert status == 0 + result = json.loads(output.out.splitlines()[0].split(": ", 1)[1]) + assert result["productionEligible"] is False + assert ( + "wheel producer signWindowsBinaries was not enabled" + in result["productionIneligibilityReasons"] + ) + + def test_invalid_publish_flag_is_not_treated_as_dry_run(monkeypatch): monkeypatch.setenv("PUBLISH_TO_CONDA", "perhaps") with pytest.raises(ValueError, match="true or false"): @@ -1251,7 +1411,8 @@ def get_json(path): with pytest.raises((ValueError, KeyError, HTTPError)): _verify(chain, get_json=get_json) assert reads[-1] == key - assert len(reads) == 8 + assert reads[4] == "build/builds/173176/timeline?api-version=7.1" + assert len(reads) == 9 @pytest.mark.parametrize(