From a0023d4a0e18407a259610cd6181481690404c1d Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 17 Nov 2026 23:33:45 +0530 Subject: [PATCH 1/7] FIX: Restore Linux ARM64 Conda builds on Ubuntu 26.04 Port the focused QEMU provisioning and Linux interpreter failure handling from ADO PR 9017. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- OneBranchPipelines/conda-build-pipeline.yml | 16 ++--- conda/README.md | 9 +++ conda/mssql-python/build.sh | 6 +- tests/test_027_conda_release_metadata.py | 65 +++++++++++++++++++++ tests/test_035_conda_macho_assert.py | 41 +++++++++++++ 5 files changed, 129 insertions(+), 8 deletions(-) diff --git a/OneBranchPipelines/conda-build-pipeline.yml b/OneBranchPipelines/conda-build-pipeline.yml index fc5598478..4469de5b5 100644 --- a/OneBranchPipelines/conda-build-pipeline.yml +++ b/OneBranchPipelines/conda-build-pipeline.yml @@ -356,13 +356,15 @@ extends: - bash: | set -euo pipefail sudo apt-get update - sudo apt-get install -y qemu-user-static binfmt-support libc6-arm64-cross - # qemu-user-static + binfmt-support register the aarch64 binfmt_misc handler - # on install (update-binfmts), so host-level qemu-user emulation works WITHOUT - # a `docker run --privileged multiarch/qemu-user-static` (an unpinned public - # image SDL would flag). Verify the handler is present and fail loudly if not, - # rather than silently emulating nothing. - test -e /proc/sys/fs/binfmt_misc/qemu-aarch64 || { echo "ERROR: aarch64 binfmt handler not registered after apt install" >&2; exit 1; } + sudo apt-get install -y qemu-user qemu-user-binfmt libc6-arm64-cross + # Ubuntu 26.04 provides static emulators in qemu-user; qemu-user-binfmt + # supplies systemd registration files, not update-binfmts entries. + sudo systemctl restart systemd-binfmt.service + qemu-aarch64 --version + test -x /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 || { echo "ERROR: ARM64 glibc loader is missing after apt install" >&2; exit 1; } + handler=/proc/sys/fs/binfmt_misc/qemu-aarch64 + grep -qx enabled "$handler" || { echo "ERROR: aarch64 binfmt handler is missing or disabled after systemd-binfmt restart" >&2; exit 1; } + grep -qx 'interpreter /usr/bin/qemu-aarch64' "$handler" || { echo "ERROR: aarch64 binfmt handler does not use the installed qemu-user emulator" >&2; exit 1; } displayName: 'Install libc6-arm64-cross + register QEMU binfmt (aarch64)' # continueOnError so a transient apt/binfmt flake fails ONLY the best-effort # aarch64 leg (which is itself continueOnError), never the shared Linux job -- a diff --git a/conda/README.md b/conda/README.md index 432fc7125..06eb06af8 100644 --- a/conda/README.md +++ b/conda/README.md @@ -78,6 +78,15 @@ version before rendering/building. The shared orchestrator derives and supplies automatically; omitted input fails recipe rendering instead of choosing a release. Both native installation and cross extraction require the bulk-copy initializer and a compatible extension filename. The separate native audits still validate binary headers. +Linux ARM64 builds on the x64 Ubuntu agent require working QEMU emulation. The pipeline +installs `qemu-user`, `qemu-user-binfmt` and `libc6-arm64-cross`, reloads the systemd binfmt +registrations, and checks the enabled aarch64 handler, its emulator and the ARM64 loader. +On Ubuntu 26.04, `qemu-user-static` is only a virtual package, not an install candidate. +The recipe sets `QEMU_LD_PREFIX` to the cross libc directory when available; failed +Linux interpreter execution stops with an emulation diagnostic, not a missing-wheel +error. Interpreter-free wheel extraction remains restricted to `osx-*` targets. +ARM64 failures still preserve native Linux artifacts, but a missing ARM64 matrix blocks release. + For RS-dependent bindings, the orchestrator also supplies the derived `MSSQL_RS_VERSION` and a validated `rs-wheel-cp.txt` selection in `WHEELS_DIR`. Recipes install or extract that entire wheel, including private libraries and distribution metadata, before diff --git a/conda/mssql-python/build.sh b/conda/mssql-python/build.sh index beecf5037..8f341a885 100644 --- a/conda/mssql-python/build.sh +++ b/conda/mssql-python/build.sh @@ -29,7 +29,7 @@ if "$PYTHON" -c "import sys" >/dev/null 2>&1; then "$PYTHON" -m pip install --no-deps --no-index "$WHEELS_DIR/$rs_whl" -vv fi core_suffix="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX"))')" -else +elif [[ "${target_platform:-${CONDA_SUBDIR:-}}" == osx-* ]]; then core_suffix=".cpython-${CONDA_PY}-darwin.so" echo "Host Python '$PYTHON' is not executable on this agent (non-emulated cross-build);" echo "extracting both wheels into \$SP_DIR without running Python." @@ -67,6 +67,10 @@ else if [ -n "$rs_ver" ]; then unzip -oq "$WHEELS_DIR/$rs_whl" -d "$SP_DIR" fi +else + echo "ERROR: target Python '$PYTHON' cannot execute for '${target_platform:-${CONDA_SUBDIR:-unknown}}'; only osx-* targets support wheel extraction without Python." >&2 + echo "For linux-aarch64, check QEMU binfmt registration and the ARM64 glibc loader/QEMU_LD_PREFIX before rebuilding." >&2 + exit 1 fi if [ -n "$rs_ver" ]; then diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py index 6af251129..bf00364ed 100644 --- a/tests/test_027_conda_release_metadata.py +++ b/tests/test_027_conda_release_metadata.py @@ -54,6 +54,71 @@ def _promoter_cli(argv): _MP_VER = "1.13.0" +@pytest.mark.parametrize( + "state", ["ready", "missing-handler", "disabled", "wrong-interpreter", "missing-loader"] +) +def test_linux_qemu_setup_requires_supported_packages_and_enabled_handler(tmp_path, state): + import os + import shutil + import subprocess + import textwrap + + bash = shutil.which("bash") + if not bash: + pytest.skip("QEMU setup regression requires bash") + source = (_ROOT / "OneBranchPipelines/conda-build-pipeline.yml").read_text() + setup = source.split( + " displayName: 'Install libc6-arm64-cross + register QEMU binfmt (aarch64)'", + 1, + )[0].rsplit(" - bash: |\n", 1)[1] + handler = tmp_path / "handler" + if state != "missing-handler": + handler.write_text( + ("disabled" if state == "disabled" else "enabled") + + "\ninterpreter " + + ("/missing/qemu-aarch64" if state == "wrong-interpreter" else "/usr/bin/qemu-aarch64") + + "\n", + newline="\n", + ) + mocks = r""" +sudo() { + case "$*" in + "apt-get update"|"apt-get install -y qemu-user qemu-user-binfmt libc6-arm64-cross"|\ + "systemctl restart systemd-binfmt.service") printf '%s\n' "$*" ;; + *) echo "Unexpected privileged command: $*" >&2; return 1 ;; + esac +} +qemu-aarch64() { echo "QEMU_VERSION_CHECK"; } +test() { + if [[ "$*" == "-x /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1" ]]; then + [[ "$TEST_STATE" != missing-loader ]] + else + builtin test "$@" + fi +} +grep() { + if [[ "${!#}" != /proc/sys/fs/binfmt_misc/qemu-aarch64 ]]; then return 1; fi + command grep "${@:1:$#-1}" "$TEST_HANDLER" +} +""" + result = subprocess.run( + [bash, "-c", mocks + textwrap.dedent(setup)], + env=dict(os.environ, TEST_STATE=state, TEST_HANDLER=handler.as_posix()), + capture_output=True, + text=True, + timeout=15, + ) + output = result.stdout + result.stderr + if state == "ready": + assert result.returncode == 0, output + assert "systemctl restart systemd-binfmt.service" in output + assert "QEMU_VERSION_CHECK" in output + else: + assert result.returncode != 0, output + assert "ERROR:" in output + assert ("ARM64 glibc loader" if state == "missing-loader" else "aarch64 binfmt") in output + + @pytest.mark.parametrize( "source_present", [False, True], ids=["wheel-tests-only", "source-checkout"] ) diff --git a/tests/test_035_conda_macho_assert.py b/tests/test_035_conda_macho_assert.py index d851a9510..194244052 100644 --- a/tests/test_035_conda_macho_assert.py +++ b/tests/test_035_conda_macho_assert.py @@ -190,6 +190,47 @@ def _realistic_payload(binding, arm64=None, x86_64=None): return payload +@pytest.mark.parametrize( + ("target", "subdir"), + [("linux-64", ""), ("linux-aarch64", ""), ("", "linux-aarch64")], +) +def test_unix_recipe_rejects_failed_linux_python_before_wheel_extraction(tmp_path, target, subdir): + bash = shutil.which("bash") + if not bash: + pytest.skip("Direct Unix recipe execution requires bash") + wheels = tmp_path / "wheels" + wheels.mkdir() + arch = "aarch64" if "aarch64" in (target or subdir) else "x86_64" + (wheels / f"mssql_python-1.18.0-cp312-cp312-manylinux_2_28_{arch}.whl").touch() + site_packages = tmp_path / "site-packages" + result = subprocess.run( + [bash, (_ROOT / "conda/mssql-python/build.sh").as_posix()], + env=dict( + os.environ, + PYTHON=(tmp_path / "nonexecutable-python").as_posix(), + PKG_NAME="mssql-python", + PKG_VERSION="1.18.0", + CONDA_PY="312", + WHEELS_DIR=wheels.as_posix(), + SP_DIR=site_packages.as_posix(), + MSSQL_ODBC_VERSION="18.6.2.2", + MSSQL_RS_VERSION="", + target_platform=target, + CONDA_SUBDIR=subdir, + ), + capture_output=True, + text=True, + timeout=15, + ) + output = result.stdout + result.stderr + assert result.returncode != 0 + assert f"cannot execute for '{target or subdir}'" in output + assert "QEMU binfmt" in output and "QEMU_LD_PREFIX" in output + assert "no mssql-python==" not in output + assert "extracting both wheels" not in output + assert not site_packages.exists() + + @pytest.mark.parametrize("cross_build", [False, True]) @pytest.mark.parametrize( ("rs_owned", "state"), From 425a707b1fa828a0df32e6172818a5a914b5acd5 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 17 Nov 2026 23:54:21 +0530 Subject: [PATCH 2/7] FIX: Gate ARM64 Conda builds on validated QEMU setup Require the setup success output before running the ARM64 leg, preserve native artifact publication, and assert prerequisite package installation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- OneBranchPipelines/conda-build-pipeline.yml | 3 ++ .../steps/conda-build-validate-step-posix.yml | 5 +++ conda/README.md | 4 ++- tests/test_027_conda_release_metadata.py | 31 +++++++++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/OneBranchPipelines/conda-build-pipeline.yml b/OneBranchPipelines/conda-build-pipeline.yml index 4469de5b5..fc1abaa2f 100644 --- a/OneBranchPipelines/conda-build-pipeline.yml +++ b/OneBranchPipelines/conda-build-pipeline.yml @@ -365,7 +365,9 @@ extends: handler=/proc/sys/fs/binfmt_misc/qemu-aarch64 grep -qx enabled "$handler" || { echo "ERROR: aarch64 binfmt handler is missing or disabled after systemd-binfmt restart" >&2; exit 1; } grep -qx 'interpreter /usr/bin/qemu-aarch64' "$handler" || { echo "ERROR: aarch64 binfmt handler does not use the installed qemu-user emulator" >&2; exit 1; } + echo "##vso[task.setvariable variable=ready;isOutput=true]true" displayName: 'Install libc6-arm64-cross + register QEMU binfmt (aarch64)' + name: setupArm64Qemu # continueOnError so a transient apt/binfmt flake fails ONLY the best-effort # aarch64 leg (which is itself continueOnError), never the shared Linux job -- a # blocking setup step here would discard the already-validated linux-64 packages @@ -376,6 +378,7 @@ extends: condaSubdir: 'linux-aarch64' condaTargetSubdir: 'linux-aarch64' continueOnError: true + condition: and(succeeded(), eq(variables['setupArm64Qemu.ready'], 'true')) odbcWheelDir: '$(Pipeline.Workspace)/odbc_wheels' odbcWheelFilter: 'mssql_python_odbc-*manylinux_2_28_aarch64.whl' pythonVersions: '${{ parameters.pythonVersions }}' diff --git a/OneBranchPipelines/steps/conda-build-validate-step-posix.yml b/OneBranchPipelines/steps/conda-build-validate-step-posix.yml index a440cae91..7adfb3db0 100644 --- a/OneBranchPipelines/steps/conda-build-validate-step-posix.yml +++ b/OneBranchPipelines/steps/conda-build-validate-step-posix.yml @@ -67,6 +67,10 @@ parameters: - name: continueOnError type: boolean default: false + # Emulated callers can require successful provisioning without blocking native artifacts. + - name: condition + type: string + default: succeeded() steps: - bash: | @@ -102,4 +106,5 @@ steps: MSSQL_CONDA_PYTHON_VERSIONS: ${{ parameters.pythonVersions }} CONDA_TARGET_SUBDIR: ${{ parameters.condaTargetSubdir }} displayName: 'Conda build + validate (${{ parameters.condaSubdir }})' + condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} diff --git a/conda/README.md b/conda/README.md index 06eb06af8..9a58278be 100644 --- a/conda/README.md +++ b/conda/README.md @@ -85,7 +85,9 @@ On Ubuntu 26.04, `qemu-user-static` is only a virtual package, not an install ca The recipe sets `QEMU_LD_PREFIX` to the cross libc directory when available; failed Linux interpreter execution stops with an emulation diagnostic, not a missing-wheel error. Interpreter-free wheel extraction remains restricted to `osx-*` targets. -ARM64 failures still preserve native Linux artifacts, but a missing ARM64 matrix blocks release. +The ARM64 build runs only after every setup check passes; failed setup skips that leg +without blocking native Linux artifact publication. ARM64 failures still preserve native +Linux artifacts, but a missing ARM64 matrix blocks release. For RS-dependent bindings, the orchestrator also supplies the derived `MSSQL_RS_VERSION` and a validated `rs-wheel-cp.txt` selection in `WHEELS_DIR`. Recipes install or diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py index bf00364ed..c289ed47e 100644 --- a/tests/test_027_conda_release_metadata.py +++ b/tests/test_027_conda_release_metadata.py @@ -111,12 +111,43 @@ def test_linux_qemu_setup_requires_supported_packages_and_enabled_handler(tmp_pa output = result.stdout + result.stderr if state == "ready": assert result.returncode == 0, output + assert ( + output.splitlines().count( + "apt-get install -y qemu-user qemu-user-binfmt libc6-arm64-cross" + ) + == 1 + ), output assert "systemctl restart systemd-binfmt.service" in output assert "QEMU_VERSION_CHECK" in output + assert "##vso[task.setvariable variable=ready;isOutput=true]true" in output else: assert result.returncode != 0, output assert "ERROR:" in output assert ("ARM64 glibc loader" if state == "missing-loader" else "aarch64 binfmt") in output + assert "##vso[task.setvariable variable=ready;isOutput=true]true" not in output + + +def test_linux_qemu_setup_gates_arm64_build_only(): + source = (_ROOT / "OneBranchPipelines/conda-build-pipeline.yml").read_text() + linux = source.split("- stage: CondaLinux\n", 1)[1].split("- stage: ConsolidateConda\n", 1)[0] + steps = re.split(r"^ - ", linux, flags=re.MULTILINE)[1:] + setup = next(step for step in steps if "register QEMU binfmt (aarch64)" in step) + arm64 = next(step for step in steps if "condaSubdir: 'linux-aarch64'" in step) + native = next(step for step in steps if "condaSubdir: 'linux-64'" in step) + publish = next(step for step in steps if "displayName: 'Publish Linux conda artifact'" in step) + assert " name: setupArm64Qemu\n" in setup + assert " continueOnError: true\n" in setup + assert " continueOnError: true\n" in arm64 + assert re.findall(r"^ condition: (.+)$", arm64, re.MULTILINE) == [ + "and(succeeded(), eq(variables['setupArm64Qemu.ready'], 'true'))" + ] + for step in (native, publish): + assert re.findall(r"^\s+condition: (.+)$", step, re.MULTILINE) in ([], ["succeeded()"]) + + template = (_ROOT / "OneBranchPipelines/steps/conda-build-validate-step-posix.yml").read_text() + parameters, build = template.split("\nsteps:\n", 1) + assert re.search(r"- name: condition\n type: string\n default: succeeded\(\)", parameters) + assert re.search(r"^ condition: \$\{\{ parameters\.condition \}\}$", build, re.MULTILINE) @pytest.mark.parametrize( From dfa71eb68b2f1e9611f701ff3f2188d42cfd0e46 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 17 Nov 2026 23:59:58 +0530 Subject: [PATCH 3/7] FEAT: Chain Official Conda builds and non-publishing rehearsals Trigger from successful main wheel builds and exact Conda producer runs. Restrict production publication to explicit manual opt-in, preserving provenance, matrix and approval gates. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- OneBranchPipelines/conda-build-pipeline.yml | 10 +++- OneBranchPipelines/conda-release-pipeline.yml | 24 +++++--- conda/README.md | 19 +++++- tests/test_027_conda_release_metadata.py | 59 ++++++++++++++++++- 4 files changed, 97 insertions(+), 15 deletions(-) diff --git a/OneBranchPipelines/conda-build-pipeline.yml b/OneBranchPipelines/conda-build-pipeline.yml index fc1abaa2f..7f2c182a9 100644 --- a/OneBranchPipelines/conda-build-pipeline.yml +++ b/OneBranchPipelines/conda-build-pipeline.yml @@ -21,6 +21,7 @@ # (drop_ConsolidateConda_ConsolidateArtifacts). Publishing is the companion # conda-release-pipeline.yml. Validated locally that the recipe builds + imports; the # per-OS legs + SDL settings need an actual ADO run to confirm. +# Successful main wheel builds trigger Official Conda builds using that exact producer run. # ========================================================================================= name: $(Year:YY)$(DayOfYear)$(Rev:.r)-CondaBuild @@ -52,7 +53,7 @@ parameters: variables: - name: effectiveOneBranchType - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + ${{ if or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.Reason'], 'ResourceTrigger')) }}: value: 'Official' ${{ else }}: value: '${{ parameters.oneBranchType }}' @@ -68,9 +69,12 @@ resources: # The wheel build pipeline whose consolidated wheel artifacts this pipeline repackages. pipelines: - pipeline: buildPipeline - source: 'Build-Release-Package-Pipeline' + source: '\OneBranch\Build-Release-Package-Pipeline' branch: main - trigger: none + trigger: + branches: + include: + - refs/heads/main extends: template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' diff --git a/OneBranchPipelines/conda-release-pipeline.yml b/OneBranchPipelines/conda-release-pipeline.yml index 340f327c5..bd21b2826 100644 --- a/OneBranchPipelines/conda-release-pipeline.yml +++ b/OneBranchPipelines/conda-release-pipeline.yml @@ -13,6 +13,8 @@ # then deliberately fails at a read-only Anaconda authentication probe with a random # invalid token. Only a confirmed HTTP 401 is the expected rehearsal outcome. # No upload, promotion or cleanup is attempted in rehearsal, even if authentication succeeds. +# Successful main Conda builds trigger this Official rehearsal. Only explicitly opted-in +# manual runs can enable publication; automatic runs never receive publishing credentials. # An administrator must enable the native Exclusive lock check on 'Anaconda Publishing'. # The production stage uses that shared resource throughout publication and recovery. # Keep its credentials, pipeline authorizations and other checks restricted. @@ -48,6 +50,11 @@ parameters: default: '' variables: + - name: effectivePublishToConda + ${{ if and(eq(variables['Build.Reason'], 'Manual'), eq(parameters.publishToConda, true)) }}: + value: 'true' + ${{ else }}: + value: 'false' - template: /OneBranchPipelines/variables/common-variables.yml@self - template: /OneBranchPipelines/variables/onebranch-variables.yml@self @@ -63,9 +70,12 @@ resources: # Registered ADO definition Conda-Build-Pipeline, backed by conda-build-pipeline.yml. Its # ConsolidateConda stage publishes drop_ConsolidateConda_ConsolidateArtifacts. # This is not the wheel pipeline (definition 2199), which no longer builds Conda. - source: 'Conda-Build-Pipeline' + source: '\OneBranch\Conda-Build-Pipeline' branch: main - trigger: none + trigger: + branches: + include: + - refs/heads/main extends: # Rehearse the actual managed container/network path, not a NonOfficial substitute. @@ -90,7 +100,7 @@ extends: publishLogs: enabled: true # Publication-only compliance outputs; both modes retain managed container isolation. - ${{ if eq(parameters.publishToConda, true) }}: + ${{ if eq(variables.effectivePublishToConda, 'true') }}: binskim: enabled: true break: true @@ -101,7 +111,7 @@ extends: stages: - stage: CondaRelease displayName: 'Validate & Publish Conda Release' - ${{ if eq(parameters.publishToConda, true) }}: + ${{ if eq(variables.effectivePublishToConda, 'true') }}: lockBehavior: sequential variables: - group: 'Anaconda Publishing' @@ -135,7 +145,7 @@ extends: CONDA_BUILD_SOURCE_BRANCH: $(resources.pipeline.buildPipeline.sourceBranch) CONDA_BUILD_SOURCE_COMMIT: $(resources.pipeline.buildPipeline.sourceCommit) RELEASE_SOURCE_BRANCH: $(Build.SourceBranch) - PUBLISH_TO_CONDA: '${{ parameters.publishToConda }}' + PUBLISH_TO_CONDA: '${{ variables.effectivePublishToConda }}' MSSQL_PYTHON_VERSION: '${{ parameters.mssqlPythonVersion }}' SYSTEM_ACCESSTOKEN: $(System.AccessToken) SYSTEM_COLLECTIONURI: $(System.CollectionUri) @@ -164,7 +174,7 @@ extends: cancelTimeoutInMinutes: 5 templateContext: type: releaseJob - isProduction: ${{ parameters.publishToConda }} + isProduction: ${{ eq(variables.effectivePublishToConda, 'true') }} pool: type: windows variables: @@ -179,7 +189,7 @@ extends: fetchDepth: 1 - template: /OneBranchPipelines/steps/conda-publish-step.yml@self parameters: - publishToConda: ${{ parameters.publishToConda }} + publishToConda: ${{ eq(variables.effectivePublishToConda, 'true') }} condaChannel: '${{ parameters.condaChannel }}' condaLabel: '${{ parameters.condaLabel }}' mssqlPythonVersion: '$(mssqlPythonVersion)' diff --git a/conda/README.md b/conda/README.md index 9a58278be..380603d6e 100644 --- a/conda/README.md +++ b/conda/README.md @@ -270,6 +270,21 @@ path is passed to the publishing task through a read-only task output. Dependenc use the publisher's target-specific hash lock and explicit package-feed-proxy `--index-url` with pip's `--isolated` mode. Both production and rehearsal use this same code. +On `refs/heads/main`, successful completion of `Build-Release-Package-Pipeline` +automatically starts an Official `Conda-Build-Pipeline` run. Successful completion +of that Conda build starts the Official `Conda-Release-Pipeline` rehearsal. +The resource triggers wait for the entire upstream pipeline, not just an individual +stage, and retain the exact triggering producer run through `buildPipeline.runID`. +Direct CI and PR triggers remain disabled on both Conda pipelines. The trigger YAML +must be present in the ADO repository's configured default branch, with the existing +pipeline-resource authorizations in place, before this chain can run. + +Automatic release runs are always non-publishing, even if `publishToConda=true` is +supplied. Only a manually queued release with that explicit opt-in can enable the +publishing credential group, production job and upload path. Manual runs still +default to non-publishing. Recorded wheel/Conda provenance, the full release matrix, +native audits and external approval/lock requirements remain unchanged. + With `publishToConda=false` (the default), the release pipeline now rehearses the publisher instead of only printing an upload plan. Both modes use the Official template, Windows 2022 managed container, network isolation, independent producer @@ -293,7 +308,7 @@ exact branch/commit, `publishToConda=false` and the intended recorded Conda prod run. Inspect actual hosted logs through the expected rejection; a compiled preview or a local interpreter check does not establish hosted image/venv/dependency coverage. Resource or policy denials must be resolved by the owner, never by weakening the -production checks. `publishToConda=true` retains the main-branch provenance guard, +production checks. A manual `publishToConda=true` run retains the main-branch provenance guard, publishing credential group and shared exclusive lock. The old upload-plan-only task and duplicate interpreter selection in the validation job are removed. Superseded MSI cleanup and temporary installer diagnostics are removed; @@ -316,7 +331,7 @@ must fail before the stage receives the publishing credential. Preserve existing permissions and approvals, and restrict permission to administer or bypass the check. Do not grant broader pipeline access as part of configuring it. -The existing `publishToConda` condition omits this group for rehearsal runs, so +The effective publication condition omits this group for all automatic and rehearsal runs, so trusted feature-branch rehearsal remains separate from credentialed publication. Neither a checkout-local branch check nor a secondary pinned checkout can replace the resource-side control: editable YAML could bypass either. These external checks diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py index c289ed47e..7e050454c 100644 --- a/tests/test_027_conda_release_metadata.py +++ b/tests/test_027_conda_release_metadata.py @@ -150,6 +150,56 @@ def test_linux_qemu_setup_gates_arm64_build_only(): assert re.search(r"^ condition: \$\{\{ parameters\.condition \}\}$", build, re.MULTILINE) +@pytest.mark.parametrize( + ("filename", "producer"), + [ + ("conda-build-pipeline.yml", r"\OneBranch\Build-Release-Package-Pipeline"), + ("conda-release-pipeline.yml", r"\OneBranch\Conda-Build-Pipeline"), + ], +) +def test_conda_pipeline_completion_triggers_only_main(filename, producer): + source = (_ROOT / "OneBranchPipelines" / filename).read_text() + assert "\ntrigger: none\npr: none\n" in source + resource = source.split("\n pipelines:\n", 1)[1].split("\nextends:\n", 1)[0] + assert resource.count("- pipeline: buildPipeline") == 1 + assert f" source: '{producer}'\n" in resource + assert " branch: main\n" in resource + assert resource.split(" trigger:\n", 1)[1].strip() == ( + "branches:\n include:\n - refs/heads/main" + ) + assert "$(resources.pipeline.buildPipeline.runID)" in source + + +def test_automatic_conda_builds_use_official_template(): + source = (_ROOT / "OneBranchPipelines/conda-build-pipeline.yml").read_text() + assert ( + " - name: effectiveOneBranchType\n" + " ${{ if or(eq(variables['Build.Reason'], 'Schedule'), " + "eq(variables['Build.Reason'], 'ResourceTrigger')) }}:\n" + " value: 'Official'\n" + " ${{ else }}:\n" + " value: '${{ parameters.oneBranchType }}'\n" + ) in source + assert "template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml" in source + + +def test_automatic_conda_release_cannot_enable_publication(): + source = (_ROOT / "OneBranchPipelines/conda-release-pipeline.yml").read_text() + assert ( + " - name: effectivePublishToConda\n" + " ${{ if and(eq(variables['Build.Reason'], 'Manual'), " + "eq(parameters.publishToConda, true)) }}:\n" + " value: 'true'\n" + " ${{ else }}:\n" + " value: 'false'\n" + ) in source + assert source.count("parameters.publishToConda") == 1 + assert "PUBLISH_TO_CONDA: '${{ variables.effectivePublishToConda }}'" in source + assert ( + " ${{ if eq(variables.effectivePublishToConda, 'true') }}:\n" " binskim:\n" + ) in source + + @pytest.mark.parametrize( "source_present", [False, True], ids=["wheel-tests-only", "source-checkout"] ) @@ -1925,7 +1975,7 @@ def test_publication_lock_covers_the_complete_release_stage(): stage = pipeline.split(" - stage: CondaRelease\n", 1)[1] configuration, jobs = stage.split(" jobs:\n", 1) assert ( - " ${{ if eq(parameters.publishToConda, true) }}:\n" + " ${{ if eq(variables.effectivePublishToConda, 'true') }}:\n" " lockBehavior: sequential\n" " variables:\n" " - group: 'Anaconda Publishing'\n" @@ -1944,12 +1994,15 @@ def test_dry_release_shares_the_official_publisher_without_credentials(): assert " dependsOn: ValidateConda\n" in guarded_job assert ( " type: releaseJob\n" - " isProduction: ${{ parameters.publishToConda }}\n" + " isProduction: ${{ eq(variables.effectivePublishToConda, 'true') }}\n" ) in guarded_job template = "/OneBranchPipelines/steps/conda-publish-step.yml@self" assert pipeline.count(template) == 1 assert template in guarded_job - assert " publishToConda: ${{ parameters.publishToConda }}" in guarded_job + assert ( + " publishToConda: ${{ eq(variables.effectivePublishToConda, 'true') }}" + in guarded_job + ) assert "path: s" in guarded_job assert ( "WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022" From 364db6c6eac29d56f55a1104f5c7ee9022d485a6 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 18 Nov 2026 13:20:43 +0530 Subject: [PATCH 4/7] FIX: Limit Conda automation to the build pipeline Remove downstream release/rehearsal completion triggering and restore the original manual release configuration. Preserve the main wheel-to-Official-Conda-build trigger and QEMU review fixes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- OneBranchPipelines/conda-release-pipeline.yml | 24 +++-------- conda/README.md | 23 ++++------ tests/test_027_conda_release_metadata.py | 43 ++++++------------- 3 files changed, 29 insertions(+), 61 deletions(-) diff --git a/OneBranchPipelines/conda-release-pipeline.yml b/OneBranchPipelines/conda-release-pipeline.yml index bd21b2826..340f327c5 100644 --- a/OneBranchPipelines/conda-release-pipeline.yml +++ b/OneBranchPipelines/conda-release-pipeline.yml @@ -13,8 +13,6 @@ # then deliberately fails at a read-only Anaconda authentication probe with a random # invalid token. Only a confirmed HTTP 401 is the expected rehearsal outcome. # No upload, promotion or cleanup is attempted in rehearsal, even if authentication succeeds. -# Successful main Conda builds trigger this Official rehearsal. Only explicitly opted-in -# manual runs can enable publication; automatic runs never receive publishing credentials. # An administrator must enable the native Exclusive lock check on 'Anaconda Publishing'. # The production stage uses that shared resource throughout publication and recovery. # Keep its credentials, pipeline authorizations and other checks restricted. @@ -50,11 +48,6 @@ parameters: default: '' variables: - - name: effectivePublishToConda - ${{ if and(eq(variables['Build.Reason'], 'Manual'), eq(parameters.publishToConda, true)) }}: - value: 'true' - ${{ else }}: - value: 'false' - template: /OneBranchPipelines/variables/common-variables.yml@self - template: /OneBranchPipelines/variables/onebranch-variables.yml@self @@ -70,12 +63,9 @@ resources: # Registered ADO definition Conda-Build-Pipeline, backed by conda-build-pipeline.yml. Its # ConsolidateConda stage publishes drop_ConsolidateConda_ConsolidateArtifacts. # This is not the wheel pipeline (definition 2199), which no longer builds Conda. - source: '\OneBranch\Conda-Build-Pipeline' + source: 'Conda-Build-Pipeline' branch: main - trigger: - branches: - include: - - refs/heads/main + trigger: none extends: # Rehearse the actual managed container/network path, not a NonOfficial substitute. @@ -100,7 +90,7 @@ extends: publishLogs: enabled: true # Publication-only compliance outputs; both modes retain managed container isolation. - ${{ if eq(variables.effectivePublishToConda, 'true') }}: + ${{ if eq(parameters.publishToConda, true) }}: binskim: enabled: true break: true @@ -111,7 +101,7 @@ extends: stages: - stage: CondaRelease displayName: 'Validate & Publish Conda Release' - ${{ if eq(variables.effectivePublishToConda, 'true') }}: + ${{ if eq(parameters.publishToConda, true) }}: lockBehavior: sequential variables: - group: 'Anaconda Publishing' @@ -145,7 +135,7 @@ extends: CONDA_BUILD_SOURCE_BRANCH: $(resources.pipeline.buildPipeline.sourceBranch) CONDA_BUILD_SOURCE_COMMIT: $(resources.pipeline.buildPipeline.sourceCommit) RELEASE_SOURCE_BRANCH: $(Build.SourceBranch) - PUBLISH_TO_CONDA: '${{ variables.effectivePublishToConda }}' + PUBLISH_TO_CONDA: '${{ parameters.publishToConda }}' MSSQL_PYTHON_VERSION: '${{ parameters.mssqlPythonVersion }}' SYSTEM_ACCESSTOKEN: $(System.AccessToken) SYSTEM_COLLECTIONURI: $(System.CollectionUri) @@ -174,7 +164,7 @@ extends: cancelTimeoutInMinutes: 5 templateContext: type: releaseJob - isProduction: ${{ eq(variables.effectivePublishToConda, 'true') }} + isProduction: ${{ parameters.publishToConda }} pool: type: windows variables: @@ -189,7 +179,7 @@ extends: fetchDepth: 1 - template: /OneBranchPipelines/steps/conda-publish-step.yml@self parameters: - publishToConda: ${{ eq(variables.effectivePublishToConda, 'true') }} + publishToConda: ${{ parameters.publishToConda }} condaChannel: '${{ parameters.condaChannel }}' condaLabel: '${{ parameters.condaLabel }}' mssqlPythonVersion: '$(mssqlPythonVersion)' diff --git a/conda/README.md b/conda/README.md index 380603d6e..ca6210384 100644 --- a/conda/README.md +++ b/conda/README.md @@ -271,19 +271,14 @@ use the publisher's target-specific hash lock and explicit package-feed-proxy `- with pip's `--isolated` mode. Both production and rehearsal use this same code. On `refs/heads/main`, successful completion of `Build-Release-Package-Pipeline` -automatically starts an Official `Conda-Build-Pipeline` run. Successful completion -of that Conda build starts the Official `Conda-Release-Pipeline` rehearsal. -The resource triggers wait for the entire upstream pipeline, not just an individual -stage, and retain the exact triggering producer run through `buildPipeline.runID`. -Direct CI and PR triggers remain disabled on both Conda pipelines. The trigger YAML +automatically starts an Official `Conda-Build-Pipeline` run. This resource trigger +waits for the entire upstream pipeline, not just an individual stage, and retains +the exact triggering wheel producer run through `buildPipeline.runID`. +Automation stops after the Conda build: `Conda-Release-Pipeline` remains manually +queued, with no build-completion trigger for release, rehearsal or publication. +Direct CI and PR triggers remain disabled on both Conda pipelines. The build-trigger YAML must be present in the ADO repository's configured default branch, with the existing -pipeline-resource authorizations in place, before this chain can run. - -Automatic release runs are always non-publishing, even if `publishToConda=true` is -supplied. Only a manually queued release with that explicit opt-in can enable the -publishing credential group, production job and upload path. Manual runs still -default to non-publishing. Recorded wheel/Conda provenance, the full release matrix, -native audits and external approval/lock requirements remain unchanged. +pipeline-resource authorizations in place, before automatic Conda builds can run. With `publishToConda=false` (the default), the release pipeline now rehearses the publisher instead of only printing an upload plan. Both modes use the Official @@ -308,7 +303,7 @@ exact branch/commit, `publishToConda=false` and the intended recorded Conda prod run. Inspect actual hosted logs through the expected rejection; a compiled preview or a local interpreter check does not establish hosted image/venv/dependency coverage. Resource or policy denials must be resolved by the owner, never by weakening the -production checks. A manual `publishToConda=true` run retains the main-branch provenance guard, +production checks. `publishToConda=true` retains the main-branch provenance guard, publishing credential group and shared exclusive lock. The old upload-plan-only task and duplicate interpreter selection in the validation job are removed. Superseded MSI cleanup and temporary installer diagnostics are removed; @@ -331,7 +326,7 @@ must fail before the stage receives the publishing credential. Preserve existing permissions and approvals, and restrict permission to administer or bypass the check. Do not grant broader pipeline access as part of configuring it. -The effective publication condition omits this group for all automatic and rehearsal runs, so +The existing `publishToConda` condition omits this group for rehearsal runs, so trusted feature-branch rehearsal remains separate from credentialed publication. Neither a checkout-local branch check nor a secondary pinned checkout can replace the resource-side control: editable YAML could bypass either. These external checks diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py index 7e050454c..3d834c562 100644 --- a/tests/test_027_conda_release_metadata.py +++ b/tests/test_027_conda_release_metadata.py @@ -150,19 +150,12 @@ def test_linux_qemu_setup_gates_arm64_build_only(): assert re.search(r"^ condition: \$\{\{ parameters\.condition \}\}$", build, re.MULTILINE) -@pytest.mark.parametrize( - ("filename", "producer"), - [ - ("conda-build-pipeline.yml", r"\OneBranch\Build-Release-Package-Pipeline"), - ("conda-release-pipeline.yml", r"\OneBranch\Conda-Build-Pipeline"), - ], -) -def test_conda_pipeline_completion_triggers_only_main(filename, producer): - source = (_ROOT / "OneBranchPipelines" / filename).read_text() +def test_conda_build_completion_trigger_only_main(): + source = (_ROOT / "OneBranchPipelines/conda-build-pipeline.yml").read_text() assert "\ntrigger: none\npr: none\n" in source resource = source.split("\n pipelines:\n", 1)[1].split("\nextends:\n", 1)[0] assert resource.count("- pipeline: buildPipeline") == 1 - assert f" source: '{producer}'\n" in resource + assert " source: '\\OneBranch\\Build-Release-Package-Pipeline'\n" in resource assert " branch: main\n" in resource assert resource.split(" trigger:\n", 1)[1].strip() == ( "branches:\n include:\n - refs/heads/main" @@ -183,21 +176,14 @@ def test_automatic_conda_builds_use_official_template(): assert "template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml" in source -def test_automatic_conda_release_cannot_enable_publication(): +def test_conda_release_remains_manual(): source = (_ROOT / "OneBranchPipelines/conda-release-pipeline.yml").read_text() - assert ( - " - name: effectivePublishToConda\n" - " ${{ if and(eq(variables['Build.Reason'], 'Manual'), " - "eq(parameters.publishToConda, true)) }}:\n" - " value: 'true'\n" - " ${{ else }}:\n" - " value: 'false'\n" - ) in source - assert source.count("parameters.publishToConda") == 1 - assert "PUBLISH_TO_CONDA: '${{ variables.effectivePublishToConda }}'" in source - assert ( - " ${{ if eq(variables.effectivePublishToConda, 'true') }}:\n" " binskim:\n" - ) in source + assert "\ntrigger: none\npr: none\n" in source + resource = source.split("\n pipelines:\n", 1)[1].split("\nextends:\n", 1)[0] + assert re.findall(r"^ trigger: (.+)$", resource, re.MULTILINE) == ["none"] + assert " trigger:\n" not in resource + assert "PUBLISH_TO_CONDA: '${{ parameters.publishToConda }}'" in source + assert (" ${{ if eq(parameters.publishToConda, true) }}:\n" " binskim:\n") in source @pytest.mark.parametrize( @@ -1975,7 +1961,7 @@ def test_publication_lock_covers_the_complete_release_stage(): stage = pipeline.split(" - stage: CondaRelease\n", 1)[1] configuration, jobs = stage.split(" jobs:\n", 1) assert ( - " ${{ if eq(variables.effectivePublishToConda, 'true') }}:\n" + " ${{ if eq(parameters.publishToConda, true) }}:\n" " lockBehavior: sequential\n" " variables:\n" " - group: 'Anaconda Publishing'\n" @@ -1994,15 +1980,12 @@ def test_dry_release_shares_the_official_publisher_without_credentials(): assert " dependsOn: ValidateConda\n" in guarded_job assert ( " type: releaseJob\n" - " isProduction: ${{ eq(variables.effectivePublishToConda, 'true') }}\n" + " isProduction: ${{ parameters.publishToConda }}\n" ) in guarded_job template = "/OneBranchPipelines/steps/conda-publish-step.yml@self" assert pipeline.count(template) == 1 assert template in guarded_job - assert ( - " publishToConda: ${{ eq(variables.effectivePublishToConda, 'true') }}" - in guarded_job - ) + assert " publishToConda: ${{ parameters.publishToConda }}" in guarded_job assert "path: s" in guarded_job assert ( "WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022" From fca66cd5c168a5ad210577f434708815902e1dd8 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 18 Nov 2026 13:49:29 +0530 Subject: [PATCH 5/7] FIX: Verify Conda wheel security and QEMU symlinks Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- OneBranchPipelines/conda-build-pipeline.yml | 10 +- conda/README.md | 10 ++ eng/conda_tools/provenance.py | 99 +++++++++++- tests/test_027_conda_release_metadata.py | 37 ++++- tests/test_036_conda_provenance.py | 163 +++++++++++++++++++- 5 files changed, 302 insertions(+), 17 deletions(-) diff --git a/OneBranchPipelines/conda-build-pipeline.yml b/OneBranchPipelines/conda-build-pipeline.yml index 7f2c182a9..50fa08ee2 100644 --- a/OneBranchPipelines/conda-build-pipeline.yml +++ b/OneBranchPipelines/conda-build-pipeline.yml @@ -366,9 +366,15 @@ extends: sudo systemctl restart systemd-binfmt.service qemu-aarch64 --version test -x /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 || { echo "ERROR: ARM64 glibc loader is missing after apt install" >&2; exit 1; } - handler=/proc/sys/fs/binfmt_misc/qemu-aarch64 + handler="/proc/sys/fs/binfmt_misc/qemu-aarch64" grep -qx enabled "$handler" || { echo "ERROR: aarch64 binfmt handler is missing or disabled after systemd-binfmt restart" >&2; exit 1; } - grep -qx 'interpreter /usr/bin/qemu-aarch64' "$handler" || { echo "ERROR: aarch64 binfmt handler does not use the installed qemu-user emulator" >&2; exit 1; } + interpreter="$(sed -n 's/^interpreter //p' "$handler")" + # Ubuntu 24.04 can register a binfmt-P symlink to the same emulator. + if [[ -z "$interpreter" ]] || ! installed="$(readlink -e "/usr/bin/qemu-aarch64")" \ + || ! registered="$(readlink -e "$interpreter")" || [[ "$registered" != "$installed" ]]; then + echo "ERROR: aarch64 binfmt handler does not resolve to the installed qemu-user emulator" >&2 + exit 1 + fi echo "##vso[task.setvariable variable=ready;isOutput=true]true" displayName: 'Install libc6-arm64-cross + register QEMU binfmt (aarch64)' name: setupArm64Qemu diff --git a/conda/README.md b/conda/README.md index ca6210384..894973867 100644 --- a/conda/README.md +++ b/conda/README.md @@ -81,6 +81,8 @@ a compatible extension filename. The separate native audits still validate binar Linux ARM64 builds on the x64 Ubuntu agent require working QEMU emulation. The pipeline installs `qemu-user`, `qemu-user-binfmt` and `libc6-arm64-cross`, reloads the systemd binfmt registrations, and checks the enabled aarch64 handler, its emulator and the ARM64 loader. +The handler's interpreter must resolve to the installed emulator; valid Ubuntu 24.04 +`binfmt-P` symlinks are accepted, but missing, dangling or different interpreters are not. On Ubuntu 26.04, `qemu-user-static` is only a virtual package, not an install candidate. The recipe sets `QEMU_LD_PREFIX` to the cross libc directory when available; failed Linux interpreter execution stops with an emulation diagnostic, not a missing-wheel @@ -274,6 +276,14 @@ On `refs/heads/main`, successful completion of `Build-Release-Package-Pipeline` automatically starts an Official `Conda-Build-Pipeline` run. This resource trigger waits for the entire upstream pipeline, not just an individual stage, and retains the exact triggering wheel producer run through `buildPipeline.runID`. +An Official Conda build does not by itself make its input wheels production-eligible. +The release provenance gate checks the recorded wheel run's effective Official mode +(including scheduled Official runs), enabled SDL and Windows signing settings, and +completed/succeeded timeline evidence for every recorded platform build job: wheel +ESRP malware scans and Component Governance, plus Windows native malware scanning, +signing, Authenticode verification and signed-wheel verification. The ODBC wheel scan +is also required. Missing, skipped, failed or mismatched evidence blocks production; +non-publishing validation reports the ineligibility without granting publication. Automation stops after the Conda build: `Conda-Release-Pipeline` remains manually queued, with no build-completion trigger for release, rehearsal or publication. Direct CI and PR triggers remain disabled on both Conda pipelines. The build-trigger YAML diff --git a/eng/conda_tools/provenance.py b/eng/conda_tools/provenance.py index af6d93035..2d0a2096e 100644 --- a/eng/conda_tools/provenance.py +++ b/eng/conda_tools/provenance.py @@ -92,6 +92,96 @@ def _verify_run(build: dict, run: dict, pipeline_id: int, run_id: int) -> dict: return {"pipeline": pipeline_id, "run": run_id, "branch": branch, "commit": commit} +def _wheel_security_reasons(get_json: Callable[[str], dict], build: dict, run: dict) -> list[str]: + parameters = run.get("templateParameters") + if not isinstance(parameters, dict): + return ["wheel producer security parameters are missing"] + reasons = [] + # Scheduled producers select Official even when the parameter defaults to NonOfficial. + if build.get("reason") != "schedule" and parameters.get("oneBranchType") != "Official": + reasons.append("wheel producer did not use Official mode") + for name in ("runSdlTasks", "signWindowsBinaries"): + if parameters.get(name) is not True: + reasons.append(f"wheel producer {name} was not enabled") + if reasons: + return reasons + + expected_jobs = {"ODBC_BuildAll.BuildWheel.__default"} + for parameter, prefix, fields, job in ( + ("windowsConfigs", "Win_py", ("pyVer", "arch"), "BuildWheel"), + ("macosConfigs", "MacOS_py", ("pyVer",), "BuildWheel"), + ("linuxConfigs", "Linux_", ("tag", "arch"), "BuildWheels"), + ): + configs = parameters.get(parameter) + if not isinstance(configs, list) or not configs: + return [f"wheel producer {parameter} evidence is missing"] + for config in configs: + if not isinstance(config, dict) or any( + not isinstance(config.get(field), str) + or not re.fullmatch(r"[a-zA-Z0-9_]+", config[field]) + for field in fields + ): + return [f"wheel producer {parameter} evidence is invalid"] + identifier = prefix + "_".join(config[field] for field in fields) + f".{job}.__default" + if identifier in expected_jobs: + return [f"wheel producer {parameter} contains duplicate jobs"] + expected_jobs.add(identifier) + + timeline = get_json(f"build/builds/{build['id']}/timeline?api-version=7.1").get("records") + if not isinstance(timeline, list) or not all(isinstance(record, dict) for record in timeline): + return ["wheel producer security timeline is missing or invalid"] + for identifier in sorted(expected_jobs): + jobs = [ + record + for record in timeline + if record.get("type") == "Job" and record.get("identifier") == identifier + ] + if ( + len(jobs) != 1 + or not isinstance(jobs[0].get("id"), str) + or not jobs[0]["id"] + or jobs[0].get("state") != "completed" + or jobs[0].get("result") != "succeeded" + or type(jobs[0].get("attempt")) is not int + or jobs[0]["attempt"] < 1 + ): + reasons.append(f"wheel producer job {identifier} is missing or did not succeed") + continue + job_record = jobs[0] + wheel_kind = "ODBC" if identifier.startswith("ODBC_") else "Python" + required_tasks = [ + (f"ESRP MalwareScanning - {wheel_kind} Wheels (Official)", "EsrpMalwareScanning"), + ("Component Governance Detection", "ComponentGovernanceComponentDetection"), + ] + if identifier.startswith("Win_"): + required_tasks.extend( + [ + ("ESRP MalwareScanning - Native extension", "EsrpMalwareScanning"), + ("ESRP CodeSigning - Native extension (.pyd)", "EsrpCodeSigning"), + ("Verify Authenticode signature (native extension)", "PowerShell"), + ("Verify signed .pyd inside built wheel", "PowerShell"), + ] + ) + for name, task_type in required_tasks: + tasks = [ + record + for record in timeline + if record.get("type") == "Task" + and record.get("parentId") == job_record["id"] + and record.get("name") == name + and record.get("attempt") == job_record["attempt"] + ] + if ( + len(tasks) != 1 + or tasks[0].get("state") != "completed" + or tasks[0].get("result") != "succeeded" + or not isinstance(tasks[0].get("task"), dict) + or tasks[0]["task"].get("name") != task_type + ): + reasons.append(f"wheel producer {identifier} lacks completed/succeeded {name}") + return reasons + + def verify_provenance( get_json: Callable[[str], dict], *, @@ -122,12 +212,8 @@ def verify_provenance( "buildNumber" ): raise ValueError("Recorded upstream wheel version differs from its Build API build number.") - wheel = _verify_run( - wheel_build, - get_json(f"pipelines/{_WHEEL_PIPELINE_ID}/runs/{wheel_run_id}?api-version=7.1"), - _WHEEL_PIPELINE_ID, - wheel_run_id, - ) + wheel_run = get_json(f"pipelines/{_WHEEL_PIPELINE_ID}/runs/{wheel_run_id}?api-version=7.1") + wheel = _verify_run(wheel_build, wheel_run, _WHEEL_PIPELINE_ID, wheel_run_id) reasons = [ f"{name} source is {branch}, not {_MAIN}" for name, branch in ( @@ -137,6 +223,7 @@ def verify_provenance( ) if branch != _MAIN ] + reasons.extend(_wheel_security_reasons(get_json, wheel_build, wheel_run)) if publish and reasons: raise ValueError("Production publication is ineligible: " + "; ".join(reasons)) return { diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py index 3d834c562..515111246 100644 --- a/tests/test_027_conda_release_metadata.py +++ b/tests/test_027_conda_release_metadata.py @@ -55,7 +55,16 @@ def _promoter_cli(argv): @pytest.mark.parametrize( - "state", ["ready", "missing-handler", "disabled", "wrong-interpreter", "missing-loader"] + "state", + [ + "ready", + "symlinked-interpreter", + "broken-symlink", + "missing-handler", + "disabled", + "wrong-interpreter", + "missing-loader", + ], ) def test_linux_qemu_setup_requires_supported_packages_and_enabled_handler(tmp_path, state): import os @@ -72,14 +81,30 @@ def test_linux_qemu_setup_requires_supported_packages_and_enabled_handler(tmp_pa 1, )[0].rsplit(" - bash: |\n", 1)[1] handler = tmp_path / "handler" + emulator = tmp_path / "qemu-aarch64" + emulator.touch() + interpreter = emulator + if state in {"symlinked-interpreter", "broken-symlink"}: + interpreter = tmp_path / "aarch64-binfmt-P" + try: + interpreter.symlink_to( + emulator if state == "symlinked-interpreter" else tmp_path / "missing-emulator" + ) + except OSError as error: + pytest.skip(f"Symlink regression requires symlink support: {error}") + elif state == "wrong-interpreter": + interpreter = tmp_path / "other-emulator" + interpreter.touch() if state != "missing-handler": handler.write_text( ("disabled" if state == "disabled" else "enabled") + "\ninterpreter " - + ("/missing/qemu-aarch64" if state == "wrong-interpreter" else "/usr/bin/qemu-aarch64") + + interpreter.as_posix() + "\n", newline="\n", ) + setup = setup.replace("/proc/sys/fs/binfmt_misc/qemu-aarch64", handler.as_posix()) + setup = setup.replace("/usr/bin/qemu-aarch64", emulator.as_posix()) mocks = r""" sudo() { case "$*" in @@ -96,20 +121,16 @@ def test_linux_qemu_setup_requires_supported_packages_and_enabled_handler(tmp_pa builtin test "$@" fi } -grep() { - if [[ "${!#}" != /proc/sys/fs/binfmt_misc/qemu-aarch64 ]]; then return 1; fi - command grep "${@:1:$#-1}" "$TEST_HANDLER" -} """ result = subprocess.run( [bash, "-c", mocks + textwrap.dedent(setup)], - env=dict(os.environ, TEST_STATE=state, TEST_HANDLER=handler.as_posix()), + env=dict(os.environ, TEST_STATE=state), capture_output=True, text=True, timeout=15, ) output = result.stdout + result.stderr - if state == "ready": + if state in {"ready", "symlinked-interpreter"}: assert result.returncode == 0, output assert ( output.splitlines().count( diff --git a/tests/test_036_conda_provenance.py b/tests/test_036_conda_provenance.py index 5bd2a2ca7..3aa655bff 100644 --- a/tests/test_036_conda_provenance.py +++ b/tests/test_036_conda_provenance.py @@ -701,11 +701,66 @@ def chain(release_sources): producer_run["resources"]["pipelines"] = { "buildPipeline": {"pipeline": {"id": 173176}, "version": "26250.2"} } + wheel["reason"] = "manual" + wheel_run["templateParameters"] = { + "oneBranchType": "Official", + "runSdlTasks": True, + "signWindowsBinaries": True, + "windowsConfigs": [{"pyVer": "312", "arch": "x64"}, {"pyVer": "312", "arch": "arm64"}], + "macosConfigs": [{"pyVer": "312"}], + "linuxConfigs": [{"tag": "manylinux_2_28", "arch": "aarch64"}], + } + timeline = [] + for identifier in ( + "Win_py312_x64.BuildWheel.__default", + "Win_py312_arm64.BuildWheel.__default", + "MacOS_py312.BuildWheel.__default", + "Linux_manylinux_2_28_aarch64.BuildWheels.__default", + "ODBC_BuildAll.BuildWheel.__default", + ): + timeline.append( + { + "id": identifier, + "identifier": identifier, + "type": "Job", + "state": "completed", + "result": "succeeded", + "attempt": 1, + } + ) + label = "ODBC" if identifier.startswith("ODBC_") else "Python" + tasks = [ + (f"ESRP MalwareScanning - {label} Wheels (Official)", "EsrpMalwareScanning"), + ("Component Governance Detection", "ComponentGovernanceComponentDetection"), + ] + if identifier.startswith("Win_"): + tasks.extend( + [ + ("ESRP MalwareScanning - Native extension", "EsrpMalwareScanning"), + ("ESRP CodeSigning - Native extension (.pyd)", "EsrpCodeSigning"), + ("Verify Authenticode signature (native extension)", "PowerShell"), + ("Verify signed .pyd inside built wheel", "PowerShell"), + ] + ) + timeline.extend( + { + "id": f"{identifier}-{index}", + "parentId": identifier, + "type": "Task", + "name": name, + "task": {"name": task}, + "state": "completed", + "result": "succeeded", + "attempt": 1, + } + for index, (name, task) in enumerate(tasks) + ) records = { "build/builds/174195?api-version=7.1": producer, "pipelines/2318/runs/174195?api-version=7.1": producer_run, "build/builds/173176?api-version=7.1": wheel, "pipelines/2199/runs/173176?api-version=7.1": wheel_run, + "build/builds/173176/timeline?api-version=7.1": {"records": timeline}, } records.update(_source_record(path, content) for path, content in release_sources.items()) return records @@ -737,6 +792,86 @@ def test_exact_recorded_wheel_run_is_verified_not_packaging_mode_or_build_number assert result["versions"] == {"mssql-python": "1.15.0", "mssql-python-odbc": "18.6.2.1"} +@pytest.mark.parametrize( + "problem", + [ + "nonofficial", + "missing-parameters", + "sdl-disabled", + "signing-disabled", + "empty-matrix", + "missing-timeline", + "missing-job", + "missing-wheel-scan", + "missing-odbc-scan", + "missing-signing", + "missing-signature-verification", + "failed-task", + "skipped-task", + "wrong-task-kind", + "wrong-job", + "previous-attempt", + ], +) +def test_wheel_security_evidence_required_for_production(chain, problem): + run = chain["pipelines/2199/runs/173176?api-version=7.1"] + timeline = chain["build/builds/173176/timeline?api-version=7.1"]["records"] + params = run["templateParameters"] + scan = next( + record + for record in timeline + if record.get("name") == "ESRP MalwareScanning - Python Wheels (Official)" + and record["parentId"].startswith("Win_py312_arm64.") + ) + if problem == "nonofficial": + params["oneBranchType"] = "NonOfficial" + elif problem == "missing-parameters": + del run["templateParameters"] + elif problem == "sdl-disabled": + params["runSdlTasks"] = False + elif problem == "signing-disabled": + params["signWindowsBinaries"] = False + elif problem == "empty-matrix": + params["windowsConfigs"] = [] + elif problem == "missing-timeline": + timeline.clear() + elif problem == "missing-job": + timeline[:] = [record for record in timeline if record.get("id") != scan["parentId"]] + elif problem == "missing-wheel-scan": + timeline.remove(scan) + elif problem in {"missing-odbc-scan", "missing-signing", "missing-signature-verification"}: + label = { + "missing-odbc-scan": "ESRP MalwareScanning - ODBC Wheels (Official)", + "missing-signing": "ESRP CodeSigning - Native extension (.pyd)", + "missing-signature-verification": "Verify signed .pyd inside built wheel", + }[problem] + timeline.remove(next(record for record in timeline if record.get("name") == label)) + elif problem in {"failed-task", "skipped-task"}: + scan["result"] = problem.removesuffix("-task") + elif problem == "wrong-task-kind": + scan["task"]["name"] = "CmdLine" + elif problem == "wrong-job": + scan["parentId"] = "unrelated-job" + elif problem == "previous-attempt": + next(record for record in timeline if record["id"] == scan["parentId"])["attempt"] = 2 + with pytest.raises(ValueError, match="Production publication is ineligible"): + _verify(chain) + result = _verify(chain, publish=False) + assert result["productionEligible"] is False + assert result["productionIneligibilityReasons"] + + +def test_scheduled_official_wheel_run_still_requires_completed_security_checks(chain): + chain["build/builds/173176?api-version=7.1"]["reason"] = "schedule" + chain["pipelines/2199/runs/173176?api-version=7.1"]["templateParameters"][ + "oneBranchType" + ] = "NonOfficial" + assert _verify(chain)["productionEligible"] is True + chain["build/builds/173176/timeline?api-version=7.1"]["records"].clear() + with pytest.raises(ValueError, match="Production publication is ineligible"): + _verify(chain) + + @pytest.mark.parametrize( "content", [ @@ -1002,6 +1137,31 @@ def test_provenance_cli_executes_same_chain_in_both_modes(monkeypatch, chain, ca ) +@pytest.mark.parametrize("publish", ["true", "false"]) +def test_provenance_cli_cannot_promote_unsigned_wheels( + monkeypatch, provenance_cli_env, chain, capsys, publish +): + monkeypatch.setenv("PUBLISH_TO_CONDA", publish) + monkeypatch.setattr(provenance, "ado_get_json", chain.__getitem__) + chain["pipelines/2199/runs/173176?api-version=7.1"]["templateParameters"][ + "signWindowsBinaries" + ] = False + status = provenance.cli() + output = capsys.readouterr() + if publish == "true": + assert status == 1 + assert "Production publication is ineligible" in output.err + assert "##vso[task.setvariable" not in output.out + else: + assert status == 0 + result = json.loads(output.out.splitlines()[0].split(": ", 1)[1]) + assert result["productionEligible"] is False + assert ( + "wheel producer signWindowsBinaries was not enabled" + in result["productionIneligibilityReasons"] + ) + + def test_invalid_publish_flag_is_not_treated_as_dry_run(monkeypatch): monkeypatch.setenv("PUBLISH_TO_CONDA", "perhaps") with pytest.raises(ValueError, match="true or false"): @@ -1251,7 +1411,8 @@ def get_json(path): with pytest.raises((ValueError, KeyError, HTTPError)): _verify(chain, get_json=get_json) assert reads[-1] == key - assert len(reads) == 8 + assert reads[4] == "build/builds/173176/timeline?api-version=7.1" + assert len(reads) == 9 @pytest.mark.parametrize( From 33362e517908f70286d6b7df9d233fd2025cd155 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 18 Nov 2026 15:15:53 +0530 Subject: [PATCH 6/7] FIX: Skip QEMU tests without readlink -e support Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_027_conda_release_metadata.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py index 515111246..28f9fdf0c 100644 --- a/tests/test_027_conda_release_metadata.py +++ b/tests/test_027_conda_release_metadata.py @@ -75,6 +75,12 @@ def test_linux_qemu_setup_requires_supported_packages_and_enabled_handler(tmp_pa bash = shutil.which("bash") if not bash: pytest.skip("QEMU setup regression requires bash") + if subprocess.run( + [bash, "-c", 'readlink -e "$1" >/dev/null 2>&1', "bash", tmp_path.as_posix()], + capture_output=True, + timeout=15, + ).returncode: + pytest.skip("QEMU setup regression requires readlink -e") source = (_ROOT / "OneBranchPipelines/conda-build-pipeline.yml").read_text() setup = source.split( " displayName: 'Install libc6-arm64-cross + register QEMU binfmt (aarch64)'", From 2098c959811757050255b3da6c883944898cc61f Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 18 Nov 2026 21:58:15 +0530 Subject: [PATCH 7/7] FIX: Run QEMU provisioning tests only on Linux Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_027_conda_release_metadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py index 28f9fdf0c..b90d6067c 100644 --- a/tests/test_027_conda_release_metadata.py +++ b/tests/test_027_conda_release_metadata.py @@ -54,6 +54,7 @@ def _promoter_cli(argv): _MP_VER = "1.13.0" +@pytest.mark.skipif(sys.platform != "linux", reason="QEMU provisioning runs on Linux") @pytest.mark.parametrize( "state", [