From 7512bad908e24a248a36cd286fa7c9ab1663da3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 09:13:33 -0700 Subject: [PATCH 01/24] Bump the actions group with 2 updates (#3070) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dist.yml | 2 +- .github/workflows/test-python.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 819261bd9d..c0856cf2d8 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -61,7 +61,7 @@ jobs: - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: # setup-qemu-action by default uses `tonistiigi/binfmt:latest` image, # which is out of date. This causes seg faults during build. diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 5e0d89f7a4..cd63f683a5 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -116,7 +116,7 @@ jobs: - name: Generate xml report run: uv tool run --with "coverage[toml]" coverage xml - name: Upload test results to Codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} doctest: From dda56d3c6815eddeff438b41da23c4721cc108aa Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 25 Sep 2026 12:24:50 -0500 Subject: [PATCH 02/24] PYTHON-6084 Streamline Evergreen Python interpreter selection for uv (#3074) --- .evergreen/generated_configs/functions.yml | 4 +- .evergreen/generated_configs/tasks.yml | 702 ++++++++++---------- .evergreen/run-tests.sh | 4 +- .evergreen/scripts/generate_config.py | 28 +- .evergreen/scripts/generate_config_utils.py | 2 +- .evergreen/scripts/install-dependencies.sh | 7 + .evergreen/scripts/setup-uv-python.sh | 123 ++-- .evergreen/scripts/setup_tests.py | 9 +- .evergreen/scripts/utils.py | 4 +- CONTRIBUTING.md | 15 +- 10 files changed, 473 insertions(+), 425 deletions(-) diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index 9fbc1f8500..57f1182377 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -102,7 +102,6 @@ functions: - SSL - ORCHESTRATION_FILE - UV_PYTHON - - TOOLCHAIN_VERSION - STORAGE_ENGINE - REQUIRE_API_VERSION - DRIVERS_TOOLS @@ -137,7 +136,6 @@ functions: - UV_PYTHON - LIBMONGOCRYPT_URL - MONGODB_URI - - TOOLCHAIN_VERSION - DISABLE_TEST_COMMANDS - GREEN_FRAMEWORK - NO_EXT @@ -249,7 +247,7 @@ functions: - test-numpy working_dir: src include_expansions_in_env: - - TOOLCHAIN_VERSION + - UV_PYTHON - COVERAGE type: test diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index e2677e7399..d4ff6ad5d5 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -20,7 +20,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: regular - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_MIN_DEPS: "1" tags: [auth-aws, auth-aws-regular] - name: test-auth-aws-5.0-assume-role-python3.11 @@ -34,7 +34,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: assume-role - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: [auth-aws, auth-aws-assume-role] - name: test-auth-aws-6.0-ec2-python3.12 commands: @@ -47,7 +47,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: ec2 - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: [auth-aws, auth-aws-ec2] - name: test-auth-aws-7.0-env-creds-python3.13 commands: @@ -60,7 +60,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: env-creds - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: [auth-aws, auth-aws-env-creds] - name: test-auth-aws-8.0-session-creds-python3.14t commands: @@ -73,7 +73,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: session-creds - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: [auth-aws, auth-aws-session-creds, free-threaded] - name: test-auth-aws-9.0-web-identity-python3.14 commands: @@ -86,7 +86,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: web-identity - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: [auth-aws, auth-aws-web-identity] - name: test-auth-aws-9.0-web-identity-session-name-python3.14 commands: @@ -100,7 +100,7 @@ tasks: TEST_NAME: auth_aws SUB_TEST_NAME: web-identity AWS_ROLE_SESSION_NAME: test - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: [auth-aws, auth-aws-web-identity] - name: test-auth-aws-rapid-regular-python3.15t commands: @@ -113,7 +113,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: regular - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: [auth-aws, auth-aws-regular, free-threaded] - name: test-auth-aws-latest-assume-role-python3.15-cov commands: @@ -126,7 +126,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: assume-role - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" COVERAGE: "1" tags: [auth-aws, auth-aws-assume-role, pr] - name: test-auth-aws-ecs @@ -322,7 +322,7 @@ tasks: - func: assume ec2 role - func: run tests vars: - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_MIN_DEPS: "1" tags: [test-no-orchestration, python-3.10] - name: test-no-orchestration-python3.15 @@ -330,14 +330,14 @@ tasks: - func: assume ec2 role - func: run tests vars: - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: [test-no-orchestration, python-3.15] - name: test-no-orchestration-pypy3.11 commands: - func: assume ec2 role - func: run tests vars: - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: [test-no-orchestration, python-pypy3.11] # No toolchain tests @@ -392,7 +392,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -403,7 +403,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -414,7 +414,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -425,7 +425,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -436,7 +436,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -447,7 +447,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -458,7 +458,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -469,7 +469,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -479,7 +479,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -490,7 +490,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -501,7 +501,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -512,7 +512,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -523,7 +523,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -534,7 +534,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -545,7 +545,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -556,7 +556,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-delegate-valid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -566,7 +566,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -577,7 +577,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -588,7 +588,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -599,7 +599,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -610,7 +610,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -621,7 +621,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -632,7 +632,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -643,7 +643,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-delegate-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -653,7 +653,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -664,7 +664,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -675,7 +675,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -686,7 +686,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -697,7 +697,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -708,7 +708,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -719,7 +719,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -730,7 +730,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-soft-fail-v4.4-python3.10-min-deps @@ -740,7 +740,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -751,7 +751,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -762,7 +762,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -773,7 +773,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -784,7 +784,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -795,7 +795,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -806,7 +806,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -817,7 +817,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-valid-cert-server-staples-v4.4-python3.10-min-deps @@ -827,7 +827,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -843,7 +843,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -858,7 +858,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -873,7 +873,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -888,7 +888,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -903,7 +903,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -918,7 +918,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -933,7 +933,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest COVERAGE: "1" tags: @@ -949,7 +949,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -964,7 +964,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -979,7 +979,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -994,7 +994,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -1009,7 +1009,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -1024,7 +1024,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -1039,7 +1039,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -1054,7 +1054,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -1068,7 +1068,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -1083,7 +1083,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -1098,7 +1098,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -1113,7 +1113,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -1128,7 +1128,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -1143,7 +1143,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -1158,7 +1158,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -1173,7 +1173,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -1187,7 +1187,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -1202,7 +1202,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -1217,7 +1217,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -1232,7 +1232,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -1247,7 +1247,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -1262,7 +1262,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -1277,7 +1277,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -1292,7 +1292,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -1306,7 +1306,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -1317,7 +1317,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -1328,7 +1328,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -1339,7 +1339,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -1350,7 +1350,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -1361,7 +1361,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -1372,7 +1372,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -1383,7 +1383,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -1393,7 +1393,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -1404,7 +1404,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -1415,7 +1415,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -1426,7 +1426,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -1437,7 +1437,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -1448,7 +1448,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -1459,7 +1459,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -1470,7 +1470,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-malicious-no-responder-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -1480,7 +1480,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -1491,7 +1491,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -1502,7 +1502,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -1513,7 +1513,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -1524,7 +1524,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -1535,7 +1535,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -1546,7 +1546,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -1557,7 +1557,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-rsa-valid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1567,7 +1567,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1578,7 +1578,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1589,7 +1589,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1600,7 +1600,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1611,7 +1611,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1622,7 +1622,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1633,7 +1633,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1644,7 +1644,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1654,7 +1654,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1665,7 +1665,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1676,7 +1676,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1687,7 +1687,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1698,7 +1698,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1709,7 +1709,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1720,7 +1720,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1731,7 +1731,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-delegate-valid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1741,7 +1741,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1752,7 +1752,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1763,7 +1763,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1774,7 +1774,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1785,7 +1785,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1796,7 +1796,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1807,7 +1807,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1818,7 +1818,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-delegate-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1828,7 +1828,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1839,7 +1839,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1850,7 +1850,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1861,7 +1861,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1872,7 +1872,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1883,7 +1883,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1894,7 +1894,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1905,7 +1905,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-soft-fail-v4.4-python3.10-min-deps @@ -1915,7 +1915,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1926,7 +1926,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1937,7 +1937,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1948,7 +1948,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1959,7 +1959,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1970,7 +1970,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1981,7 +1981,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1992,7 +1992,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-valid-cert-server-staples-v4.4-python3.10-min-deps @@ -2002,7 +2002,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2017,7 +2017,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2032,7 +2032,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2047,7 +2047,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2062,7 +2062,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2077,7 +2077,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2092,7 +2092,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2107,7 +2107,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2121,7 +2121,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2136,7 +2136,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2151,7 +2151,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2166,7 +2166,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2181,7 +2181,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2196,7 +2196,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2211,7 +2211,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2226,7 +2226,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2240,7 +2240,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2255,7 +2255,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2270,7 +2270,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2285,7 +2285,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2300,7 +2300,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2315,7 +2315,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2330,7 +2330,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2345,7 +2345,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2359,7 +2359,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2374,7 +2374,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2389,7 +2389,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2404,7 +2404,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2419,7 +2419,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2434,7 +2434,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2449,7 +2449,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2464,7 +2464,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2478,7 +2478,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -2489,7 +2489,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -2500,7 +2500,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -2511,7 +2511,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -2522,7 +2522,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -2533,7 +2533,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -2544,7 +2544,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -2555,7 +2555,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -2565,7 +2565,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -2576,7 +2576,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -2587,7 +2587,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -2598,7 +2598,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -2609,7 +2609,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -2620,7 +2620,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -2631,7 +2631,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -2642,7 +2642,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-malicious-no-responder-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -2652,7 +2652,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -2663,7 +2663,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -2674,7 +2674,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -2685,7 +2685,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -2696,7 +2696,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -2707,7 +2707,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -2718,7 +2718,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -2729,7 +2729,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] @@ -2841,7 +2841,7 @@ tasks: - func: run tests vars: TEST_NAME: search_index - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: [search_index] # Server version tests @@ -2857,7 +2857,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - server-version @@ -2876,7 +2876,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - server-version @@ -2897,7 +2897,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - server-version @@ -2916,7 +2916,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - server-version @@ -2937,7 +2937,7 @@ tasks: SSL: nossl TOPOLOGY: replica_set COVERAGE: "1" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - server-version @@ -2960,7 +2960,7 @@ tasks: SSL: nossl TOPOLOGY: replica_set COVERAGE: "1" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - server-version @@ -2980,7 +2980,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - server-version @@ -3000,7 +3000,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - server-version @@ -3019,7 +3019,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - server-version @@ -3040,7 +3040,7 @@ tasks: SSL: nossl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - server-version @@ -3063,7 +3063,7 @@ tasks: TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" COVERAGE: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - server-version @@ -3083,7 +3083,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - server-version @@ -3102,7 +3102,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - server-version @@ -3121,7 +3121,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - server-version @@ -3140,7 +3140,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - server-version @@ -3159,7 +3159,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - server-version @@ -3179,7 +3179,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - server-version @@ -3198,7 +3198,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - server-version @@ -3218,7 +3218,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - server-version @@ -3241,7 +3241,7 @@ tasks: TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" COVERAGE: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - server-version @@ -3261,7 +3261,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - server-version @@ -3280,7 +3280,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - server-version @@ -3299,7 +3299,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - server-version @@ -3318,7 +3318,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - server-version @@ -3337,7 +3337,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - server-version @@ -3357,7 +3357,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - server-version @@ -3376,7 +3376,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - server-version @@ -3396,7 +3396,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - server-version @@ -3415,7 +3415,7 @@ tasks: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - server-version @@ -3434,7 +3434,7 @@ tasks: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - server-version @@ -3454,7 +3454,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - server-version @@ -3473,7 +3473,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - server-version @@ -3492,7 +3492,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - server-version @@ -3511,7 +3511,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - server-version @@ -3530,7 +3530,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - server-version @@ -3551,7 +3551,7 @@ tasks: SSL: ssl TOPOLOGY: standalone TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - server-version @@ -3572,7 +3572,7 @@ tasks: SSL: nossl TOPOLOGY: standalone COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - server-version @@ -3594,7 +3594,7 @@ tasks: SSL: nossl TOPOLOGY: standalone COVERAGE: "1" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - server-version @@ -3615,7 +3615,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - server-version @@ -3634,7 +3634,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: standalone - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - server-version @@ -3658,7 +3658,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "4.4" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - test-standard @@ -3680,7 +3680,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "4.4" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - test-standard @@ -3702,7 +3702,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - test-standard @@ -3724,7 +3724,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - test-standard @@ -3749,7 +3749,7 @@ tasks: TOPOLOGY: standalone VERSION: "4.4" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - test-standard @@ -3771,7 +3771,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "4.4" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - test-standard @@ -3794,7 +3794,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "4.4" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - test-standard @@ -3817,7 +3817,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard @@ -3839,7 +3839,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - test-standard @@ -3861,7 +3861,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - test-standard @@ -3883,7 +3883,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - test-standard @@ -3908,7 +3908,7 @@ tasks: TOPOLOGY: standalone VERSION: "5.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - test-standard @@ -3930,7 +3930,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "5.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - test-standard @@ -3953,7 +3953,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "5.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - test-standard @@ -3978,7 +3978,7 @@ tasks: TOPOLOGY: replica_set VERSION: "6.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - test-standard @@ -4000,7 +4000,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "6.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - test-standard @@ -4023,7 +4023,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "6.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - test-standard @@ -4046,7 +4046,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - test-standard @@ -4068,7 +4068,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - test-standard @@ -4090,7 +4090,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "6.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - test-standard @@ -4112,7 +4112,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "6.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - test-standard @@ -4136,7 +4136,7 @@ tasks: TOPOLOGY: replica_set VERSION: "7.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - test-standard @@ -4158,7 +4158,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "7.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - test-standard @@ -4181,7 +4181,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "7.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - test-standard @@ -4204,7 +4204,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard @@ -4226,7 +4226,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - test-standard @@ -4248,7 +4248,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - test-standard @@ -4270,7 +4270,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard @@ -4292,7 +4292,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - test-standard @@ -4314,7 +4314,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - test-standard @@ -4338,7 +4338,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "8.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - test-standard @@ -4360,7 +4360,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - test-standard @@ -4383,7 +4383,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - test-standard @@ -4406,7 +4406,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "8.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - test-standard @@ -4428,7 +4428,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "8.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - test-standard @@ -4451,7 +4451,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "9.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - test-standard @@ -4473,7 +4473,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "9.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard @@ -4497,7 +4497,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "9.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - test-standard @@ -4519,7 +4519,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - test-standard @@ -4542,7 +4542,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - test-standard @@ -4565,7 +4565,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "9.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - test-standard @@ -4587,7 +4587,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "9.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - test-standard @@ -4612,7 +4612,7 @@ tasks: TOPOLOGY: replica_set VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - test-standard @@ -4637,7 +4637,7 @@ tasks: TOPOLOGY: replica_set VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - test-standard @@ -4663,7 +4663,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - test-standard @@ -4688,7 +4688,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard @@ -4713,7 +4713,7 @@ tasks: TOPOLOGY: standalone VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard @@ -4738,7 +4738,7 @@ tasks: TOPOLOGY: standalone VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - test-standard @@ -4761,7 +4761,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: rapid - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - test-standard @@ -4783,7 +4783,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: rapid - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - test-standard @@ -4806,7 +4806,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - test-standard @@ -4828,7 +4828,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - test-standard @@ -4850,7 +4850,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - test-standard @@ -4872,7 +4872,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - test-standard @@ -4896,7 +4896,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.2" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-string-query-preview - server-8.2 @@ -4919,7 +4919,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "4.4" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-non-standard - server-4.4 @@ -4940,7 +4940,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-non-standard - server-4.4 @@ -4963,7 +4963,7 @@ tasks: TOPOLOGY: standalone VERSION: "4.4" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-non-standard - server-4.4 @@ -4984,7 +4984,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "4.4" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-4.4 @@ -5006,7 +5006,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-non-standard - server-5.0 @@ -5028,7 +5028,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-5.0 @@ -5050,7 +5050,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-non-standard - server-5.0 @@ -5071,7 +5071,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "5.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-non-standard - server-5.0 @@ -5092,7 +5092,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "6.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard - server-6.0 @@ -5115,7 +5115,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "6.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-non-standard - server-6.0 @@ -5136,7 +5136,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-6.0 @@ -5158,7 +5158,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "6.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-non-standard - server-6.0 @@ -5180,7 +5180,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "7.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-non-standard - server-7.0 @@ -5201,7 +5201,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-non-standard - server-7.0 @@ -5222,7 +5222,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-non-standard - server-7.0 @@ -5243,7 +5243,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-7.0 @@ -5265,7 +5265,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-non-standard - server-8.0 @@ -5286,7 +5286,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-8.0 @@ -5308,7 +5308,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-non-standard - server-8.0 @@ -5330,7 +5330,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "8.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-non-standard - server-8.0 @@ -5354,7 +5354,7 @@ tasks: TOPOLOGY: replica_set VERSION: "9.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-non-standard - server-9.0 @@ -5375,7 +5375,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-non-standard - server-9.0 @@ -5396,7 +5396,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-9.0 @@ -5418,7 +5418,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "9.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard - server-9.0 @@ -5439,7 +5439,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-non-standard - server-latest @@ -5461,7 +5461,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-latest @@ -5485,7 +5485,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard - server-latest @@ -5508,7 +5508,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard-no-cov - server-latest @@ -5530,7 +5530,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-non-standard - server-latest @@ -5551,7 +5551,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: rapid - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-non-standard - server-rapid @@ -5572,7 +5572,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-non-standard - server-rapid @@ -5594,7 +5594,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-non-standard - server-rapid @@ -5615,7 +5615,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-rapid @@ -5629,7 +5629,7 @@ tasks: commands: - func: test numpy vars: - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - binary - vector @@ -5639,7 +5639,7 @@ tasks: commands: - func: test numpy vars: - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" COVERAGE: "1" tags: - binary @@ -5665,7 +5665,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "4.4" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-standard-auth - server-4.4 @@ -5686,7 +5686,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-4.4 @@ -5708,7 +5708,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-standard-auth - server-5.0 @@ -5729,7 +5729,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-5.0 @@ -5751,7 +5751,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-standard-auth - server-6.0 @@ -5772,7 +5772,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-6.0 @@ -5794,7 +5794,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-standard-auth - server-7.0 @@ -5815,7 +5815,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-7.0 @@ -5837,7 +5837,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-standard-auth - server-8.0 @@ -5859,7 +5859,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-8.0 @@ -5881,7 +5881,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-standard-auth - server-9.0 @@ -5902,7 +5902,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-9.0 @@ -5926,7 +5926,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-standard-auth - server-latest @@ -5948,7 +5948,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-latest @@ -5970,7 +5970,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-standard-auth - server-rapid @@ -5992,7 +5992,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-rapid diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 4ec527c941..358b3e532e 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -27,9 +27,9 @@ else fi # Start the test runner. -echo "Running tests with UV_PYTHON=${UV_PYTHON:-}..." +echo "Running tests with UV_PYTHON=${UV_PYTHON:-} UV_PYTHON_SEARCH_PATH=${UV_PYTHON_SEARCH_PATH:-}..." echo "UV_ARGS=${UV_ARGS}" uv run ${UV_ARGS} --reinstall-package pymongo .evergreen/scripts/run_tests.py "$@" -echo "Running tests with UV_PYTHON=${UV_PYTHON:-}... done." +echo "Running tests with UV_PYTHON=${UV_PYTHON:-} UV_PYTHON_SEARCH_PATH=${UV_PYTHON_SEARCH_PATH:-}... done." cd $PREV_DIR diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 97fd9b0176..e4251141be 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -412,7 +412,7 @@ def create_test_numpy_tasks(): tasks = [] for python in MIN_MAX_PYTHON: tags = ["binary", "vector", f"python-{python}", "test-numpy"] - vars = dict(TOOLCHAIN_VERSION=python) + vars = dict(UV_PYTHON=python) if python == MIN_MAX_PYTHON[-1]: tags.append("pr") vars["COVERAGE"] = "1" @@ -692,7 +692,7 @@ def create_server_version_tasks(): ) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_vars["TEST_NAME"] = f"default_{sync}" test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) @@ -754,7 +754,7 @@ def create_test_non_standard_tasks(): name = get_task_name("test-non-standard", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) # For each coverage task, also emit a non-coverage companion so that @@ -772,7 +772,7 @@ def create_test_non_standard_tasks(): nc_name = get_task_name("test-non-standard", python=python, **nc_expansions) nc_server_func = FunctionCall(func="run server", vars=nc_expansions) nc_test_vars = nc_expansions.copy() - nc_test_vars["TOOLCHAIN_VERSION"] = python + nc_test_vars["UV_PYTHON"] = python nc_test_func = FunctionCall(func="run tests", vars=nc_test_vars) tasks.append( EvgTask(name=nc_name, tags=nc_tags, commands=[nc_server_func, nc_test_func]) @@ -803,7 +803,7 @@ def create_string_query_preview_tasks(): name = get_task_name("test-string-query-preview", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_func = FunctionCall(func="run tests", vars=test_vars) return [EvgTask(name=name, tags=tags, commands=[server_func, test_func])] @@ -846,7 +846,7 @@ def create_test_standard_auth_tasks(): name = get_task_name("test-standard-auth", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) return tasks @@ -886,7 +886,7 @@ def create_standard_tasks(): name = get_task_name("test-standard", python=python, sync=sync, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_vars["TEST_NAME"] = f"default_{sync}" test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) @@ -901,7 +901,7 @@ def create_no_orchestration_tasks(): f"python-{python}", ] assume_func = FunctionCall(func="assume ec2 role") - test_vars = dict(TOOLCHAIN_VERSION=python) + test_vars = dict(UV_PYTHON=python) if python == ALL_PYTHONS[0]: test_vars["TEST_MIN_DEPS"] = "1" name = get_task_name("test-no-orchestration", **test_vars) @@ -951,7 +951,7 @@ def create_aws_tasks(): tags = [*base_tags, f"auth-aws-{test_type}"] if "t" in python: tags.append("free-threaded") - test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, TOOLCHAIN_VERSION=python) + test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, UV_PYTHON=python) if python == MIN_MAX_PYTHON[0]: test_vars["TEST_MIN_DEPS"] = "1" elif python == MIN_MAX_PYTHON[-1]: @@ -969,7 +969,7 @@ def create_aws_tasks(): TEST_NAME="auth_aws", SUB_TEST_NAME="web-identity", AWS_ROLE_SESSION_NAME="test", - TOOLCHAIN_VERSION=python, + UV_PYTHON=python, ) if "t" in python: tags.append("free-threaded") @@ -1018,7 +1018,7 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): ORCHESTRATION_FILE=file_name, OCSP_SERVER_TYPE=server_type, TEST_NAME="ocsp", - TOOLCHAIN_VERSION=python, + UV_PYTHON=python, VERSION=version, ) if python == ALL_PYTHONS[0]: @@ -1081,7 +1081,7 @@ def create_aws_lambda_tasks(): def create_search_index_tasks(): assume_func = FunctionCall(func="assume ec2 role") server_func = FunctionCall(func="run server", vars=dict(TEST_NAME="search_index")) - vars = dict(TEST_NAME="search_index", TOOLCHAIN_VERSION=CPYTHONS[0]) + vars = dict(TEST_NAME="search_index", UV_PYTHON=CPYTHONS[0]) test_func = FunctionCall(func="run tests", vars=vars) task_name = "test-search-index-helpers" tags = ["search_index"] @@ -1313,7 +1313,6 @@ def create_run_server_func(): "SSL", "ORCHESTRATION_FILE", "UV_PYTHON", - "TOOLCHAIN_VERSION", "STORAGE_ENGINE", "REQUIRE_API_VERSION", "DRIVERS_TOOLS", @@ -1340,7 +1339,6 @@ def create_run_tests_func(): "UV_PYTHON", "LIBMONGOCRYPT_URL", "MONGODB_URI", - "TOOLCHAIN_VERSION", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", @@ -1363,7 +1361,7 @@ def create_run_tests_func(): def create_test_numpy_func(): - includes = ["TOOLCHAIN_VERSION", "COVERAGE"] + includes = ["UV_PYTHON", "COVERAGE"] test_cmd = get_subprocess_exec( include_expansions_in_env=includes, args=[".evergreen/just.sh", "test-numpy"] ) diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index efb8d6a34b..67be0dbec6 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -178,7 +178,7 @@ def get_common_name(base: str, sep: str, **kwargs) -> str: display_name = f"{display_name}{sep}{version}" for key, value in kwargs.items(): name = value - if key.lower() in ["python", "toolchain_version"]: + if key.lower() in ["python", "uv_python"]: if not value.startswith("pypy"): name = f"Python{value}" else: diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index 36e7866735..a5f947f88e 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -2,6 +2,13 @@ # Install the necessary dependencies. set -euo pipefail +# The uv tool installs below (pinning uv, rust-just) just need a working +# interpreter, so drop the task's Python selection: it may not exist on this +# host (e.g. 3.15 with a 3.14 toolchain), and uv would fail on a too-old +# interpreter found in the search path (e.g. /usr/bin/python3 on RHEL) rather +# than skip it. Later steps re-source env.sh, restoring the selection. +unset UV_PYTHON UV_PYTHON_SEARCH_PATH UV_PYTHON_PREFERENCE + HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" pushd "$(dirname "$(dirname $HERE)")" > /dev/null diff --git a/.evergreen/scripts/setup-uv-python.sh b/.evergreen/scripts/setup-uv-python.sh index e83abce4b9..2ebfe04d9d 100755 --- a/.evergreen/scripts/setup-uv-python.sh +++ b/.evergreen/scripts/setup-uv-python.sh @@ -1,13 +1,21 @@ #!/bin/bash -# Set up the UV_PYTHON variable. -set -eu +# Choose the Python that uv uses, and make sure uv can provide it. +# +# Input: +# UV_PYTHON - a version ("3.10", "3.14t"), an implementation ("pypy3.11"), +# or an interpreter path ("/usr/bin/python3.11"). +# +# Exports: +# UV_PYTHON_SEARCH_PATH - the Python toolchain bin dir for the request, so uv +# uses the toolchain Python instead of downloading one. +# UV_PYTHON_PREFERENCE - "system" so the toolchain wins over managed installs. +# UV_PYTHON - the Python interpreter uv uses; defaults to CPython +# 3.10 when the task does not set one. +set -euo pipefail HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" -# Use min supported version by default. -_python="3.10" - # Source the env files to pick up common variables. if [ -f $HERE/env.sh ]; then . $HERE/env.sh @@ -18,43 +26,74 @@ if [ -f $HERE/test-env.sh ]; then . $HERE/test-env.sh fi -if [ -z "${UV_PYTHON:-}" ]; then - set -x - # Translate a TOOLCHAIN_VERSION to UV_PYTHON. - if [ -n "${TOOLCHAIN_VERSION:-}" ]; then - _python=$TOOLCHAIN_VERSION - if [ "$(uname -s)" = "Darwin" ]; then - if [[ "$_python" == *"t"* ]]; then - binary_name="python3t" - framework_dir="PythonT" - else - binary_name="python3" - framework_dir="Python" - fi - _python=$(echo "$_python" | sed 's/t//g') - _python="/Library/Frameworks/$framework_dir.Framework/Versions/$_python/bin/$binary_name" - elif [ "Windows_NT" = "${OS:-}" ]; then - _python=$(echo $_python | cut -d. -f1,2 | sed 's/\.//g; s/t//g') - if [[ "$TOOLCHAIN_VERSION" == *"t"* ]]; then - _exe="python${TOOLCHAIN_VERSION}.exe" - else - _exe="python.exe" - fi - if [ -n "${IS_WIN32:-}" ]; then - _win_python="C:/python/32/Python${_python}/${_exe}" - else - _win_python="C:/python/Python${_python}/${_exe}" - fi - # Fall back to the version string if the toolchain path doesn't exist. - # uv will then download and manage Python itself. - if [ -f "$_win_python" ]; then - _python="$_win_python" - else - _python="$TOOLCHAIN_VERSION" - fi - elif [ -d "/opt/python/$_python/bin" ]; then - _python="/opt/python/$_python/bin/python3" +# Default to a known-good Python so behavior is deterministic when a task does +# not pin one. Selecting the uv binary is separate and driven by the required +# uv version in pyproject.toml. +export UV_PYTHON="${UV_PYTHON:-3.10}" + +# Print the Python toolchain bin dir for a version like "3.10" or "3.14t". +function _toolchain_dir() { + local version="$1" dir + # Only plain CPython versions live in the toolchain. + [[ "$version" =~ ^[0-9]+\.[0-9]+t?$ ]] || return 1 + if [ "Windows_NT" = "${OS:-}" ]; then + local winver="${version%t}" + winver="${winver//./}" + if [ -n "${IS_WIN32:-}" ]; then + dir="C:/python/32/Python${winver}" + else + dir="C:/python/Python${winver}" fi + elif [ "$(uname -s)" = "Darwin" ]; then + if [[ "$version" == *t ]]; then + dir="/Library/Frameworks/PythonT.Framework/Versions/${version%t}/bin" + else + dir="/Library/Frameworks/Python.Framework/Versions/${version}/bin" + fi + else + dir="/opt/python/${version}/bin" fi - export UV_PYTHON="$_python" + [ -d "$dir" ] || return 1 + echo "$dir" +} + +# Make sure uv can provide the requested Python, downloading it if needed. +function _ensure_python() { + local request="$1" + if uv python find "$request" > /dev/null 2>&1; then + return 0 + fi + echo "Python \"$request\" was not found on this host; asking uv to install it..." + if uv python install "$request"; then + return 0 + fi + echo "ERROR: uv could not find or install Python \"$request\"." >&2 + return 1 +} + +_search_path="" +_skip_ensure="" + +if [[ "$UV_PYTHON" == */* || "$UV_PYTHON" == *\\* ]]; then + # An explicit interpreter path; uv uses it as-is, so probe/install is skipped. + if [ ! -x "$UV_PYTHON" ]; then + echo "ERROR: UV_PYTHON=$UV_PYTHON does not exist or is not executable." >&2 + exit 1 + fi + _skip_ensure=1 +elif _dir=$(_toolchain_dir "$UV_PYTHON"); then + _search_path="$_dir" +fi + +# Point uv at the toolchain Python when there is one. On CI the toolchain dir +# is already first on PATH (configure-env.sh), so this mainly benefits local +# hosts and later steps, keeping `uv sync` and `uv tool install` on the +# toolchain interpreter instead of downloading a managed one. +if [ -n "$_search_path" ]; then + export UV_PYTHON_SEARCH_PATH="$_search_path" + export UV_PYTHON_PREFERENCE="system" +fi + +if [ -z "$_skip_ensure" ]; then + _ensure_python "${UV_PYTHON:-}" fi diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 5a3f441f58..4f8b03b5ec 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -30,6 +30,8 @@ "MONGODB_API_VERSION", "DEBUG_LOG", "UV_PYTHON", + "UV_PYTHON_SEARCH_PATH", + "UV_PYTHON_PREFERENCE", "REQUIRE_FIPS", "IS_WIN32", ] @@ -511,8 +513,13 @@ def handle_test_env() -> None: run_command("tar xf single_and_multi_document.tgz", cwd=data_dir) write_env("TEST_PATH", str(data_dir)) write_env("OUTPUT_FILE", str(ROOT / "results.json")) - # Overwrite the UV_PYTHON from the env.sh file. + # Overwrite the UV_PYTHON value from env.sh, and unset the toolchain + # search-path variables: an empty value would make uv reject the request, + # and a toolchain path would miss the exact patch version requested. write_env("UV_PYTHON", "") + with ENV_FILE.open("a", newline="\n") as fid: + fid.write("unset UV_PYTHON_SEARCH_PATH\n") + fid.write("unset UV_PYTHON_PREFERENCE\n") UV_ARGS.append(f"--python={PERF_PYTHON_VERSION}") diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index a7decdfaa8..3f948dd292 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -297,8 +297,8 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None: kwargs.setdefault("check", True) # Prevent overriding the python used by other tools. env = kwargs.pop("env", os.environ).copy() - if "UV_PYTHON" in env: - del env["UV_PYTHON"] + for var in ["UV_PYTHON", "UV_PYTHON_SEARCH_PATH", "UV_PYTHON_PREFERENCE"]: + env.pop(var, None) kwargs["env"] = env try: subprocess.run(shlex.split(cmd), **kwargs) # noqa: PLW1510, S603 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 20b7210736..42a8606cd9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -403,15 +403,14 @@ To run any of the test suites with minimum supported dependencies, pass `--test- - If there are any services or atlas clusters to teardown, handle them in `.evergreen/scripts/teardown_tests.py`. - Add functions to generate the test variant(s) and task(s) to the `.evergreen/scripts/generate_config.py`. - There are some considerations about the Python version used in the test: - - If a specific version of Python is needed in a task that is running on variants with a toolchain, use -``TOOLCHAIN_VERSION`` (e.g. `TOOLCHAIN_VERSION=3.10`). The actual path lookup needs to be done on the host, since -tasks are host-agnostic. + - To request a specific Python, set `UV_PYTHON` (e.g. `UV_PYTHON=3.10`, `UV_PYTHON=3.14t`, or +`UV_PYTHON=pypy3.11`). Tasks are host-agnostic, so the interpreter lookup happens on the host: for a plain +CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at it, and for anything else +(including PyPy, or a version the toolchain lacks) uv downloads it. - If a specific Python binary is needed (for example on the FIPS host), set `UV_PYTHON=/path/to/python`. - - If a specific Python version is needed and the toolchain will not be available, use `UV_PYTHON` (e.g. `UV_PYTHON=3.11`). - - The default if neither ``TOOLCHAIN_VERSION`` or ``UV_PYTHON`` is set is to use UV to install the minimum - supported version of Python and use that. This ensures a consistent behavior across host types that do not - have the Python toolchain (e.g. Azure VMs), by having a known version of Python with the build headers (`Python.h`) - needed to build the C extensions. + - The default if `UV_PYTHON` is not set is CPython 3.10. This is deterministic across host types, so a task + that does not pin a version always gets the same Python (with the build headers (`Python.h`) needed to build + the C extensions). - The uv binary version is pinned once in `[tool.uv] required-version` in `pyproject.toml`. `.evergreen/scripts/install-dependencies.sh` installs it with `uv tool install`, uv enforces it locally, and `astral-sh/setup-uv` reads it on GitHub. Bump it manually when a newer uv is needed. If uv cannot find the From ac8ae76fc110aafb8b8dcd761e025858c394d6d0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 28 Sep 2026 10:25:03 -0500 Subject: [PATCH 03/24] PYTHON-6114 Run the mod_wsgi tests on GitHub Actions (#3069) --- .evergreen/scripts/dispatch.sh | 46 +++ .evergreen/scripts/mod_wsgi.sh | 262 ++++++++++++++ .evergreen/scripts/mod_wsgi_matrix.py | 29 ++ .evergreen/scripts/mod_wsgi_tester.py | 312 +++++++++++++++++ .github/actions/mod-wsgi-versions/action.yml | 23 ++ .github/workflows/test-mod-wsgi.yml | 176 ++++++++++ .gitignore | 2 + CONTRIBUTING.md | 18 + justfile | 36 +- pyproject.toml | 3 + test/mod_wsgi_test/README.rst | 77 +++-- test/mod_wsgi_test/apache22amazon.conf | 34 -- test/mod_wsgi_test/apache22ubuntu1204.conf | 29 -- test/mod_wsgi_test/apache24ubuntu.conf | 47 +++ test/mod_wsgi_test/apache24ubuntu161404.conf | 28 -- test/mod_wsgi_test/httpd24fedora.conf | 58 ++++ test/mod_wsgi_test/mod_wsgi_test.conf | 2 - .../mod_wsgi_test/mod_wsgi_test_embedded.conf | 2 - uv.lock | 325 ++++++++++-------- 19 files changed, 1225 insertions(+), 284 deletions(-) create mode 100755 .evergreen/scripts/dispatch.sh create mode 100755 .evergreen/scripts/mod_wsgi.sh create mode 100644 .evergreen/scripts/mod_wsgi_matrix.py create mode 100644 .evergreen/scripts/mod_wsgi_tester.py create mode 100644 .github/actions/mod-wsgi-versions/action.yml create mode 100644 .github/workflows/test-mod-wsgi.yml delete mode 100644 test/mod_wsgi_test/apache22amazon.conf delete mode 100644 test/mod_wsgi_test/apache22ubuntu1204.conf create mode 100644 test/mod_wsgi_test/apache24ubuntu.conf delete mode 100644 test/mod_wsgi_test/apache24ubuntu161404.conf create mode 100644 test/mod_wsgi_test/httpd24fedora.conf diff --git a/.evergreen/scripts/dispatch.sh b/.evergreen/scripts/dispatch.sh new file mode 100755 index 0000000000..372cf6535c --- /dev/null +++ b/.evergreen/scripts/dispatch.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Route the just test recipes to the mod_wsgi tests or the Evergreen test +# runner. The mod_wsgi setup writes TEST_NAME to the test env file. +set -eu + +SCRIPT_DIR=$(dirname "${BASH_SOURCE:-$0}") +RECIPE="${1:-}" +if [ "$#" -gt 0 ]; then + shift +fi + +mod_wsgi_env() { + [ -f "$SCRIPT_DIR/test-env.sh" ] \ + && grep -q 'TEST_NAME="mod_wsgi"' "$SCRIPT_DIR/test-env.sh" \ + && bash "$SCRIPT_DIR/mod_wsgi.sh" active +} + +case "$RECIPE" in + setup-tests) + if [ "${1:-}" = "mod_wsgi" ]; then + shift + exec bash "$SCRIPT_DIR/mod_wsgi.sh" setup "${1:-}" + fi + exec bash "$SCRIPT_DIR/setup-tests.sh" "$@" + ;; + run-tests) + if mod_wsgi_env; then + if [ "$#" -gt 0 ]; then + echo "Ignoring arguments for the mod_wsgi tests: $*" >&2 + fi + exec bash "$SCRIPT_DIR/mod_wsgi.sh" test + fi + exec bash "$(dirname "$SCRIPT_DIR")/run-tests.sh" "$@" + ;; + teardown-tests) + if [ -f "$SCRIPT_DIR/test-env.sh" ] \ + && grep -q 'TEST_NAME="mod_wsgi"' "$SCRIPT_DIR/test-env.sh"; then + exec bash "$SCRIPT_DIR/mod_wsgi.sh" teardown + fi + exec bash "$SCRIPT_DIR/teardown-tests.sh" "$@" + ;; + *) + echo "Unknown recipe: $RECIPE" + exit 1 + ;; +esac diff --git a/.evergreen/scripts/mod_wsgi.sh b/.evergreen/scripts/mod_wsgi.sh new file mode 100755 index 0000000000..61aed24b73 --- /dev/null +++ b/.evergreen/scripts/mod_wsgi.sh @@ -0,0 +1,262 @@ +#!/bin/bash +# Set up, run, or tear down the mod_wsgi tests. On Linux the test runs against +# the host's Apache; on other hosts it runs in an ubuntu:24.04 container with +# its own MongoDB replica set. +# +# Usage: mod_wsgi.sh setup | test | teardown | active +set -euo pipefail + +SCRIPT_DIR=$(dirname "${BASH_SOURCE:-$0}") +ROOT=$(dirname "$(dirname "$SCRIPT_DIR")") +CMD="${1:-}" +MODE="${2:-}" +CONTAINER=pymongo-mod-wsgi +HOME_SRC=/home/smoke/src +# The venv, MongoDB install, and the recorded version selection live outside +# the source copy, so refreshing the checkout never reinstalls them. +VENV=/home/smoke/venv +VERSIONS=/home/smoke/work/versions + +cd "$ROOT" + +case "$CMD" in + setup) + if [ "$MODE" != "standalone" ] && [ "$MODE" != "embedded" ]; then + echo "Usage: mod_wsgi.sh setup " + exit 1 + fi + # The harness recipes (run-tests, teardown-tests) dispatch on this. + echo "export TEST_NAME=\"mod_wsgi\"" > .evergreen/scripts/test-env.sh + echo "export SUB_TEST_NAME=\"$MODE\"" >> .evergreen/scripts/test-env.sh + # A failed setup leaves a partially started session behind. While the + # marker still routes to this session, tear it down; then remove the + # marker, so later just recipes do not route to the half-set-up state. + # The teardown is what stops the server or container setup may have + # started, and the next setup attempt stops any leftovers as well. + cleanup_marker() { + status=$? + if [ "$status" -ne 0 ]; then + bash "$SCRIPT_DIR/mod_wsgi.sh" teardown || true + rm -f .evergreen/scripts/test-env.sh + fi + } + trap cleanup_marker EXIT + ;; + test|teardown|active) ;; + *) + echo "Usage: mod_wsgi.sh setup | test | teardown | active" + exit 1 + ;; +esac + +if [ "$CMD" = "active" ]; then + # Exit 0 when a mod_wsgi session is active in this environment, so the + # dispatch script can route to us without a stale marker. Uses the system + # python and the tester's own STATE_FILE constant, so a missing or + # mismatched uv cannot break the dispatch and the path cannot drift. + if [ "$(uname -s)" = "Linux" ] && [ -z "${MOD_WSGI_DOCKER:-}" ]; then + if python3 -c " +import sys +sys.path.insert(0, '.evergreen/scripts') +from mod_wsgi_tester import STATE_FILE +sys.exit(0 if STATE_FILE.exists() else 1) +"; then + exit 0 + fi + exit 1 + fi + # The docker branch below handles the container case. +fi + +if [ "$(uname -s)" = "Linux" ] && [ -z "${MOD_WSGI_DOCKER:-}" ]; then + # Native: Apache runs on the host; MongoDB is already running, started by + # "just run-server" or the CI workflow. Set MOD_WSGI_DOCKER to force the + # container path on Linux. + if [ "$CMD" != "teardown" ]; then + # Other recipes' exact uv syncs — including run-tests' resync — prune the + # mod_wsgi group; restore it before setup or test when missing. Checking + # first keeps the sync from reconciling dependencies that a caller + # resolved differently, like the min-deps job's lowest-direct install. + if ! uv run --no-sync python -c "import mod_wsgi" 2>/dev/null; then + uv sync --group mod_wsgi --quiet + fi + fi + if [ "$CMD" = "setup" ]; then + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py setup "$MODE" + else + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py "$CMD" + fi + # Clear the dispatch marker so later just recipes use the harness. + if [ "$CMD" = "teardown" ]; then + rm -f .evergreen/scripts/test-env.sh + fi + exit 0 +fi + +# Docker path for hosts that cannot run Apache and mongod natively. +container_exists() { docker container inspect "$CONTAINER" >/dev/null 2>&1; } +container_running() { + [ "$(docker container inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null)" = "true" ] +} + +if [ "$CMD" = "active" ]; then + if container_exists; then + exit 0 + fi + exit 1 +fi + +# Copy the checkout into the container's home directory, so root-owned test +# artifacts never pollute it. +sync_source() { + docker exec "$CONTAINER" rm -rf "$HOME_SRC" + docker exec -u smoke "$CONTAINER" mkdir -p "$HOME_SRC" + tar -C "$ROOT" -cf - --exclude=.git --exclude=.venv \ + --exclude=access_log --exclude=error_log . | docker exec -i -u smoke "$CONTAINER" tar -C "$HOME_SRC" -xf - +} + +# Start a single-node replica set, if it is not already running. +ensure_mongod() { + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + mkdir -p "$HOME/work/db" + if ! pgrep -x mongod >/dev/null; then + "$HOME/work/bin/mongod" --replSet rs0 --bind_ip 127.0.0.1 --port 27017 --dbpath "$HOME/work/db" --fork --logpath "$HOME/work/mongod.log" + fi + ' +} + +# Create the container and install the environment (idempotent). +bootstrap() { + if ! container_exists; then + docker run --name "$CONTAINER" -d ubuntu:24.04 sleep infinity + # A transient failure during initialization (for example in apt-get) + # would leave the half-created container behind, and later bootstraps + # would skip initialization and fail when sync_source expects the + # missing smoke user. Remove the container on failure, so the next + # attempt initializes a fresh one. + initialize() { + docker exec "$CONTAINER" apt-get update -qq + docker exec "$CONTAINER" apt-get install -y -qq apache2 apache2-dev build-essential curl jq git tar gzip ca-certificates + docker exec "$CONTAINER" useradd -m smoke + } + if ! initialize; then + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + exit 1 + fi + fi + if ! container_running; then + docker start "$CONTAINER" + fi + # Always refresh the checkout copy, so reused containers test the current + # source. The venv and versions file live outside it and survive the + # refresh. + sync_source + # Re-evaluate the newest Python and MongoDB on every bootstrap, so a reused + # container tracks matrix and server releases, but only reinstall when the + # recorded selection differs, since the downloads are slow. + docker exec --user smoke -w "$HOME_SRC" \ + -e VERSIONS="$VERSIONS" -e VENV="$VENV" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PYTHON_INSTALL_DIR="$HOME/uv-python" + export UV_PROJECT_ENVIRONMENT="$VENV" + if ! command -v uv >/dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh >/dev/null + fi + # Mirror the CI jobs and test the newest supported CPython. + LATEST_PYTHON=$(uv run --no-project --with "shrub.py>=3.10.0" python .evergreen/scripts/mod_wsgi_matrix.py | jq -r ".[-1].\"python-version\"") + case "$(uname -m)" in + aarch64|arm64) MARCH=aarch64 ;; + x86_64|amd64) MARCH=x86_64 ;; + *) echo "Unsupported architecture: $(uname -m)"; exit 1 ;; + esac + # MongoDB builds are per-distro; the container is ubuntu:24.04. + MONGODB_URL=$(curl -fsSL https://downloads.mongodb.org/current.json | jq -r " + .versions[] | select(.current and .production_release) | .downloads[] | + select(.target==\"ubuntu2404\" and .arch==\"$MARCH\") | .archive.url" | grep -v enterprise | head -1) + DESIRED="python=$LATEST_PYTHON mongodb=$MONGODB_URL" + if [ "$(cat "$VERSIONS" 2>/dev/null || true)" != "$DESIRED" ]; then + uv python install "$LATEST_PYTHON" >/dev/null + mkdir -p "$HOME/work/bin" + curl -fsSL "$MONGODB_URL" -o "$HOME/work/mongo.tgz" + MEMBER=$(tar -tzf "$HOME/work/mongo.tgz" | grep "bin/mongod$") + # Strip the two leading path components, so ensure_mongod finds the + # binary at "$HOME/work/bin/mongod". + tar -xz -C "$HOME/work/bin" --strip-components=2 -f "$HOME/work/mongo.tgz" "$MEMBER" + # A new interpreter needs a fresh venv; the sync below repopulates it. + rm -rf "$VENV" + printf "%s\n" "$DESIRED" > "$VERSIONS" + fi + # Rebuild the in-place C extensions on every bootstrap; the source + # refresh may carry artifacts built by a different interpreter. + export PYMONGO_C_EXT_MUST_BUILD=1 + uv sync --group mod_wsgi --refresh-package pymongo + ' + ensure_mongod + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + # Probe replSetGetStatus over a direct connection: it returns immediately + # for an uninitialized replica set, while the default hello-based server + # selection would block for the full server selection timeout first. + if ! uv run --no-sync python -c " +from pymongo import MongoClient +client = MongoClient(\"127.0.0.1:27017\", directConnection=True) +try: + client.admin.command(\"replSetGetStatus\") +except Exception: + # NotYetInitialized (or a transient failure): initiation is still needed. + raise SystemExit(1) +" 2>/dev/null; then + uv run --no-sync python -c " +from pymongo import MongoClient +client = MongoClient(\"127.0.0.1:27017\", directConnection=True) +client.admin.command(\"replSetInitiate\", {\"_id\": \"rs0\", \"members\": [{\"_id\": 0, \"host\": \"127.0.0.1:27017\"}]}) +" + fi + ' +} + +# Bring a running session to readiness for a test run without refreshing the +# checkout copy: Apache may be live in it, and sync_source would unlink its +# pid file and error log. Only a missing, stopped, or never-bootstrapped +# container needs the full bootstrap, since then nothing is live to disturb. +ensure_ready() { + if container_exists && container_running \ + && docker exec "$CONTAINER" test -e "$VERSIONS"; then + ensure_mongod + else + bootstrap + fi +} + +case "$CMD" in + setup) + bootstrap + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PYTHON_INSTALL_DIR="$HOME/uv-python" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py setup '"$MODE"' + ' + ;; + test) + ensure_ready + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py test + ' + ;; + teardown) + if container_running; then + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py teardown + ' || true + docker stop -t 1 "$CONTAINER" + fi + rm -f .evergreen/scripts/test-env.sh + ;; +esac diff --git a/.evergreen/scripts/mod_wsgi_matrix.py b/.evergreen/scripts/mod_wsgi_matrix.py new file mode 100644 index 0000000000..2b8bb6611b --- /dev/null +++ b/.evergreen/scripts/mod_wsgi_matrix.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import json +import sys + +from generate_config_utils import ALL_VERSIONS, CPYTHONS + +# MongoDB 6.0 with the oldest supported CPython and minimum dependencies, and +# the latest MongoDB with the newest supported CPython. MongoDB 6.0 is the +# oldest version that installs on the GitHub runners. +VERSIONS = [ + { + "python-version": CPYTHONS[0], + "mongodb-version": "6.0", + }, + { + "python-version": CPYTHONS[-1], + "mongodb-version": ALL_VERSIONS[-1], + }, +] + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--index": + # Emit one version pair in GITHUB_OUTPUT format. + entry = VERSIONS[int(sys.argv[2])] + print(f"python-version={entry['python-version']}") + print(f"mongodb-version={entry['mongodb-version']}") + else: + print(json.dumps(VERSIONS)) diff --git a/.evergreen/scripts/mod_wsgi_tester.py b/.evergreen/scripts/mod_wsgi_tester.py new file mode 100644 index 0000000000..c326f6b476 --- /dev/null +++ b/.evergreen/scripts/mod_wsgi_tester.py @@ -0,0 +1,312 @@ +"""Setup, run, and tear down the mod_wsgi tests. + +Usage: + python .evergreen/scripts/mod_wsgi_tester.py setup + python .evergreen/scripts/mod_wsgi_tester.py test + python .evergreen/scripts/mod_wsgi_tester.py teardown + +The commands run as separate workflow steps, so state is passed between them +in a 0600 file in a user-owned cache directory. +""" + +from __future__ import annotations + +import json +import logging +import os +import signal +import socket +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") +LOGGER = logging.getLogger("mod_wsgi_tester") + +# Resolve __file__, so ROOT is correct when the script is invoked with a +# relative path, as mod_wsgi.sh does. +ROOT = Path(__file__).resolve().parents[2] +# State lives in a user-owned cache directory rather than shared /tmp: another +# local user must not be able to read or plant it, since teardown feeds its +# values into the commands that stop Apache. +STATE_DIR = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "pymongo-mod-wsgi" +STATE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True) +STATE_FILE = STATE_DIR / "state.json" +PORT = 8080 + + +def run_command( + cmd: str | list[str], env: dict[str, str] | None = None, check: bool = True +) -> None: + LOGGER.info(f"Running command: {cmd}") + try: + result = subprocess.run( # noqa: S603 + cmd, + shell=isinstance(cmd, str), + env={**os.environ, **(env or {})}, + cwd=str(ROOT), + capture_output=True, + text=True, + check=False, + ) + except FileNotFoundError: + # A missing binary is a failed command like any other: raised when + # required, logged and swallowed for best-effort calls. The shell + # path used to behave the same way via the exit code, and hosts like + # the Fedora job's container do not ship pkill. + if check: + raise + LOGGER.warning(f"Command not found: {cmd if isinstance(cmd, str) else cmd[0]}") + return + if result.stdout: + LOGGER.info(result.stdout) + if result.returncode != 0: + LOGGER.error(result.stderr) + if check: + raise RuntimeError(f"Command failed with code {result.returncode}: {cmd}") + + +def make_request(url: str, timeout: int = 10) -> None: + for _ in range(timeout): + try: + # Per-request timeout, so an Apache that accepts the connection + # but never answers fails after roughly timeout seconds instead + # of hanging forever. + urllib.request.urlopen(url, timeout=1) # noqa: S310 + return + except OSError: + # URLError (and its HTTPError subclass) plus a socket timeout. + pass + time.sleep(1) + raise TimeoutError(f"Failed to access {url}") + + +def find_mod_wsgi_so() -> str: + # The .so is built against the installing interpreter and lives in the + # installed package. + import mod_wsgi + + so_files = list((Path(mod_wsgi.__file__).parent / "server").glob("mod_wsgi*.so")) + if len(so_files) != 1: + raise ValueError(f"Expected one mod_wsgi.so in {mod_wsgi.__file__}, found {so_files}") + return str(so_files[0]) + + +def find_apache() -> tuple[str, str]: + """Return the Apache binary and the config file matching its distribution.""" + from shutil import which + + apache = which("apache2") + if not apache: + # apache2 lives in sbin, which is not always on PATH for non-root users. + for candidate in ("/usr/lib/apache2/mpm-prefork/apache2", "/usr/sbin/apache2"): + if Path(candidate).exists(): + apache = candidate + break + if apache: + return apache, "apache24ubuntu.conf" + apache = which("httpd") + if not apache: + for candidate in ("/usr/sbin/httpd", "/usr/local/apache2/bin/httpd"): + if Path(candidate).exists(): + apache = candidate + break + if not apache: + raise ValueError("Could not find apache2 or httpd") + return apache, "httpd24fedora.conf" + + +def port_is_open() -> bool: + try: + socket.create_connection(("127.0.0.1", PORT), timeout=1).close() + return True + except OSError: + return False + + +def pid_file(apache_config: str) -> Path: + # The test configs write the master process's pid to the checkout root. + return ROOT / ("httpd.pid" if "httpd" in apache_config else "apache2.pid") + + +def pid_is_our_apache(pid: int, apache_config: str) -> bool: + # A stale pid file may name a pid the OS has reused for an unrelated + # process, so only signal an Apache whose command line carries this + # checkout's config argument. + try: + cmdline = (Path("/proc") / str(pid) / "cmdline").read_bytes() + except OSError: + return False + argv = cmdline.decode("utf-8", "replace").split("\0") + return f"{ROOT}/test/mod_wsgi_test/{apache_config}" in argv + + +def stop_apache(apache: str, apache_config: str, env: dict[str, str] | None = None) -> None: + # Idempotent: safe to call when Apache is not running. + # Prefer the pid file: -k stop re-parses the config, which fails when the + # mod_wsgi .so or the config's env vars are unavailable. Fall back to a + # validated pgrep for a server whose pid file was removed under it. The + # commands run as argument lists, so state-derived values never reach a + # shell. + from shutil import which + + pid = pid_file(apache_config) + if pid.exists(): + try: + victim = int(pid.read_text().strip()) + except ValueError: + victim = 0 + if victim and pid_is_our_apache(victim, apache_config): + try: + os.kill(victim, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + LOGGER.warning("Could not stop Apache via the pid file") + else: + # Stale, reused, or unreadable pid: remove the file and stop + # through the config-validated path instead. + LOGGER.warning(f"Removing stale pid file {pid}") + pid.unlink() + if not pid.exists(): + run_command( + [apache, "-k", "stop", "-f", f"{ROOT}/test/mod_wsgi_test/{apache_config}"], + env=env, + check=False, + ) + if port_is_open(): + # Last resort: find pids whose full command line mentions this + # checkout's absolute config path, then signal only the ones whose + # command line carries it as an exact argument, so a server from + # another checkout (or an unrelated process) is never killed. + config_path = f"{ROOT}/test/mod_wsgi_test/{apache_config}" + pgrep = which("pgrep") + if pgrep: + result = subprocess.run( # noqa: S603 + [pgrep, "-f", config_path], capture_output=True, text=True, check=False + ) + for line in result.stdout.split(): + try: + victim = int(line) + except ValueError: + continue + if pid_is_our_apache(victim, apache_config): + try: + os.kill(victim, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + LOGGER.warning(f"Could not stop Apache pid {victim}") + + # Stopping returns before the port is released, which the next mode needs + # to rebind. + deadline = time.time() + 10 + while time.time() < deadline: + if not port_is_open(): + return + time.sleep(0.5) + raise ValueError(f"Apache did not release port {PORT}") + + +def write_state(state: dict) -> None: + # The cache directory is user-only (0700) and the file is created 0600, so + # no other local user can read or plant this state. + fd = os.open(STATE_FILE, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + json.dump(state, f) + + +def read_state() -> dict: + # Lost or corrupt state falls back to discovery in teardown. + try: + return json.loads(STATE_FILE.read_text()) + except (OSError, ValueError): + return {} + + +def setup(sub_test_name: str) -> None: + if sub_test_name == "embedded": + conf = "mod_wsgi_test_embedded.conf" + elif sub_test_name == "standalone": + conf = "mod_wsgi_test.conf" + else: + raise ValueError("mod_wsgi sub test must be either 'standalone' or 'embedded'") + apache, apache_config = find_apache() + # On Fedora/RHEL, httpd embeds the interpreter mod_wsgi was built with and + # needs to be told where the Python installation lives. + env = { + "MOD_WSGI_CONF": conf, + "MOD_WSGI_SO": find_mod_wsgi_so(), + "MOD_WSGI_PYTHON_HOME": sys.base_prefix, + "PROJECT_DIRECTORY": str(ROOT), + } + # Stop any server left behind by a previous failed run so the port is free. + stop_apache(apache, apache_config, env) + run_command( + [apache, "-k", "start", "-f", f"{ROOT}/test/mod_wsgi_test/{apache_config}"], + env=env, + ) + write_state({"apache": apache, "config": apache_config, "env": env}) + + try: + for interp in ("interpreter1", "interpreter2"): + make_request(f"http://localhost:{PORT}/{interp}{ROOT}", 10) + except Exception: + # The Apache error log holds the actual failure. Stop the server so a + # failed setup does not leave it bound to the port. + stop_apache(apache, apache_config, env) + error_log = Path("error_log") + if error_log.exists(): + LOGGER.error(error_log.read_text()) + raise + + +def test() -> None: + sys.path.insert(0, str(ROOT)) + from test.mod_wsgi_test.test_client import main, parse_args + + uri1 = f"http://localhost:{PORT}/interpreter1{ROOT}" + uri2 = f"http://localhost:{PORT}/interpreter2{ROOT}" + try: + main(*parse_args(f"-n 25000 -t 100 parallel {uri1} {uri2}".split())) + main(*parse_args(f"-n 25000 serial {uri1} {uri2}".split())) + except BaseException: + # The test client raises KeyboardInterrupt in the failing path. + error_log = Path("error_log") + if error_log.exists(): + LOGGER.error(error_log.read_text()) + raise + + +def teardown() -> None: + state = read_state() + apache = state.get("apache") + apache_config = state.get("config") + env = state.get("env") + if not apache or not apache_config: + # State lost; fall back to discovery to stop a leftover server. The + # pid file in stop_apache usually makes this unnecessary. + try: + apache, apache_config = find_apache() + except ValueError: + return + env = { + "MOD_WSGI_CONF": "mod_wsgi_test.conf", + "MOD_WSGI_PYTHON_HOME": sys.base_prefix, + "PROJECT_DIRECTORY": str(ROOT), + } + try: + env["MOD_WSGI_SO"] = find_mod_wsgi_so() + except (ImportError, ValueError): + pass + stop_apache(apache, apache_config, env) + + +if __name__ == "__main__": + command = sys.argv[1] if len(sys.argv) > 1 else "" + if command == "setup": + setup(sys.argv[2]) + elif command == "test": + test() + elif command == "teardown": + teardown() + else: + raise SystemExit(f"Usage: {sys.argv[0]} setup |test|teardown") diff --git a/.github/actions/mod-wsgi-versions/action.yml b/.github/actions/mod-wsgi-versions/action.yml new file mode 100644 index 0000000000..63994a3c14 --- /dev/null +++ b/.github/actions/mod-wsgi-versions/action.yml @@ -0,0 +1,23 @@ +name: Compute Mod WSGI versions +description: Compute the CPython and MongoDB versions for a Mod WSGI job. +inputs: + index: + description: Row of the version matrix to use. + required: true +outputs: + python-version: + description: The CPython version to test with. + value: ${{ steps.versions.outputs.python-version }} + mongodb-version: + description: The MongoDB version to test against. + value: ${{ steps.versions.outputs.mongodb-version }} +runs: + using: composite + steps: + - id: versions + # The versions come from the Evergreen config's version lists. + shell: bash + env: + INDEX: ${{ inputs.index }} + run: | + uv run --no-project --with 'shrub.py>=3.10.0' python .evergreen/scripts/mod_wsgi_matrix.py --index "$INDEX" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/test-mod-wsgi.yml b/.github/workflows/test-mod-wsgi.yml new file mode 100644 index 0000000000..4985161986 --- /dev/null +++ b/.github/workflows/test-mod-wsgi.yml @@ -0,0 +1,176 @@ +name: Mod WSGI Tests + +on: + push: + branches: ["main", "v**"] + paths: + - ".evergreen/scripts/mod_wsgi_*" + - ".evergreen/scripts/mod_wsgi.sh" + - ".evergreen/scripts/dispatch.sh" + - ".evergreen/scripts/generate_config_utils.py" + - ".github/workflows/test-mod-wsgi.yml" + - ".github/actions/mod-wsgi-*/**" + - "test/mod_wsgi_test/**" + - "pymongo/**" + - "bson/**" + - "justfile" + - "pyproject.toml" + - "uv.lock" + pull_request: + paths: + - ".evergreen/scripts/mod_wsgi_*" + - ".evergreen/scripts/mod_wsgi.sh" + - ".evergreen/scripts/dispatch.sh" + - ".evergreen/scripts/generate_config_utils.py" + - ".github/workflows/test-mod-wsgi.yml" + - ".github/actions/mod-wsgi-*/**" + - "test/mod_wsgi_test/**" + - "pymongo/**" + - "bson/**" + - "justfile" + - "pyproject.toml" + - "uv.lock" + workflow_dispatch: + +concurrency: + group: mod-wsgi-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -eux {0} + +permissions: + contents: read + +jobs: + mod_wsgi_min_deps: + # Exercise the minimum dependency versions against Fedora's httpd, so the + # mod_wsgi tests cover httpd in addition to Ubuntu's apache2. + runs-on: ubuntu-latest + name: Mod WSGI Min Deps + container: + # fedora:latest, pinned by digest; bump periodically. + image: fedora@sha256:43b29f65a41eb9c35e1cd5323e3bdf3b655c2357a9f4f1ff2f9c2798e5045d80 + env: + # Install the interpreter where the Apache user can read it. + UV_PYTHON_INSTALL_DIR: /opt/uv-python + # The test rounds run against the lowest direct dependency versions + # installed below; a resync would reconcile them back to the lockfile. + JUST_NO_RESYNC: "1" + steps: + # actions/checkout needs git, which fedora:latest does not include. + - name: Install git + run: dnf install -y git + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - name: Install just + run: | + uv tool install rust-just + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Install Apache + # jq and curl are needed to pick the MongoDB download. + run: dnf install -y httpd httpd-devel jq curl + - id: versions + uses: $/.github/actions/mod-wsgi-versions + with: + index: 0 + - name: Install Python + env: + PYTHON_VERSION: ${{ steps.versions.outputs.python-version }} + run: | + uv python install "$PYTHON_VERSION" + echo "UV_PYTHON=$PYTHON_VERSION" >> "$GITHUB_ENV" + - name: Install dependencies + run: | + export PYMONGO_C_EXT_MUST_BUILD=1 + just install + uv sync --group mod_wsgi --refresh-package pymongo + uv run --no-sync python tools/fail_if_no_c.py + # Re-resolve at the lowest direct versions, including the mod_wsgi + # group: uv sync above installed the locked version, but this job + # must exercise the group's mod_wsgi>=4.9.4 lower bound. + uv pip install -e ".[test]" --group mod_wsgi --resolution=lowest-direct --force-reinstall + - name: Start MongoDB + # Fedora is not a supported MongoDB platform, so install the RHEL build + # directly instead of using the drivers-evergreen-tools action, which + # relies on actions/setup-python and does not run on Fedora. + env: + MONGODB_VERSION: ${{ steps.versions.outputs.mongodb-version }} + run: | + ARCH=$(uname -m) + URL=$(curl -fsSL https://downloads.mongodb.org/current.json | jq -r --arg v "$MONGODB_VERSION." --arg a "$ARCH" ' + .versions[] | select(.current and .production_release) | select(.version | startswith($v)) | + .downloads[] | select(.target=="rhel90" and .arch==$a) | .archive.url' | grep -v enterprise | head -1) + curl -fsSL "$URL" -o /tmp/mongo.tgz + tar -xz -C /tmp --strip-components=1 -f /tmp/mongo.tgz + mkdir -p /tmp/db + /tmp/bin/mongod --replSet rs0 --bind_ip 127.0.0.1 --port 27017 --dbpath /tmp/db --fork --logpath /tmp/mongod.log + # --no-sync: a sync would reconcile the lowest-direct dependency + # versions installed above back to the lockfile. + uv run --no-sync python -c " + from pymongo import MongoClient + client = MongoClient('127.0.0.1:27017', directConnection=True) + client.admin.command('replSetInitiate', {'_id': 'rs0', 'members': [{'_id': 0, 'host': '127.0.0.1:27017'}]}) + MongoClient().admin.command('hello') + " + - name: Run standalone tests + run: | + just setup-tests mod_wsgi standalone + just run-tests + - name: Teardown standalone + if: always() + run: just teardown-tests + - name: Run embedded tests + run: | + just setup-tests mod_wsgi embedded + just run-tests + - name: Teardown embedded + if: always() + run: just teardown-tests + + mod_wsgi_latest: + runs-on: ubuntu-22.04 + name: Mod WSGI Latest + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - id: versions + uses: $/.github/actions/mod-wsgi-versions + with: + index: 1 + - name: Install Python tooling + uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 + with: + python-version: ${{ steps.versions.outputs.python-version }} + - name: Install Apache + run: sudo apt-get install -y apache2 apache2-dev + - name: Install dependencies + run: | + export PYMONGO_C_EXT_MUST_BUILD=1 + just install + uv sync --group mod_wsgi --refresh-package pymongo + uv run --no-sync python tools/fail_if_no_c.py + - id: setup-mongodb + uses: mongodb-labs/drivers-evergreen-tools@49da51cebc0060b4d699d57bdef30ee7e071de44 + with: + version: ${{ steps.versions.outputs.mongodb-version }} + topology: "replica_set" + - name: Run standalone tests + run: | + just setup-tests mod_wsgi standalone + just run-tests + - name: Teardown standalone + if: always() + run: just teardown-tests + - name: Run embedded tests + run: | + just setup-tests mod_wsgi embedded + just run-tests + - name: Teardown embedded + if: always() + run: just teardown-tests diff --git a/.gitignore b/.gitignore index 8546aeabb7..222e030ad0 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,5 @@ xunit-results/ coverage.xml server.log .coverage +access_log +error_log diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 42a8606cd9..61e369767a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -353,6 +353,24 @@ You will need to set up access to the `drivers-test-secrets-role`, see the [Wiki - Run `just setup-tests aws_lambda`. - Run `just run-tests`. +### mod_wsgi tests + +Continuous integration runs the tests on pull requests that change +mod_wsgi-relevant files, in the `test-mod-wsgi.yml` workflow. + +To run the tests by hand, install Apache and mod_wsgi (`sudo apt-get install -y apache2 +apache2-dev` and `uv sync --group mod_wsgi` on Ubuntu), then: + +- On Linux, run `TOPOLOGY=replica_set just run-server`. +- Run `just setup-tests mod_wsgi `. +- Run `just run-tests`. +- Run `just teardown-tests`. + +The `mode` can be `standalone` or `embedded`. On non-Linux hosts the same +commands run inside an ubuntu container, which bootstraps its own replica +set (so the `run-server` step does not apply), or use `just smoke-mod-wsgi` +to run both modes. + ### OCSP tests - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`. diff --git a/justfile b/justfile index df678cdbac..9b37109c74 100644 --- a/justfile +++ b/justfile @@ -12,9 +12,12 @@ mypy_args := "--install-types --non-interactive" default: @just --list +# Restore the default development environment. Set JUST_NO_RESYNC to skip the +# sync when a caller resolves dependencies differently, such as the +# minimum-dependencies job in test-mod-wsgi.yml. [private] resync: - @uv sync --quiet + @if [ -z "${JUST_NO_RESYNC:-}" ]; then uv sync --quiet; fi # Set up the development environment install: @@ -85,17 +88,17 @@ test-numpy *args="": && resync # Run tests via the Evergreen test runner script [group('test')] run-tests *args: && resync - bash ./.evergreen/run-tests.sh {{args}} + bash .evergreen/scripts/dispatch.sh run-tests {{args}} # Set up the test environment (auth, TLS, etc.) [group('test')] setup-tests *args="": - bash .evergreen/scripts/setup-tests.sh {{args}} + bash .evergreen/scripts/dispatch.sh setup-tests {{args}} # Tear down resources created by setup-tests [group('test')] -teardown-tests: - bash .evergreen/scripts/teardown-tests.sh +teardown-tests *args="": + bash .evergreen/scripts/dispatch.sh teardown-tests {{args}} [group('test')] integration-tests: @@ -124,6 +127,29 @@ coverage-xml: uv tool run --with "coverage[toml]" coverage xml @echo "Coverage report generated in coverage.xml" +# Run the mod_wsgi tests (natively on Linux, else in an ubuntu container) +[group('test')] +smoke-mod-wsgi: + #!/usr/bin/env bash + set -euo pipefail + STATUS=0 + # Tear down on an early exit (a failed setup or teardown), so a partially + # started Apache or container and the dispatch marker never outlive the + # recipe; the per-mode teardown below clears the guard. + CLEANED=1 + cleanup() { + [ "$CLEANED" -eq 1 ] || just teardown-tests || true + } + trap cleanup EXIT + for mode in standalone embedded; do + CLEANED=0 + just setup-tests mod_wsgi $mode + just run-tests || STATUS=$? + just teardown-tests + CLEANED=1 + [ "$STATUS" -eq 0 ] || exit "$STATUS" + done + # Start a MongoDB server via drivers-evergreen-tools [group('server')] run-server *args="": diff --git a/pyproject.toml b/pyproject.toml index 2f7a68c837..8083b03b46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,9 @@ unasync = [ ] pip = ["pip>=20.2"] gevent = ["gevent>=21.12"] +# mod_wsgi is built against the interpreter it is installed with and is used +# only by the mod_wsgi test suite. +mod_wsgi = ["mod_wsgi>=4.9.4"] coverage = [ "coverage[toml]>=5,<=7.15.0" ] diff --git a/test/mod_wsgi_test/README.rst b/test/mod_wsgi_test/README.rst index e96db9406c..3dddb3e5f7 100644 --- a/test/mod_wsgi_test/README.rst +++ b/test/mod_wsgi_test/README.rst @@ -13,44 +13,38 @@ mod_wsgi creates. Test Matrix ----------- -PyMongo should be tested with several versions of mod_wsgi and a selection -of Python versions. Each combination of mod_wsgi and Python version should -be tested with a standalone and a replica set. ``mod_wsgi_test.py`` -detects if the deployment is a replica set and connects to the whole set. +Continuous integration tests the oldest supported CPython against MongoDB +6.0, the oldest version that installs on the GitHub runners, with minimum +dependencies, and the newest supported CPython against the latest MongoDB, in +both daemon and embedded mode against a replica set. The Python and MongoDB +versions come from ``generate_config_utils.py`` in ``.evergreen/scripts``. +Other combinations can be tested manually. Setup ----- -Compile Python +Install Apache .............. -We need a Python interpreter built as a shared library. Download the -source tarball for each Python version tested, untar it, and run:: +On Ubuntu, install Apache and the headers used to build mod_wsgi:: - ./configure --prefix=/some/directory --enable-shared LDFLAGS="-Wl,--rpath=/some/directory/lib" - make - make install + sudo apt-get install -y apache2 apache2-dev -This results in an executable named "python" or "python3" and a shared -library named something like "libpython2.7.so.1.0" or "libpython3.3m.so.1.0". +On Fedora/RHEL, install httpd and its headers instead:: -Compile mod_wsgi -................ + sudo dnf install -y httpd httpd-devel + +The test harness detects ``apache2`` or ``httpd`` automatically and starts it +with the matching config file (``apache24ubuntu.conf`` or +``httpd24fedora.conf``). -Compile mod_wsgi for each combination for Python and mod_wsgi version in the -test matrix. For example, to compile mod_wsgi 3.4 for Python 2.7 on a -RedHat-like Linux:: +Install mod_wsgi +................ - sudo yum install -y httpd httpd-devel - wget https://modwsgi.googlecode.com/files/mod_wsgi-3.4.tar.gz - tar xzf mod_wsgi-3.4.tar.gz - cd mod_wsgi-3.4 - ./configure --with-python=/some/directory/bin/python LDFLAGS="-Wl,--rpath=/some/directory/lib" - make - make install +The project defines a ``mod_wsgi`` dependency group used for testing. pip +builds mod_wsgi against the interpreter it is installed with:: -To ease testing of several matrix combinations, copy the resulting -``mod_wsgi.so`` to a safe place. + uv sync --group mod_wsgi Start mongod ............ @@ -61,12 +55,21 @@ listening on port 27017. Configure Apache ................ -Set a MOD_WSGI_SO environment variable so our ``mod_wsgi_test.conf`` -can find and load mod_wsgi.so:: +The Apache configs expand environment variables, so export them before +starting Apache directly. All configs need ``MOD_WSGI_SO``, the path to +mod_wsgi.so, and include the mode config named by ``MOD_WSGI_CONF`` +(``mod_wsgi_test.conf`` for standalone or ``mod_wsgi_test_embedded.conf`` +for embedded mode) from the checkout at ``PROJECT_DIRECTORY``. The Fedora +config also needs ``MOD_WSGI_PYTHON_HOME``, the prefix of the interpreter +mod_wsgi was built with:: - export MOD_WSGI_SO=/path/to/mod_wsgi.so + export MOD_WSGI_SO=$(find .venv -name "mod_wsgi*.so") + export MOD_WSGI_CONF=mod_wsgi_test.conf + export PROJECT_DIRECTORY=$PWD + export MOD_WSGI_PYTHON_HOME=$(.venv/bin/python -c "import sys; print(sys.base_prefix)") -Start Apache with one of the config files in this directory. +``mod_wsgi_tester.py`` exports these variables itself, so ``just setup-tests +mod_wsgi `` (see `Automation`_) is the easier path. Run the test ------------ @@ -104,7 +107,13 @@ the workaround added in `PYTHON-569 Automation ---------- -At MongoDB, Inc. we use a continuous integration job that tests each -combination in the matrix. The job starts up Apache, starts a single server -or replica set, and runs ``test_client.py`` with the proper arguments. -See `run-mod-wsgi-tests.sh `_ +Continuous integration runs the test on pull requests that change +mod_wsgi-relevant files. The minimum dependency job runs in a Fedora +container against ``httpd`` and the latest dependency job runs on Ubuntu +against ``apache2`` (see the `test-mod-wsgi workflow +`_). +To run the same steps locally, use ``just setup-tests mod_wsgi ``, +``just run-tests``, and ``just teardown-tests``. On Linux they run against the +host's Apache, either ``apache2`` or ``httpd``; on other hosts they run in an +ubuntu container with Apache, a single-node replica set, and both test modes. +``just smoke-mod-wsgi`` runs both modes with one command. diff --git a/test/mod_wsgi_test/apache22amazon.conf b/test/mod_wsgi_test/apache22amazon.conf deleted file mode 100644 index 7755336b07..0000000000 --- a/test/mod_wsgi_test/apache22amazon.conf +++ /dev/null @@ -1,34 +0,0 @@ -# This is a minimal httpd.conf file written for Apache 2.2 on Amazon Linux - -# The modules directory is here by default. -# ServerRoot "/etc/httpd" -DocumentRoot ${PWD} -PidFile ${PWD}/apache2.pid - -User nobody -Group nobody - -# Bind to localhost only, don't require sudo. -Listen 127.0.0.1:8080 - -# Required modules. -LoadModule authz_host_module modules/mod_authz_host.so -# Needed so we can set a custom log location. -LoadModule log_config_module modules/mod_log_config.so - -ErrorLog ${PWD}/error_log -CustomLog ${PWD}/access_log combined - - - AllowOverride None - Order Deny,Allow - Deny from All - - - - AllowOverride None - Order Allow,Deny - Allow from All - - -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/apache22ubuntu1204.conf b/test/mod_wsgi_test/apache22ubuntu1204.conf deleted file mode 100644 index 9fa4b2060b..0000000000 --- a/test/mod_wsgi_test/apache22ubuntu1204.conf +++ /dev/null @@ -1,29 +0,0 @@ -# This is a minimal httpd.conf file written for Apache 2.2 on Ubuntu 12.04 - -# The modules directory is here on Ubuntu. -ServerRoot "/usr/lib/apache2" -DocumentRoot ${PWD} -PidFile ${PWD}/apache2.pid - -# Bind to localhost only, don't require sudo. -Listen 127.0.0.1:8080 - -# Required modules. -LoadModule authz_host_module modules/mod_authz_host.so - -ErrorLog ${PWD}/error_log -CustomLog ${PWD}/access_log combined - - - AllowOverride None - Order Deny,Allow - Deny from All - - - - AllowOverride None - Order Allow,Deny - Allow from All - - -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/apache24ubuntu.conf b/test/mod_wsgi_test/apache24ubuntu.conf new file mode 100644 index 0000000000..63161de86b --- /dev/null +++ b/test/mod_wsgi_test/apache24ubuntu.conf @@ -0,0 +1,47 @@ +# This is a minimal httpd.conf file written for Apache 2.4 on Ubuntu. + +# The modules directory is here on Ubuntu. +ServerRoot "/usr/lib/apache2" + +# mod_wsgi 6 destroys the interpreter as child processes exit. pymongo shuts +# its monitor threads down during interpreter finalization, so the default +# lifecycle (WSGIDestroyInterpreter On) is safe to use. +DocumentRoot ${PWD} +PidFile ${PWD}/apache2.pid + +# Bind to localhost only, don't require sudo. +Listen 127.0.0.1:8080 + +# Required modules. +LoadModule mpm_prefork_module modules/mod_mpm_prefork.so +LoadModule authz_core_module modules/mod_authz_core.so +# Loaded before the mod_wsgi directives in the included test config. +LoadModule wsgi_module ${MOD_WSGI_SO} + +# Bound the number of worker processes. mod_wsgi embedded mode loads a full +# Python interpreter and PyMongo in every worker, so the parallel test's +# concurrent requests would otherwise ramp prefork up to its default +# MaxRequestWorkers (256) and exhaust the container's memory. + + StartServers 2 + MinSpareServers 2 + MaxSpareServers 4 + ServerLimit 16 + MaxRequestWorkers 16 + MaxConnectionsPerChild 0 + + +ErrorLog ${PWD}/error_log +CustomLog ${PWD}/access_log combined + + + AllowOverride None + Require all denied + + + + AllowOverride None + Require all granted + + +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/apache24ubuntu161404.conf b/test/mod_wsgi_test/apache24ubuntu161404.conf deleted file mode 100644 index eb5414f0f7..0000000000 --- a/test/mod_wsgi_test/apache24ubuntu161404.conf +++ /dev/null @@ -1,28 +0,0 @@ -# This is a minimal httpd.conf file written for Apache 2.4 on Ubuntu 14.04/16.04 - -# The modules directory is here on Ubuntu. -ServerRoot "/usr/lib/apache2" -DocumentRoot ${PWD} -PidFile ${PWD}/apache2.pid - -# Bind to localhost only, don't require sudo. -Listen 127.0.0.1:8080 - -# Required modules. -LoadModule mpm_prefork_module modules/mod_mpm_prefork.so -LoadModule authz_core_module modules/mod_authz_core.so - -ErrorLog ${PWD}/error_log -CustomLog ${PWD}/access_log combined - - - AllowOverride None - Require all denied - - - - AllowOverride None - Require all granted - - -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/httpd24fedora.conf b/test/mod_wsgi_test/httpd24fedora.conf new file mode 100644 index 0000000000..6693b75447 --- /dev/null +++ b/test/mod_wsgi_test/httpd24fedora.conf @@ -0,0 +1,58 @@ +# This is a minimal httpd.conf file written for Apache 2.4 on Fedora/RHEL. + +# The modules directory is here on Fedora/RHEL. +ServerRoot "/etc/httpd" +DefaultRuntimeDir ${PWD} + +DocumentRoot ${PWD} +PidFile ${PWD}/httpd.pid + +# Bind to localhost only, don't require sudo. +Listen 127.0.0.1:8080 + +# Required modules. +LoadModule mpm_prefork_module modules/mod_mpm_prefork.so +LoadModule authz_core_module modules/mod_authz_core.so +LoadModule unixd_module modules/mod_unixd.so +LoadModule log_config_module modules/mod_log_config.so +# Loaded before any mod_wsgi directive below. +LoadModule wsgi_module ${MOD_WSGI_SO} + +# mod_wsgi is built against the interpreter used to install it. When Apache is +# started by a different executable than that interpreter, the embedded Python +# cannot locate its standard library, so point it at the installation. +WSGIPythonHome "${MOD_WSGI_PYTHON_HOME}" + +# Bound the number of worker processes. mod_wsgi embedded mode loads a full +# Python interpreter and PyMongo in every worker, so the parallel test's +# concurrent requests would otherwise ramp prefork up to its default +# MaxRequestWorkers (256) and exhaust the container's memory. + + StartServers 2 + MinSpareServers 2 + MaxSpareServers 4 + ServerLimit 16 + MaxRequestWorkers 16 + MaxConnectionsPerChild 0 + + +ServerName localhost + +ErrorLog ${PWD}/error_log +CustomLog ${PWD}/access_log combined + +# Apache must not run as root. The httpd package creates this user. +User apache +Group apache + + + AllowOverride None + Require all denied + + + + AllowOverride None + Require all granted + + +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/mod_wsgi_test.conf b/test/mod_wsgi_test/mod_wsgi_test.conf index a5b09e437f..b4cbaa083e 100644 --- a/test/mod_wsgi_test/mod_wsgi_test.conf +++ b/test/mod_wsgi_test/mod_wsgi_test.conf @@ -14,8 +14,6 @@ # Minimal test of PyMongo in a WSGI application, see bug PYTHON-353 -LoadModule wsgi_module ${MOD_WSGI_SO} - # Avoid permissions issues WSGISocketPrefix /tmp/ diff --git a/test/mod_wsgi_test/mod_wsgi_test_embedded.conf b/test/mod_wsgi_test/mod_wsgi_test_embedded.conf index 306dab4ab6..970803e586 100644 --- a/test/mod_wsgi_test/mod_wsgi_test_embedded.conf +++ b/test/mod_wsgi_test/mod_wsgi_test_embedded.conf @@ -14,8 +14,6 @@ # Minimal test of PyMongo in an *Embedded mode* WSGI application. -LoadModule wsgi_module ${MOD_WSGI_SO} - # Avoid permissions issues WSGISocketPrefix /tmp/ diff --git a/uv.lock b/uv.lock index f2ae10151d..f7fe85a6ca 100644 --- a/uv.lock +++ b/uv.lock @@ -58,9 +58,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "idna", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "exceptiongroup" }, + { name = "idna" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" } wheels = [ @@ -76,9 +76,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "idna", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.15'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } wheels = [ @@ -358,9 +358,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "jmespath", marker = "python_full_version < '3.10'" }, - { name = "s3transfer", version = "0.16.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" } }, + { name = "jmespath" }, + { name = "s3transfer", version = "0.16.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/7d/5c6fa0bb9fd5caf865b9356411793900304328bcd0bc1eda96a32a1368a6/boto3-1.42.97.tar.gz", hash = "sha256:2833dbeda3670ea610ad48dff7d27cdc829dbbfcdfbc6b750b673948e949b6f0", size = 113217, upload-time = "2026-04-27T20:39:17.646Z" } wheels = [ @@ -376,9 +376,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "jmespath", marker = "python_full_version >= '3.10'" }, - { name = "s3transfer", version = "0.19.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" } }, + { name = "jmespath" }, + { name = "s3transfer", version = "0.19.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/d5/0f/a1d691e9c51ae5dce5a7394e550d7e36a995494e1c2795ce210d3520b257/boto3-1.43.94.tar.gz", hash = "sha256:aa832d88e85035cebb83f04700c65623328b88fe828ff7ed9539483bd9d608c5", size = 112663, upload-time = "2026-09-14T20:10:05.902Z" } wheels = [ @@ -394,9 +394,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "jmespath", marker = "python_full_version < '3.10'" }, - { name = "python-dateutil", marker = "python_full_version < '3.10'" }, - { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c6/95/c37edb602948fad2253ffd1bb3dba5b938645bd1845ee4160350136a0f41/botocore-1.42.97.tar.gz", hash = "sha256:5c0bb00e32d16ff6d278cc8c9e10dc3672d9c1d569031635ac3c908a60de8310", size = 15269348, upload-time = "2026-04-27T20:39:05.625Z" } wheels = [ @@ -412,9 +412,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "jmespath", marker = "python_full_version >= '3.10'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.10'" }, - { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/5b/5a7cc096db8ddb5f6c221112d5a4a46888d9e38c1f5d5a06ae7e3e0755f2/botocore-1.43.94.tar.gz", hash = "sha256:cb97cedec5fceee3261f7fff75d0b0aa2cf68b32eeb2e8c2649e4657419773a6", size = 16104907, upload-time = "2026-09-14T20:10:00.315Z" } wheels = [ @@ -439,7 +439,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "pycparser", version = "2.23", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and implementation_name != 'PyPy'" }, + { name = "pycparser", version = "2.23", source = { registry = "https://pypi.org/simple" }, marker = "implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -537,7 +537,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "pycparser", version = "3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and implementation_name != 'PyPy'" }, + { name = "pycparser", version = "3.0", source = { registry = "https://pypi.org/simple" }, marker = "implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } wheels = [ @@ -830,7 +830,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } wheels = [ @@ -976,7 +976,7 @@ wheels = [ [package.optional-dependencies] toml = [ - { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "tomli" }, ] [[package]] @@ -1083,7 +1083,7 @@ wheels = [ [package.optional-dependencies] toml = [ - { name = "tomli", marker = "python_full_version >= '3.10' and python_full_version <= '3.11'" }, + { name = "tomli", marker = "python_full_version <= '3.11'" }, ] [[package]] @@ -1353,8 +1353,8 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version <= '3.9' and platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version <= '3.9'" }, + { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation != 'PyPy'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/b2/7ffa7fe8207a8c42147ffe70c3e360b228160c1d85dc3faff16aaa3244c0/cryptography-47.0.0.tar.gz", hash = "sha256:9f8e55fe4e63613a5e1cc5819030f27b97742d720203a087802ce4ce9ceb52bb", size = 830863, upload-time = "2026-04-24T19:54:57.056Z" } wheels = [ @@ -1418,9 +1418,9 @@ resolution-markers = [ "python_full_version > '3.9' and python_full_version < '3.10'", ] dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.9' and python_full_version < '3.10' and platform_python_implementation != 'PyPy'" }, + { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and platform_python_implementation != 'PyPy'" }, { name = "cffi", version = "2.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version > '3.9' and python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bb/ad/5d6702db60b1e40b41ef513b6967ff5848f307d50f8449baf1634f5908f1/cryptography-50.0.1.tar.gz", hash = "sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20", size = 880381, upload-time = "2026-08-25T19:45:45.499Z" } wheels = [ @@ -1520,7 +1520,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1554,10 +1554,10 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, - { name = "greenlet", version = "3.2.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and platform_python_implementation == 'CPython'" }, - { name = "zope-event", version = "6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "zope-interface", version = "8.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, + { name = "greenlet", version = "3.2.5", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython'" }, + { name = "zope-event", version = "6.0", source = { registry = "https://pypi.org/simple" } }, + { name = "zope-interface", version = "8.0.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/9f/5c/92002455a57cb3634383e2b822e3bccf409f43cde34528e46428971475cf/gevent-26.7.0.tar.gz", hash = "sha256:5b333a556e38a302b1b8c80525bef16d437e16f1e7767947789406841856a102", size = 6729213, upload-time = "2026-07-22T20:16:04.713Z" } wheels = [ @@ -1617,10 +1617,10 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "cffi", version = "2.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, - { name = "greenlet", version = "3.5.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and platform_python_implementation == 'CPython'" }, - { name = "zope-event", version = "6.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "zope-interface", version = "8.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "cffi", version = "2.1.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, + { name = "greenlet", version = "3.5.6", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython'" }, + { name = "zope-event", version = "6.2", source = { registry = "https://pypi.org/simple" } }, + { name = "zope-interface", version = "8.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/b8/eb/5f2db8013f1a4a6df2c23201f384a066f13ff5764a9f62a608c8a50ac8cc/gevent-26.8.0.tar.gz", hash = "sha256:96039f41bbde6dcd72559e5ffbd408a04f46774b47d991d4cf032da8fa79e5a0", size = 6625998, upload-time = "2026-08-10T18:02:28.038Z" } wheels = [ @@ -1903,7 +1903,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "zipp", version = "3.23.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "zipp", version = "3.23.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } wheels = [ @@ -1919,7 +1919,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "zipp", version = "4.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "zipp", version = "4.1.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/7e/1e7e8dc30634b93ebb3d58a3dea569ad146e656218d3960ab04f62047b29/importlib_metadata-9.0.1.tar.gz", hash = "sha256:ab830580bc0ef3db61ce8fae716389e5462b67e033018bab6d8f80ef17172f99", size = 59124, upload-time = "2026-08-28T15:30:34.646Z" } wheels = [ @@ -2225,6 +2225,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/c0/bc0c3e82bf91993bf2337b125c4d49c3cadae39f5a4c642baada958f976e/mockupdb-1.8.1-py2.py3-none-any.whl", hash = "sha256:8eb76212963bad5df7f3ea8f26af916087c2dcc69a0457c75328abad44956dbb", size = 27578, upload-time = "2021-10-15T19:02:55.489Z" }, ] +[[package]] +name = "mod-wsgi" +version = "5.0.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version > '3.9' and python_full_version < '3.10'", + "python_full_version <= '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/ca/4d/275853efa4d2a4efe1614aa0df06295e71f2082cebccc4d40d0f0a551c24/mod_wsgi-5.0.2.tar.gz", hash = "sha256:6fe5557c3ef25c184c2c994905fe52be1c06ae5c46235ecafe558e84789e2159", size = 498627, upload-time = "2024-11-22T08:53:22.752Z" } + +[[package]] +name = "mod-wsgi" +version = "6.0.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/ce/fdbfe51f596e37c657176ea64d637737074214bfcd21c2177aa6b1f1da14/mod_wsgi-6.0.6.tar.gz", hash = "sha256:897f58ee961be7dcfe527074dd64d43d18ca108cf919d4c3bcf10fa6a4c12921", size = 607559, upload-time = "2026-08-19T09:51:18.52Z" } + [[package]] name = "mypy" version = "1.19.1" @@ -2463,6 +2483,10 @@ lint = [ mockupdb = [ { name = "mockupdb" }, ] +mod-wsgi = [ + { name = "mod-wsgi", version = "5.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "mod-wsgi", version = "6.0.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] perf = [ { name = "simplejson" }, ] @@ -2516,6 +2540,7 @@ dev = [] gevent = [{ name = "gevent", specifier = ">=21.12" }] lint = [{ name = "ruff", specifier = "==0.15.20" }] mockupdb = [{ name = "mockupdb", specifier = ">=1.8.1" }] +mod-wsgi = [{ name = "mod-wsgi", specifier = ">=4.9.4" }] perf = [{ name = "simplejson", specifier = ">=3.17.0" }] pip = [{ name = "pip", specifier = ">=20.2" }] typing = [ @@ -2572,8 +2597,8 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "cryptography", version = "47.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version <= '3.9'" }, - { name = "typing-extensions", marker = "python_full_version <= '3.9'" }, + { name = "cryptography", version = "47.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1a/51/27a5ad5f939d08f690a326ef9582cda7140555180db71695f6fb747d6a36/pyopenssl-26.2.0.tar.gz", hash = "sha256:8c6fcecd1183a7fc897548dfe388b0cdb7f37e018200d8409cf33959dbe35387", size = 182195, upload-time = "2026-05-04T23:06:09.72Z" } wheels = [ @@ -2590,8 +2615,8 @@ resolution-markers = [ "python_full_version > '3.9' and python_full_version < '3.10'", ] dependencies = [ - { name = "cryptography", version = "50.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.9'" }, - { name = "typing-extensions", marker = "python_full_version > '3.9' and python_full_version < '3.13'" }, + { name = "cryptography", version = "50.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3f/e8/7325d258199b159eb2c03fe32107533e2832e70e63f4fb88a6aa00023201/pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7", size = 182046, upload-time = "2026-08-01T19:50:50.512Z" } wheels = [ @@ -2620,13 +2645,13 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "pluggy", marker = "python_full_version < '3.10'" }, - { name = "pygments", marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup" }, + { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } wheels = [ @@ -2642,13 +2667,13 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "pluggy", marker = "python_full_version >= '3.10'" }, - { name = "pygments", marker = "python_full_version >= '3.10'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ @@ -2664,9 +2689,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "backports-asyncio-runner", marker = "python_full_version < '3.10'" }, - { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "backports-asyncio-runner" }, + { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/86/9e3c5f48f7b7b638b216e4b9e645f54d199d7abbbab7a64a13b4e12ba10f/pytest_asyncio-1.2.0.tar.gz", hash = "sha256:c609a64a2a8768462d0c99811ddb8bd2583c33fd33cf7f21af1c142e824ffb57", size = 50119, upload-time = "2025-09-12T07:33:53.816Z" } wheels = [ @@ -2682,9 +2707,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "backports-asyncio-runner", marker = "python_full_version == '3.10.*'" }, - { name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" }, + { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, + { name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } wheels = [ @@ -2734,10 +2759,10 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "certifi", marker = "python_full_version < '3.10'" }, - { name = "charset-normalizer", marker = "python_full_version < '3.10'" }, - { name = "idna", marker = "python_full_version < '3.10'" }, - { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } wheels = [ @@ -2753,10 +2778,10 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "certifi", marker = "python_full_version >= '3.10'" }, - { name = "charset-normalizer", marker = "python_full_version >= '3.10'" }, - { name = "idna", marker = "python_full_version >= '3.10'" }, - { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ @@ -2777,7 +2802,7 @@ name = "roman-numerals-py" version = "4.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "roman-numerals", marker = "python_full_version >= '3.11'" }, + { name = "roman-numerals" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/b5/de96fca640f4f656eb79bbee0e79aeec52e3e0e359f8a3e6a0d366378b64/roman_numerals_py-4.1.0.tar.gz", hash = "sha256:f5d7b2b4ca52dd855ef7ab8eb3590f428c0b1ea480736ce32b01fef2a5f8daf9", size = 4274, upload-time = "2025-12-17T18:25:41.153Z" } wheels = [ @@ -2818,7 +2843,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/29/af14f4ef3c11a50435308660e2cc68761c9a7742475e0585cd4396b91777/s3transfer-0.16.1.tar.gz", hash = "sha256:8e424355754b9ccb32467bdc568edf55be82692ef2002d934b1311dbb3b9e524", size = 154801, upload-time = "2026-04-22T20:36:06.475Z" } wheels = [ @@ -2834,7 +2859,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } wheels = [ @@ -3020,24 +3045,24 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "alabaster", version = "0.7.16", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "babel", marker = "python_full_version < '3.10'" }, - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version < '3.10'" }, - { name = "imagesize", version = "1.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "importlib-metadata", version = "8.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "jinja2", marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "pygments", marker = "python_full_version < '3.10'" }, - { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "snowballstemmer", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "alabaster", version = "0.7.16", source = { registry = "https://pypi.org/simple" } }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize", version = "1.5.0", source = { registry = "https://pypi.org/simple" } }, + { name = "importlib-metadata", version = "8.7.1", source = { registry = "https://pypi.org/simple" } }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" } }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/be/50e50cb4f2eff47df05673d361095cafd95521d2a22521b920c67a372dcb/sphinx-7.4.7.tar.gz", hash = "sha256:242f92a7ea7e6c5b406fdc2615413890ba9f699114a9c09192d7dfead2ee9cfe", size = 8067911, upload-time = "2024-07-20T14:46:56.059Z" } wheels = [ @@ -3052,23 +3077,23 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "babel", marker = "python_full_version == '3.10.*'" }, - { name = "colorama", marker = "python_full_version == '3.10.*' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version == '3.10.*'" }, - { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "jinja2", marker = "python_full_version == '3.10.*'" }, - { name = "packaging", marker = "python_full_version == '3.10.*'" }, - { name = "pygments", marker = "python_full_version == '3.10.*'" }, - { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "snowballstemmer", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version == '3.10.*'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" } }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/be0b61178fe2cdcb67e2a92fc9ebb488e3c51c4f74a36a7824c0adf23425/sphinx-8.1.3.tar.gz", hash = "sha256:43c1911eecb0d3e161ad78611bc905d1ad0e523e4ddc202a58a821773dc4c927", size = 8184611, upload-time = "2024-10-13T20:27:13.93Z" } wheels = [ @@ -3083,23 +3108,23 @@ resolution-markers = [ "python_full_version >= '3.11'", ] dependencies = [ - { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "babel", marker = "python_full_version >= '3.11'" }, - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version >= '3.11'" }, - { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "jinja2", marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "roman-numerals-py", marker = "python_full_version >= '3.11'" }, - { name = "snowballstemmer", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.11'" }, + { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" } }, + { name = "roman-numerals-py" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/38/ad/4360e50ed56cb483667b8e6dadf2d3fda62359593faabbe749a27c4eaca6/sphinx-8.2.3.tar.gz", hash = "sha256:398ad29dee7f63a75888314e9424d40f52ce5a6a87ae88e7071e80af296ec348", size = 8321876, upload-time = "2025-03-02T22:31:59.658Z" } wheels = [ @@ -3116,16 +3141,16 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "colorama" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "starlette", version = "0.49.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "starlette", version = "0.49.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "starlette", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "uvicorn", version = "0.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "uvicorn", version = "0.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "uvicorn", version = "0.53.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "watchfiles", version = "1.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "watchfiles", version = "1.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "watchfiles", version = "1.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "websockets", version = "16.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a5/2c/155e1de2c1ba96a72e5dba152c509a8b41e047ee5c2def9e9f0d812f8be7/sphinx_autobuild-2024.10.3.tar.gz", hash = "sha256:248150f8f333e825107b6d4b86113ab28fa51750e5f9ae63b59dc339be951fb1", size = 14023, upload-time = "2024-10-02T23:15:30.172Z" } @@ -3141,12 +3166,12 @@ resolution-markers = [ "python_full_version >= '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "starlette", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "uvicorn", version = "0.53.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "watchfiles", version = "1.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "websockets", version = "17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "colorama" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" } }, + { name = "starlette", version = "1.6.0", source = { registry = "https://pypi.org/simple" } }, + { name = "uvicorn", version = "0.53.0", source = { registry = "https://pypi.org/simple" } }, + { name = "watchfiles", version = "1.2.0", source = { registry = "https://pypi.org/simple" } }, + { name = "websockets", version = "17.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/e0/3c/a59a3a453d4133777f7ed2e83c80b7dc817d43c74b74298ca0af869662ad/sphinx_autobuild-2025.8.25.tar.gz", hash = "sha256:9cf5aab32853c8c31af572e4fecdc09c997e2b8be5a07daf2a389e270e85b213", size = 15200, upload-time = "2025-08-25T18:44:55.436Z" } wheels = [ @@ -3280,8 +3305,8 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/de/1a/608df0b10b53b0beb96a37854ee05864d182ddd4b1156a22f1ad3860425a/starlette-0.49.3.tar.gz", hash = "sha256:1c14546f299b5901a1ea0e34410575bc33bbd741377a10484a54445588d00284", size = 2655031, upload-time = "2025-11-01T15:12:26.13Z" } wheels = [ @@ -3297,8 +3322,8 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" }, + { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b5/b4/205b0d5241d934e8add0c38aa924c4f9fb7330834ff11e5444db964ec3f9/starlette-1.6.0.tar.gz", hash = "sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b", size = 2716969, upload-time = "2026-08-08T18:27:57.512Z" } wheels = [ @@ -3426,9 +3451,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "h11", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" } }, + { name = "h11" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ae/4f/f9fdac7cf6dd79790eb165639b5c452ceeabc7bbabbba4569155470a287d/uvicorn-0.39.0.tar.gz", hash = "sha256:610512b19baa93423d2892d7823741f6d27717b642c8964000d7194dded19302", size = 82001, upload-time = "2025-12-21T13:05:17.973Z" } wheels = [ @@ -3444,9 +3469,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "click", version = "8.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "h11", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, + { name = "click", version = "8.5.0", source = { registry = "https://pypi.org/simple" } }, + { name = "h11" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5d/ad/04bbb797c84fc1f26cb171f7394716f4865ffb8d8c5e1eef42565c2dfa6b/uvicorn-0.53.0.tar.gz", hash = "sha256:a9356f0cb89b3b8621529c5d5eebd69bfe154f4c3f68b4cf2de47e45fa855c2e", size = 110881, upload-time = "2026-09-14T07:44:23.815Z" } wheels = [ @@ -3462,7 +3487,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440, upload-time = "2025-10-14T15:06:21.08Z" } wheels = [ @@ -3585,7 +3610,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } wheels = [ @@ -4135,7 +4160,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "setuptools", version = "82.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "setuptools", version = "82.0.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/d8/9c8b0c6bb1db09725395618f68d3b8a08089fca0aed28437500caaf713ee/zope_event-6.0.tar.gz", hash = "sha256:0ebac894fa7c5f8b7a89141c272133d8c1de6ddc75ea4b1f327f00d1f890df92", size = 18731, upload-time = "2025-09-12T07:10:13.551Z" } wheels = [ From 63c7a81fbc66eb4ceccb9c955c4a34c64734b584 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 28 Sep 2026 12:20:28 -0500 Subject: [PATCH 04/24] PYTHON-6040 Preserve version-to-name mapping in Client Metadata (#3054) Co-authored-by: Noah Stapp --- doc/changelog.rst | 15 ++ pymongo/driver_info.py | 12 +- pymongo/pool_options.py | 196 ++++++++++++++++------ test/asynchronous/test_client.py | 143 ++++++++++++++-- test/asynchronous/test_client_metadata.py | 177 ++++++++++++++++--- test/mockupdb/test_handshake.py | 4 +- test/test_client.py | 143 ++++++++++++++-- test/test_client_metadata.py | 177 ++++++++++++++++--- test/utils_shared.py | 36 ++++ 9 files changed, 768 insertions(+), 135 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 08d3908e62..6082f30dc5 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -23,8 +23,23 @@ Bug fixes - Fixed a bug where the synchronous client could permanently deadlock under gevent when a greenlet was killed while checking a connection back into the pool (`PYTHON-6074`_). +- ``MongoClient.append_metadata()`` and ``AsyncMongoClient.append_metadata()`` + now detect duplicates by comparing the whole + :class:`~pymongo.driver_info.DriverInfo` instead of only its name. The + comparison is exact, so drivers that differ in name case or platform are no + longer treated as duplicates (`PYTHON-6040`_). +- ``driver.name`` and ``driver.version`` in the handshake metadata are now + ``|``-delimited lists with 1:1 index correspondence, including empty version + entries for the built-in ``|c`` and ``|async`` name segments + (`PYTHON-6040`_). +- Fixed a bug where truncating the handshake metadata to 512 bytes could leave + ``driver.name`` and ``driver.version`` with different numbers of ``|`` + delimiters (`PYTHON-6040`_). +- :class:`~pymongo.driver_info.DriverInfo` now raises :class:`ValueError` when + any field contains the reserved ``|`` delimiter (`PYTHON-6040`_). .. _PYTHON-6074: https://jira.mongodb.org/browse/PYTHON-6074 +.. _PYTHON-6040: https://jira.mongodb.org/browse/PYTHON-6040 Changes in Version 4.18.2 (2026/09/24) -------------------------------------- diff --git a/pymongo/driver_info.py b/pymongo/driver_info.py index 18a51ae638..a325ea1493 100644 --- a/pymongo/driver_info.py +++ b/pymongo/driver_info.py @@ -29,8 +29,14 @@ class DriverInfo(namedtuple("DriverInfo", ["name", "version", "platform"])): The MongoDB server logs PyMongo's name, version, and platform whenever PyMongo establishes a connection. A driver implemented on top of PyMongo can add its own info to this log message. Initialize with three strings - like 'MyDriver', '1.2.3', 'some platform info'. Any of these strings may be - None to accept PyMongo's default. + like 'MyDriver', '1.2.3', 'some platform info'. Any of these strings may + be None. A None or empty name or version appends an empty metadata + segment, keeping the ``driver.name`` and ``driver.version`` segment + counts aligned. A None or empty platform omits the platform. + + The ``|`` character is the reserved delimiter used to join appended + metadata, so it must not appear in any of the fields. A + :class:`ValueError` is raised if it does. """ def __new__( @@ -42,5 +48,7 @@ def __new__( raise TypeError( f"Wrong type for DriverInfo {key} option, value must be an instance of str, not {type(value)}" ) + if value and "|" in value: + raise ValueError(f"DriverInfo {key} must not contain the '|' delimiter") return self diff --git a/pymongo/pool_options.py b/pymongo/pool_options.py index 8b26b4baf2..dd95db525e 100644 --- a/pymongo/pool_options.py +++ b/pymongo/pool_options.py @@ -24,6 +24,7 @@ import platform import sys from collections.abc import MutableMapping +from contextlib import AbstractContextManager, nullcontext from pathlib import Path from typing import TYPE_CHECKING, Any, Optional @@ -37,6 +38,7 @@ WAIT_QUEUE_TIMEOUT, has_c, ) +from pymongo.lock import _create_lock if TYPE_CHECKING: from pymongo.auth_shared import MongoCredential @@ -200,60 +202,117 @@ def _metadata_env() -> dict[str, Any]: _MAX_METADATA_SIZE = 512 +def _truncate_utf8(content: str, overflow: int) -> str: + """Trim `overflow` UTF-8 bytes from the end of content, keeping a valid prefix.""" + if overflow <= 0: + return content + data = content.encode("utf-8") + if len(data) <= overflow: + return "" + return data[: len(data) - overflow].decode("utf-8", errors="ignore") + + +def _normalize_driver(driver: DriverInfo) -> DriverInfo: + """Treat None and "" as equivalent unset fields for deduplication.""" + return driver._replace( + name=driver.name or "", + version=driver.version or "", + platform=driver.platform or "", + ) + + +def _element_size(key: str, value: Any) -> int: + """Size in bytes of the BSON element for ``key``, excluding doc overhead.""" + # bson.encode({key: value}) is 4 bytes of document header plus 1 byte of + # document terminator on top of the element itself. + return len(bson.encode({key: value})) - 5 + + # See: https://github.com/mongodb/specifications/blob/master/source/mongodb-handshake/handshake.md#limitations def _truncate_metadata(metadata: MutableMapping[str, Any]) -> None: """Perform metadata truncation.""" - if len(bson.encode(metadata)) <= _MAX_METADATA_SIZE: + # The only full encode is the initial size check; each step then shrinks + # the tracked size by the exact bytes its change removes. + size = len(bson.encode(metadata)) + if size <= _MAX_METADATA_SIZE: return # 1. Omit fields from env except env.name. env_name = metadata.get("env", {}).get("name") if env_name: - metadata["env"] = {"name": env_name} - if len(bson.encode(metadata)) <= _MAX_METADATA_SIZE: + env = {"name": env_name} + size += _element_size("env", env) - _element_size("env", metadata["env"]) + metadata["env"] = env + if size <= _MAX_METADATA_SIZE: return # 2. Omit fields from os except os.type. os_type = metadata.get("os", {}).get("type") if os_type: - metadata["os"] = {"type": os_type} - if len(bson.encode(metadata)) <= _MAX_METADATA_SIZE: + old_os = metadata["os"] + new_os = {"type": os_type} + size += _element_size("os", new_os) - _element_size("os", old_os) + metadata["os"] = new_os + if size <= _MAX_METADATA_SIZE: return # 3. Omit the env document entirely. - metadata.pop("env", None) - encoded_size = len(bson.encode(metadata)) - if encoded_size <= _MAX_METADATA_SIZE: + env = metadata.pop("env", None) + if env is not None: + size -= _element_size("env", env) + if size <= _MAX_METADATA_SIZE: return # 4. Truncate platform. - overflow = encoded_size - _MAX_METADATA_SIZE + overflow = size - _MAX_METADATA_SIZE plat = metadata.get("platform", "") if plat: - plat = plat[:-overflow] - if plat: - metadata["platform"] = plat + truncated = _truncate_utf8(plat, overflow) + if truncated: + size += _element_size("platform", truncated) - _element_size("platform", plat) + else: + size -= _element_size("platform", plat) + if truncated: + metadata["platform"] = truncated + else: + del metadata["platform"] else: - metadata.pop("platform", None) - encoded_size = len(bson.encode(metadata)) - if encoded_size <= _MAX_METADATA_SIZE: - return - # 5. Truncate driver info. - overflow = encoded_size - _MAX_METADATA_SIZE + # The platform field may be present but empty. + plat = metadata.pop("platform", None) + if plat is not None: + size -= _element_size("platform", plat) + # 5. Truncate driver info, keeping name and version 1:1 index-aligned. driver = metadata.get("driver", {}) if driver: - # Truncate driver version. - driver_version = driver.get("version")[:-overflow] - if len(driver_version) >= len(_METADATA["driver"]["version"]): - metadata["driver"]["version"] = driver_version - else: - metadata["driver"]["version"] = _METADATA["driver"]["version"] - encoded_size = len(bson.encode(metadata)) - if encoded_size <= _MAX_METADATA_SIZE: - return - # Truncate driver name. - overflow = encoded_size - _MAX_METADATA_SIZE - driver_name = driver.get("name")[:-overflow] - if len(driver_name) >= len(_METADATA["driver"]["name"]): - metadata["driver"]["name"] = driver_name - else: - metadata["driver"]["name"] = _METADATA["driver"]["name"] + # Keep the name and version segments paired so they stay 1:1 aligned, + # trimming wrapper content and dropping paired segments only as a + # last resort. Blank pairs (for example from a platform-only appended + # driver) are dropped first since they cost nothing. + name_str = driver.get("name", "") + version_str = driver.get("version", "") + raw_pairs = list(zip(name_str.split("|"), version_str.split("|"))) + pairs = [(name, version) for name, version in raw_pairs if name or version] + new_name = "|".join(name for name, _ in pairs) + new_version = "|".join(version for _, version in pairs) + size += (len(new_name.encode("utf-8")) - len(name_str.encode("utf-8"))) + ( + len(new_version.encode("utf-8")) - len(version_str.encode("utf-8")) + ) + driver["name"] = new_name + driver["version"] = new_version + overflow = size - _MAX_METADATA_SIZE + while overflow > 0 and len(pairs) > 1: + # A single remaining pair never exceeds the limit: it is the base + # driver pair, which steps 1-4 left small enough to fit. + last_name, last_version = pairs[-1] + if last_version: + new_version = _truncate_utf8(last_version, overflow) + overflow -= len(last_version.encode("utf-8")) - len(new_version.encode("utf-8")) + pairs[-1] = (last_name, new_version) + elif last_name: + new_name = _truncate_utf8(last_name, overflow) + overflow -= len(last_name.encode("utf-8")) - len(new_name.encode("utf-8")) + pairs[-1] = (new_name, last_version) + else: + pairs.pop() + overflow -= len(f"|{last_name}|{last_version}".encode()) + driver["name"] = "|".join(name for name, _ in pairs) + driver["version"] = "|".join(version for _, version in pairs) # If the first getaddrinfo call of this interpreter's life is on a thread, @@ -277,6 +336,7 @@ class PoolOptions: """ __slots__ = ( + "__appended_drivers", "__appname", "__compression_settings", "__connect_timeout", @@ -288,6 +348,7 @@ class PoolOptions: "__max_idle_time_seconds", "__max_pool_size", "__metadata", + "__metadata_lock", "__min_pool_size", "__pause_enabled", "__server_api", @@ -336,6 +397,11 @@ def __init__( self.__load_balanced = load_balanced self.__credentials = credentials self.__metadata = copy.deepcopy(_METADATA) + self.__appended_drivers: set[DriverInfo] = set() + # Only the synchronous client can append metadata from multiple threads. + self.__metadata_lock: AbstractContextManager[bool | None] = ( + _create_lock() if is_sync else nullcontext() + ) if appname: self.__metadata["application"] = {"name": appname} @@ -353,11 +419,19 @@ def __init__( self.__metadata["driver"]["name"], "c", ) + self.__metadata["driver"]["version"] = "{}|{}".format( + self.__metadata["driver"]["version"], + "", + ) if not is_sync: self.__metadata["driver"]["name"] = "{}|{}".format( self.__metadata["driver"]["name"], "async", ) + self.__metadata["driver"]["version"] = "{}|{}".format( + self.__metadata["driver"]["version"], + "", + ) if driver: self._update_metadata(driver) @@ -368,28 +442,42 @@ def __init__( _truncate_metadata(self.__metadata) def _update_metadata(self, driver: DriverInfo) -> None: - """Updates the client's metadata""" - if driver.name and driver.name.lower() in self.__metadata["driver"]["name"].lower().split( - "|" - ): - return - - metadata = copy.deepcopy(self.__metadata) - - if driver.name: - metadata["driver"]["name"] = "{}|{}".format( - metadata["driver"]["name"], - driver.name, - ) - if driver.version: + """Updates the client's metadata.""" + with self.__metadata_lock: + driver = _normalize_driver(driver) + if driver in self.__appended_drivers: + return + + # Only the top-level keys and the "driver" document are mutated, + # so shallow copies of those two are enough. + metadata = {**self.__metadata, "driver": dict(self.__metadata["driver"])} + + metadata["driver"]["name"] = "{}|{}".format(metadata["driver"]["name"], driver.name) metadata["driver"]["version"] = "{}|{}".format( - metadata["driver"]["version"], - driver.version, + metadata["driver"]["version"], driver.version ) - if driver.platform: - metadata["platform"] = "{}|{}".format(metadata["platform"], driver.platform) - - self.__metadata = metadata + if driver.platform: + if "platform" in metadata: + metadata["platform"] = "{}|{}".format(metadata["platform"], driver.platform) + else: + metadata["platform"] = driver.platform + + _truncate_metadata(metadata) + + self.__metadata = metadata + + # Only track drivers whose appended name/version pair survived + # truncation, so __appended_drivers stays bounded. Truncation + # keeps name and version index-aligned, so strict=True never + # raises. + names = metadata["driver"]["name"].split("|") + versions = metadata["driver"]["version"].split("|") + if sys.version_info >= (3, 10): + pairs = zip(names, versions, strict=True) + else: + pairs = zip(names, versions) + if (driver.name, driver.version) in pairs: + self.__appended_drivers.add(driver) @property def _credentials(self) -> Optional[MongoCredential]: diff --git a/test/asynchronous/test_client.py b/test/asynchronous/test_client.py index 0c95ba978b..56810b1433 100644 --- a/test/asynchronous/test_client.py +++ b/test/asynchronous/test_client.py @@ -90,7 +90,13 @@ WriteConcernError, ) from pymongo.monitoring import ServerHeartbeatListener, ServerHeartbeatStartedEvent -from pymongo.pool_options import _MAX_METADATA_SIZE, _METADATA, ENV_VAR_K8S, PoolOptions +from pymongo.pool_options import ( + _MAX_METADATA_SIZE, + _METADATA, + ENV_VAR_K8S, + PoolOptions, + _truncate_metadata, +) from pymongo.read_preferences import ReadPreference from pymongo.server_description import ServerDescription from pymongo.server_selectors import readable_server_selector, writable_server_selector @@ -124,6 +130,8 @@ NTHREADS, CMAPListener, FunctionCallRecorder, + _driver_version, + _metadata_with_appended_driver, delay, gevent_monkey_patched, is_greenthread_patched, @@ -387,6 +395,9 @@ async def test_metadata(self): metadata["driver"]["name"] = "PyMongo|c|async" else: metadata["driver"]["name"] = "PyMongo|async" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) metadata["application"] = {"name": "foobar"} client = self.simple_client("mongodb://foo:27017/?appname=foobar&connect=false") options = client.options @@ -398,6 +409,8 @@ async def test_metadata(self): self.simple_client(appname="x" * 128) with self.assertRaises(ValueError): self.simple_client(appname="x" * 129) + + async def test_metadata_bad_driver_options(self): # Bad "driver" options. self.assertRaises(TypeError, DriverInfo, "Foo", 1, "a") self.assertRaises(TypeError, DriverInfo, version="1", platform="a") @@ -408,12 +421,9 @@ async def test_metadata(self): self.simple_client(driver="abc") with self.assertRaises(TypeError): self.simple_client(driver=("Foo", "1", "a")) - # Test appending to driver info. - if has_c(): - metadata["driver"]["name"] = "PyMongo|c|async|FooDriver" - else: - metadata["driver"]["name"] = "PyMongo|async|FooDriver" - metadata["driver"]["version"] = "{}|1.2.3".format(_METADATA["driver"]["version"]) + + async def test_metadata_appends_driver_info(self): + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3") client = self.simple_client( "foo", 27017, @@ -421,9 +431,9 @@ async def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", None), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - metadata["platform"] = "{}|FooPlatform".format(_METADATA["platform"]) + self.assertEqual(client.options.pool_options.metadata, metadata) + + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3", "FooPlatform") client = self.simple_client( "foo", 27017, @@ -431,27 +441,131 @@ async def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", "FooPlatform"), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - # Test truncating driver info metadata. + self.assertEqual(client.options.pool_options.metadata, metadata) + + async def test_metadata_truncates_driver_info(self): + # Truncated driver info must stay within the limit and keep name and + # version index-aligned. client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE, version="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + # An oversized wrapper name with no version must retain a truncated + # name rather than collapse to the base entry. + client = self.simple_client( + driver=DriverInfo(name="x" * (_MAX_METADATA_SIZE * 2), version=None), + connect=False, + ) + truncated = client.options.pool_options.metadata["driver"] + self.assertLessEqual( + len(bson.encode(client.options.pool_options.metadata)), + _MAX_METADATA_SIZE, + ) + self.assertIn("xxxx", truncated["name"]) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + + async def test_metadata_truncation_drops_blank_pairs_first(self): + # A platform-only driver appends a blank name/version pair. Truncation + # must drop that pair before trimming a real driver's content. + def metadata_for(name_len: int) -> dict[str, Any]: + name = "PyMongo||" + "W" * name_len # blank pair in the middle + return {"driver": {"name": name, "version": "1.0||1.0"}} + + # Size the name so the document is exactly two bytes over the limit: + # dropping the blank pair alone must make it fit. + name_len = next( + n + for n in range(1, 2 * _MAX_METADATA_SIZE) + if len(bson.encode(metadata_for(n))) == _MAX_METADATA_SIZE + 2 + ) + metadata = metadata_for(name_len) + _truncate_metadata(metadata) + self.assertEqual(len(bson.encode(metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + metadata["driver"], + {"name": "PyMongo|" + "W" * name_len, "version": "1.0|1.0"}, + ) + + async def test_metadata_append_is_bounded(self): + # Successive appends must stay within the limit and keep name and + # version index-aligned after truncation. Once the metadata saturates, + # further appends must not grow the dedup tracking list. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + # Platform-only appends (empty name/version) stay bounded the same way. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + + async def test_metadata_recreates_platform_after_truncation(self): + # Appending a platform after truncation has dropped it recreates the field. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"Q{i}")) + pool = client.options.pool_options + self.assertLess(len(pool._PoolOptions__appended_drivers), 300) + client.append_metadata(DriverInfo(name="Wrapper", version="1.0", platform="Recreated")) + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + + async def test_metadata_deduplicates_none_and_empty(self): + # Empty strings are treated as unset, so a duplicate differing only in + # None vs "" is a no-op. + client = self.simple_client(connect=False) + client.append_metadata(DriverInfo("library", None, "Library Platform")) + names = client.options.pool_options.metadata["driver"]["name"] + vers = client.options.pool_options.metadata["driver"]["version"] + client.append_metadata(DriverInfo("library", "", "Library Platform")) + metadata = client.options.pool_options.metadata + self.assertEqual(metadata["driver"]["name"], names) + self.assertEqual(metadata["driver"]["version"], vers) @mock.patch.dict("os.environ", {ENV_VAR_K8S: "1"}) def test_container_metadata(self): @@ -2226,6 +2340,9 @@ async def _test_handshake(self, env_vars, expected_env): metadata["driver"]["name"] = "PyMongo|c|async" else: metadata["driver"]["name"] = "PyMongo|async" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) if expected_env is not None: metadata["env"] = expected_env diff --git a/test/asynchronous/test_client_metadata.py b/test/asynchronous/test_client_metadata.py index 1a07e835a8..a0438fb72a 100644 --- a/test/asynchronous/test_client_metadata.py +++ b/test/asynchronous/test_client_metadata.py @@ -18,7 +18,7 @@ import pathlib import time import unittest -from typing import Any, Optional +from typing import Any, Optional, cast import pytest @@ -99,20 +99,14 @@ async def check_metadata_added( new_name, new_version, new_platform, new_metadata = await self.send_ping_and_get_metadata( client, True ) - if add_name is not None and add_name.lower() in name.lower().split("|"): - self.assertEqual(name, new_name) - self.assertEqual(version, new_version) - self.assertEqual(platform, new_platform) - else: - self.assertEqual(new_name, f"{name}|{add_name}" if add_name is not None else name) - self.assertEqual( - new_version, - f"{version}|{add_version}" if add_version is not None else version, - ) - self.assertEqual( - new_platform, - f"{platform}|{add_platform}" if add_platform is not None else platform, - ) + # Name and version always get a delimiter (empty string if None) to + # preserve 1:1 index correspondence. + self.assertEqual(new_name, f"{name}|{add_name or ''}") + self.assertEqual(new_version, f"{version}|{add_version or ''}") + self.assertEqual( + new_platform, + f"{platform}|{add_platform}" if add_platform is not None else platform, + ) metadata.pop("driver") metadata.pop("platform") @@ -120,7 +114,7 @@ async def check_metadata_added( new_metadata.pop("platform") self.assertEqual(metadata, new_metadata) - async def test_append_metadata(self): + async def test_1_test_that_the_driver_updates_metadata(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -128,7 +122,7 @@ async def test_append_metadata(self): ) await self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - async def test_append_metadata_platform_none(self): + async def test_1_test_that_the_driver_updates_metadata_platform_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -136,7 +130,7 @@ async def test_append_metadata_platform_none(self): ) await self.check_metadata_added(client, "framework", "2.0", None) - async def test_append_metadata_version_none(self): + async def test_1_test_that_the_driver_updates_metadata_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -144,7 +138,7 @@ async def test_append_metadata_version_none(self): ) await self.check_metadata_added(client, "framework", None, "Framework Platform") - async def test_append_metadata_platform_version_none(self): + async def test_1_test_that_the_driver_updates_metadata_platform_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -152,14 +146,14 @@ async def test_append_metadata_platform_version_none(self): ) await self.check_metadata_added(client, "framework", None, None) - async def test_multiple_successive_metadata_updates(self): + async def test_2_multiple_successive_metadata_updates(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, connect=False ) client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) await self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - async def test_multiple_successive_metadata_updates_platform_none(self): + async def test_2_multiple_successive_metadata_updates_platform_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -167,7 +161,7 @@ async def test_multiple_successive_metadata_updates_platform_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) await self.check_metadata_added(client, "framework", "2.0", None) - async def test_multiple_successive_metadata_updates_version_none(self): + async def test_2_multiple_successive_metadata_updates_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -175,7 +169,7 @@ async def test_multiple_successive_metadata_updates_version_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) await self.check_metadata_added(client, "framework", None, "Framework Platform") - async def test_multiple_successive_metadata_updates_platform_version_none(self): + async def test_2_multiple_successive_metadata_updates_platform_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -216,10 +210,16 @@ async def test_duplicate_driver_name_no_op(self): await self.check_metadata_added(client, "framework", None, None) # wait for connection to become idle await asyncio.sleep(0.005) - # add same metadata again - await self.check_metadata_added(client, "Framework", None, None) + # Append the exact same DriverInfo again: no-op. + name, version, platform, _ = await self.send_ping_and_get_metadata(client, True) + await asyncio.sleep(0.005) + client.append_metadata(DriverInfo("framework", None, None)) + new_name, new_version, new_platform, _ = await self.send_ping_and_get_metadata(client, True) + self.assertEqual(new_name, name) + self.assertEqual(new_version, version) + self.assertEqual(new_platform, platform) - async def test_handshake_documents_include_backpressure(self): + async def test_9_handshake_documents_include_backpressure(self): # Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of # connection establishment. client = await self.async_rs_or_single_client("mongodb://" + self.server.address_string) @@ -232,6 +232,131 @@ async def test_handshake_documents_include_backpressure(self): # the document has a field `backpressure` whose value is `"2"`. self.assertEqual(self.handshake_req["backpressure"], "2") + async def test_10_entries_in_driver_name_and_driver_version_correspond_by_index(self): + cases = [ + ("Gap in middle (name)", [(None, None), ("F2", None)], "||F2", "||"), + ("Gap in middle (version)", [("F1", None), ("F2", "2.0")], "|F1|F2", "||2.0"), + ("Trailing delimiter retained", [("F1", None)], "|F1", "|"), + ( + "Equal versions do not collapse", + [("F1", "{driver_version}")], + "|F1", + "|{driver_version}", + ), + ( + "Equal names do not collapse", + [("{driver_name}", "1.0")], + "|{driver_name}", + "|1.0", + ), + ("Duplicates deduplicate", [("F1", "1.0"), ("F1", "1.0")], "|F1", "|1.0"), + ("All versions absent", [("F1", None), ("F2", None)], "|F1|F2", "||"), + ("All names absent", [(None, "1.0"), (None, "2.0")], "||", "|1.0|2.0"), + ( + "Non-adjacent duplicate", + [("F1", "1.0"), ("F2", "2.0"), ("F1", "1.0")], + "|F1|F2", + "|1.0|2.0", + ), + ( + "Platform-only difference is not a duplicate", + [("F1", "1.0", "P1"), ("F1", "1.0", "P2")], + "|F1|F1", + "|1.0|1.0", + ), + ( + "Wrapper matching the driver's own identity", + [("{driver_name}", "{driver_version}")], + "|{driver_name}", + "|{driver_version}", + ), + ( + "Duplicates with an unset field deduplicate", + [("F1", None), ("F1", None)], + "|F1", + "|", + ), + ] + for ( + description, + appended, + expected_name_suffix, + expected_version_suffix, + ) in cases: + with self.subTest(description=description): + client = await self.async_rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + ) + self.addAsyncCleanup(client.close) + # Capture the driver's own name and version from the first handshake. + name0, version0, _, _ = await self.send_ping_and_get_metadata(client, True) + await asyncio.sleep(0.005) + + self.assertIsNotNone(name0) + self.assertIsNotNone(version0) + version0 = cast(str, version0) + driver_name = name0.split("|")[0] + driver_version = version0.split("|")[0] + + def resolve(value: Optional[str]) -> Optional[str]: + if value is None: + return None + return value.format(driver_name=driver_name, driver_version=driver_version) + + # Append each DriverInfo in order. + for opts in appended: + d_name = resolve(opts[0]) if len(opts) > 0 else None + d_version = resolve(opts[1]) if len(opts) > 1 else None + d_platform = resolve(opts[2]) if len(opts) > 2 else None + client.append_metadata(DriverInfo(d_name or "", d_version, d_platform)) + + # New handshake with the appended metadata. + name1, version1, _, _ = await self.send_ping_and_get_metadata(client, True) + + self.assertEqual( + name1, + name0 + + expected_name_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + self.assertEqual( + version1, + version0 + + expected_version_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + + async def test_11_appending_metadata_containing_the_delimiter_raises_an_error(self): + cases = [ + ("frame|work", "2.0", "Framework Platform"), + ("framework", "2|0", "Framework Platform"), + ("framework", "2.0", "Framework|Platform"), + ] + for name, version, platform in cases: + with self.subTest(name=name, version=version, platform=platform): + client = await self.async_rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + driver=DriverInfo("library", "1.2", "Library Platform"), + ) + self.addAsyncCleanup(client.close) + # Send initial handshake. + name0, version0, platform0, _metadata = await self.send_ping_and_get_metadata( + client, True + ) + await asyncio.sleep(0.005) + # Constructing metadata containing the delimiter raises. + with self.assertRaises(ValueError): + DriverInfo(name, version, platform) + # Metadata is unchanged on the next handshake. + name1, version1, platform1, _ = await self.send_ping_and_get_metadata(client, True) + self.assertEqual(name1, name0) + self.assertEqual(version1, version0) + self.assertEqual(platform1, platform0) + if __name__ == "__main__": unittest.main() diff --git a/test/mockupdb/test_handshake.py b/test/mockupdb/test_handshake.py index 2772e6f77a..e3a3fc0562 100644 --- a/test/mockupdb/test_handshake.py +++ b/test/mockupdb/test_handshake.py @@ -49,9 +49,11 @@ def _check_handshake_data(request): assert data["application"] == {"name": "my app"} if has_c(): name = "PyMongo|c" + version = pymongo_version + "|" else: name = "PyMongo" - assert data["driver"] == {"name": name, "version": pymongo_version} + version = pymongo_version + assert data["driver"] == {"name": name, "version": version} # Keep it simple, just check these fields exist. assert "os" in data diff --git a/test/test_client.py b/test/test_client.py index ee492e9b9b..ebd1e670ee 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -81,7 +81,13 @@ WriteConcernError, ) from pymongo.monitoring import ServerHeartbeatListener, ServerHeartbeatStartedEvent -from pymongo.pool_options import _MAX_METADATA_SIZE, _METADATA, ENV_VAR_K8S, PoolOptions +from pymongo.pool_options import ( + _MAX_METADATA_SIZE, + _METADATA, + ENV_VAR_K8S, + PoolOptions, + _truncate_metadata, +) from pymongo.read_preferences import ReadPreference from pymongo.server_description import ServerDescription from pymongo.server_selectors import readable_server_selector, writable_server_selector @@ -123,6 +129,8 @@ NTHREADS, CMAPListener, FunctionCallRecorder, + _driver_version, + _metadata_with_appended_driver, delay, gevent_monkey_patched, is_greenthread_patched, @@ -380,6 +388,9 @@ def test_metadata(self): metadata["driver"]["name"] = "PyMongo|c" else: metadata["driver"]["name"] = "PyMongo" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) metadata["application"] = {"name": "foobar"} client = self.simple_client("mongodb://foo:27017/?appname=foobar&connect=false") options = client.options @@ -391,6 +402,8 @@ def test_metadata(self): self.simple_client(appname="x" * 128) with self.assertRaises(ValueError): self.simple_client(appname="x" * 129) + + def test_metadata_bad_driver_options(self): # Bad "driver" options. self.assertRaises(TypeError, DriverInfo, "Foo", 1, "a") self.assertRaises(TypeError, DriverInfo, version="1", platform="a") @@ -401,12 +414,9 @@ def test_metadata(self): self.simple_client(driver="abc") with self.assertRaises(TypeError): self.simple_client(driver=("Foo", "1", "a")) - # Test appending to driver info. - if has_c(): - metadata["driver"]["name"] = "PyMongo|c|FooDriver" - else: - metadata["driver"]["name"] = "PyMongo|FooDriver" - metadata["driver"]["version"] = "{}|1.2.3".format(_METADATA["driver"]["version"]) + + def test_metadata_appends_driver_info(self): + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3") client = self.simple_client( "foo", 27017, @@ -414,9 +424,9 @@ def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", None), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - metadata["platform"] = "{}|FooPlatform".format(_METADATA["platform"]) + self.assertEqual(client.options.pool_options.metadata, metadata) + + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3", "FooPlatform") client = self.simple_client( "foo", 27017, @@ -424,27 +434,131 @@ def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", "FooPlatform"), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - # Test truncating driver info metadata. + self.assertEqual(client.options.pool_options.metadata, metadata) + + def test_metadata_truncates_driver_info(self): + # Truncated driver info must stay within the limit and keep name and + # version index-aligned. client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE, version="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + # An oversized wrapper name with no version must retain a truncated + # name rather than collapse to the base entry. + client = self.simple_client( + driver=DriverInfo(name="x" * (_MAX_METADATA_SIZE * 2), version=None), + connect=False, + ) + truncated = client.options.pool_options.metadata["driver"] + self.assertLessEqual( + len(bson.encode(client.options.pool_options.metadata)), + _MAX_METADATA_SIZE, + ) + self.assertIn("xxxx", truncated["name"]) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + + def test_metadata_truncation_drops_blank_pairs_first(self): + # A platform-only driver appends a blank name/version pair. Truncation + # must drop that pair before trimming a real driver's content. + def metadata_for(name_len: int) -> dict[str, Any]: + name = "PyMongo||" + "W" * name_len # blank pair in the middle + return {"driver": {"name": name, "version": "1.0||1.0"}} + + # Size the name so the document is exactly two bytes over the limit: + # dropping the blank pair alone must make it fit. + name_len = next( + n + for n in range(1, 2 * _MAX_METADATA_SIZE) + if len(bson.encode(metadata_for(n))) == _MAX_METADATA_SIZE + 2 + ) + metadata = metadata_for(name_len) + _truncate_metadata(metadata) + self.assertEqual(len(bson.encode(metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + metadata["driver"], + {"name": "PyMongo|" + "W" * name_len, "version": "1.0|1.0"}, + ) + + def test_metadata_append_is_bounded(self): + # Successive appends must stay within the limit and keep name and + # version index-aligned after truncation. Once the metadata saturates, + # further appends must not grow the dedup tracking list. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + # Platform-only appends (empty name/version) stay bounded the same way. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + + def test_metadata_recreates_platform_after_truncation(self): + # Appending a platform after truncation has dropped it recreates the field. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"Q{i}")) + pool = client.options.pool_options + self.assertLess(len(pool._PoolOptions__appended_drivers), 300) + client.append_metadata(DriverInfo(name="Wrapper", version="1.0", platform="Recreated")) + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + + def test_metadata_deduplicates_none_and_empty(self): + # Empty strings are treated as unset, so a duplicate differing only in + # None vs "" is a no-op. + client = self.simple_client(connect=False) + client.append_metadata(DriverInfo("library", None, "Library Platform")) + names = client.options.pool_options.metadata["driver"]["name"] + vers = client.options.pool_options.metadata["driver"]["version"] + client.append_metadata(DriverInfo("library", "", "Library Platform")) + metadata = client.options.pool_options.metadata + self.assertEqual(metadata["driver"]["name"], names) + self.assertEqual(metadata["driver"]["version"], vers) @mock.patch.dict("os.environ", {ENV_VAR_K8S: "1"}) def test_container_metadata(self): @@ -2179,6 +2293,9 @@ def _test_handshake(self, env_vars, expected_env): metadata["driver"]["name"] = "PyMongo|c" else: metadata["driver"]["name"] = "PyMongo" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) if expected_env is not None: metadata["env"] = expected_env diff --git a/test/test_client_metadata.py b/test/test_client_metadata.py index f5ec92f2f3..b35e3f0bc7 100644 --- a/test/test_client_metadata.py +++ b/test/test_client_metadata.py @@ -18,7 +18,7 @@ import pathlib import time import unittest -from typing import Any, Optional +from typing import Any, Optional, cast import pytest @@ -99,20 +99,14 @@ def check_metadata_added( new_name, new_version, new_platform, new_metadata = self.send_ping_and_get_metadata( client, True ) - if add_name is not None and add_name.lower() in name.lower().split("|"): - self.assertEqual(name, new_name) - self.assertEqual(version, new_version) - self.assertEqual(platform, new_platform) - else: - self.assertEqual(new_name, f"{name}|{add_name}" if add_name is not None else name) - self.assertEqual( - new_version, - f"{version}|{add_version}" if add_version is not None else version, - ) - self.assertEqual( - new_platform, - f"{platform}|{add_platform}" if add_platform is not None else platform, - ) + # Name and version always get a delimiter (empty string if None) to + # preserve 1:1 index correspondence. + self.assertEqual(new_name, f"{name}|{add_name or ''}") + self.assertEqual(new_version, f"{version}|{add_version or ''}") + self.assertEqual( + new_platform, + f"{platform}|{add_platform}" if add_platform is not None else platform, + ) metadata.pop("driver") metadata.pop("platform") @@ -120,7 +114,7 @@ def check_metadata_added( new_metadata.pop("platform") self.assertEqual(metadata, new_metadata) - def test_append_metadata(self): + def test_1_test_that_the_driver_updates_metadata(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -128,7 +122,7 @@ def test_append_metadata(self): ) self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - def test_append_metadata_platform_none(self): + def test_1_test_that_the_driver_updates_metadata_platform_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -136,7 +130,7 @@ def test_append_metadata_platform_none(self): ) self.check_metadata_added(client, "framework", "2.0", None) - def test_append_metadata_version_none(self): + def test_1_test_that_the_driver_updates_metadata_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -144,7 +138,7 @@ def test_append_metadata_version_none(self): ) self.check_metadata_added(client, "framework", None, "Framework Platform") - def test_append_metadata_platform_version_none(self): + def test_1_test_that_the_driver_updates_metadata_platform_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -152,14 +146,14 @@ def test_append_metadata_platform_version_none(self): ) self.check_metadata_added(client, "framework", None, None) - def test_multiple_successive_metadata_updates(self): + def test_2_multiple_successive_metadata_updates(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, connect=False ) client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - def test_multiple_successive_metadata_updates_platform_none(self): + def test_2_multiple_successive_metadata_updates_platform_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -167,7 +161,7 @@ def test_multiple_successive_metadata_updates_platform_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) self.check_metadata_added(client, "framework", "2.0", None) - def test_multiple_successive_metadata_updates_version_none(self): + def test_2_multiple_successive_metadata_updates_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -175,7 +169,7 @@ def test_multiple_successive_metadata_updates_version_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) self.check_metadata_added(client, "framework", None, "Framework Platform") - def test_multiple_successive_metadata_updates_platform_version_none(self): + def test_2_multiple_successive_metadata_updates_platform_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -216,10 +210,16 @@ def test_duplicate_driver_name_no_op(self): self.check_metadata_added(client, "framework", None, None) # wait for connection to become idle time.sleep(0.005) - # add same metadata again - self.check_metadata_added(client, "Framework", None, None) + # Append the exact same DriverInfo again: no-op. + name, version, platform, _ = self.send_ping_and_get_metadata(client, True) + time.sleep(0.005) + client.append_metadata(DriverInfo("framework", None, None)) + new_name, new_version, new_platform, _ = self.send_ping_and_get_metadata(client, True) + self.assertEqual(new_name, name) + self.assertEqual(new_version, version) + self.assertEqual(new_platform, platform) - def test_handshake_documents_include_backpressure(self): + def test_9_handshake_documents_include_backpressure(self): # Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of # connection establishment. client = self.rs_or_single_client("mongodb://" + self.server.address_string) @@ -232,6 +232,131 @@ def test_handshake_documents_include_backpressure(self): # the document has a field `backpressure` whose value is `"2"`. self.assertEqual(self.handshake_req["backpressure"], "2") + def test_10_entries_in_driver_name_and_driver_version_correspond_by_index(self): + cases = [ + ("Gap in middle (name)", [(None, None), ("F2", None)], "||F2", "||"), + ("Gap in middle (version)", [("F1", None), ("F2", "2.0")], "|F1|F2", "||2.0"), + ("Trailing delimiter retained", [("F1", None)], "|F1", "|"), + ( + "Equal versions do not collapse", + [("F1", "{driver_version}")], + "|F1", + "|{driver_version}", + ), + ( + "Equal names do not collapse", + [("{driver_name}", "1.0")], + "|{driver_name}", + "|1.0", + ), + ("Duplicates deduplicate", [("F1", "1.0"), ("F1", "1.0")], "|F1", "|1.0"), + ("All versions absent", [("F1", None), ("F2", None)], "|F1|F2", "||"), + ("All names absent", [(None, "1.0"), (None, "2.0")], "||", "|1.0|2.0"), + ( + "Non-adjacent duplicate", + [("F1", "1.0"), ("F2", "2.0"), ("F1", "1.0")], + "|F1|F2", + "|1.0|2.0", + ), + ( + "Platform-only difference is not a duplicate", + [("F1", "1.0", "P1"), ("F1", "1.0", "P2")], + "|F1|F1", + "|1.0|1.0", + ), + ( + "Wrapper matching the driver's own identity", + [("{driver_name}", "{driver_version}")], + "|{driver_name}", + "|{driver_version}", + ), + ( + "Duplicates with an unset field deduplicate", + [("F1", None), ("F1", None)], + "|F1", + "|", + ), + ] + for ( + description, + appended, + expected_name_suffix, + expected_version_suffix, + ) in cases: + with self.subTest(description=description): + client = self.rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + ) + self.addCleanup(client.close) + # Capture the driver's own name and version from the first handshake. + name0, version0, _, _ = self.send_ping_and_get_metadata(client, True) + time.sleep(0.005) + + self.assertIsNotNone(name0) + self.assertIsNotNone(version0) + version0 = cast(str, version0) + driver_name = name0.split("|")[0] + driver_version = version0.split("|")[0] + + def resolve(value: Optional[str]) -> Optional[str]: + if value is None: + return None + return value.format(driver_name=driver_name, driver_version=driver_version) + + # Append each DriverInfo in order. + for opts in appended: + d_name = resolve(opts[0]) if len(opts) > 0 else None + d_version = resolve(opts[1]) if len(opts) > 1 else None + d_platform = resolve(opts[2]) if len(opts) > 2 else None + client.append_metadata(DriverInfo(d_name or "", d_version, d_platform)) + + # New handshake with the appended metadata. + name1, version1, _, _ = self.send_ping_and_get_metadata(client, True) + + self.assertEqual( + name1, + name0 + + expected_name_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + self.assertEqual( + version1, + version0 + + expected_version_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + + def test_11_appending_metadata_containing_the_delimiter_raises_an_error(self): + cases = [ + ("frame|work", "2.0", "Framework Platform"), + ("framework", "2|0", "Framework Platform"), + ("framework", "2.0", "Framework|Platform"), + ] + for name, version, platform in cases: + with self.subTest(name=name, version=version, platform=platform): + client = self.rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + driver=DriverInfo("library", "1.2", "Library Platform"), + ) + self.addCleanup(client.close) + # Send initial handshake. + name0, version0, platform0, _metadata = self.send_ping_and_get_metadata( + client, True + ) + time.sleep(0.005) + # Constructing metadata containing the delimiter raises. + with self.assertRaises(ValueError): + DriverInfo(name, version, platform) + # Metadata is unchanged on the next handshake. + name1, version1, platform1, _ = self.send_ping_and_get_metadata(client, True) + self.assertEqual(name1, name0) + self.assertEqual(version1, version0) + self.assertEqual(platform1, platform0) + if __name__ == "__main__": unittest.main() diff --git a/test/utils_shared.py b/test/utils_shared.py index 96f3b1f6ed..fb209d69a0 100644 --- a/test/utils_shared.py +++ b/test/utils_shared.py @@ -31,9 +31,11 @@ from collections import abc, defaultdict from functools import partial from inspect import iscoroutinefunction +from typing import Any from bson.objectid import ObjectId from pymongo import monitoring, operations, read_preferences +from pymongo.common import has_c from pymongo.cursor_shared import CursorType from pymongo.errors import OperationFailure from pymongo.helpers_shared import _SENSITIVE_COMMANDS @@ -51,6 +53,7 @@ PoolCreatedEvent, PoolReadyEvent, ) +from pymongo.pool_options import _METADATA from pymongo.pool_shared import _CancellationContext, _PoolGeneration from pymongo.read_concern import ReadConcern from pymongo.server_type import SERVER_TYPE @@ -791,3 +794,36 @@ def pack_msg_header(length: int, request_id: int, response_to: int, op_code: int production header-packing never does. """ return struct.pack(" str: + """Build a metadata driver version aligned 1:1 with ``name`` segments. + + The ``|c`` and ``|async`` name segments always have an empty version entry, + so the version string has one delimiter per name delimiter. ``last_version`` + is used when the final segment carries a wrapped driver's version. + """ + segments = [""] * name.count("|") + if last_version is not None: + segments[-1] = last_version + return "|".join([base_version, *segments]) + + +def _metadata_with_appended_driver( + is_sync: bool, name: str, version: str, platform: str | None = None +) -> dict[str, Any]: + """Build the expected client metadata after appending a driver.""" + driver_name = "PyMongo" + if has_c(): + driver_name += "|c" + if not is_sync: + driver_name += "|async" + metadata = copy.deepcopy(_METADATA) + metadata["driver"]["name"] = driver_name + f"|{name}" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"], last_version=version + ) + metadata["application"] = {"name": "foobar"} + if platform is not None: + metadata["platform"] = "{}|{}".format(_METADATA["platform"], platform) + return metadata From 916f32076edeb8d6c41912344eecfffa495c94c4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 30 Sep 2026 05:24:56 -0500 Subject: [PATCH 05/24] PYTHON-6083 Latest server binary downloads require private S3 (#3076) --- .evergreen/generated_configs/functions.yml | 3 +++ .evergreen/scripts/generate_config.py | 5 +++++ .evergreen/scripts/setup_tests.py | 4 +++- CONTRIBUTING.md | 4 +++- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index 57f1182377..e442973c30 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -110,6 +110,9 @@ functions: - LOAD_BALANCER - LOCAL_ATLAS - NO_EXT + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN type: test - command: expansions.update params: diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index e4251141be..9b4313c423 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -1321,6 +1321,11 @@ def create_run_server_func(): "LOAD_BALANCER", "LOCAL_ATLAS", "NO_EXT", + # The AWS credentials are needed to download the "latest" nightly + # build from the private S3 bucket, see DRIVERS-3628. + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", ] args = [".evergreen/just.sh", "run-server", "${TEST_NAME}"] sub_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 4f8b03b5ec..b57188eee1 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -360,7 +360,9 @@ def handle_test_env() -> None: # The mock OCSP responder MUST BE started before the mongod as the mongod expects that # a responder will be available upon startup. - version = os.environ.get("VERSION", "latest") + # Default to the newest stable release; the "latest" nightly build is + # downloaded from a private S3 bucket and needs AWS credentials. + version = os.environ.get("VERSION", "latest-stable") cmd = [ "bash", f"{DRIVERS_TOOLS}/.evergreen/run-mongodb.sh", diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 61e369767a..14e3ddfd26 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -206,7 +206,9 @@ the pages will re-render and the browser will automatically refresh. - Ensure you have started the appropriate Mongo Server(s). You can run `just run-server` with optional args to set up the server. All given options will be passed to [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `$DRIVERS_TOOLS/.evergreen/run-mongodb.sh start -h` - for a full list of options. + for a full list of options. By default the newest stable server release is used. To test against the + nightly build instead, pass `--version latest`, which is downloaded from a private S3 bucket and requires + an `AWS_PROFILE` with [Drivers test secrets](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/secrets_handling#secrets-handling) credentials. - Run `just test` or `pytest` to run all of the tests. - Append `test/.py::::` to run specific tests. You can omit the `` to test a full class From 23f4a128aaacebae6228e9859f706a1a595ce414 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 30 Sep 2026 05:26:23 -0500 Subject: [PATCH 06/24] PYTHON-6125 Fix spec PR creation due to malformed command (#3078) --- .evergreen/scripts/create-spec-pr.sh | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/create-spec-pr.sh b/.evergreen/scripts/create-spec-pr.sh index 1de1a7cae1..0867ae5487 100755 --- a/.evergreen/scripts/create-spec-pr.sh +++ b/.evergreen/scripts/create-spec-pr.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -eu tools="$(realpath -s "../drivers-tools")" pushd $tools/.evergreen/github_app || exit @@ -37,14 +38,32 @@ git commit -am "resyncing specs $(date '+%m-%d-%Y')" echo "Creating the git checkout... done." git push origin $branch + +# Build the payload as a file so the body content is always properly +# JSON-escaped, rather than interpolated into the command line as text. +payload_file=$(mktemp) +trap 'rm -f "$payload_file"' EXIT +jq -n \ + --arg title "[Spec Resync] $(date '+%m-%d-%Y')" \ + --arg head "${branch}" \ + --rawfile body "$1" \ + '{title: $title, body: $body, head: $head, base: "main"}' > "$payload_file" + resp=$(curl -L \ -X POST \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $token" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - -d "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"$(cat "$1")\",\"head\":\"${branch}\",\"base\":\"main\"}" \ + -d "@${payload_file}" \ --url https://api.github.com/repos/$owner/$repo/pulls) -echo $resp | jq '.html_url' + +pr_url=$(echo "$resp" | jq -r '.html_url // empty') +if [ -z "$pr_url" ]; then + echo "Failed to create PR! API response:" + echo "$resp" | jq . + exit 1 +fi +echo "$pr_url" echo "Creating the PR... done." rm -rf $tools From d90b247b269acc399bab10265ed9b19b77af6560 Mon Sep 17 00:00:00 2001 From: Iris <58442094+sleepyStick@users.noreply.github.com> Date: Wed, 30 Sep 2026 10:11:46 -0700 Subject: [PATCH 07/24] PYTHON-6080 Introduce Custom Callback logic in Configurable DNS (#3051) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Jeffrey 'Alex' Clark --- doc/changelog.rst | 15 +++ pymongo/_psl.py | 22 +++- pymongo/_telemetry.py | 11 ++ pymongo/asynchronous/mongo_client.py | 57 +++++++++- pymongo/asynchronous/monitor.py | 1 + pymongo/asynchronous/settings.py | 4 +- pymongo/asynchronous/srv_resolver.py | 105 ++++++++++++------ pymongo/asynchronous/uri_parser.py | 12 ++- pymongo/common.py | 1 + pymongo/public_suffix_list.dat | 15 ++- pymongo/settings_shared.py | 9 +- pymongo/synchronous/mongo_client.py | 57 +++++++++- pymongo/synchronous/monitor.py | 1 + pymongo/synchronous/settings.py | 4 +- pymongo/synchronous/srv_resolver.py | 105 ++++++++++++------ pymongo/synchronous/uri_parser.py | 12 ++- test/asynchronous/test_dns.py | 150 ++++++++++++++++++++++---- test/asynchronous/test_srv_polling.py | 111 ++++++++++++++++++- test/asynchronous/unified_format.py | 6 ++ test/test_dns.py | 148 +++++++++++++++++++++---- test/test_srv_polling.py | 111 ++++++++++++++++++- test/unified_format.py | 6 ++ 22 files changed, 836 insertions(+), 127 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 6082f30dc5..cb3ded580a 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -16,6 +16,19 @@ PyMongo 4.19 brings a number of changes including: interpreter remain daemon threads and shutdown behavior is unchanged. Note that because these threads are non-daemon, a subinterpreter may block on teardown until any in-flight monitor work completes. +- Added the ``srv_host_validator`` keyword argument to + :class:`~pymongo.synchronous.mongo_client.MongoClient` and + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient`, an alternative to + ``srvAllowedHostsSuffix`` for deployments whose acceptable SRV hosts cannot be + expressed as a single suffix. The callback is invoked once per SRV-returned + host and returns ``True`` to accept it. It is mutually exclusive with + ``srvAllowedHostsSuffix`` and, because it takes a callable, cannot be set in a + connection string. See the + :class:`~pymongo.synchronous.mongo_client.MongoClient` and + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` documentation for + security considerations. +- ``srvAllowedHostsSuffix`` may now be set to a single label reserved for + private or special use, such as ``localhost``, ``test``, or ``internal``. Bug fixes ......... @@ -37,6 +50,8 @@ Bug fixes delimiters (`PYTHON-6040`_). - :class:`~pymongo.driver_info.DriverInfo` now raises :class:`ValueError` when any field contains the reserved ``|`` delimiter (`PYTHON-6040`_). +- Fixed a bug in SRV polling where invalid hosts where topology would not be + updated if one returned host was invalid. .. _PYTHON-6074: https://jira.mongodb.org/browse/PYTHON-6074 .. _PYTHON-6040: https://jira.mongodb.org/browse/PYTHON-6040 diff --git a/pymongo/_psl.py b/pymongo/_psl.py index b8f11f5bd7..e0130188ff 100644 --- a/pymongo/_psl.py +++ b/pymongo/_psl.py @@ -21,6 +21,26 @@ _PUBLIC_SUFFIXES: Optional[tuple[set[str], set[str], set[str]]] = None +# Single labels that srvAllowedHostsSuffix may be set to. See the +# srvAllowedHostsSuffix section of the Initial DNS Seedlist Discovery spec. +SPECIAL_USE_LABELS = frozenset( + [ + # RFC 6761 special use names. + "test", + "localhost", + "invalid", + "example", + # RFC 6762 multicast DNS. + "local", + # Reserved by ICANN for private use. + "internal", + # Not officially reserved by ICANN but commonly used privately. + "corp", + "home", + "mail", + ] +) + def _to_punycode(string: str) -> str: """Convert a string to Punycode.""" @@ -28,7 +48,7 @@ def _to_punycode(string: str) -> str: return string.encode("idna").decode("ascii") except UnicodeError: # Not every input is a valid IDNA label (e.g. empty or over-long - # labels). Leave it as-is and let the caller reject it. + # labels). Return it unchanged -- SRV host will fail verification. return string diff --git a/pymongo/_telemetry.py b/pymongo/_telemetry.py index 499309b336..b8c8fa00ff 100644 --- a/pymongo/_telemetry.py +++ b/pymongo/_telemetry.py @@ -746,6 +746,17 @@ def log_srv_monitor_failure(failure: Exception) -> None: _debug_log(_SDAM_LOGGER, message="SRV monitor check failed", failure=repr(failure)) +def log_srv_monitor_invalid_host(host: str, failure: Exception) -> None: + """Emit a log entry when the SRV monitor skips a host that failed verification.""" + if _is_debug_enabled(_SDAM_LOGGER): + _debug_log( + _SDAM_LOGGER, + message="SRV monitor skipped invalid host", + serverHost=host, + failure=repr(failure), + ) + + def log_command_retry( topology_id: Optional[ObjectId], command_name: str, diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index a0f1397a4d..65aecce446 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -470,6 +470,31 @@ def __init__( srvAllowedHostsSuffix=".internal.example.com", ) + - `srv_host_validator`: (callable or None) A callback used in place of the + default parent-domain check for hosts returned by SRV DNS records. It is + called once per returned host with the lowercased hostname as its only + argument, and must return ``True`` to accept the host or ``False`` to + reject it. During initial DNS discovery, rejecting a host raises + :exc:`~pymongo.errors.ConfigurationError`, as does an exception raised by + the callback itself. Use this when the set of acceptable hosts cannot be + expressed as a single suffix:: + + def validator(host: str) -> bool: + return host.endswith((".a.example.com", ".b.example.com")) + + AsyncMongoClient( + "mongodb+srv://cluster.example.com/", + srv_host_validator=validator, + ) + + The callback must not block. It is mutually exclusive with + ``srvAllowedHostsSuffix``. Because this option is a callable, it can + only be passed in as a keyword argument, not through the connection string. + + .. warning:: + + This option replaces the built-in DNS spoofing safeguards. + Please use with caution. | **Write Concern options:** | (Only set if passed. No default values.) @@ -637,6 +662,12 @@ def __init__( .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.19 + Added the ``srv_host_validator`` keyword argument. + + .. versionchanged:: 4.18 + Added the ``srvAllowedHostsSuffix`` URI and keyword argument. + .. versionchanged:: 4.17 Added the ``max_adaptive_retries`` and ``enable_overload_retargeting`` URI and keyword arguments. @@ -823,6 +854,7 @@ def __init__( srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") if len([h for h in self._host if "/" in h]) > 1: raise ConfigurationError("host must not contain multiple MongoDB URIs") for entity in self._host: @@ -915,7 +947,11 @@ def __init__( ) self._init_based_on_options( - self._seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + self._seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) self._opened = False @@ -935,6 +971,7 @@ async def _resolve_srv(self) -> None: srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") for entity in self._host: # A hostname can only include a-z, 0-9, '-' and '.'. If we find a '/' # it must be a URI, @@ -956,6 +993,7 @@ async def _resolve_srv(self) -> None: srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, ) seeds.update(res["nodelist"]) opts = res["options"] @@ -1000,7 +1038,11 @@ async def _resolve_srv(self) -> None: ) self._init_based_on_options( - seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) def _init_based_on_options( @@ -1009,7 +1051,17 @@ def _init_based_on_options( srv_max_hosts: Any, srv_service_name: Any, srv_allowed_hosts_suffix: Any, + srv_host_validator: Any = None, ) -> None: + if srv_host_validator is not None: + if srv_allowed_hosts_suffix is not None: + raise ConfigurationError( + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" + ) + if not self._resolve_srv_info["is_srv"]: + raise ConfigurationError( + "The srv_host_validator option is only allowed with 'mongodb+srv://' URIs" + ) self._event_listeners = self._options.pool_options._event_listeners self._topology_settings = TopologySettings( seeds=seeds, @@ -1028,6 +1080,7 @@ def _init_based_on_options( srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, server_monitoring_mode=self._options.server_monitoring_mode, topology_id=self._topology_settings._topology_id if self._topology_settings else None, ) diff --git a/pymongo/asynchronous/monitor.py b/pymongo/asynchronous/monitor.py index da261e37ea..8ed4078fd7 100644 --- a/pymongo/asynchronous/monitor.py +++ b/pymongo/asynchronous/monitor.py @@ -368,6 +368,7 @@ async def _get_seedlist(self) -> Optional[list[tuple[str, Any]]]: self._settings.pool_options.connect_timeout, self._settings.srv_service_name, srv_allowed_hosts_suffix=self._settings.srv_allowed_hosts_suffix, + srv_host_validator=self._settings.srv_host_validator, ) seedlist, ttl = await resolver.get_hosts_and_min_ttl() if len(seedlist) == 0: diff --git a/pymongo/asynchronous/settings.py b/pymongo/asynchronous/settings.py index bdfb334ab7..4bc5f05f28 100644 --- a/pymongo/asynchronous/settings.py +++ b/pymongo/asynchronous/settings.py @@ -18,7 +18,7 @@ import threading from collections.abc import Collection -from typing import Optional +from typing import Callable, Optional from bson.objectid import ObjectId from pymongo import common @@ -51,6 +51,7 @@ def __init__( srv_service_name: str = common.SRV_SERVICE_NAME, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, server_monitoring_mode: str = common.SERVER_MONITORING_MODE, topology_id: Optional[ObjectId] = None, ): @@ -77,6 +78,7 @@ def __init__( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, server_monitoring_mode, topology_id, ) diff --git a/pymongo/asynchronous/srv_resolver.py b/pymongo/asynchronous/srv_resolver.py index 5a068dbe72..554d226acf 100644 --- a/pymongo/asynchronous/srv_resolver.py +++ b/pymongo/asynchronous/srv_resolver.py @@ -18,9 +18,10 @@ import ipaddress import random -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union -from pymongo._psl import is_public_suffix +from pymongo._psl import SPECIAL_USE_LABELS, _to_punycode, is_public_suffix +from pymongo._telemetry import log_srv_monitor_invalid_host from pymongo.common import CONNECT_TIMEOUT from pymongo.errors import ConfigurationError @@ -64,21 +65,40 @@ def __init__( srv_service_name: str, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ): self.__fqdn = fqdn.lower() self.__srv = srv_service_name self.__connect_timeout = connect_timeout or CONNECT_TIMEOUT self.__srv_max_hosts = srv_max_hosts or 0 - self.__srv_allowed_hosts_suffix = ( - "." + srv_allowed_hosts_suffix.lower().strip(".") if srv_allowed_hosts_suffix else None - ) # ensure there's a . at the beginning of the domain - if self.__srv_allowed_hosts_suffix is not None and is_public_suffix( - self.__srv_allowed_hosts_suffix - ): + self.__srv_host_validator = srv_host_validator + # parse_uri() can call this constructor independently of AsyncMongoClient. + if srv_host_validator is not None and srv_allowed_hosts_suffix is not None: raise ConfigurationError( - f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" ) - # Validate the fully qualified domain name. + self.__srv_allowed_hosts_suffix = None + if srv_allowed_hosts_suffix is not None: + suffix = srv_allowed_hosts_suffix.strip(".") + if not suffix: + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be empty, got: {srv_allowed_hosts_suffix!r}" + ) + suffix = _to_punycode(suffix).lower() + + is_special_use = suffix in SPECIAL_USE_LABELS + if len(suffix.split(".")) < 2 and not is_special_use: + raise ConfigurationError( + "srvAllowedHostsSuffix must contain at least two '.' separated labels, " + f"got: {srv_allowed_hosts_suffix}" + ) + + if not is_special_use and is_public_suffix(suffix): + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + ) + self.__srv_allowed_hosts_suffix = "." + suffix + try: ipaddress.ip_address(fqdn) raise ConfigurationError(_INVALID_HOST_MSG % ("an IP address",)) @@ -106,23 +126,51 @@ async def get_options(self) -> Optional[str]: raise ConfigurationError("Only one TXT record is supported") return (b"&".join([b"".join(res.strings) for res in results])).decode("utf-8") # type: ignore[attr-defined] - async def _resolve_uri(self, encapsulate_errors: bool) -> resolver.Answer: + async def _resolve_uri(self, is_polling: bool) -> resolver.Answer: try: results = await _resolve( "_" + self.__srv + "._tcp." + self.__fqdn, "SRV", lifetime=self.__connect_timeout ) except Exception as exc: - if not encapsulate_errors: + if is_polling: # Raise the original error. raise # Else, raise all errors as ConfigurationError. raise ConfigurationError(str(exc)) from exc return results + def _validate_host(self, srv_host: str) -> None: + """Raise ConfigurationError if an SRV-returned host fails verification.""" + if self.__srv_host_validator is not None: + try: + allowed = self.__srv_host_validator(srv_host) + except Exception as exc: + raise ConfigurationError( + f"srv_host_validator raised an exception for SRV host {srv_host}: {exc}" + ) from exc + if not allowed: + raise ConfigurationError( + f"Invalid SRV host: {srv_host} was rejected by srv_host_validator" + ) + elif self.__srv_allowed_hosts_suffix is not None: + if not srv_host.endswith(self.__srv_allowed_hosts_suffix): + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + else: + if self.__fqdn == srv_host and self.nparts < 3: + raise ConfigurationError( + "Invalid SRV host: return address is identical to SRV hostname" + ) + try: + nlist = srv_host.split(".")[1:][-self.__slen :] + except Exception as exc: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") from exc + if self.__plist != nlist: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + async def _get_srv_response_and_hosts( - self, encapsulate_errors: bool + self, is_polling: bool ) -> tuple[resolver.Answer, list[tuple[str, Any]]]: - results = await self._resolve_uri(encapsulate_errors) + results = await self._resolve_uri(is_polling) # Construct address tuples nodes = [ @@ -130,33 +178,28 @@ async def _get_srv_response_and_hosts( for res in results ] - # Validate hosts + # Validate hosts. + valid_nodes = [] for node in nodes: - srv_host = node[0].lower() - if self.__fqdn == srv_host and self.nparts < 3: - raise ConfigurationError( - "Invalid SRV host: return address is identical to SRV hostname" - ) - if self.__srv_allowed_hosts_suffix is not None: - if not srv_host.endswith(self.__srv_allowed_hosts_suffix): - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + try: + self._validate_host(node[0]) + except ConfigurationError as exc: + if not is_polling: + raise + log_srv_monitor_invalid_host(node[0], exc) else: - try: - nlist = srv_host.split(".")[1:][-self.__slen :] - except Exception as exc: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") from exc - if self.__plist != nlist: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + valid_nodes.append(node) + nodes = valid_nodes if self.__srv_max_hosts: nodes = random.sample(nodes, min(self.__srv_max_hosts, len(nodes))) return results, nodes async def get_hosts(self) -> list[tuple[str, Any]]: - _, nodes = await self._get_srv_response_and_hosts(True) + _, nodes = await self._get_srv_response_and_hosts(is_polling=False) return nodes async def get_hosts_and_min_ttl(self) -> tuple[list[tuple[str, Any]], int]: - results, nodes = await self._get_srv_response_and_hosts(False) + results, nodes = await self._get_srv_response_and_hosts(is_polling=True) rrset = results.rrset ttl = rrset.ttl if rrset else 0 return nodes, ttl diff --git a/pymongo/asynchronous/uri_parser.py b/pymongo/asynchronous/uri_parser.py index 24d3427843..c42add6ae8 100644 --- a/pymongo/asynchronous/uri_parser.py +++ b/pymongo/asynchronous/uri_parser.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Callable, Optional from pymongo.asynchronous.srv_resolver import _SrvResolver from pymongo.common import SRV_SERVICE_NAME, _CaseInsensitiveDictionary @@ -48,6 +48,7 @@ async def parse_uri( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: """Parse and validate a MongoDB URI. @@ -117,6 +118,7 @@ async def parse_uri( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, ) ) result["options"] = _make_options_case_sensitive(result["options"]) @@ -133,6 +135,7 @@ async def _parse_srv( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -168,7 +171,12 @@ async def _parse_srv( # argument overrides the same option passed in the connection string. connect_timeout = connect_timeout or options.get("connectTimeoutMS") dns_resolver = _SrvResolver( - fqdn, connect_timeout, srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix + fqdn, + connect_timeout, + srv_service_name, + srv_max_hosts, + srv_allowed_hosts_suffix, + srv_host_validator, ) nodes = await dns_resolver.get_hosts() dns_options = await dns_resolver.get_options() diff --git a/pymongo/common.py b/pymongo/common.py index dc18b937e5..9fa3bb15db 100644 --- a/pymongo/common.py +++ b/pymongo/common.py @@ -784,6 +784,7 @@ def validate_server_monitoring_mode(option: str, value: str) -> str: "username": validate_string_or_none, "password": validate_string_or_none, "server_selector": validate_is_callable_or_none, + "srv_host_validator": validate_is_callable_or_none, "auto_encryption_opts": validate_auto_encryption_opts_or_none, "authoidcallowedhosts": validate_list, "max_adaptive_retries": validate_non_negative_integer, diff --git a/pymongo/public_suffix_list.dat b/pymongo/public_suffix_list.dat index 78da70c4d4..0f09a04291 100644 --- a/pymongo/public_suffix_list.dat +++ b/pymongo/public_suffix_list.dat @@ -6962,7 +6962,6 @@ ltd.ua a2hosted.com cpserver.com activetrail.biz -adaptable.app myaddr.dev myaddr.io dyn.addr.tools @@ -6982,7 +6981,7 @@ africa.com *.auiusercontent.com beep.pl aiven.app -aivencloud.com +*.aivencloud.com akadns.net akamai.net akamai-staging.net @@ -7857,6 +7856,9 @@ eero-stage.online opentunnel.xyz antagonist.cloud claude.app +claudeusercontent.com +frame.claudeusercontent.com +*.cursorusercontent.com apigee.io panel.dev siiites.com @@ -8069,6 +8071,8 @@ sch.ac dev.cv store.cv codeberg.page +codepen.app +codepen.dev csb.app preview.csb.app co.nl @@ -8948,6 +8952,7 @@ hepforge.org onhercules.app hercules-app.com hercules-dev.com +here.now herokuapp.com heyflow.page heyflow.site @@ -9043,6 +9048,7 @@ botdash.gg botdash.net botda.sh botdash.xyz +online-server.cloud apps-1and1.com live-website.com webspace-host.com @@ -9882,6 +9888,9 @@ scbl.pl-waw.scw.cloud scalebook.scw.cloud smartlabeling.scw.cloud dedibox.fr +scw.site +ams.scw.site +waw.scw.site schokokeks.net gov.scot service.gov.scot @@ -10175,9 +10184,9 @@ toolforge.org wmcloud.org beta.wmcloud.org wmflabs.org -vps.hrsn.au hrsn.dev is-a.dev +vps.hrsn.net localcert.net windsurf.app windsurf.build diff --git a/pymongo/settings_shared.py b/pymongo/settings_shared.py index fbe87697c1..bf45c40334 100644 --- a/pymongo/settings_shared.py +++ b/pymongo/settings_shared.py @@ -19,7 +19,7 @@ import threading import traceback from collections.abc import Collection -from typing import Any, Generic, Optional, TypeVar, Union +from typing import Any, Callable, Generic, Optional, TypeVar, Union from bson.objectid import ObjectId from pymongo import common @@ -52,6 +52,7 @@ def __init__( srv_service_name: str = common.SRV_SERVICE_NAME, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, server_monitoring_mode: str = common.SERVER_MONITORING_MODE, topology_id: Optional[ObjectId] = None, ): @@ -79,6 +80,7 @@ def __init__( self._srv_service_name = srv_service_name self._srv_max_hosts = srv_max_hosts or 0 self._srv_allowed_hosts_suffix = srv_allowed_hosts_suffix + self._srv_host_validator = srv_host_validator self._server_monitoring_mode = server_monitoring_mode if topology_id is not None: self._topology_id = topology_id @@ -161,6 +163,11 @@ def srv_allowed_hosts_suffix(self) -> Optional[str]: """The srvAllowedHostsSuffix.""" return self._srv_allowed_hosts_suffix + @property + def srv_host_validator(self) -> Optional[Callable[[str], bool]]: + """The srv_host_validator callback.""" + return self._srv_host_validator + @property def server_monitoring_mode(self) -> str: """The serverMonitoringMode.""" diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 574fbc00de..df09b03b20 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -471,6 +471,31 @@ def __init__( srvAllowedHostsSuffix=".internal.example.com", ) + - `srv_host_validator`: (callable or None) A callback used in place of the + default parent-domain check for hosts returned by SRV DNS records. It is + called once per returned host with the lowercased hostname as its only + argument, and must return ``True`` to accept the host or ``False`` to + reject it. During initial DNS discovery, rejecting a host raises + :exc:`~pymongo.errors.ConfigurationError`, as does an exception raised by + the callback itself. Use this when the set of acceptable hosts cannot be + expressed as a single suffix:: + + def validator(host: str) -> bool: + return host.endswith((".a.example.com", ".b.example.com")) + + MongoClient( + "mongodb+srv://cluster.example.com/", + srv_host_validator=validator, + ) + + The callback must not block. It is mutually exclusive with + ``srvAllowedHostsSuffix``. Because this option is a callable, it can + only be passed in as a keyword argument, not through the connection string. + + .. warning:: + + This option replaces the built-in DNS spoofing safeguards. + Please use with caution. | **Write Concern options:** | (Only set if passed. No default values.) @@ -638,6 +663,12 @@ def __init__( .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.19 + Added the ``srv_host_validator`` keyword argument. + + .. versionchanged:: 4.18 + Added the ``srvAllowedHostsSuffix`` URI and keyword argument. + .. versionchanged:: 4.17 Added the ``max_adaptive_retries`` and ``enable_overload_retargeting`` URI and keyword arguments. @@ -824,6 +855,7 @@ def __init__( srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") if len([h for h in self._host if "/" in h]) > 1: raise ConfigurationError("host must not contain multiple MongoDB URIs") for entity in self._host: @@ -916,7 +948,11 @@ def __init__( ) self._init_based_on_options( - self._seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + self._seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) self._opened = False @@ -936,6 +972,7 @@ def _resolve_srv(self) -> None: srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") for entity in self._host: # A hostname can only include a-z, 0-9, '-' and '.'. If we find a '/' # it must be a URI, @@ -957,6 +994,7 @@ def _resolve_srv(self) -> None: srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, ) seeds.update(res["nodelist"]) opts = res["options"] @@ -1001,7 +1039,11 @@ def _resolve_srv(self) -> None: ) self._init_based_on_options( - seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) def _init_based_on_options( @@ -1010,7 +1052,17 @@ def _init_based_on_options( srv_max_hosts: Any, srv_service_name: Any, srv_allowed_hosts_suffix: Any, + srv_host_validator: Any = None, ) -> None: + if srv_host_validator is not None: + if srv_allowed_hosts_suffix is not None: + raise ConfigurationError( + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" + ) + if not self._resolve_srv_info["is_srv"]: + raise ConfigurationError( + "The srv_host_validator option is only allowed with 'mongodb+srv://' URIs" + ) self._event_listeners = self._options.pool_options._event_listeners self._topology_settings = TopologySettings( seeds=seeds, @@ -1029,6 +1081,7 @@ def _init_based_on_options( srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, server_monitoring_mode=self._options.server_monitoring_mode, topology_id=self._topology_settings._topology_id if self._topology_settings else None, ) diff --git a/pymongo/synchronous/monitor.py b/pymongo/synchronous/monitor.py index 3a1e1a49d0..cbc5796a8a 100644 --- a/pymongo/synchronous/monitor.py +++ b/pymongo/synchronous/monitor.py @@ -366,6 +366,7 @@ def _get_seedlist(self) -> Optional[list[tuple[str, Any]]]: self._settings.pool_options.connect_timeout, self._settings.srv_service_name, srv_allowed_hosts_suffix=self._settings.srv_allowed_hosts_suffix, + srv_host_validator=self._settings.srv_host_validator, ) seedlist, ttl = resolver.get_hosts_and_min_ttl() if len(seedlist) == 0: diff --git a/pymongo/synchronous/settings.py b/pymongo/synchronous/settings.py index 2e8c6ddcd8..3416bbd130 100644 --- a/pymongo/synchronous/settings.py +++ b/pymongo/synchronous/settings.py @@ -18,7 +18,7 @@ import threading from collections.abc import Collection -from typing import Optional +from typing import Callable, Optional from bson.objectid import ObjectId from pymongo import common @@ -51,6 +51,7 @@ def __init__( srv_service_name: str = common.SRV_SERVICE_NAME, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, server_monitoring_mode: str = common.SERVER_MONITORING_MODE, topology_id: Optional[ObjectId] = None, ): @@ -77,6 +78,7 @@ def __init__( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, server_monitoring_mode, topology_id, ) diff --git a/pymongo/synchronous/srv_resolver.py b/pymongo/synchronous/srv_resolver.py index 0e9745eb08..18083c8a0c 100644 --- a/pymongo/synchronous/srv_resolver.py +++ b/pymongo/synchronous/srv_resolver.py @@ -18,9 +18,10 @@ import ipaddress import random -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union -from pymongo._psl import is_public_suffix +from pymongo._psl import SPECIAL_USE_LABELS, _to_punycode, is_public_suffix +from pymongo._telemetry import log_srv_monitor_invalid_host from pymongo.common import CONNECT_TIMEOUT from pymongo.errors import ConfigurationError @@ -64,21 +65,40 @@ def __init__( srv_service_name: str, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ): self.__fqdn = fqdn.lower() self.__srv = srv_service_name self.__connect_timeout = connect_timeout or CONNECT_TIMEOUT self.__srv_max_hosts = srv_max_hosts or 0 - self.__srv_allowed_hosts_suffix = ( - "." + srv_allowed_hosts_suffix.lower().strip(".") if srv_allowed_hosts_suffix else None - ) # ensure there's a . at the beginning of the domain - if self.__srv_allowed_hosts_suffix is not None and is_public_suffix( - self.__srv_allowed_hosts_suffix - ): + self.__srv_host_validator = srv_host_validator + # parse_uri() can call this constructor independently of MongoClient. + if srv_host_validator is not None and srv_allowed_hosts_suffix is not None: raise ConfigurationError( - f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" ) - # Validate the fully qualified domain name. + self.__srv_allowed_hosts_suffix = None + if srv_allowed_hosts_suffix is not None: + suffix = srv_allowed_hosts_suffix.strip(".") + if not suffix: + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be empty, got: {srv_allowed_hosts_suffix!r}" + ) + suffix = _to_punycode(suffix).lower() + + is_special_use = suffix in SPECIAL_USE_LABELS + if len(suffix.split(".")) < 2 and not is_special_use: + raise ConfigurationError( + "srvAllowedHostsSuffix must contain at least two '.' separated labels, " + f"got: {srv_allowed_hosts_suffix}" + ) + + if not is_special_use and is_public_suffix(suffix): + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + ) + self.__srv_allowed_hosts_suffix = "." + suffix + try: ipaddress.ip_address(fqdn) raise ConfigurationError(_INVALID_HOST_MSG % ("an IP address",)) @@ -106,23 +126,51 @@ def get_options(self) -> Optional[str]: raise ConfigurationError("Only one TXT record is supported") return (b"&".join([b"".join(res.strings) for res in results])).decode("utf-8") # type: ignore[attr-defined] - def _resolve_uri(self, encapsulate_errors: bool) -> resolver.Answer: + def _resolve_uri(self, is_polling: bool) -> resolver.Answer: try: results = _resolve( "_" + self.__srv + "._tcp." + self.__fqdn, "SRV", lifetime=self.__connect_timeout ) except Exception as exc: - if not encapsulate_errors: + if is_polling: # Raise the original error. raise # Else, raise all errors as ConfigurationError. raise ConfigurationError(str(exc)) from exc return results + def _validate_host(self, srv_host: str) -> None: + """Raise ConfigurationError if an SRV-returned host fails verification.""" + if self.__srv_host_validator is not None: + try: + allowed = self.__srv_host_validator(srv_host) + except Exception as exc: + raise ConfigurationError( + f"srv_host_validator raised an exception for SRV host {srv_host}: {exc}" + ) from exc + if not allowed: + raise ConfigurationError( + f"Invalid SRV host: {srv_host} was rejected by srv_host_validator" + ) + elif self.__srv_allowed_hosts_suffix is not None: + if not srv_host.endswith(self.__srv_allowed_hosts_suffix): + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + else: + if self.__fqdn == srv_host and self.nparts < 3: + raise ConfigurationError( + "Invalid SRV host: return address is identical to SRV hostname" + ) + try: + nlist = srv_host.split(".")[1:][-self.__slen :] + except Exception as exc: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") from exc + if self.__plist != nlist: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + def _get_srv_response_and_hosts( - self, encapsulate_errors: bool + self, is_polling: bool ) -> tuple[resolver.Answer, list[tuple[str, Any]]]: - results = self._resolve_uri(encapsulate_errors) + results = self._resolve_uri(is_polling) # Construct address tuples nodes = [ @@ -130,33 +178,28 @@ def _get_srv_response_and_hosts( for res in results ] - # Validate hosts + # Validate hosts. + valid_nodes = [] for node in nodes: - srv_host = node[0].lower() - if self.__fqdn == srv_host and self.nparts < 3: - raise ConfigurationError( - "Invalid SRV host: return address is identical to SRV hostname" - ) - if self.__srv_allowed_hosts_suffix is not None: - if not srv_host.endswith(self.__srv_allowed_hosts_suffix): - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + try: + self._validate_host(node[0]) + except ConfigurationError as exc: + if not is_polling: + raise + log_srv_monitor_invalid_host(node[0], exc) else: - try: - nlist = srv_host.split(".")[1:][-self.__slen :] - except Exception as exc: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") from exc - if self.__plist != nlist: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + valid_nodes.append(node) + nodes = valid_nodes if self.__srv_max_hosts: nodes = random.sample(nodes, min(self.__srv_max_hosts, len(nodes))) return results, nodes def get_hosts(self) -> list[tuple[str, Any]]: - _, nodes = self._get_srv_response_and_hosts(True) + _, nodes = self._get_srv_response_and_hosts(is_polling=False) return nodes def get_hosts_and_min_ttl(self) -> tuple[list[tuple[str, Any]], int]: - results, nodes = self._get_srv_response_and_hosts(False) + results, nodes = self._get_srv_response_and_hosts(is_polling=True) rrset = results.rrset ttl = rrset.ttl if rrset else 0 return nodes, ttl diff --git a/pymongo/synchronous/uri_parser.py b/pymongo/synchronous/uri_parser.py index e4f521e3a9..92608b5efe 100644 --- a/pymongo/synchronous/uri_parser.py +++ b/pymongo/synchronous/uri_parser.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Callable, Optional from pymongo.common import SRV_SERVICE_NAME, _CaseInsensitiveDictionary from pymongo.errors import ConfigurationError, InvalidURI @@ -48,6 +48,7 @@ def parse_uri( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: """Parse and validate a MongoDB URI. @@ -117,6 +118,7 @@ def parse_uri( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, ) ) result["options"] = _make_options_case_sensitive(result["options"]) @@ -133,6 +135,7 @@ def _parse_srv( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -168,7 +171,12 @@ def _parse_srv( # argument overrides the same option passed in the connection string. connect_timeout = connect_timeout or options.get("connectTimeoutMS") dns_resolver = _SrvResolver( - fqdn, connect_timeout, srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix + fqdn, + connect_timeout, + srv_service_name, + srv_max_hosts, + srv_allowed_hosts_suffix, + srv_host_validator, ) nodes = dns_resolver.get_hosts() dns_options = dns_resolver.get_options() diff --git a/test/asynchronous/test_dns.py b/test/asynchronous/test_dns.py index 6796ecff55..befa119b46 100644 --- a/test/asynchronous/test_dns.py +++ b/test/asynchronous/test_dns.py @@ -209,30 +209,40 @@ async def test_connect_case_insensitive(self): class TestInitialDnsSeedlistDiscovery(AsyncPyMongoTestCase): """ Initial DNS Seedlist Discovery prose tests - https://github.com/mongodb/specifications/blob/0a7a8b5/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + https://github.com/mongodb/specifications/blob/5036f26/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + + Numbered tests correspond to the numbered prose tests in the spec. The + unnumbered tests are PyMongo-specific additions with no spec counterpart. """ + async def _parse(self, srv_hostname, mock_target, **kwargs): + """Resolve mongodb+srv:// with SRV records naming mock_target.""" + with patch("dns.asyncresolver.resolve") as mock_resolver: + + async def mock_resolve(query, record_type, *args, **kwargs): + mock_srv = MagicMock() + # Mirror dnspython: the wire form keeps the root label, and the + # caller strips it via omit_final_dot. + mock_srv.target.to_text.side_effect = lambda omit_final_dot=False: ( + mock_target.rstrip(".") if omit_final_dot else mock_target + ) + return [mock_srv] + + mock_resolver.side_effect = mock_resolve + return await parse_uri(f"mongodb+srv://{srv_hostname}", **kwargs) + async def run_initial_dns_seedlist_discovery_prose_tests(self, test_cases): for case in test_cases: - with patch("dns.asyncresolver.resolve") as mock_resolver: - - async def mock_resolve(query, record_type, *args, **kwargs): - mock_srv = MagicMock() - mock_srv.target.to_text.return_value = case["mock_target"] - return [mock_srv] - - mock_resolver.side_effect = mock_resolve - domain = case["query"].split("._tcp.")[1] - connection_string = f"mongodb+srv://{domain}" - if "expected_error" not in case: - await parse_uri(connection_string) + domain = case["query"].split("._tcp.")[1] + if "expected_error" not in case: + await self._parse(domain, case["mock_target"]) + else: + try: + await self._parse(domain, case["mock_target"]) + except ConfigurationError as e: + self.assertIn(case["expected_error"], str(e)) else: - try: - await parse_uri(connection_string) - except ConfigurationError as e: - self.assertIn(case["expected_error"], str(e)) - else: - self.fail(f"ConfigurationError was not raised for query: {case['query']}") + self.fail(f"ConfigurationError was not raised for query: {case['query']}") async def test_1_allow_srv_hosts_with_fewer_than_three_dot_separated_parts(self): with patch("dns.asyncresolver.resolve"): @@ -252,8 +262,8 @@ async def test_2_throw_when_return_address_does_not_end_with_srv_domain(self): "expected_error": "Invalid SRV host", }, { - "query": "_mongodb._tcp.blogs.mongo.local", - "mock_target": "test_1.evil.com", + "query": "_mongodb._tcp.mongo.local", + "mock_target": "test_1.evil.local", "expected_error": "Invalid SRV host", }, ] @@ -296,7 +306,103 @@ async def test_4_throw_when_return_address_does_not_contain_dot_separating_share ] await self.run_initial_dns_seedlist_discovery_prose_tests(test_cases) - async def test_5_when_srv_hostname_has_three_or_more_dot_separated_parts_it_is_valid_for_the_returned_hostname_to_be_identical( + async def test_5_srv_host_validator_accepts_a_host_the_default_verification_would_reject(self): + # "blogs.evil.com" does not share a parent domain with the seed, so the + # default check rejects it; the callback overrides that decision. + res = await self._parse( + "blogs.mongodb.com", "blogs.evil.com", srv_host_validator=lambda host: True + ) + self.assertEqual(["blogs.evil.com"], [node[0] for node in res["nodelist"]]) + + # "mongo.local" does not add a domain level to an SRV hostname with fewer + # than three "." separated parts, which the default check also rejects. + res = await self._parse("mongo.local", "mongo.local", srv_host_validator=lambda host: True) + self.assertEqual(["mongo.local"], [node[0] for node in res["nodelist"]]) + + async def test_6_reject_a_host_the_default_verification_would_accept(self): + with self.assertRaisesRegex(ConfigurationError, "rejected by srv_host_validator"): + await self._parse( + "blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=lambda host: False + ) + + async def test_7_the_validator_receives_the_normalized_host_name(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + await self._parse("blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_host_validator=validator) + self.assertEqual(["cluster.mongodb.com"], seen) + + async def test_8_wrap_an_error_raised_by_the_validator(self): + original_exc = Exception("validator_error") + + def validator(host): + raise original_exc + + with self.assertRaisesRegex( + ConfigurationError, "srv_host_validator raised an exception" + ) as ctx: + await self._parse( + "blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=validator + ) + # The wrapping error must retain the error raised by the validator. + self.assertIs(original_exc, ctx.exception.__cause__) + self.assertIn("validator_error", str(ctx.exception)) + + async def test_9_throw_when_both_srv_allowed_hosts_suffix_and_srv_host_validator_are_configured( + self, + ): + # Rejected by the client + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + self.simple_client( + "mongodb+srv://blogs.mongodb.com", + srv_host_validator=lambda host: True, + srvAllowedHostsSuffix=".mongodb.com", + connect=False, + ) + + # Rejected by the resolver + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + await self._parse( + "blogs.mongodb.com", + "cluster.mongodb.com", + srv_host_validator=lambda host: True, + srv_allowed_hosts_suffix=".mongodb.com", + ) + + async def test_10_accept_a_mixed_case_returned_address_with_srv_allowed_hosts_suffix(self): + # Returned addresses are normalized before the suffix comparison, so + # the case DNS happens to use must not affect the result. + res = await self._parse( + "blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_allowed_hosts_suffix=".mongodb.com" + ) + self.assertEqual(["cluster.mongodb.com"], [node[0] for node in res["nodelist"]]) + + async def test_11_throw_when_srv_host_validator_is_not_callable(self): + with self.assertRaisesRegex(ValueError, "must be a callable"): + self.simple_client("mongodb+srv://blogs.mongodb.com", srv_host_validator="notacallable") + + async def test_12_accept_a_reserved_single_label_as_srv_allowed_hosts_suffix(self): + # A single label is a public suffix under the Public Suffix List's "*" + # rule, but the reserved names are accepted despite that. + res = await self._parse( + "cluster.localhost", "db.cluster.localhost", srv_allowed_hosts_suffix="localhost" + ) + self.assertEqual(["db.cluster.localhost"], [node[0] for node in res["nodelist"]]) + + async def test_13_throw_when_srv_host_validator_is_used_with_a_non_srv_uri(self): + with self.assertRaisesRegex( + ConfigurationError, "only allowed with 'mongodb\\+srv://' URIs" + ): + self.simple_client( + "mongodb://localhost:27017", + srv_host_validator=lambda host: True, + connect=False, + ) + + async def test_srv_hostname_with_three_or_more_parts_may_equal_the_returned_hostname( self, ): test_cases = [ diff --git a/test/asynchronous/test_srv_polling.py b/test/asynchronous/test_srv_polling.py index b8b9129925..3d98a6fdc8 100644 --- a/test/asynchronous/test_srv_polling.py +++ b/test/asynchronous/test_srv_polling.py @@ -19,7 +19,9 @@ import asyncio import sys import time +from types import SimpleNamespace from typing import Any +from unittest.mock import patch from test.utils_shared import FunctionCallRecorder @@ -37,6 +39,19 @@ WAIT_TIME = 0.1 +class _FakeSrvAnswer(list): + """A minimal stand-in for the dnspython answer to an SRV query.""" + + def __init__(self, hosts, ttl=60): + import dns.name + + hosts = [(h, 27017) if isinstance(h, str) else h for h in hosts] + super().__init__( + SimpleNamespace(target=dns.name.from_text(h), port=port) for h, port in hosts + ) + self.rrset = SimpleNamespace(ttl=ttl) + + class SrvPollingKnobs: def __init__( self, @@ -64,9 +79,15 @@ def enable(self): async def mock_get_hosts_and_min_ttl(resolver, *args): assert self.old_dns_resolver_response is not None - nodes, ttl = await self.old_dns_resolver_response(resolver) - if self.nodelist_callback is not None: - nodes = self.nodelist_callback() + if self.nodelist_callback is None: + nodes, ttl = await self.old_dns_resolver_response(resolver) + else: + + async def mock_resolve_uri(is_polling): + return _FakeSrvAnswer(self.nodelist_callback()) + + with patch.object(resolver, "_resolve_uri", mock_resolve_uri): + nodes, ttl = await self.old_dns_resolver_response(resolver) if self.ttl_time is not None: ttl = self.ttl_time return nodes, ttl @@ -357,8 +378,92 @@ def nodelist_callback(): ) await client.aconnect() with SrvPollingKnobs(nodelist_callback=nodelist_callback): + expected = [(host.rstrip("."), port) for host, port in response] + await self.assert_nodelist_change(expected, client) + + async def test_14_the_validator_is_consulted_when_srv_records_are_rescanned(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client(self.CONNECTION_STRING, srv_host_validator=validator) + await client.aconnect() + await self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + seen.clear() + with SrvPollingKnobs(nodelist_callback=lambda: response): await self.assert_nodelist_change(response, client) + self.assertIn("localhost.test.build.10gen.cc", seen) + + async def test_15_a_rejecting_or_raising_validator_does_not_raise_or_stop_polling(self): + def rejecting(host): + return False + + def raising(host): + raise RuntimeError("boom") + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + for failing in (rejecting, raising): + with self.subTest(validator=failing.__name__): + # Accept everything until the client is connected, then start failing. + state = {"validator": lambda host: True} + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client( + self.CONNECTION_STRING, + srv_host_validator=lambda host: state["validator"](host), + ) + await client.aconnect() + await self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + + # The rescan sees the new record but the validator fails: no + # error reaches the application and the topology is unchanged. + state["validator"] = failing + with SrvPollingKnobs( + nodelist_callback=lambda: response, count_resolver_calls=True + ): + await self.assert_nodelist_nochange(self.BASE_SRV_RESPONSE, client) + + # Polling was not stopped by the failures: once the validator + # accepts again, the new host is picked up. + state["validator"] = lambda host: True + with SrvPollingKnobs(nodelist_callback=lambda: response): + await self.assert_nodelist_change(response, client) + + await client.close() + + async def test_rescan_skips_only_the_hosts_that_fail_verification(self): + # Per the polling spec, a host that fails verification is left out of + # the topology without discarding the other hosts from the same rescan. + good_host = "localhost.test.build.10gen.cc" + bad_host = "bad.evil.com" + + def raising(host): + if host == bad_host: + raise RuntimeError("some error") + return True + + def rejecting(host): + return host != bad_host + + for validator in (rejecting, raising): + with self.subTest(validator=validator.__name__): + resolver = pymongo.asynchronous.srv_resolver._SrvResolver( + "test1.test.build.10gen.cc", None, "mongodb", srv_host_validator=validator + ) + answer = _FakeSrvAnswer([good_host, bad_host]) + with patch("dns.asyncresolver.resolve", return_value=answer): + nodes, _ = await resolver.get_hosts_and_min_ttl() + self.assertEqual([(good_host, 27017)], nodes) + async def test_srv_waits_to_poll(self): modified = [("localhost.test.build.10gen.cc", 27019)] diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 5d16bc5918..2ee4e68657 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -23,6 +23,7 @@ import binascii import copy import functools +import gc import os import platform import re @@ -539,6 +540,11 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: cls.knobs.disable() + # Workaround for sockets leaked by tests (PYTHON-3923): force garbage + # collection so they are closed here, rather than during a later test + # where their ResourceWarnings could cause failures. Remove once the + # leaks are fixed. + gc.collect() async def asyncSetUp(self): # super call creates internal client cls.client diff --git a/test/test_dns.py b/test/test_dns.py index e631420643..89dee6f3a7 100644 --- a/test/test_dns.py +++ b/test/test_dns.py @@ -207,30 +207,40 @@ def test_connect_case_insensitive(self): class TestInitialDnsSeedlistDiscovery(PyMongoTestCase): """ Initial DNS Seedlist Discovery prose tests - https://github.com/mongodb/specifications/blob/0a7a8b5/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + https://github.com/mongodb/specifications/blob/5036f26/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + + Numbered tests correspond to the numbered prose tests in the spec. The + unnumbered tests are PyMongo-specific additions with no spec counterpart. """ + def _parse(self, srv_hostname, mock_target, **kwargs): + """Resolve mongodb+srv:// with SRV records naming mock_target.""" + with patch("dns.resolver.resolve") as mock_resolver: + + def mock_resolve(query, record_type, *args, **kwargs): + mock_srv = MagicMock() + # Mirror dnspython: the wire form keeps the root label, and the + # caller strips it via omit_final_dot. + mock_srv.target.to_text.side_effect = lambda omit_final_dot=False: ( + mock_target.rstrip(".") if omit_final_dot else mock_target + ) + return [mock_srv] + + mock_resolver.side_effect = mock_resolve + return parse_uri(f"mongodb+srv://{srv_hostname}", **kwargs) + def run_initial_dns_seedlist_discovery_prose_tests(self, test_cases): for case in test_cases: - with patch("dns.resolver.resolve") as mock_resolver: - - def mock_resolve(query, record_type, *args, **kwargs): - mock_srv = MagicMock() - mock_srv.target.to_text.return_value = case["mock_target"] - return [mock_srv] - - mock_resolver.side_effect = mock_resolve - domain = case["query"].split("._tcp.")[1] - connection_string = f"mongodb+srv://{domain}" - if "expected_error" not in case: - parse_uri(connection_string) + domain = case["query"].split("._tcp.")[1] + if "expected_error" not in case: + self._parse(domain, case["mock_target"]) + else: + try: + self._parse(domain, case["mock_target"]) + except ConfigurationError as e: + self.assertIn(case["expected_error"], str(e)) else: - try: - parse_uri(connection_string) - except ConfigurationError as e: - self.assertIn(case["expected_error"], str(e)) - else: - self.fail(f"ConfigurationError was not raised for query: {case['query']}") + self.fail(f"ConfigurationError was not raised for query: {case['query']}") def test_1_allow_srv_hosts_with_fewer_than_three_dot_separated_parts(self): with patch("dns.resolver.resolve"): @@ -250,8 +260,8 @@ def test_2_throw_when_return_address_does_not_end_with_srv_domain(self): "expected_error": "Invalid SRV host", }, { - "query": "_mongodb._tcp.blogs.mongo.local", - "mock_target": "test_1.evil.com", + "query": "_mongodb._tcp.mongo.local", + "mock_target": "test_1.evil.local", "expected_error": "Invalid SRV host", }, ] @@ -294,7 +304,101 @@ def test_4_throw_when_return_address_does_not_contain_dot_separating_shared_part ] self.run_initial_dns_seedlist_discovery_prose_tests(test_cases) - def test_5_when_srv_hostname_has_three_or_more_dot_separated_parts_it_is_valid_for_the_returned_hostname_to_be_identical( + def test_5_srv_host_validator_accepts_a_host_the_default_verification_would_reject(self): + # "blogs.evil.com" does not share a parent domain with the seed, so the + # default check rejects it; the callback overrides that decision. + res = self._parse( + "blogs.mongodb.com", "blogs.evil.com", srv_host_validator=lambda host: True + ) + self.assertEqual(["blogs.evil.com"], [node[0] for node in res["nodelist"]]) + + # "mongo.local" does not add a domain level to an SRV hostname with fewer + # than three "." separated parts, which the default check also rejects. + res = self._parse("mongo.local", "mongo.local", srv_host_validator=lambda host: True) + self.assertEqual(["mongo.local"], [node[0] for node in res["nodelist"]]) + + def test_6_reject_a_host_the_default_verification_would_accept(self): + with self.assertRaisesRegex(ConfigurationError, "rejected by srv_host_validator"): + self._parse( + "blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=lambda host: False + ) + + def test_7_the_validator_receives_the_normalized_host_name(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + self._parse("blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_host_validator=validator) + self.assertEqual(["cluster.mongodb.com"], seen) + + def test_8_wrap_an_error_raised_by_the_validator(self): + original_exc = Exception("validator_error") + + def validator(host): + raise original_exc + + with self.assertRaisesRegex( + ConfigurationError, "srv_host_validator raised an exception" + ) as ctx: + self._parse("blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=validator) + # The wrapping error must retain the error raised by the validator. + self.assertIs(original_exc, ctx.exception.__cause__) + self.assertIn("validator_error", str(ctx.exception)) + + def test_9_throw_when_both_srv_allowed_hosts_suffix_and_srv_host_validator_are_configured( + self, + ): + # Rejected by the client + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + self.simple_client( + "mongodb+srv://blogs.mongodb.com", + srv_host_validator=lambda host: True, + srvAllowedHostsSuffix=".mongodb.com", + connect=False, + ) + + # Rejected by the resolver + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + self._parse( + "blogs.mongodb.com", + "cluster.mongodb.com", + srv_host_validator=lambda host: True, + srv_allowed_hosts_suffix=".mongodb.com", + ) + + def test_10_accept_a_mixed_case_returned_address_with_srv_allowed_hosts_suffix(self): + # Returned addresses are normalized before the suffix comparison, so + # the case DNS happens to use must not affect the result. + res = self._parse( + "blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_allowed_hosts_suffix=".mongodb.com" + ) + self.assertEqual(["cluster.mongodb.com"], [node[0] for node in res["nodelist"]]) + + def test_11_throw_when_srv_host_validator_is_not_callable(self): + with self.assertRaisesRegex(ValueError, "must be a callable"): + self.simple_client("mongodb+srv://blogs.mongodb.com", srv_host_validator="notacallable") + + def test_12_accept_a_reserved_single_label_as_srv_allowed_hosts_suffix(self): + # A single label is a public suffix under the Public Suffix List's "*" + # rule, but the reserved names are accepted despite that. + res = self._parse( + "cluster.localhost", "db.cluster.localhost", srv_allowed_hosts_suffix="localhost" + ) + self.assertEqual(["db.cluster.localhost"], [node[0] for node in res["nodelist"]]) + + def test_13_throw_when_srv_host_validator_is_used_with_a_non_srv_uri(self): + with self.assertRaisesRegex( + ConfigurationError, "only allowed with 'mongodb\\+srv://' URIs" + ): + self.simple_client( + "mongodb://localhost:27017", + srv_host_validator=lambda host: True, + connect=False, + ) + + def test_srv_hostname_with_three_or_more_parts_may_equal_the_returned_hostname( self, ): test_cases = [ diff --git a/test/test_srv_polling.py b/test/test_srv_polling.py index e01ae1da7f..11ea739a55 100644 --- a/test/test_srv_polling.py +++ b/test/test_srv_polling.py @@ -19,7 +19,9 @@ import asyncio import sys import time +from types import SimpleNamespace from typing import Any +from unittest.mock import patch from test.utils_shared import FunctionCallRecorder @@ -37,6 +39,19 @@ WAIT_TIME = 0.1 +class _FakeSrvAnswer(list): + """A minimal stand-in for the dnspython answer to an SRV query.""" + + def __init__(self, hosts, ttl=60): + import dns.name + + hosts = [(h, 27017) if isinstance(h, str) else h for h in hosts] + super().__init__( + SimpleNamespace(target=dns.name.from_text(h), port=port) for h, port in hosts + ) + self.rrset = SimpleNamespace(ttl=ttl) + + class SrvPollingKnobs: def __init__( self, @@ -64,9 +79,15 @@ def enable(self): def mock_get_hosts_and_min_ttl(resolver, *args): assert self.old_dns_resolver_response is not None - nodes, ttl = self.old_dns_resolver_response(resolver) - if self.nodelist_callback is not None: - nodes = self.nodelist_callback() + if self.nodelist_callback is None: + nodes, ttl = self.old_dns_resolver_response(resolver) + else: + + def mock_resolve_uri(is_polling): + return _FakeSrvAnswer(self.nodelist_callback()) + + with patch.object(resolver, "_resolve_uri", mock_resolve_uri): + nodes, ttl = self.old_dns_resolver_response(resolver) if self.ttl_time is not None: ttl = self.ttl_time return nodes, ttl @@ -357,8 +378,92 @@ def nodelist_callback(): ) client._connect() with SrvPollingKnobs(nodelist_callback=nodelist_callback): + expected = [(host.rstrip("."), port) for host, port in response] + self.assert_nodelist_change(expected, client) + + def test_14_the_validator_is_consulted_when_srv_records_are_rescanned(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client(self.CONNECTION_STRING, srv_host_validator=validator) + client._connect() + self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + seen.clear() + with SrvPollingKnobs(nodelist_callback=lambda: response): self.assert_nodelist_change(response, client) + self.assertIn("localhost.test.build.10gen.cc", seen) + + def test_15_a_rejecting_or_raising_validator_does_not_raise_or_stop_polling(self): + def rejecting(host): + return False + + def raising(host): + raise RuntimeError("boom") + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + for failing in (rejecting, raising): + with self.subTest(validator=failing.__name__): + # Accept everything until the client is connected, then start failing. + state = {"validator": lambda host: True} + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client( + self.CONNECTION_STRING, + srv_host_validator=lambda host: state["validator"](host), + ) + client._connect() + self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + + # The rescan sees the new record but the validator fails: no + # error reaches the application and the topology is unchanged. + state["validator"] = failing + with SrvPollingKnobs( + nodelist_callback=lambda: response, count_resolver_calls=True + ): + self.assert_nodelist_nochange(self.BASE_SRV_RESPONSE, client) + + # Polling was not stopped by the failures: once the validator + # accepts again, the new host is picked up. + state["validator"] = lambda host: True + with SrvPollingKnobs(nodelist_callback=lambda: response): + self.assert_nodelist_change(response, client) + + client.close() + + def test_rescan_skips_only_the_hosts_that_fail_verification(self): + # Per the polling spec, a host that fails verification is left out of + # the topology without discarding the other hosts from the same rescan. + good_host = "localhost.test.build.10gen.cc" + bad_host = "bad.evil.com" + + def raising(host): + if host == bad_host: + raise RuntimeError("some error") + return True + + def rejecting(host): + return host != bad_host + + for validator in (rejecting, raising): + with self.subTest(validator=validator.__name__): + resolver = pymongo.synchronous.srv_resolver._SrvResolver( + "test1.test.build.10gen.cc", None, "mongodb", srv_host_validator=validator + ) + answer = _FakeSrvAnswer([good_host, bad_host]) + with patch("dns.resolver.resolve", return_value=answer): + nodes, _ = resolver.get_hosts_and_min_ttl() + self.assertEqual([(good_host, 27017)], nodes) + def test_srv_waits_to_poll(self): modified = [("localhost.test.build.10gen.cc", 27019)] diff --git a/test/unified_format.py b/test/unified_format.py index 56ea563e6d..657d7c0f22 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -23,6 +23,7 @@ import binascii import copy import functools +import gc import os import platform import re @@ -538,6 +539,11 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: cls.knobs.disable() + # Workaround for sockets leaked by tests (PYTHON-3923): force garbage + # collection so they are closed here, rather than during a later test + # where their ResourceWarnings could cause failures. Remove once the + # leaks are fixed. + gc.collect() def setUp(self): # super call creates internal client cls.client From 6cbedaf02d0aa7853508f2929c4d3db26cd704d8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 30 Sep 2026 13:10:19 -0500 Subject: [PATCH 08/24] PYTHON-6132 Re-sync specs PR improvements (#3082) --- .evergreen/resync-specs.sh | 69 +++++++++++++++++++------- .evergreen/scripts/create-spec-pr.sh | 55 +++++++++++++++++++- .evergreen/scripts/resync-all-specs.py | 9 +++- .evergreen/scripts/resync-all-specs.sh | 7 ++- 4 files changed, 116 insertions(+), 24 deletions(-) diff --git a/.evergreen/resync-specs.sh b/.evergreen/resync-specs.sh index 4feca0aa5b..840d05bf1e 100755 --- a/.evergreen/resync-specs.sh +++ b/.evergreen/resync-specs.sh @@ -85,28 +85,59 @@ cp_psl () { echo "Could not find the public suffix list at $src" >&2 return 1 fi - python - "$src" "$PYMONGO"/pymongo/public_suffix_list.dat <<'EOF' + # Use the newest mongodbtoolchain python on CI hosts; python3 fallback locally. + PY=$(ls -1d /opt/mongodbtoolchain/v*/bin/python3 2>/dev/null | sort -V | tail -1) + [ -n "$PY" ] || PY=python3 + [ "$("$PY" -c 'import sys; print(sys.version_info >= (3, 7))' 2>/dev/null)" = True ] \ + || { echo "Error: $PY is not Python 3.7+, which this conversion requires." >&2; return 1; } + "$PY" - "$src" "$PYMONGO"/pymongo/public_suffix_list.dat <<'EOF' +import os import sys src, dst = sys.argv[1], sys.argv[2] -with open(src, encoding="utf-8") as f, open(dst, "w", encoding="utf-8") as out: - for line in f: - rule = line.strip() - if rule and not rule.startswith("//") and not rule.isascii(): - # "!" and "*." prefixes are not valid IDNA, so encode only the - # domain part and re-attach the prefix afterwards. - prefix = "" - for p in ("!", "*."): - if rule.startswith(p): - prefix, rule = p, rule[len(p) :] - break - try: - rule = rule.encode("idna").decode("ascii") - except UnicodeError: - print(f"Could not convert rule to Punycode: {prefix}{rule}", file=sys.stderr) - sys.exit(1) - line = f"{prefix}{rule.lower()}\n" - out.write(line) + +# Convert in memory and write atomically so failures can't truncate the list. +with open(src, encoding="utf-8") as f: + lines = f.readlines() +if not lines: + print(f"Source public suffix list is empty: {src}", file=sys.stderr) + sys.exit(1) + +converted = [] +for line in lines: + rule = line.strip() + if rule and not rule.startswith("//") and not rule.isascii(): + # "!" and "*." prefixes are not valid IDNA, so encode only the + # domain part and re-attach the prefix afterwards. + prefix = "" + for p in ("!", "*."): + if rule.startswith(p): + prefix, rule = p, rule[len(p) :] + break + try: + rule = rule.encode("idna").decode("ascii") + except UnicodeError: + print(f"Could not convert rule to Punycode: {prefix}{rule}", file=sys.stderr) + sys.exit(1) + line = f"{prefix}{rule.lower()}\n" + converted.append(line) + +if len(converted) != len(lines): + print( + f"Conversion lost lines: {len(lines)} in, {len(converted)} out", + file=sys.stderr, + ) + sys.exit(1) + +# A blank or comment-only source must not replace the real list. +if not any(ln.strip() and not ln.strip().startswith("//") for ln in converted): + print(f"Source public suffix list has no rules: {src}", file=sys.stderr) + sys.exit(1) + +tmp = dst + ".tmp" +with open(tmp, "w", encoding="utf-8") as out: + out.writelines(converted) +os.replace(tmp, dst) EOF } diff --git a/.evergreen/scripts/create-spec-pr.sh b/.evergreen/scripts/create-spec-pr.sh index 0867ae5487..cffd8dc4de 100755 --- a/.evergreen/scripts/create-spec-pr.sh +++ b/.evergreen/scripts/create-spec-pr.sh @@ -32,12 +32,38 @@ echo "Creating the git checkout..." branch="spec-resync-"$(date '+%m-%d-%Y') git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git -git checkout -b $branch "origin/main" +# -B resets the branch if a reused checkout already has it, keeping reruns idempotent. +git checkout -B "$branch" "origin/main" + +# Attribute the commit to the bot user; fall back to the host identity. +bot_login="mongodb-drivers-pr-bot[bot]" +bot_id=$(curl -gsm 10 "https://api.github.com/users/${bot_login}" | jq -r '.id // empty' || true) +if [ -n "$bot_id" ]; then + git config user.name "$bot_login" + git config user.email "${bot_id}+${bot_login}@users.noreply.github.com" +fi + git add ./test git commit -am "resyncing specs $(date '+%m-%d-%Y')" echo "Creating the git checkout... done." -git push origin $branch +# Date-stamped branch: force-push makes same-day reruns idempotent; retry +# absorbs GitHub's intermittent 403s on the receive-pack endpoint. +push_ok=false +for _attempt in 1 2 3; do + if git push --force origin $branch; then + push_ok=true + break + fi + if [ "$_attempt" -lt 3 ]; then + echo "Push attempt ${_attempt} failed; retrying in 30s..." + sleep 30 + fi +done +if [ "$push_ok" != true ]; then + echo "Failed to push $branch after 3 attempts!" >&2 + exit 1 +fi # Build the payload as a file so the body content is always properly # JSON-escaped, rather than interpolated into the command line as text. @@ -49,6 +75,31 @@ jq -n \ --rawfile body "$1" \ '{title: $title, body: $body, head: $head, base: "main"}' > "$payload_file" +# Reuse an open PR for this branch (same-day rerun) instead of duplicating it. +existing_pr_json=$(curl -gs \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $token" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + --url "https://api.github.com/repos/$owner/$repo/pulls?head=$owner:$branch&state=open") +existing_pr_url=$(echo "$existing_pr_json" | jq -r '.[0].html_url // empty') +if [ -n "$existing_pr_url" ]; then + existing_pr_number=$(echo "$existing_pr_json" | jq -r '.[0].number // empty') + if ! curl -sfX PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $token" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -d "{\"title\": \"[Spec Resync] $(date '+%m-%d-%Y')\", \"body\": $(jq -Rs . < "$1")}" \ + --url "https://api.github.com/repos/$owner/$repo/pulls/${existing_pr_number}" > /dev/null + then + echo "Failed to update PR summary!" >&2 + exit 1 + fi + echo "$existing_pr_url" + echo "Creating the PR... done. (PR already existed; branch and summary were updated)" + rm -rf $tools + exit 0 +fi + resp=$(curl -L \ -X POST \ -H "Accept: application/vnd.github+json" \ diff --git a/.evergreen/scripts/resync-all-specs.py b/.evergreen/scripts/resync-all-specs.py index 78afe8a4d8..52c276b8b8 100644 --- a/.evergreen/scripts/resync-all-specs.py +++ b/.evergreen/scripts/resync-all-specs.py @@ -28,6 +28,8 @@ def resync_specs(directory: pathlib.Path, errored: dict[str, str]) -> None: ) except CalledProcessError as exc: errored[spec.name] = exc.stderr + # The PR-body summary is lost if PR creation fails. + print(f"Spec sync failed for {spec.name}:\n{exc.stderr}", flush=True) print("Done syncing specs") @@ -57,9 +59,11 @@ def apply_patches(errored): ], check=True, stderr=subprocess.PIPE, + text=True, ) except CalledProcessError as exc: errored[f"{patch}"] = exc.stderr + print(f"Patch application failed for {patch}:\n{exc.stderr}", flush=True) def check_new_spec_directories(directory: pathlib.Path) -> list[str]: @@ -127,9 +131,10 @@ def write_summary(errored: dict[str, str], new: list[str], filename: str | None) if filename is None: print(f"\n{pr_body}") else: + # Plain text: the consumer embeds this file via jq --rawfile, + # which handles the JSON escaping itself. with open(filename, "w") as f: - # replacements made for proper json - f.write(pr_body.replace("\n", "\\n").replace("\t", "\\t")) + f.write(pr_body) def main(args: Namespace): diff --git a/.evergreen/scripts/resync-all-specs.sh b/.evergreen/scripts/resync-all-specs.sh index 41e4a2bc73..11dd2e65ba 100755 --- a/.evergreen/scripts/resync-all-specs.sh +++ b/.evergreen/scripts/resync-all-specs.sh @@ -34,7 +34,12 @@ then # we're running locally python3 ./.evergreen/scripts/resync-all-specs.py else - /opt/devtools/bin/python3.11 ./.evergreen/scripts/resync-all-specs.py --filename "$PR_DESC" + # Use the newest mongodbtoolchain python; the default python3 may be too old. + PY=$(ls -1d /opt/mongodbtoolchain/v*/bin/python3 2>/dev/null | sort -V | tail -1) + [ -n "$PY" ] || PY=python3 + [ "$("$PY" -c 'import sys; print(sys.version_info >= (3, 7))' 2>/dev/null)" = True ] \ + || { echo "Error: $PY is not Python 3.7+, which resync-all-specs.py requires." >&2; exit 1; } + "$PY" ./.evergreen/scripts/resync-all-specs.py --filename "$PR_DESC" if [[ -f $PR_DESC ]]; then # changes were made -> call scrypt to create PR for us .evergreen/scripts/create-spec-pr.sh "$PR_DESC" From 621d94e4e950e184dd0857438b498344ed582526 Mon Sep 17 00:00:00 2001 From: "mongodb-drivers-pr-bot[bot]" <147046816+mongodb-drivers-pr-bot[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:25:10 -0500 Subject: [PATCH 09/24] [Spec Resync] 09-30-2026 (#3081) Co-authored-by: mongodb-drivers-pr-bot[bot] <147046816+mongodb-drivers-pr-bot[bot]@users.noreply.github.com> Co-authored-by: Steven Silvester --- .evergreen/spec-patch/PYTHON-5993.patch | 94 +++++++++++++++++++++++++ test/bson_corpus/dbref.json | 15 ++++ 2 files changed, 109 insertions(+) create mode 100644 .evergreen/spec-patch/PYTHON-5993.patch diff --git a/.evergreen/spec-patch/PYTHON-5993.patch b/.evergreen/spec-patch/PYTHON-5993.patch new file mode 100644 index 0000000000..3880924e83 --- /dev/null +++ b/.evergreen/spec-patch/PYTHON-5993.patch @@ -0,0 +1,94 @@ +diff --git a/test/unified-test-format/valid-pass/operator-gte.json b/test/unified-test-format/valid-pass/operator-gte.json +new file mode 100644 +index 000000000..ae69da5c9 +--- /dev/null ++++ b/test/unified-test-format/valid-pass/operator-gte.json +@@ -0,0 +1,88 @@ ++{ ++ "description": "operator-gte", ++ "schemaVersion": "1.29", ++ "createEntities": [ ++ { ++ "client": { ++ "id": "client0", ++ "observeEvents": [ ++ "commandStartedEvent" ++ ] ++ } ++ }, ++ { ++ "database": { ++ "id": "database0", ++ "client": "client0", ++ "databaseName": "database0Name" ++ } ++ }, ++ { ++ "collection": { ++ "id": "collection0", ++ "database": "database0", ++ "collectionName": "coll0" ++ } ++ } ++ ], ++ "initialData": [ ++ { ++ "collectionName": "coll0", ++ "databaseName": "database0Name", ++ "documents": [] ++ } ++ ], ++ "tests": [ ++ { ++ "description": "special gte matching operator", ++ "operations": [ ++ { ++ "name": "insertOne", ++ "object": "collection0", ++ "arguments": { ++ "document": { ++ "_id": 1, ++ "x": 2, ++ "y": 3, ++ "z": 4 ++ } ++ } ++ } ++ ], ++ "expectEvents": [ ++ { ++ "client": "client0", ++ "events": [ ++ { ++ "commandStartedEvent": { ++ "command": { ++ "insert": "coll0", ++ "documents": [ ++ { ++ "_id": { ++ "$$gte": 1 ++ }, ++ "x": { ++ "$$gte": 1.9 ++ }, ++ "y": { ++ "$$gte": { ++ "$numberLong": "3" ++ } ++ }, ++ "z": { ++ "$$gte": 4 ++ } ++ } ++ ] ++ }, ++ "commandName": "insert", ++ "databaseName": "database0Name" ++ } ++ } ++ ] ++ } ++ ] ++ } ++ ] ++} diff --git a/test/bson_corpus/dbref.json b/test/bson_corpus/dbref.json index 41c0b09d0e..cf54ccf538 100644 --- a/test/bson_corpus/dbref.json +++ b/test/bson_corpus/dbref.json @@ -22,6 +22,21 @@ "canonical_bson": "4400000003646272656600380000000224726566000b000000636f6c6c656374696f6e00072469640058921b3e6e32ab156a22b59e02666f6f0004000000626172000000", "canonical_extjson": "{\"dbref\": {\"$ref\": \"collection\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}, \"foo\": \"bar\"}}" }, + { + "description": "DBRef with dotted collection name", + "canonical_bson": "3a000000036462726566002e0000000224726566000e00000064622e636f6c6c656374696f6e00072469640058921b3e6e32ab156a22b59e0000", + "canonical_extjson": "{\"dbref\": {\"$ref\": \"db.collection\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}}}" + }, + { + "description": "DBRef with dotted collection name and database", + "canonical_bson": "4b000000036462726566003f0000000224726566000e00000064622e636f6c6c656374696f6e00072469640058921b3e6e32ab156a22b59e0224646200080000006f746865726462000000", + "canonical_extjson": "{\"dbref\": {\"$ref\": \"db.collection\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}, \"$db\": \"otherdb\"}}" + }, + { + "description": "DBRef with multiple dots in collection name", + "canonical_bson": "32000000036462726566002600000002247265660006000000612e622e6300072469640058921b3e6e32ab156a22b59e0000", + "canonical_extjson": "{\"dbref\": {\"$ref\": \"a.b.c\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}}}" + }, { "description": "Document with key names similar to those of a DBRef", "canonical_bson": "3e0000000224726566000c0000006e6f742d612d646272656600072469640058921b3e6e32ab156a22b59e022462616e616e6100050000007065656c0000", From e1527edb18f550bb7b4b5735c5fb2aed65555320 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 1 Oct 2026 10:59:15 -0500 Subject: [PATCH 10/24] PYTHON-6114 Exclude mod_wsgi from Windows dependency resolution (#3085) --- .evergreen/scripts/mod_wsgi.sh | 13 +++++++------ .github/dependabot.yml | 17 +++++++++++++++++ .github/workflows/sbom.yml | 1 + .github/workflows/test-mod-wsgi.yml | 15 +++++++++------ CONTRIBUTING.md | 2 +- pyproject.toml | 3 --- requirements/mod_wsgi.txt | 3 +++ test/mod_wsgi_test/README.rst | 7 ++++--- tools/generate_sbom_requirements.py | 4 ++-- uv.lock | 25 ------------------------- 10 files changed, 44 insertions(+), 46 deletions(-) create mode 100644 requirements/mod_wsgi.txt diff --git a/.evergreen/scripts/mod_wsgi.sh b/.evergreen/scripts/mod_wsgi.sh index 61aed24b73..78233a4ff2 100755 --- a/.evergreen/scripts/mod_wsgi.sh +++ b/.evergreen/scripts/mod_wsgi.sh @@ -73,12 +73,11 @@ if [ "$(uname -s)" = "Linux" ] && [ -z "${MOD_WSGI_DOCKER:-}" ]; then # "just run-server" or the CI workflow. Set MOD_WSGI_DOCKER to force the # container path on Linux. if [ "$CMD" != "teardown" ]; then - # Other recipes' exact uv syncs — including run-tests' resync — prune the - # mod_wsgi group; restore it before setup or test when missing. Checking - # first keeps the sync from reconciling dependencies that a caller - # resolved differently, like the min-deps job's lowest-direct install. + # Other recipes' exact uv syncs prune the on-demand mod_wsgi install; + # restore it when missing. Checking first avoids reconciling a caller's + # differently-resolved dependencies. if ! uv run --no-sync python -c "import mod_wsgi" 2>/dev/null; then - uv sync --group mod_wsgi --quiet + uv pip install -r requirements/mod_wsgi.txt --quiet fi fi if [ "$CMD" = "setup" ]; then @@ -190,7 +189,9 @@ bootstrap() { # Rebuild the in-place C extensions on every bootstrap; the source # refresh may carry artifacts built by a different interpreter. export PYMONGO_C_EXT_MUST_BUILD=1 - uv sync --group mod_wsgi --refresh-package pymongo + uv sync --refresh-package pymongo + # The sync above prunes mod_wsgi, so install it after. + VIRTUAL_ENV="$VENV" uv pip install -r requirements/mod_wsgi.txt --quiet ' ensure_mongod docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 72d1e1e084..d9f19ad27e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,3 +19,20 @@ updates: schedule: interval: "weekly" open-pull-requests-limit: 0 + # Python (pip). Manages requirements/mod_wsgi.txt only; the other + # requirements files are resolved in uv.lock. + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + exclude-paths: + - "/pyproject.toml" + - "/requirements.txt" + - "/requirements/aws.txt" + - "/requirements/docs.txt" + - "/requirements/encryption.txt" + - "/requirements/gssapi.txt" + - "/requirements/ocsp.txt" + - "/requirements/snappy.txt" + - "/requirements/test.txt" + - "/requirements/zstd.txt" diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml index 91af8d840c..62b0ca3ceb 100644 --- a/.github/workflows/sbom.yml +++ b/.github/workflows/sbom.yml @@ -14,6 +14,7 @@ on: - 'requirements/**.txt' - '!requirements/docs.txt' - '!requirements/test.txt' + - '!requirements/mod_wsgi.txt' permissions: contents: write diff --git a/.github/workflows/test-mod-wsgi.yml b/.github/workflows/test-mod-wsgi.yml index 4985161986..2ae2310ab6 100644 --- a/.github/workflows/test-mod-wsgi.yml +++ b/.github/workflows/test-mod-wsgi.yml @@ -16,6 +16,7 @@ on: - "justfile" - "pyproject.toml" - "uv.lock" + - "requirements/mod_wsgi.txt" pull_request: paths: - ".evergreen/scripts/mod_wsgi_*" @@ -30,6 +31,7 @@ on: - "justfile" - "pyproject.toml" - "uv.lock" + - "requirements/mod_wsgi.txt" workflow_dispatch: concurrency: @@ -87,12 +89,11 @@ jobs: run: | export PYMONGO_C_EXT_MUST_BUILD=1 just install - uv sync --group mod_wsgi --refresh-package pymongo + # just install skips the sync when uv is already set up. + uv sync --refresh-package pymongo uv run --no-sync python tools/fail_if_no_c.py - # Re-resolve at the lowest direct versions, including the mod_wsgi - # group: uv sync above installed the locked version, but this job - # must exercise the group's mod_wsgi>=4.9.4 lower bound. - uv pip install -e ".[test]" --group mod_wsgi --resolution=lowest-direct --force-reinstall + # Exercise the mod_wsgi>=4.9.4 lower bound. + uv pip install -e ".[test]" -r requirements/mod_wsgi.txt --resolution=lowest-direct --force-reinstall - name: Start MongoDB # Fedora is not a supported MongoDB platform, so install the RHEL build # directly instead of using the drivers-evergreen-tools action, which @@ -153,7 +154,9 @@ jobs: run: | export PYMONGO_C_EXT_MUST_BUILD=1 just install - uv sync --group mod_wsgi --refresh-package pymongo + # just install skips the sync when uv is already set up. + uv sync --refresh-package pymongo + uv pip install -r requirements/mod_wsgi.txt uv run --no-sync python tools/fail_if_no_c.py - id: setup-mongodb uses: mongodb-labs/drivers-evergreen-tools@49da51cebc0060b4d699d57bdef30ee7e071de44 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 14e3ddfd26..e1b0c88db6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -361,7 +361,7 @@ Continuous integration runs the tests on pull requests that change mod_wsgi-relevant files, in the `test-mod-wsgi.yml` workflow. To run the tests by hand, install Apache and mod_wsgi (`sudo apt-get install -y apache2 -apache2-dev` and `uv sync --group mod_wsgi` on Ubuntu), then: +apache2-dev` and `uv pip install -r requirements/mod_wsgi.txt` on Ubuntu), then: - On Linux, run `TOPOLOGY=replica_set just run-server`. - Run `just setup-tests mod_wsgi `. diff --git a/pyproject.toml b/pyproject.toml index 8083b03b46..2f7a68c837 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,9 +70,6 @@ unasync = [ ] pip = ["pip>=20.2"] gevent = ["gevent>=21.12"] -# mod_wsgi is built against the interpreter it is installed with and is used -# only by the mod_wsgi test suite. -mod_wsgi = ["mod_wsgi>=4.9.4"] coverage = [ "coverage[toml]>=5,<=7.15.0" ] diff --git a/requirements/mod_wsgi.txt b/requirements/mod_wsgi.txt new file mode 100644 index 0000000000..31bb5dfda8 --- /dev/null +++ b/requirements/mod_wsgi.txt @@ -0,0 +1,3 @@ +# Test-only; kept out of pyproject.toml so the uv lock never has to +# resolve it (building its sdist needs Apache). +mod_wsgi>=4.9.4 diff --git a/test/mod_wsgi_test/README.rst b/test/mod_wsgi_test/README.rst index 3dddb3e5f7..5a9348feb2 100644 --- a/test/mod_wsgi_test/README.rst +++ b/test/mod_wsgi_test/README.rst @@ -41,10 +41,11 @@ with the matching config file (``apache24ubuntu.conf`` or Install mod_wsgi ................ -The project defines a ``mod_wsgi`` dependency group used for testing. pip -builds mod_wsgi against the interpreter it is installed with:: +mod_wsgi is only needed by these tests, so it is not a project dependency. +Its constraint lives in ``requirements/mod_wsgi.txt``. pip builds mod_wsgi +against the interpreter it is installed with:: - uv sync --group mod_wsgi + uv pip install -r requirements/mod_wsgi.txt Start mongod ............ diff --git a/tools/generate_sbom_requirements.py b/tools/generate_sbom_requirements.py index 59389bee5a..aa8d5579ca 100644 --- a/tools/generate_sbom_requirements.py +++ b/tools/generate_sbom_requirements.py @@ -3,7 +3,7 @@ - Includes repo_root/requirements.txt if present - Includes all files matching repo_root/requirements/**/*.txt -- Excludes docs.txt and test.txt in the requirements folder +- Excludes docs.txt, test.txt, and mod_wsgi.txt in the requirements folder - Writes output to sbom-requirements.txt (overwrites) """ @@ -11,7 +11,7 @@ from pathlib import Path -EXCLUDED_NAMES = {"docs.txt", "test.txt"} +EXCLUDED_NAMES = {"docs.txt", "test.txt", "mod_wsgi.txt"} def collect_files(root: Path) -> list[Path]: diff --git a/uv.lock b/uv.lock index f7fe85a6ca..ee008e88d4 100644 --- a/uv.lock +++ b/uv.lock @@ -2225,26 +2225,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/c0/bc0c3e82bf91993bf2337b125c4d49c3cadae39f5a4c642baada958f976e/mockupdb-1.8.1-py2.py3-none-any.whl", hash = "sha256:8eb76212963bad5df7f3ea8f26af916087c2dcc69a0457c75328abad44956dbb", size = 27578, upload-time = "2021-10-15T19:02:55.489Z" }, ] -[[package]] -name = "mod-wsgi" -version = "5.0.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version > '3.9' and python_full_version < '3.10'", - "python_full_version <= '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/ca/4d/275853efa4d2a4efe1614aa0df06295e71f2082cebccc4d40d0f0a551c24/mod_wsgi-5.0.2.tar.gz", hash = "sha256:6fe5557c3ef25c184c2c994905fe52be1c06ae5c46235ecafe558e84789e2159", size = 498627, upload-time = "2024-11-22T08:53:22.752Z" } - -[[package]] -name = "mod-wsgi" -version = "6.0.6" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.11'", - "python_full_version == '3.10.*'", -] -sdist = { url = "https://files.pythonhosted.org/packages/c2/ce/fdbfe51f596e37c657176ea64d637737074214bfcd21c2177aa6b1f1da14/mod_wsgi-6.0.6.tar.gz", hash = "sha256:897f58ee961be7dcfe527074dd64d43d18ca108cf919d4c3bcf10fa6a4c12921", size = 607559, upload-time = "2026-08-19T09:51:18.52Z" } - [[package]] name = "mypy" version = "1.19.1" @@ -2483,10 +2463,6 @@ lint = [ mockupdb = [ { name = "mockupdb" }, ] -mod-wsgi = [ - { name = "mod-wsgi", version = "5.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "mod-wsgi", version = "6.0.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, -] perf = [ { name = "simplejson" }, ] @@ -2540,7 +2516,6 @@ dev = [] gevent = [{ name = "gevent", specifier = ">=21.12" }] lint = [{ name = "ruff", specifier = "==0.15.20" }] mockupdb = [{ name = "mockupdb", specifier = ">=1.8.1" }] -mod-wsgi = [{ name = "mod-wsgi", specifier = ">=4.9.4" }] perf = [{ name = "simplejson", specifier = ">=3.17.0" }] pip = [{ name = "pip", specifier = ">=20.2" }] typing = [ From 1bb24c934407bead62045c5497af5b6f75b32e6a Mon Sep 17 00:00:00 2001 From: Iris <58442094+sleepyStick@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:53:50 -0700 Subject: [PATCH 11/24] PYTHON-6024 Consolidate all non-IO code in _AsyncClientBulk/_AsyncBulk into _AgnosticClientBulkBase/_AgnosticBulkBase (#3087) --- pymongo/_client_bulk_shared.py | 155 ++++++++++++++++++++++++- pymongo/asynchronous/bulk.py | 168 +-------------------------- pymongo/asynchronous/client_bulk.py | 151 +----------------------- pymongo/bulk_shared.py | 171 +++++++++++++++++++++++++++- pymongo/synchronous/bulk.py | 168 +-------------------------- pymongo/synchronous/client_bulk.py | 151 +----------------------- pymongo/typings.py | 1 - 7 files changed, 337 insertions(+), 628 deletions(-) diff --git a/pymongo/_client_bulk_shared.py b/pymongo/_client_bulk_shared.py index f4f7f8fe7f..0019d0c0bb 100644 --- a/pymongo/_client_bulk_shared.py +++ b/pymongo/_client_bulk_shared.py @@ -18,13 +18,22 @@ from __future__ import annotations from collections.abc import Mapping, MutableMapping -from typing import TYPE_CHECKING, Any, NoReturn +from typing import TYPE_CHECKING, Any, NoReturn, Optional, Union +from bson.objectid import ObjectId +from bson.raw_bson import RawBSONDocument +from pymongo.common import ( + validate_is_document_type, + validate_ok_for_replace, + validate_ok_for_update, +) from pymongo.errors import ClientBulkWriteException, OperationFailure from pymongo.helpers_shared import _get_wce_doc +from pymongo.message import _ClientBulkWriteContext +from pymongo.write_concern import WriteConcern if TYPE_CHECKING: - from pymongo.typings import _DocumentOut + from pymongo.typings import _AgnosticMongoClient, _DocumentOut, _Pipeline def _merge_command( @@ -79,3 +88,145 @@ def _throw_client_bulk_write_exception( if isinstance(full_result["error"], BaseException): raise ClientBulkWriteException(full_result, verbose_results) from full_result["error"] raise ClientBulkWriteException(full_result, verbose_results) + + +class _AgnosticClientBulkBase: + """The private guts of the client-level bulk write API.""" + + def __init__( + self, + client: _AgnosticMongoClient, + write_concern: WriteConcern, + ordered: bool = True, + bypass_document_validation: Optional[bool] = None, + comment: Optional[str] = None, + let: Optional[Any] = None, + verbose_results: bool = False, + ) -> None: + """Initialize a _AgnosticClientBulkBase instance.""" + self.client = client + self.write_concern = write_concern + self.let = let + if self.let is not None: + validate_is_document_type("let", self.let) + self.ordered = ordered + self.bypass_doc_val = bypass_document_validation + self.comment = comment + self.verbose_results = verbose_results + self.ops: list[tuple[str, Mapping[str, Any]]] = [] + self.namespaces: list[str] = [] + self.idx_offset: int = 0 + self.total_ops: int = 0 + self.executed = False + self.uses_collation = False + self.uses_array_filters = False + self.is_retryable = self.client.options.retry_writes + self.retrying = False + self.started_retryable_write = False + + @property + def bulk_ctx_class(self) -> type[_ClientBulkWriteContext]: + return _ClientBulkWriteContext + + def add_insert(self, namespace: str, document: _DocumentOut) -> None: + """Add an insert document to the list of ops.""" + validate_is_document_type("document", document) + # Generate ObjectId client side. + if not (isinstance(document, RawBSONDocument) or "_id" in document): + document["_id"] = ObjectId() + cmd = {"insert": -1, "document": document} + self.ops.append(("insert", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 + + def add_update( + self, + namespace: str, + selector: Mapping[str, Any], + update: Union[Mapping[str, Any], _Pipeline], + multi: bool, + upsert: Optional[bool] = None, + collation: Optional[Mapping[str, Any]] = None, + array_filters: Optional[list[Mapping[str, Any]]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create an update document and add it to the list of ops.""" + validate_ok_for_update(update) + cmd = { + "update": -1, + "filter": selector, + "updateMods": update, + "multi": multi, + } + if upsert is not None: + cmd["upsert"] = upsert + if array_filters is not None: + self.uses_array_filters = True + cmd["arrayFilters"] = array_filters + if hint is not None: + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if sort is not None: + cmd["sort"] = sort + if multi: + # A bulk_write containing an update_many is not retryable. + self.is_retryable = False + self.ops.append(("update", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 + + def add_replace( + self, + namespace: str, + selector: Mapping[str, Any], + replacement: Mapping[str, Any], + upsert: Optional[bool] = None, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create a replace document and add it to the list of ops.""" + validate_ok_for_replace(replacement) + cmd = { + "update": -1, + "filter": selector, + "updateMods": replacement, + "multi": False, + } + if upsert is not None: + cmd["upsert"] = upsert + if hint is not None: + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if sort is not None: + cmd["sort"] = sort + self.ops.append(("replace", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 + + def add_delete( + self, + namespace: str, + selector: Mapping[str, Any], + multi: bool, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create a delete document and add it to the list of ops.""" + cmd = {"delete": -1, "filter": selector, "multi": multi} + if hint is not None: + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if multi: + # A bulk_write containing a delete_many is not retryable. + self.is_retryable = False + self.ops.append(("delete", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 diff --git a/pymongo/asynchronous/bulk.py b/pymongo/asynchronous/bulk.py index 86b9528b6e..81186499f6 100644 --- a/pymongo/asynchronous/bulk.py +++ b/pymongo/asynchronous/bulk.py @@ -29,9 +29,7 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.asynchronous.client_session import AsyncClientSession from pymongo.asynchronous.command_runner import ( @@ -40,17 +38,11 @@ from pymongo.asynchronous.helpers import _handle_reauth from pymongo.bulk_shared import ( _COMMANDS, - _DELETE_ALL, + _AgnosticBulkBase, _merge_command, _raise_bulk_write_error, - _Run, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, InvalidOperation, @@ -59,7 +51,6 @@ from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES from pymongo.message import ( _DELETE, - _INSERT, _UPDATE, _BulkWriteContext, _EncryptedBulkWriteContext, @@ -71,165 +62,14 @@ from pymongo.asynchronous.collection import AsyncCollection from pymongo.asynchronous.mongo_client import AsyncMongoClient from pymongo.asynchronous.pool import AsyncConnection - from pymongo.typings import _DocumentOut, _DocumentType, _Pipeline _IS_SYNC = False -class _AsyncBulk: +class _AsyncBulk(_AgnosticBulkBase): """The private guts of the bulk write API.""" - def __init__( - self, - collection: AsyncCollection[_DocumentType], - ordered: bool, - bypass_document_validation: Optional[bool], - comment: Optional[str] = None, - let: Optional[Any] = None, - ) -> None: - """Initialize a _AsyncBulk instance.""" - self.collection = collection.with_options( - codec_options=collection.codec_options._replace( - unicode_decode_error_handler="replace", document_class=dict - ) - ) - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.comment: Optional[str] = comment - self.ordered = ordered - self.ops: list[tuple[int, Mapping[str, Any]]] = [] - self.executed = False - self.bypass_doc_val = bypass_document_validation - self.uses_collation = False - self.uses_array_filters = False - self.uses_sort = False - self.is_retryable = True - self.retrying = False - self.started_retryable_write = False - # Extra state so that we know where to pick up on a retry attempt. - self.current_run = None - self.next_run = None - self.is_encrypted = False - - @property - def bulk_ctx_class(self) -> type[_BulkWriteContext]: - encrypter = self.collection.database.client._encrypter - if encrypter and not encrypter._bypass_auto_encryption: - self.is_encrypted = True - return _EncryptedBulkWriteContext - else: - self.is_encrypted = False - return _BulkWriteContext - - def add_insert(self, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - self.ops.append((_INSERT, document)) - - def add_update( - self, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd: dict[str, Any] = {"q": selector, "u": update, "multi": multi} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append((_UPDATE, cmd)) - - def add_replace( - self, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd: dict[str, Any] = {"q": selector, "u": replacement} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - self.ops.append((_UPDATE, cmd)) - - def add_delete( - self, - selector: Mapping[str, Any], - limit: int, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd: dict[str, Any] = {"q": selector, "limit": limit} - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if limit == _DELETE_ALL: - # A bulk_write containing a delete_many is not retryable. - self.is_retryable = False - self.ops.append((_DELETE, cmd)) - - def gen_ordered(self) -> Iterator[Optional[_Run]]: - """Generate batches of operations, batched by type of - operation, in the order **provided**. - """ - run = None - for idx, (op_type, operation) in enumerate(self.ops): - if run is None: - run = _Run(op_type) - elif run.op_type != op_type: - yield run - run = _Run(op_type) - run.add(idx, operation) - yield run - - def gen_unordered(self) -> Iterator[_Run]: - """Generate batches of operations, batched by type of - operation, in arbitrary order. - """ - operations = [_Run(_INSERT), _Run(_UPDATE), _Run(_DELETE)] - for idx, (op_type, operation) in enumerate(self.ops): - operations[op_type].add(idx, operation) - - for run in operations: - if run.ops: - yield run + collection: AsyncCollection[Any] @_handle_reauth async def write_command( diff --git a/pymongo/asynchronous/client_bulk.py b/pymongo/asynchronous/client_bulk.py index 2847dcb2c0..bfbe9e2c6b 100644 --- a/pymongo/asynchronous/client_bulk.py +++ b/pymongo/asynchronous/client_bulk.py @@ -29,9 +29,7 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.asynchronous.client_session import AsyncClientSession from pymongo.asynchronous.collection import AsyncCollection @@ -46,15 +44,11 @@ from pymongo.asynchronous.mongo_client import AsyncMongoClient from pymongo.asynchronous.pool import AsyncConnection from pymongo._client_bulk_shared import ( + _AgnosticClientBulkBase, _merge_command, _throw_client_bulk_write_exception, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, ConnectionFailure, @@ -76,152 +70,15 @@ InsertOneResult, UpdateResult, ) -from pymongo.typings import _DocumentOut, _Pipeline from pymongo.write_concern import WriteConcern _IS_SYNC = False -class _AsyncClientBulk: +class _AsyncClientBulk(_AgnosticClientBulkBase): """The private guts of the client-level bulk write API.""" - def __init__( - self, - client: AsyncMongoClient[Any], - write_concern: WriteConcern, - ordered: bool = True, - bypass_document_validation: Optional[bool] = None, - comment: Optional[str] = None, - let: Optional[Any] = None, - verbose_results: bool = False, - ) -> None: - """Initialize a _AsyncClientBulk instance.""" - self.client = client - self.write_concern = write_concern - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.ordered = ordered - self.bypass_doc_val = bypass_document_validation - self.comment = comment - self.verbose_results = verbose_results - self.ops: list[tuple[str, Mapping[str, Any]]] = [] - self.namespaces: list[str] = [] - self.idx_offset: int = 0 - self.total_ops: int = 0 - self.executed = False - self.uses_collation = False - self.uses_array_filters = False - self.is_retryable = self.client.options.retry_writes - self.retrying = False - self.started_retryable_write = False - - @property - def bulk_ctx_class(self) -> type[_ClientBulkWriteContext]: - return _ClientBulkWriteContext - - def add_insert(self, namespace: str, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - cmd = {"insert": -1, "document": document} - self.ops.append(("insert", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_update( - self, - namespace: str, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd = { - "update": -1, - "filter": selector, - "updateMods": update, - "multi": multi, - } - if upsert is not None: - cmd["upsert"] = upsert - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("update", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_replace( - self, - namespace: str, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd = { - "update": -1, - "filter": selector, - "updateMods": replacement, - "multi": False, - } - if upsert is not None: - cmd["upsert"] = upsert - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - self.ops.append(("replace", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_delete( - self, - namespace: str, - selector: Mapping[str, Any], - multi: bool, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd = {"delete": -1, "filter": selector, "multi": multi} - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("delete", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 + client: AsyncMongoClient[Any] @_handle_reauth async def write_command( diff --git a/pymongo/bulk_shared.py b/pymongo/bulk_shared.py index a086611be5..54f0d4cd22 100644 --- a/pymongo/bulk_shared.py +++ b/pymongo/bulk_shared.py @@ -17,19 +17,28 @@ from __future__ import annotations -from collections.abc import Mapping, MutableMapping -from typing import TYPE_CHECKING, Any, NoReturn +from collections.abc import Iterator, Mapping, MutableMapping +from typing import TYPE_CHECKING, Any, NoReturn, Optional, Union +from bson.objectid import ObjectId +from bson.raw_bson import RawBSONDocument +from pymongo.common import ( + validate_is_document_type, + validate_ok_for_replace, + validate_ok_for_update, +) from pymongo.errors import BulkWriteError, OperationFailure from pymongo.helpers_shared import _get_wce_doc from pymongo.message import ( _DELETE, _INSERT, _UPDATE, + _BulkWriteContext, + _EncryptedBulkWriteContext, ) if TYPE_CHECKING: - from pymongo.typings import _DocumentOut + from pymongo.typings import _AgnosticCollection, _DocumentOut, _DocumentType, _Pipeline _DELETE_ALL: int = 0 @@ -131,3 +140,159 @@ def _raise_bulk_write_error(full_result: _DocumentOut) -> NoReturn: ) raise OperationFailure(errmsg, code, full_result) raise BulkWriteError(full_result) + + +class _AgnosticBulkBase: + """The private guts of the bulk write API.""" + + def __init__( + self, + collection: _AgnosticCollection[_DocumentType], + ordered: bool, + bypass_document_validation: Optional[bool], + comment: Optional[str] = None, + let: Optional[Any] = None, + ) -> None: + """Initialize an _AgnosticBulkBase instance.""" + self.collection = collection.with_options( + codec_options=collection.codec_options._replace( + unicode_decode_error_handler="replace", document_class=dict + ) + ) + self.let = let + if self.let is not None: + validate_is_document_type("let", self.let) + self.comment: Optional[str] = comment + self.ordered = ordered + self.ops: list[tuple[int, Mapping[str, Any]]] = [] + self.executed = False + self.bypass_doc_val = bypass_document_validation + self.uses_collation = False + self.uses_array_filters = False + self.uses_sort = False + self.is_retryable = True + self.retrying = False + self.started_retryable_write = False + # Extra state so that we know where to pick up on a retry attempt. + self.current_run = None + self.next_run = None + self.is_encrypted = False + + @property + def bulk_ctx_class(self) -> type[_BulkWriteContext]: + encrypter = self.collection.database.client._encrypter + if encrypter and not encrypter._bypass_auto_encryption: + self.is_encrypted = True + return _EncryptedBulkWriteContext + else: + self.is_encrypted = False + return _BulkWriteContext + + def add_insert(self, document: _DocumentOut) -> None: + """Add an insert document to the list of ops.""" + validate_is_document_type("document", document) + # Generate ObjectId client side. + if not (isinstance(document, RawBSONDocument) or "_id" in document): + document["_id"] = ObjectId() + self.ops.append((_INSERT, document)) + + def add_update( + self, + selector: Mapping[str, Any], + update: Union[Mapping[str, Any], _Pipeline], + multi: bool, + upsert: Optional[bool], + collation: Optional[Mapping[str, Any]] = None, + array_filters: Optional[list[Mapping[str, Any]]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create an update document and add it to the list of ops.""" + validate_ok_for_update(update) + cmd: dict[str, Any] = {"q": selector, "u": update, "multi": multi} + if upsert is not None: + cmd["upsert"] = upsert + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if array_filters is not None: + self.uses_array_filters = True + cmd["arrayFilters"] = array_filters + if hint is not None: + cmd["hint"] = hint + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort + if multi: + # A bulk_write containing an update_many is not retryable. + self.is_retryable = False + self.ops.append((_UPDATE, cmd)) + + def add_replace( + self, + selector: Mapping[str, Any], + replacement: Mapping[str, Any], + upsert: Optional[bool], + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create a replace document and add it to the list of ops.""" + validate_ok_for_replace(replacement) + cmd: dict[str, Any] = {"q": selector, "u": replacement} + if upsert is not None: + cmd["upsert"] = upsert + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if hint is not None: + cmd["hint"] = hint + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort + self.ops.append((_UPDATE, cmd)) + + def add_delete( + self, + selector: Mapping[str, Any], + limit: int, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create a delete document and add it to the list of ops.""" + cmd: dict[str, Any] = {"q": selector, "limit": limit} + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if hint is not None: + cmd["hint"] = hint + if limit == _DELETE_ALL: + # A bulk_write containing a delete_many is not retryable. + self.is_retryable = False + self.ops.append((_DELETE, cmd)) + + def gen_ordered(self) -> Iterator[Optional[_Run]]: + """Generate batches of operations, batched by type of + operation, in the order **provided**. + """ + run = None + for idx, (op_type, operation) in enumerate(self.ops): + if run is None: + run = _Run(op_type) + elif run.op_type != op_type: + yield run + run = _Run(op_type) + run.add(idx, operation) + yield run + + def gen_unordered(self) -> Iterator[_Run]: + """Generate batches of operations, batched by type of + operation, in arbitrary order. + """ + operations = [_Run(_INSERT), _Run(_UPDATE), _Run(_DELETE)] + for idx, (op_type, operation) in enumerate(self.ops): + operations[op_type].add(idx, operation) + + for run in operations: + if run.ops: + yield run diff --git a/pymongo/synchronous/bulk.py b/pymongo/synchronous/bulk.py index be12a72127..291965fca1 100644 --- a/pymongo/synchronous/bulk.py +++ b/pymongo/synchronous/bulk.py @@ -29,23 +29,15 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.bulk_shared import ( _COMMANDS, - _DELETE_ALL, + _AgnosticBulkBase, _merge_command, _raise_bulk_write_error, - _Run, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, InvalidOperation, @@ -54,7 +46,6 @@ from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES from pymongo.message import ( _DELETE, - _INSERT, _UPDATE, _BulkWriteContext, _EncryptedBulkWriteContext, @@ -71,165 +62,14 @@ from pymongo.synchronous.collection import Collection from pymongo.synchronous.mongo_client import MongoClient from pymongo.synchronous.pool import Connection - from pymongo.typings import _DocumentOut, _DocumentType, _Pipeline _IS_SYNC = True -class _Bulk: +class _Bulk(_AgnosticBulkBase): """The private guts of the bulk write API.""" - def __init__( - self, - collection: Collection[_DocumentType], - ordered: bool, - bypass_document_validation: Optional[bool], - comment: Optional[str] = None, - let: Optional[Any] = None, - ) -> None: - """Initialize a _Bulk instance.""" - self.collection = collection.with_options( - codec_options=collection.codec_options._replace( - unicode_decode_error_handler="replace", document_class=dict - ) - ) - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.comment: Optional[str] = comment - self.ordered = ordered - self.ops: list[tuple[int, Mapping[str, Any]]] = [] - self.executed = False - self.bypass_doc_val = bypass_document_validation - self.uses_collation = False - self.uses_array_filters = False - self.uses_sort = False - self.is_retryable = True - self.retrying = False - self.started_retryable_write = False - # Extra state so that we know where to pick up on a retry attempt. - self.current_run = None - self.next_run = None - self.is_encrypted = False - - @property - def bulk_ctx_class(self) -> type[_BulkWriteContext]: - encrypter = self.collection.database.client._encrypter - if encrypter and not encrypter._bypass_auto_encryption: - self.is_encrypted = True - return _EncryptedBulkWriteContext - else: - self.is_encrypted = False - return _BulkWriteContext - - def add_insert(self, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - self.ops.append((_INSERT, document)) - - def add_update( - self, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd: dict[str, Any] = {"q": selector, "u": update, "multi": multi} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append((_UPDATE, cmd)) - - def add_replace( - self, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd: dict[str, Any] = {"q": selector, "u": replacement} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - self.ops.append((_UPDATE, cmd)) - - def add_delete( - self, - selector: Mapping[str, Any], - limit: int, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd: dict[str, Any] = {"q": selector, "limit": limit} - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if limit == _DELETE_ALL: - # A bulk_write containing a delete_many is not retryable. - self.is_retryable = False - self.ops.append((_DELETE, cmd)) - - def gen_ordered(self) -> Iterator[Optional[_Run]]: - """Generate batches of operations, batched by type of - operation, in the order **provided**. - """ - run = None - for idx, (op_type, operation) in enumerate(self.ops): - if run is None: - run = _Run(op_type) - elif run.op_type != op_type: - yield run - run = _Run(op_type) - run.add(idx, operation) - yield run - - def gen_unordered(self) -> Iterator[_Run]: - """Generate batches of operations, batched by type of - operation, in arbitrary order. - """ - operations = [_Run(_INSERT), _Run(_UPDATE), _Run(_DELETE)] - for idx, (op_type, operation) in enumerate(self.ops): - operations[op_type].add(idx, operation) - - for run in operations: - if run.ops: - yield run + collection: Collection[Any] @_handle_reauth def write_command( diff --git a/pymongo/synchronous/client_bulk.py b/pymongo/synchronous/client_bulk.py index 5e5d1a76e8..549c9d59ca 100644 --- a/pymongo/synchronous/client_bulk.py +++ b/pymongo/synchronous/client_bulk.py @@ -29,9 +29,7 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.synchronous.client_session import ClientSession from pymongo.synchronous.collection import Collection @@ -46,15 +44,11 @@ from pymongo.synchronous.mongo_client import MongoClient from pymongo.synchronous.pool import Connection from pymongo._client_bulk_shared import ( + _AgnosticClientBulkBase, _merge_command, _throw_client_bulk_write_exception, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, ConnectionFailure, @@ -76,152 +70,15 @@ InsertOneResult, UpdateResult, ) -from pymongo.typings import _DocumentOut, _Pipeline from pymongo.write_concern import WriteConcern _IS_SYNC = True -class _ClientBulk: +class _ClientBulk(_AgnosticClientBulkBase): """The private guts of the client-level bulk write API.""" - def __init__( - self, - client: MongoClient[Any], - write_concern: WriteConcern, - ordered: bool = True, - bypass_document_validation: Optional[bool] = None, - comment: Optional[str] = None, - let: Optional[Any] = None, - verbose_results: bool = False, - ) -> None: - """Initialize a _ClientBulk instance.""" - self.client = client - self.write_concern = write_concern - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.ordered = ordered - self.bypass_doc_val = bypass_document_validation - self.comment = comment - self.verbose_results = verbose_results - self.ops: list[tuple[str, Mapping[str, Any]]] = [] - self.namespaces: list[str] = [] - self.idx_offset: int = 0 - self.total_ops: int = 0 - self.executed = False - self.uses_collation = False - self.uses_array_filters = False - self.is_retryable = self.client.options.retry_writes - self.retrying = False - self.started_retryable_write = False - - @property - def bulk_ctx_class(self) -> type[_ClientBulkWriteContext]: - return _ClientBulkWriteContext - - def add_insert(self, namespace: str, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - cmd = {"insert": -1, "document": document} - self.ops.append(("insert", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_update( - self, - namespace: str, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd = { - "update": -1, - "filter": selector, - "updateMods": update, - "multi": multi, - } - if upsert is not None: - cmd["upsert"] = upsert - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("update", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_replace( - self, - namespace: str, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd = { - "update": -1, - "filter": selector, - "updateMods": replacement, - "multi": False, - } - if upsert is not None: - cmd["upsert"] = upsert - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - self.ops.append(("replace", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_delete( - self, - namespace: str, - selector: Mapping[str, Any], - multi: bool, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd = {"delete": -1, "filter": selector, "multi": multi} - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("delete", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 + client: MongoClient[Any] @_handle_reauth def write_command( diff --git a/pymongo/typings.py b/pymongo/typings.py index 0281c57c59..e0777e07e1 100644 --- a/pymongo/typings.py +++ b/pymongo/typings.py @@ -70,7 +70,6 @@ def strip_optional(elem: Optional[_T]) -> _T: __all__ = [ "_Address", - "_AgnosticMongoClient", "_CollationIn", "_DocumentOut", "_DocumentType", From 5fb089b63c43c670cc59e168a6f846212287ce4e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 1 Oct 2026 20:01:56 -0500 Subject: [PATCH 12/24] PYTHON-6135 Fix perf regression by explicitly choosing the python binary (#3086) --- .evergreen/generated_configs/functions.yml | 2 ++ .evergreen/generated_configs/tasks.yml | 16 ++++++++++++++++ .evergreen/run-tests.sh | 3 +++ .evergreen/scripts/generate_config.py | 10 ++++++++-- .evergreen/scripts/setup-uv-python.sh | 11 ++++++----- .evergreen/scripts/setup_tests.py | 13 +++++-------- .evergreen/scripts/utils.py | 3 +++ 7 files changed, 43 insertions(+), 15 deletions(-) diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index e442973c30..df7393de4c 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -102,6 +102,7 @@ functions: - SSL - ORCHESTRATION_FILE - UV_PYTHON + - UV_PYTHON_PREFERENCE - STORAGE_ENGINE - REQUIRE_API_VERSION - DRIVERS_TOOLS @@ -137,6 +138,7 @@ functions: - AWS_SESSION_TOKEN - COVERAGE - UV_PYTHON + - UV_PYTHON_PREFERENCE - LIBMONGOCRYPT_URL - MONGODB_URI - DISABLE_TEST_COMMANDS diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index d4ff6ad5d5..169f8ce85f 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -2784,10 +2784,14 @@ tasks: vars: VERSION: v8.0-perf SSL: ssl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: sync + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] @@ -2797,10 +2801,14 @@ tasks: vars: VERSION: v8.0-perf SSL: ssl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: async + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] @@ -2810,10 +2818,14 @@ tasks: vars: VERSION: v8.0-perf SSL: nossl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: sync + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] @@ -2823,10 +2835,14 @@ tasks: vars: VERSION: v8.0-perf SSL: nossl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: async + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 358b3e532e..aa1fe97610 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -26,6 +26,9 @@ else exit 1 fi +# No-op on Evergreen (just.sh already ran this); needed for direct/local runs (PYTHON-6135). +. $SCRIPT_DIR/scripts/setup-uv-python.sh + # Start the test runner. echo "Running tests with UV_PYTHON=${UV_PYTHON:-} UV_PYTHON_SEARCH_PATH=${UV_PYTHON_SEARCH_PATH:-}..." echo "UV_ARGS=${UV_ARGS}" diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 9b4313c423..9ec13d5037 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -43,6 +43,7 @@ git_get_project, ) from shrub.v3.evg_task import EvgTask, EvgTaskDependency, EvgTaskRef +from utils import PERF_PYTHON_VERSION ############## # Variants @@ -1092,9 +1093,12 @@ def create_search_index_tasks(): def create_perf_tasks(): tasks = [] for version, ssl, sync in product(["8.0"], ["ssl", "nossl"], ["sync", "async"]): - vars = dict(VERSION=f"v{version}-perf", SSL=ssl) + # The toolchain build may be this same patch version but is not + # optimized, which regressed CPU-bound benchmarks (PYTHON-6135). + perf_vars = dict(UV_PYTHON=PERF_PYTHON_VERSION, UV_PYTHON_PREFERENCE="only-managed") + vars = dict(VERSION=f"v{version}-perf", SSL=ssl, **perf_vars) server_func = FunctionCall(func="run server", vars=vars) - vars = dict(TEST_NAME="perf", SUB_TEST_NAME=sync) + vars = dict(TEST_NAME="perf", SUB_TEST_NAME=sync, **perf_vars) test_func = FunctionCall(func="run tests", vars=vars) attach_func = FunctionCall(func="attach benchmark test results") send_func = FunctionCall(func="send dashboard data") @@ -1313,6 +1317,7 @@ def create_run_server_func(): "SSL", "ORCHESTRATION_FILE", "UV_PYTHON", + "UV_PYTHON_PREFERENCE", "STORAGE_ENGINE", "REQUIRE_API_VERSION", "DRIVERS_TOOLS", @@ -1342,6 +1347,7 @@ def create_run_tests_func(): "AWS_SESSION_TOKEN", "COVERAGE", "UV_PYTHON", + "UV_PYTHON_PREFERENCE", "LIBMONGOCRYPT_URL", "MONGODB_URI", "DISABLE_TEST_COMMANDS", diff --git a/.evergreen/scripts/setup-uv-python.sh b/.evergreen/scripts/setup-uv-python.sh index 2ebfe04d9d..09397803f5 100755 --- a/.evergreen/scripts/setup-uv-python.sh +++ b/.evergreen/scripts/setup-uv-python.sh @@ -9,6 +9,7 @@ # UV_PYTHON_SEARCH_PATH - the Python toolchain bin dir for the request, so uv # uses the toolchain Python instead of downloading one. # UV_PYTHON_PREFERENCE - "system" so the toolchain wins over managed installs. +# A preference set by the task is left alone. # UV_PYTHON - the Python interpreter uv uses; defaults to CPython # 3.10 when the task does not set one. set -euo pipefail @@ -85,11 +86,11 @@ elif _dir=$(_toolchain_dir "$UV_PYTHON"); then _search_path="$_dir" fi -# Point uv at the toolchain Python when there is one. On CI the toolchain dir -# is already first on PATH (configure-env.sh), so this mainly benefits local -# hosts and later steps, keeping `uv sync` and `uv tool install` on the -# toolchain interpreter instead of downloading a managed one. -if [ -n "$_search_path" ]; then +# Point uv at the toolchain Python when there is one and the task did not set +# its own preference: a task-level UV_PYTHON_PREFERENCE wins (PYTHON-6135). +# On CI the toolchain dir is already first on PATH (configure-env.sh), so this +# mainly benefits local hosts and later steps. +if [ -n "$_search_path" ] && [ -z "${UV_PYTHON_PREFERENCE:-}" ]; then export UV_PYTHON_SEARCH_PATH="$_search_path" export UV_PYTHON_PREFERENCE="system" fi diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index b57188eee1..6c77ecabf9 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -13,6 +13,7 @@ ENV_FILE, HERE, LOGGER, + PERF_PYTHON_VERSION, PLATFORM, ROOT, TEST_SUITE_MAP, @@ -51,9 +52,6 @@ # Map the test name to test group. GROUP_MAP = dict(mockupdb="mockupdb", perf="perf") -# The python version used for perf tests. -PERF_PYTHON_VERSION = "3.10.11" - # The libmongocrypt release used when LIBMONGOCRYPT_URL is not set. Must be at # least 1.20.0 for the GA "substring" query type. LIBMONGOCRYPT_VERSION = "1.20.2" @@ -515,13 +513,12 @@ def handle_test_env() -> None: run_command("tar xf single_and_multi_document.tgz", cwd=data_dir) write_env("TEST_PATH", str(data_dir)) write_env("OUTPUT_FILE", str(ROOT / "results.json")) - # Overwrite the UV_PYTHON value from env.sh, and unset the toolchain - # search-path variables: an empty value would make uv reject the request, - # and a toolchain path would miss the exact patch version requested. - write_env("UV_PYTHON", "") + # The toolchain build may be this same patch version but is not + # optimized, which regressed CPU-bound benchmarks (PYTHON-6135). + write_env("UV_PYTHON", PERF_PYTHON_VERSION) + write_env("UV_PYTHON_PREFERENCE", "only-managed") with ENV_FILE.open("a", newline="\n") as fid: fid.write("unset UV_PYTHON_SEARCH_PATH\n") - fid.write("unset UV_PYTHON_PREFERENCE\n") UV_ARGS.append(f"--python={PERF_PYTHON_VERSION}") diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 3f948dd292..75d4fc89b3 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -81,6 +81,9 @@ class Distro: EXTRA_TESTS = ["aws_lambda", "doctest"] +# The managed Python version used for perf tests (PYTHON-6135). +PERF_PYTHON_VERSION = "3.10.11" + # Tests that do not use run-mongodb directly. NO_RUN_ORCHESTRATION = [ "auth_oidc", From 43a4f1034693a659e55ec15abbe5486ddc5245e9 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 2 Oct 2026 14:50:43 -0500 Subject: [PATCH 13/24] PYTHON-6140 [Build Failure] TestBSON.test_encode_size_limit MemoryError on Windows (#3092) --- test/test_bson.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/test/test_bson.py b/test/test_bson.py index 191a706388..0800806b84 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -75,6 +75,25 @@ _NUMPY_AVAILABLE = importlib.util.find_spec("numpy") is not None +def _total_memory(): + """Return total physical memory in bytes, or None if it cannot be determined.""" + try: + return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") + except (ValueError, OSError, AttributeError): + return None + + +# PYTHON-6140: on Linux, skip if the host cannot allocate the ~4GiB this test +# needs. On other platforms, run the test and skip on MemoryError. +_TOTAL_MEMORY = _total_memory() +_ENCODE_SIZE_LIMIT_MIN_MEMORY = 6 << 30 +_SKIP_ENCODE_SIZE_LIMIT = ( + sys.platform == "linux" + and _TOTAL_MEMORY is not None + and _TOTAL_MEMORY < _ENCODE_SIZE_LIMIT_MIN_MEMORY +) + + class NotADict(abc.MutableMapping): """Non-dict type that implements the mapping protocol.""" @@ -692,12 +711,22 @@ def test_overflow(self): self.assertRaises(OverflowError, encode, {"x": -9223372036854775809}) @unittest.skipUnless(bson.has_c(), "This test requires the C extension") + @unittest.skipIf( + _SKIP_ENCODE_SIZE_LIMIT, + "Test requires at least 6GiB of memory (PYTHON-6140)", + ) def test_encode_size_limit(self): # PYTHON-5996: encoding must raise when a document's encoded size # exceeds the BSON size limit. - big_value = "a" * (1 << 30) - with self.assertRaises(ValueError): - encode({"a": big_value, "b": big_value, "c": big_value}) + try: + big_value = "a" * (1 << 30) + with self.assertRaises(ValueError): + encode({"a": big_value, "b": big_value, "c": big_value}) + except MemoryError: + # PYTHON-6140: some hosts report enough installed memory but + # cannot actually commit it (pagefile limits, mongod running on + # the same host). Skip rather than fail. + self.skipTest("Could not allocate enough memory (PYTHON-6140)") def test_small_long_encode_decode(self): encoded1 = encode({"x": 256}) From 07a7b8e0066ae409bcdcaf23e83b4b72f9f97b35 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 2 Oct 2026 16:07:14 -0500 Subject: [PATCH 14/24] PYTHON-6136 Make pool checkout accounting transactional (#3089) --- pymongo/asynchronous/pool.py | 128 +++++++++++++++++++++--------- pymongo/synchronous/pool.py | 128 +++++++++++++++++++++--------- test/asynchronous/test_client.py | 59 ++++++++++---- test/asynchronous/test_pooling.py | 95 ++++++++++++++++++++++ test/test_client.py | 59 ++++++++++---- test/test_pooling.py | 95 ++++++++++++++++++++++ 6 files changed, 462 insertions(+), 102 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 69ac14e7e4..b7f7a1f6a0 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -106,6 +106,13 @@ _IS_SYNC = False +# Flags recording the counters a checkout has incremented, so a failed +# checkout can restore them exactly once (PYTHON-6136). +_UNDO_OPERATION_COUNT = 1 +_UNDO_REQUESTS = 2 +_UNDO_SOCKETS = 4 +_UNDO_PENDING = 8 + class AsyncConnection(_ConnectionTelemetryInfo): """Store a connection with some metadata. @@ -969,8 +976,12 @@ async def _get_conn( "Attempted to check out a connection from closed connection pool" ) + # Every counter increment sets a flag in ``applied`` so a failed + # checkout can restore them exactly once (PYTHON-6136). + applied = 0 async with self.lock: self.operation_count += 1 + applied |= _UNDO_OPERATION_COUNT # Get a free socket or create one. if _csot.get_timeout(): @@ -980,28 +991,32 @@ async def _get_conn( else: deadline = None - async with self.size_cond: - self._raise_if_not_ready(checkout_started_time, emit_event=True) - while not (self.requests < self.max_pool_size): - timeout = deadline - time.monotonic() if deadline else None - if not await _async_cond_wait(self.size_cond, timeout): - # Timed out, notify the next thread to ensure a - # timeout doesn't consume the condition. - if self.requests < self.max_pool_size: - self.size_cond.notify() - self._raise_wait_queue_timeout(checkout_started_time) + try: + async with self.size_cond: self._raise_if_not_ready(checkout_started_time, emit_event=True) - self.requests += 1 + while not (self.requests < self.max_pool_size): + timeout = deadline - time.monotonic() if deadline else None + if not await _async_cond_wait(self.size_cond, timeout): + # Timed out, notify the next thread to ensure a + # timeout doesn't consume the condition. + if self.requests < self.max_pool_size: + self.size_cond.notify() + self._raise_wait_queue_timeout(checkout_started_time) + self._raise_if_not_ready(checkout_started_time, emit_event=True) + self.requests += 1 + applied |= _UNDO_REQUESTS + except BaseException: + await self._restore_counters(applied) + raise # We've now acquired the semaphore and must release it on error. conn = None - incremented = False emitted_event = False is_new_conn = False try: async with self.lock: self.active_sockets += 1 - incremented = True + applied |= _UNDO_SOCKETS while conn is None: # CMAP: we MUST wait for either maxConnecting OR for a socket # to be checked back into the pool. @@ -1022,6 +1037,7 @@ async def _get_conn( conn = self.conns.popleft() except IndexError: self._pending += 1 + applied |= _UNDO_PENDING if conn: # We got a socket from the pool if await self._perished(conn): conn = None @@ -1033,7 +1049,17 @@ async def _get_conn( finally: async with self._max_connecting_cond: self._pending -= 1 - self._max_connecting_cond.notify() + applied &= ~_UNDO_PENDING + notified = False + try: + self._max_connecting_cond.notify() + notified = True + finally: + if not notified: + # A kill landed inside notify() (a gevent + # yield point); retry so a waiting + # checkout is not stranded (PYTHON-6136). + self._max_connecting_cond.notify() conn.active = True # connect() already adds cancel_context for new connections; only add @@ -1043,27 +1069,14 @@ async def _get_conn( self.active_contexts.add(conn.cancel_context) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. except BaseException: - if conn: - # We checked out a socket but authentication failed. - await conn.close_conn(ConnectionClosedReason.ERROR) - # Re-apply the accounting if a GreenletExit interrupts - # during the size_cond acquisition; during unwind gevent - # lets the re-acquire complete (PYTHON-6074). - accounted = False try: - async with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - accounted = True - self.size_cond.notify() + if conn is not None: + # We checked out a socket but authentication failed. + await conn.close_conn(ConnectionClosedReason.ERROR) finally: - if not accounted: - async with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - self.size_cond.notify() + # Restore the counters even if the cleanup above was + # interrupted (PYTHON-6136). + await self._restore_counters(applied) if not emitted_event: self._telemetry.checkout_failed( @@ -1075,6 +1088,51 @@ async def _get_conn( return conn + def _restore_applied(self, applied: int) -> None: + """Restore the counters flagged in ``applied``. Caller holds ``size_cond``.""" + if applied & _UNDO_OPERATION_COUNT: + self.operation_count -= 1 + if applied & _UNDO_REQUESTS: + self.requests -= 1 + if applied & _UNDO_SOCKETS: + self.active_sockets -= 1 + if applied & _UNDO_PENDING: + self._pending -= 1 + + async def _restore_counters(self, applied: int) -> None: + """Restore the counters a failed checkout incremented (PYTHON-6136). + + Gevent grants the re-acquire during unwind (PYTHON-6074). A kill can + also land inside notify() (a yield point), so notifications are + tracked and retried separately from the counter restore. + """ + accounted = False + notified = 0 + try: + async with self.size_cond: + self._restore_applied(applied) + accounted = True + if applied & _UNDO_REQUESTS: + # A pool slot was freed; wake the next waiting thread. + self.size_cond.notify() + notified |= _UNDO_REQUESTS + if applied & _UNDO_PENDING: + # A maxConnecting slot was freed; wake the next waiting thread. + self._max_connecting_cond.notify() + notified |= _UNDO_PENDING + finally: + # Always reacquired: `applied` keeps restore-only flags + # `notified` never tracks, so skipping needs a mask synced + # to the notify sites; uncontended acquires don't yield (PYTHON-6136). + async with self.size_cond: + if not accounted: + self._restore_applied(applied) + missing = applied & ~notified + if missing & _UNDO_REQUESTS: + self.size_cond.notify() + if missing & _UNDO_PENDING: + self._max_connecting_cond.notify() + def _checkin_apply( self, conn: AsyncConnection, txn: bool, cursor: bool, forked: bool ) -> tuple[Optional[str], bool, bool]: @@ -1121,9 +1179,7 @@ async def checkin(self, conn: AsyncConnection) -> None: conn.pinned_cursor = False self._pinned_sockets.discard(conn) forked = self.pid != os.getpid() - # Re-apply the accounting if a gevent GreenletExit interrupts during - # the size_cond acquisition; gevent lets the re-acquire complete while - # unwinding (PYTHON-6074). + # Re-apply the accounting if a BaseException interrupts here (PYTHON-6074). close_conn_reason: Optional[str] = None emit_closed = False accounted = False diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index c977729ba8..dd562ac68e 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -106,6 +106,13 @@ _IS_SYNC = True +# Flags recording the counters a checkout has incremented, so a failed +# checkout can restore them exactly once (PYTHON-6136). +_UNDO_OPERATION_COUNT = 1 +_UNDO_REQUESTS = 2 +_UNDO_SOCKETS = 4 +_UNDO_PENDING = 8 + class Connection(_ConnectionTelemetryInfo): """Store a connection with some metadata. @@ -965,8 +972,12 @@ def _get_conn( "Attempted to check out a connection from closed connection pool" ) + # Every counter increment sets a flag in ``applied`` so a failed + # checkout can restore them exactly once (PYTHON-6136). + applied = 0 with self.lock: self.operation_count += 1 + applied |= _UNDO_OPERATION_COUNT # Get a free socket or create one. if _csot.get_timeout(): @@ -976,28 +987,32 @@ def _get_conn( else: deadline = None - with self.size_cond: - self._raise_if_not_ready(checkout_started_time, emit_event=True) - while not (self.requests < self.max_pool_size): - timeout = deadline - time.monotonic() if deadline else None - if not _cond_wait(self.size_cond, timeout): - # Timed out, notify the next thread to ensure a - # timeout doesn't consume the condition. - if self.requests < self.max_pool_size: - self.size_cond.notify() - self._raise_wait_queue_timeout(checkout_started_time) + try: + with self.size_cond: self._raise_if_not_ready(checkout_started_time, emit_event=True) - self.requests += 1 + while not (self.requests < self.max_pool_size): + timeout = deadline - time.monotonic() if deadline else None + if not _cond_wait(self.size_cond, timeout): + # Timed out, notify the next thread to ensure a + # timeout doesn't consume the condition. + if self.requests < self.max_pool_size: + self.size_cond.notify() + self._raise_wait_queue_timeout(checkout_started_time) + self._raise_if_not_ready(checkout_started_time, emit_event=True) + self.requests += 1 + applied |= _UNDO_REQUESTS + except BaseException: + self._restore_counters(applied) + raise # We've now acquired the semaphore and must release it on error. conn = None - incremented = False emitted_event = False is_new_conn = False try: with self.lock: self.active_sockets += 1 - incremented = True + applied |= _UNDO_SOCKETS while conn is None: # CMAP: we MUST wait for either maxConnecting OR for a socket # to be checked back into the pool. @@ -1018,6 +1033,7 @@ def _get_conn( conn = self.conns.popleft() except IndexError: self._pending += 1 + applied |= _UNDO_PENDING if conn: # We got a socket from the pool if self._perished(conn): conn = None @@ -1029,7 +1045,17 @@ def _get_conn( finally: with self._max_connecting_cond: self._pending -= 1 - self._max_connecting_cond.notify() + applied &= ~_UNDO_PENDING + notified = False + try: + self._max_connecting_cond.notify() + notified = True + finally: + if not notified: + # A kill landed inside notify() (a gevent + # yield point); retry so a waiting + # checkout is not stranded (PYTHON-6136). + self._max_connecting_cond.notify() conn.active = True # connect() already adds cancel_context for new connections; only add @@ -1039,27 +1065,14 @@ def _get_conn( self.active_contexts.add(conn.cancel_context) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. except BaseException: - if conn: - # We checked out a socket but authentication failed. - conn.close_conn(ConnectionClosedReason.ERROR) - # Re-apply the accounting if a GreenletExit interrupts - # during the size_cond acquisition; during unwind gevent - # lets the re-acquire complete (PYTHON-6074). - accounted = False try: - with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - accounted = True - self.size_cond.notify() + if conn is not None: + # We checked out a socket but authentication failed. + conn.close_conn(ConnectionClosedReason.ERROR) finally: - if not accounted: - with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - self.size_cond.notify() + # Restore the counters even if the cleanup above was + # interrupted (PYTHON-6136). + self._restore_counters(applied) if not emitted_event: self._telemetry.checkout_failed( @@ -1071,6 +1084,51 @@ def _get_conn( return conn + def _restore_applied(self, applied: int) -> None: + """Restore the counters flagged in ``applied``. Caller holds ``size_cond``.""" + if applied & _UNDO_OPERATION_COUNT: + self.operation_count -= 1 + if applied & _UNDO_REQUESTS: + self.requests -= 1 + if applied & _UNDO_SOCKETS: + self.active_sockets -= 1 + if applied & _UNDO_PENDING: + self._pending -= 1 + + def _restore_counters(self, applied: int) -> None: + """Restore the counters a failed checkout incremented (PYTHON-6136). + + Gevent grants the re-acquire during unwind (PYTHON-6074). A kill can + also land inside notify() (a yield point), so notifications are + tracked and retried separately from the counter restore. + """ + accounted = False + notified = 0 + try: + with self.size_cond: + self._restore_applied(applied) + accounted = True + if applied & _UNDO_REQUESTS: + # A pool slot was freed; wake the next waiting thread. + self.size_cond.notify() + notified |= _UNDO_REQUESTS + if applied & _UNDO_PENDING: + # A maxConnecting slot was freed; wake the next waiting thread. + self._max_connecting_cond.notify() + notified |= _UNDO_PENDING + finally: + # Always reacquired: `applied` keeps restore-only flags + # `notified` never tracks, so skipping needs a mask synced + # to the notify sites; uncontended acquires don't yield (PYTHON-6136). + with self.size_cond: + if not accounted: + self._restore_applied(applied) + missing = applied & ~notified + if missing & _UNDO_REQUESTS: + self.size_cond.notify() + if missing & _UNDO_PENDING: + self._max_connecting_cond.notify() + def _checkin_apply( self, conn: Connection, txn: bool, cursor: bool, forked: bool ) -> tuple[Optional[str], bool, bool]: @@ -1117,9 +1175,7 @@ def checkin(self, conn: Connection) -> None: conn.pinned_cursor = False self._pinned_sockets.discard(conn) forked = self.pid != os.getpid() - # Re-apply the accounting if a gevent GreenletExit interrupts during - # the size_cond acquisition; gevent lets the re-acquire complete while - # unwinding (PYTHON-6074). + # Re-apply the accounting if a BaseException interrupts here (PYTHON-6074). close_conn_reason: Optional[str] = None emit_closed = False accounted = False diff --git a/test/asynchronous/test_client.py b/test/asynchronous/test_client.py index 56810b1433..563d29cbad 100644 --- a/test/asynchronous/test_client.py +++ b/test/asynchronous/test_client.py @@ -2744,20 +2744,18 @@ def test_gevent_kill_churn_deadlock(self): import gevent.thread as _gthread from gevent import Timeout, spawn - # AMPLIFY_RACE=1 widens gevent's brief sleep on a contended lock - # so a kill lands there reliably. Only the bare sleep() is - # widened; timed sleeps pass through. The unfixed test then - # deadlocks within seconds. + # AMPLIFY_RACE=1 widens the kill windows below; keep AMPLIFY_SECONDS + # small or ops starve. + amplify_seconds = 0.0 if os.environ.get("AMPLIFY_RACE", "0") == "1": - _AMPLIFY_SECONDS = float(os.environ.get("AMPLIFY_SECONDS", "0.02")) + amplify_seconds = float(os.environ.get("AMPLIFY_SECONDS", "0.002")) _orig_thread_sleep = _gthread.sleep def _amplified_sleep(*args): - if not args: # bare sleep(): the courtesy yield on a failed - # non-blocking acquire (Condition.notify -> _is_owned -> acquire(False)) - _orig_thread_sleep(_AMPLIFY_SECONDS) - else: # sleep(0.001), sleep(2), etc.: passthrough + if not args: # bare sleep(): checkin courtesy yield + _orig_thread_sleep(amplify_seconds) + else: # timed sleeps pass through _orig_thread_sleep(*args) _gthread.sleep = _amplified_sleep @@ -2767,6 +2765,31 @@ def _amplified_sleep(*args): coll = client.pymongo_test.coll coll.insert_one({}) + pool = async_get_pool(client) # type:ignore + # Widen the post-gate checkout windows (PYTHON-6136). + if amplify_seconds: + + class _AmplifiedCondition: + def __init__(self, cond, seconds): + self._cond = cond + self._seconds = seconds + + def __enter__(self): + self._cond.__enter__() + return self + + def __exit__(self, *args): + self._cond.__exit__(*args) + time.sleep(self._seconds) + + def __getattr__(self, name): + return getattr(self._cond, name) + + pool.size_cond = _AmplifiedCondition(pool.size_cond, amplify_seconds) + pool._max_connecting_cond = _AmplifiedCondition( + pool._max_connecting_cond, amplify_seconds + ) + op_count = [0] running = [True] workers: list = [] @@ -2780,9 +2803,12 @@ def worker(): except Exception: return + # Scale the kill cadence or ops starve. + reaper_interval = max(0.003, amplify_seconds * 8) + def reaper(): while running[0]: - time.sleep(0.003) + time.sleep(reaper_interval) if not workers: continue idx = random.randrange(len(workers)) @@ -2823,11 +2849,6 @@ def reaper(): coll.find_one({}) except Timeout: self.fail("Pool gate saturated (PYTHON-6074)") - # Deterministic check: a saturated size gate pins the pool's - # checkout counters at maxPoolSize (PYTHON-6074). - pool = async_get_pool(client) # type:ignore - self.assertLess(pool.requests, pool.max_pool_size) - self.assertLess(pool.active_sockets, pool.max_pool_size) self.assertGreater(op_count[0], 0) finally: running[0] = False @@ -2845,6 +2866,14 @@ def reaper(): client.close() except Timeout: pass + # Post-settle: the counters must have fully drained. Note close() + # does not zero operation_count (only a fork does), so a leaked + # increment survives and is caught here. + time.sleep(1.0) + self.assertEqual(pool.requests, 0) + self.assertEqual(pool._pending, 0) + self.assertEqual(pool.active_sockets, 0) + self.assertEqual(pool.operation_count, 0) class TestClientLazyConnect(AsyncIntegrationTest): diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index 661bd4e3d2..25da5134b9 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -306,6 +306,7 @@ def notify(): # Accounting was applied exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) async def test_checkout_error_accounting_on_kill_during_acquire(self): # PYTHON-6074: an exception delivered while the checkout error @@ -341,6 +342,98 @@ async def __aexit__(self, *args): # The fallback applied the accounting exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) + + async def test_checkout_error_accounting_on_connect_keyboard_interrupt(self): + # PYTHON-6136: a KeyboardInterrupt from connect() must roll back the + # size gate, the maxConnecting gate, and active_sockets. + cx_pool = await self.create_pool(max_pool_size=1) + + with patch.object(cx_pool, "connect", side_effect=KeyboardInterrupt()): + with self.assertRaises(KeyboardInterrupt): + async with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + + async def test_checkout_error_accounting_on_kill_during_pending_cleanup(self): + # PYTHON-6136: an interruption during the pending-gate cleanup must + # not skip the counter restore, which must wake threads waiting at + # the maxConnecting gate. + cx_pool = await self.create_pool(max_pool_size=1) + + class _InterruptOnSecondEnter(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.enters = 0 + self.notifies = 0 + + async def __aenter__(self): + self.enters += 1 + if self.enters == 2: + # First enter is the checkout wait, second is connect()'s + # cleanup. Simulate a kill delivered while waiting there. + raise KeyboardInterrupt() + return await super().__aenter__() + + async def __aexit__(self, *args): + return await super().__aexit__(*args) + + def notify(self, n=1): + # The counter restore wakes the maxConnecting gate, then is + # itself killed. + self.notifies += 1 + raise KeyboardInterrupt() + + cond = _InterruptOnSecondEnter(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + async with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + # The restore notified the maxConnecting gate; the notify interrupted + # by the kill was retried. + self.assertEqual(2, cond.notifies) + + async def test_checkout_error_accounting_on_kill_during_pending_notify(self): + # PYTHON-6136: a kill landing inside the cleanup notify must not + # strand a checkout waiting at the maxConnecting gate. + cx_pool = await self.create_pool(max_pool_size=1) + + class _InterruptOnFirstNotify(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.notifies = 0 + + def notify(self, n=1): + self.notifies += 1 + if self.notifies == 1: + # Simulate a kill delivered inside notify(). + raise KeyboardInterrupt() + + cond = _InterruptOnFirstNotify(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + async with cx_pool.checkout(): + pass + + # The cleanup notify was interrupted, then retried. + self.assertEqual(2, cond.notifies) + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) async def test_pool_removes_closed_socket(self): # Test that Pool removes explicitly closed socket. @@ -463,6 +556,8 @@ async def test_wait_queue_timeout(self): 1, f"Waited {duration:.2f} seconds for a socket, expected {wait_queue_timeout:f}", ) + # The load metric must not be inflated by the failed checkout. + self.assertEqual(0, pool.operation_count) async def test_no_wait_queue_timeout(self): # Verify get_socket() with no wait_queue_timeout blocks forever. diff --git a/test/test_client.py b/test/test_client.py index ebd1e670ee..0c8d0e7595 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -2695,20 +2695,18 @@ def test_gevent_kill_churn_deadlock(self): import gevent.thread as _gthread from gevent import Timeout, spawn - # AMPLIFY_RACE=1 widens gevent's brief sleep on a contended lock - # so a kill lands there reliably. Only the bare sleep() is - # widened; timed sleeps pass through. The unfixed test then - # deadlocks within seconds. + # AMPLIFY_RACE=1 widens the kill windows below; keep AMPLIFY_SECONDS + # small or ops starve. + amplify_seconds = 0.0 if os.environ.get("AMPLIFY_RACE", "0") == "1": - _AMPLIFY_SECONDS = float(os.environ.get("AMPLIFY_SECONDS", "0.02")) + amplify_seconds = float(os.environ.get("AMPLIFY_SECONDS", "0.002")) _orig_thread_sleep = _gthread.sleep def _amplified_sleep(*args): - if not args: # bare sleep(): the courtesy yield on a failed - # non-blocking acquire (Condition.notify -> _is_owned -> acquire(False)) - _orig_thread_sleep(_AMPLIFY_SECONDS) - else: # sleep(0.001), sleep(2), etc.: passthrough + if not args: # bare sleep(): checkin courtesy yield + _orig_thread_sleep(amplify_seconds) + else: # timed sleeps pass through _orig_thread_sleep(*args) _gthread.sleep = _amplified_sleep @@ -2718,6 +2716,31 @@ def _amplified_sleep(*args): coll = client.pymongo_test.coll coll.insert_one({}) + pool = get_pool(client) # type:ignore + # Widen the post-gate checkout windows (PYTHON-6136). + if amplify_seconds: + + class _AmplifiedCondition: + def __init__(self, cond, seconds): + self._cond = cond + self._seconds = seconds + + def __enter__(self): + self._cond.__enter__() + return self + + def __exit__(self, *args): + self._cond.__exit__(*args) + time.sleep(self._seconds) + + def __getattr__(self, name): + return getattr(self._cond, name) + + pool.size_cond = _AmplifiedCondition(pool.size_cond, amplify_seconds) + pool._max_connecting_cond = _AmplifiedCondition( + pool._max_connecting_cond, amplify_seconds + ) + op_count = [0] running = [True] workers: list = [] @@ -2731,9 +2754,12 @@ def worker(): except Exception: return + # Scale the kill cadence or ops starve. + reaper_interval = max(0.003, amplify_seconds * 8) + def reaper(): while running[0]: - time.sleep(0.003) + time.sleep(reaper_interval) if not workers: continue idx = random.randrange(len(workers)) @@ -2774,11 +2800,6 @@ def reaper(): coll.find_one({}) except Timeout: self.fail("Pool gate saturated (PYTHON-6074)") - # Deterministic check: a saturated size gate pins the pool's - # checkout counters at maxPoolSize (PYTHON-6074). - pool = get_pool(client) # type:ignore - self.assertLess(pool.requests, pool.max_pool_size) - self.assertLess(pool.active_sockets, pool.max_pool_size) self.assertGreater(op_count[0], 0) finally: running[0] = False @@ -2796,6 +2817,14 @@ def reaper(): client.close() except Timeout: pass + # Post-settle: the counters must have fully drained. Note close() + # does not zero operation_count (only a fork does), so a leaked + # increment survives and is caught here. + time.sleep(1.0) + self.assertEqual(pool.requests, 0) + self.assertEqual(pool._pending, 0) + self.assertEqual(pool.active_sockets, 0) + self.assertEqual(pool.operation_count, 0) class TestClientLazyConnect(IntegrationTest): diff --git a/test/test_pooling.py b/test/test_pooling.py index a3f0eaf589..3a81bc796f 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -306,6 +306,7 @@ def notify(): # Accounting was applied exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) def test_checkout_error_accounting_on_kill_during_acquire(self): # PYTHON-6074: an exception delivered while the checkout error @@ -341,6 +342,98 @@ def __exit__(self, *args): # The fallback applied the accounting exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) + + def test_checkout_error_accounting_on_connect_keyboard_interrupt(self): + # PYTHON-6136: a KeyboardInterrupt from connect() must roll back the + # size gate, the maxConnecting gate, and active_sockets. + cx_pool = self.create_pool(max_pool_size=1) + + with patch.object(cx_pool, "connect", side_effect=KeyboardInterrupt()): + with self.assertRaises(KeyboardInterrupt): + with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + + def test_checkout_error_accounting_on_kill_during_pending_cleanup(self): + # PYTHON-6136: an interruption during the pending-gate cleanup must + # not skip the counter restore, which must wake threads waiting at + # the maxConnecting gate. + cx_pool = self.create_pool(max_pool_size=1) + + class _InterruptOnSecondEnter(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.enters = 0 + self.notifies = 0 + + def __enter__(self): + self.enters += 1 + if self.enters == 2: + # First enter is the checkout wait, second is connect()'s + # cleanup. Simulate a kill delivered while waiting there. + raise KeyboardInterrupt() + return super().__enter__() + + def __exit__(self, *args): + return super().__exit__(*args) + + def notify(self, n=1): + # The counter restore wakes the maxConnecting gate, then is + # itself killed. + self.notifies += 1 + raise KeyboardInterrupt() + + cond = _InterruptOnSecondEnter(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + # The restore notified the maxConnecting gate; the notify interrupted + # by the kill was retried. + self.assertEqual(2, cond.notifies) + + def test_checkout_error_accounting_on_kill_during_pending_notify(self): + # PYTHON-6136: a kill landing inside the cleanup notify must not + # strand a checkout waiting at the maxConnecting gate. + cx_pool = self.create_pool(max_pool_size=1) + + class _InterruptOnFirstNotify(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.notifies = 0 + + def notify(self, n=1): + self.notifies += 1 + if self.notifies == 1: + # Simulate a kill delivered inside notify(). + raise KeyboardInterrupt() + + cond = _InterruptOnFirstNotify(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + with cx_pool.checkout(): + pass + + # The cleanup notify was interrupted, then retried. + self.assertEqual(2, cond.notifies) + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) def test_pool_removes_closed_socket(self): # Test that Pool removes explicitly closed socket. @@ -463,6 +556,8 @@ def test_wait_queue_timeout(self): 1, f"Waited {duration:.2f} seconds for a socket, expected {wait_queue_timeout:f}", ) + # The load metric must not be inflated by the failed checkout. + self.assertEqual(0, pool.operation_count) def test_no_wait_queue_timeout(self): # Verify get_socket() with no wait_queue_timeout blocks forever. From 045ee2af723b84b8656f78948703fa84d0193a5f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 5 Oct 2026 06:00:02 -0500 Subject: [PATCH 15/24] PYTHON-6051 Convert drivers-evergreen-tools to a git submodule and pin the GitHub Actions reference (#3075) --- .evergreen/scripts/cleanup.sh | 7 ++- .evergreen/scripts/configure-env.sh | 41 ++++++++++++--- .evergreen/scripts/create-spec-pr.sh | 6 +-- .evergreen/scripts/install-dependencies.sh | 5 +- .evergreen/scripts/run-getdata.sh | 7 +++ .evergreen/scripts/run_server.py | 19 +++---- .evergreen/scripts/setup-dev-env.sh | 23 +++++++++ .evergreen/scripts/setup_tests.py | 10 ++-- .evergreen/scripts/stop-server.sh | 4 ++ .evergreen/scripts/utils.py | 18 ++++++- .github/dependabot.yml | 9 ++++ .github/workflows/test-python.yml | 17 ++++--- .github/zizmor.yml | 2 +- .gitignore | 6 ++- .gitmodules | 3 ++ CONTRIBUTING.md | 58 +++++++++++++++++++--- drivers-evergreen-tools | 1 + pyproject.toml | 5 ++ 18 files changed, 194 insertions(+), 47 deletions(-) create mode 100644 .gitmodules create mode 160000 drivers-evergreen-tools diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh index f04a936fd2..44b36d3569 100755 --- a/.evergreen/scripts/cleanup.sh +++ b/.evergreen/scripts/cleanup.sh @@ -10,5 +10,10 @@ if [ -f $HERE/env.sh ]; then source $HERE/env.sh fi -rm -rf "${DRIVERS_TOOLS}" || true +# Don't delete the in-tree submodule; clean the ignored credential and state +# files (secrets-export.sh, AWS creds, token files) that `git submodule update` +# leaves behind, so they can't carry into later tasks on a reused host. A +# caller-provided DRIVERS_TOOLS wins, so the checkout actually used is cleaned. +: "${DRIVERS_TOOLS:=$HERE/../../drivers-evergreen-tools}" rm -f $HERE/../../secrets-export.sh || true +git -C "$DRIVERS_TOOLS" clean -fdx 2>/dev/null || true diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index fec3cdadae..5e36125838 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -12,7 +12,8 @@ else fi PROJECT_DIRECTORY="$(pwd)" -DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools" +# Default to the submodule; an env var override wins. +DRIVERS_TOOLS="${DRIVERS_TOOLS:-$PROJECT_DIRECTORY/drivers-evergreen-tools}" CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" @@ -93,12 +94,40 @@ export PROJECT="${project:-mongo-python-driver}" export PIP_QUIET=1 EOT -# Write the .env file for drivers-tools. -rm -rf $DRIVERS_TOOLS -BRANCH=master -ORG=mongodb-labs -git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS +# Only touch the in-tree submodule when it is the checkout actually in use; +# an overridden DRIVERS_TOOLS is a checkout we do not own. +if [ "$DRIVERS_TOOLS" = "$PROJECT_DIRECTORY/drivers-evergreen-tools" ]; then + # Initialize the submodule (Evergreen's git.get_project does not); tolerate + # non-git hosts with a warning. + if ! git -C "$PROJECT_DIRECTORY" submodule update --init --recursive; then + echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \ + "using the existing checkout contents instead." + fi + # Write a uv config boundary into the submodule: it is vendored inside this + # project, so uv run by the tools' own scripts would otherwise walk up to + # pyproject.toml and enforce our required-version pin. Write-if-absent, so an + # upstream uv.toml fails the submodule update loudly instead of being clobbered. + if [ -d "${DRIVERS_TOOLS}" ] && [ ! -f "${DRIVERS_TOOLS}/uv.toml" ]; then + cat < "${DRIVERS_TOOLS}/uv.toml" +# Written by mongo-python-driver to stop uv's config discovery here; see +# .evergreen/scripts/configure-env.sh. +EOT + fi + + # Keep the boundary out of git status via the submodule's local exclude; + # no-op without git. + if _git_dir=$(git -C "${DRIVERS_TOOLS}" rev-parse --absolute-git-dir 2>/dev/null); then + mkdir -p "${_git_dir}/info" + grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null || + printf "uv.toml\n" >> "${_git_dir}/info/exclude" + fi +fi + +# Write the .env file for drivers-tools. Create the checkout if it is missing so +# a failed submodule init does not stop this script before setup-tests.py can +# reach check_drivers_tools() and report the problem actionably. +mkdir -p "${DRIVERS_TOOLS}" cat < ${DRIVERS_TOOLS}/.env SKIP_LEGACY_SHELL=1 DRIVERS_TOOLS="$DRIVERS_TOOLS" diff --git a/.evergreen/scripts/create-spec-pr.sh b/.evergreen/scripts/create-spec-pr.sh index cffd8dc4de..daa68d0db6 100755 --- a/.evergreen/scripts/create-spec-pr.sh +++ b/.evergreen/scripts/create-spec-pr.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash set -eu -tools="$(realpath -s "../drivers-tools")" +# Default to the drivers-evergreen-tools submodule when unset; a +# caller-provided DRIVERS_TOOLS wins, as in the other consumers. +tools="$(realpath -s "${DRIVERS_TOOLS:-./drivers-evergreen-tools}")" pushd $tools/.evergreen/github_app || exit owner="mongodb" @@ -116,5 +118,3 @@ if [ -z "$pr_url" ]; then fi echo "$pr_url" echo "Creating the PR... done." - -rm -rf $tools diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index a5f947f88e..93b62ce6b6 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -67,8 +67,9 @@ fi # Set up uv if needed. if [ "$_need_setup" = "1" ]; then - # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its env. - if [ -n "${DRIVERS_TOOLS:-}" ] && [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then + # Default DRIVERS_TOOLS to the submodule; an env var override wins. + : "${DRIVERS_TOOLS:=$(dirname "$(dirname "$HERE")")/drivers-evergreen-tools}" + if [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then . "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ensure_uv || exit 1 fi diff --git a/.evergreen/scripts/run-getdata.sh b/.evergreen/scripts/run-getdata.sh index 9435a5fcc3..416744ac03 100755 --- a/.evergreen/scripts/run-getdata.sh +++ b/.evergreen/scripts/run-getdata.sh @@ -2,6 +2,13 @@ # Get the debug data for an evergreen task. set -eu +HERE=$(dirname ${BASH_SOURCE:-$0}) +HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" +ROOT=$(dirname "$(dirname $HERE)") + +# Default to the drivers-evergreen-tools submodule when unset. +: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}" + . ${DRIVERS_TOOLS}/.evergreen/get-distro.sh || true get_distro || true echo $DISTRO diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py index f81964f367..38f817c15d 100644 --- a/.evergreen/scripts/run_server.py +++ b/.evergreen/scripts/run_server.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from utils import DRIVERS_TOOLS, ROOT, get_test_options, run_command +from utils import DRIVERS_TOOLS, ROOT, check_drivers_tools, get_test_options, run_command def set_env(name: str, value: Any = "1") -> None: @@ -14,11 +14,9 @@ def set_env(name: str, value: Any = "1") -> None: def start_server(): - run_mongodb_script = ( - Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" if DRIVERS_TOOLS else None - ) + run_mongodb_script = Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" want_help = bool({"-h", "--help"} & set(sys.argv[1:])) - if want_help and run_mongodb_script and run_mongodb_script.is_file(): + if want_help and run_mongodb_script.is_file(): # Forward straight to run-mongodb.sh's own help, without run_command's # "Running command..." logging noise. subprocess.run( # noqa: S603 @@ -28,13 +26,10 @@ def start_server(): ) return - # DRIVERS_TOOLS is only needed to actually start a server. When it's unset and - # -h/--help was requested, fall through to get_test_options' own argparse help below. - if not want_help and not DRIVERS_TOOLS: - raise ValueError( - "DRIVERS_TOOLS is not set; run `just run-server` from an Evergreen task " - "or set DRIVERS_TOOLS to a drivers-evergreen-tools checkout." - ) + # DRIVERS_TOOLS is only needed to start a server; allow -h/--help to fall + # through to argparse when the submodule is missing. + if not want_help: + check_drivers_tools() opts, extra_opts = get_test_options( "Run a MongoDB server. All given flags will be passed to run-mongodb.sh in DRIVERS_TOOLS.", diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index 508eb099bc..d9fbef2b04 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -53,6 +53,29 @@ if [ "${CI:-}" != "true" ] && [ "${GITHUB_ACTIONS:-}" != "true" ]; then printf 'export PATH="%s:$PATH"\n' "$PYMONGO_BIN_DIR_POSIX" >> "$_rc" fi +# Initialize the submodule (Evergreen's git.get_project does not); tolerate +# non-git hosts with a warning. +if ! git -C "$ROOT" submodule update --init --recursive; then + echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \ + "set DRIVERS_TOOLS to a drivers-evergreen-tools checkout instead." +fi + +# Mirror configure-env.sh's uv config boundary; see it for the full explanation. +if [ -d "$ROOT/drivers-evergreen-tools" ] && [ ! -f "$ROOT/drivers-evergreen-tools/uv.toml" ]; then + cat < "$ROOT/drivers-evergreen-tools/uv.toml" +# Written by mongo-python-driver to stop uv's config discovery here; see +# .evergreen/scripts/configure-env.sh. +EOT +fi + +# Keep the boundary out of git status via the submodule's local exclude; +# no-op without git. +if _git_dir=$(git -C "$ROOT/drivers-evergreen-tools" rev-parse --absolute-git-dir 2>/dev/null); then + mkdir -p "${_git_dir}/info" + grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null || + printf "uv.toml\n" >> "${_git_dir}/info/exclude" +fi + # Ensure dependencies are installed. bash $HERE/install-dependencies.sh diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 6c77ecabf9..c8a15ad4a6 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -18,6 +18,7 @@ ROOT, TEST_SUITE_MAP, Distro, + check_drivers_tools, get_test_options, read_env, run_command, @@ -327,8 +328,7 @@ def handle_test_env() -> None: MULTI_MONGOS_LB_URI += "&tls=true" write_env("SINGLE_MONGOS_LB_URI", SINGLE_MONGOS_LB_URI) write_env("MULTI_MONGOS_LB_URI", MULTI_MONGOS_LB_URI) - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start' run_command(cmd) @@ -376,8 +376,7 @@ def handle_test_env() -> None: run_command(cmd, cwd=DRIVERS_TOOLS) if SSL != "nossl": - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem") write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem") @@ -429,8 +428,7 @@ def handle_test_env() -> None: # PATH is updated by configure-env.sh for access to mongocryptd. if test_name == "encryption": - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle") # Opt in to corporate Azure credentials (DRIVERS-3392) os.environ["FLE_AZURE_USE_CORPORATE"] = "YES" diff --git a/.evergreen/scripts/stop-server.sh b/.evergreen/scripts/stop-server.sh index 045a655cbd..a254a2715a 100755 --- a/.evergreen/scripts/stop-server.sh +++ b/.evergreen/scripts/stop-server.sh @@ -4,6 +4,7 @@ set -eu HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" +ROOT=$(dirname "$(dirname $HERE)") # Try to source the env file. if [ -f $HERE/env.sh ]; then @@ -11,4 +12,7 @@ if [ -f $HERE/env.sh ]; then source $HERE/env.sh fi +# Default to the drivers-evergreen-tools submodule when unset. +: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}" + bash ${DRIVERS_TOOLS}/.evergreen/run-mongodb.sh stop diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 75d4fc89b3..57c460617e 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -12,7 +12,11 @@ HERE = Path(__file__).absolute().parent ROOT = HERE.parent.parent -DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") +# DRIVERS_TOOLS defaults to the drivers-evergreen-tools submodule; an env var +# override wins. +DRIVERS_TOOLS = (os.environ.get("DRIVERS_TOOLS") or str(ROOT / "drivers-evergreen-tools")).replace( + os.sep, "/" +) TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 LOGGER = logging.getLogger("test") @@ -312,6 +316,18 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None: LOGGER.info("Running command '%s'... done.", cmd) +def check_drivers_tools() -> None: + """Raise a clear error when the drivers-evergreen-tools checkout is missing.""" + # An uninitialized submodule can exist as an empty directory, so a bare + # is_dir() check passes. Require a script every consumer needs instead. + if not (Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh").is_file(): + raise RuntimeError( + "The drivers-evergreen-tools checkout is missing or empty; run `just " + "install` to initialize the submodule, or set DRIVERS_TOOLS to a " + "drivers-evergreen-tools checkout." + ) + + def create_archive() -> str: run_command("git add .", cwd=ROOT) run_command('git commit --no-verify -m "add files"', check=False, cwd=ROOT) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d9f19ad27e..603c47d7fe 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,6 +19,15 @@ updates: schedule: interval: "weekly" open-pull-requests-limit: 0 + # drivers-evergreen-tools submodule + - package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + allow: + - dependency-name: "drivers-evergreen-tools" # Python (pip). Manages requirements/mod_wsgi.txt only; the other # requirements files are resolved in uv.lock. - package-ecosystem: "pip" diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index cd63f683a5..f5b1069e30 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -85,7 +85,7 @@ jobs: # The beta Python here relies on the action's prerelease default. python-version: ${{ matrix.python-version }} - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "${{ matrix.mongodb-version }}" - name: Run tests @@ -101,12 +101,13 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: python-version: "3.10" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Setup tests @@ -126,6 +127,7 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: @@ -133,7 +135,7 @@ jobs: - name: Install dependencies run: just install - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Run tests @@ -184,6 +186,7 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: @@ -191,12 +194,12 @@ jobs: - name: Install dependencies run: just install - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 - name: Run tests run: | just integration-tests - id: setup-mongodb-ssl - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: ssl: true - name: Run tests @@ -251,7 +254,7 @@ jobs: # Test sdist on lowest supported Python python-version: "3.9" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 - name: Run connect test from sdist shell: bash run: | @@ -275,7 +278,7 @@ jobs: with: python-version: "3.9" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Run tests diff --git a/.github/zizmor.yml b/.github/zizmor.yml index af9f715dd4..2a69f0dd1a 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -4,4 +4,4 @@ rules: policies: actions/*: ref-pin mongodb-labs/drivers-github-tools/*: hash-pin - mongodb-labs/drivers-evergreen-tools: ref-pin + mongodb-labs/drivers-evergreen-tools: hash-pin diff --git a/.gitignore b/.gitignore index 222e030ad0..63813daf02 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ build/ doc/_build/ dist/ tools/settings.py -drivers-evergreen-tools pymongo.egg-info/ *.so *.egg* @@ -47,5 +46,10 @@ xunit-results/ coverage.xml server.log .coverage +test-results.json access_log error_log + +# AI-assistant review workflow logs (kept untracked) +/REVIEW_STATE.md +/REVIEW.md diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..d89986aa20 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "drivers-evergreen-tools"] + path = drivers-evergreen-tools + url = https://github.com/mongodb-labs/drivers-evergreen-tools.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1b0c88db6..7f24a3aca3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -203,9 +203,12 @@ the pages will re-render and the browser will automatically refresh. `just install` installs the pinned version of `uv` (from `[tool.uv] required-version`) into `$HOME/.local/bin`, and adds that directory to your shell rc file when missing, so the pinned `uv` takes effect in new shells. If a project `uv` command (e.g. `just test`) runs with a different `uv` version, `uv` fails fast and tells you how to update. + Scripts inside the vendored `drivers-evergreen-tools` submodule are exempt: the setup scripts write a `uv.toml` + boundary file into that checkout (see `.evergreen/scripts/configure-env.sh`), which stops uv's config discovery + there, since the tools' scripts run uv versions this project does not pin. - Ensure you have started the appropriate Mongo Server(s). You can run `just run-server` with optional args to set up the server. All given options will be passed to - [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `$DRIVERS_TOOLS/.evergreen/run-mongodb.sh start -h` + [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `${DRIVERS_TOOLS:-drivers-evergreen-tools}/.evergreen/run-mongodb.sh start -h` for a full list of options. By default the newest stable server release is used. To test against the nightly build instead, pass `--version latest`, which is downloaded from a private S3 bucket and requires an `AWS_PROFILE` with [Drivers test secrets](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/secrets_handling#secrets-handling) credentials. @@ -222,10 +225,10 @@ the pages will re-render and the browser will automatically refresh. ### Prerequisites -- Clone `drivers-evergreen-tools`: - `git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`. -- Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools`. This can be put into a `.bashrc` file - for convenience. +- The `drivers-evergreen-tools` submodule (see + [The drivers-evergreen-tools submodule](#the-drivers-evergreen-tools-submodule)). + `just install` initializes it; no manual clone or `export DRIVERS_TOOLS` is needed. + `DRIVERS_TOOLS` remains an optional override for an existing local checkout. - Some tests require access to [Drivers test secrets](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/secrets_handling#secrets-handling). ### Usage @@ -376,7 +379,7 @@ to run both modes. ### OCSP tests - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`. -This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`. +This corresponds to a config file in `${DRIVERS_TOOLS:-drivers-evergreen-tools}/.evergreen/orchestration/configs/servers`. MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA. NOTE: because the mock ocsp responder MUST be started prior to the server starting, the ocsp tests start the server as part of `setup-tests`. @@ -392,6 +395,44 @@ If you are running one of the `no-responder` tests, omit the `run-server` step. - Set up the tests with `sync` or `async`: `just setup-tests perf sync`. - Run the tests: `just run-tests`. +## The drivers-evergreen-tools submodule + +The `drivers-evergreen-tools` repository is consumed as a git submodule at the repo root, +pinned to a specific commit. Dependabot bumps the pin weekly. + +### Daily flow + +Nothing to do: `just install` initializes the submodule, and Dependabot keeps it fresh. + +### Manually bumping the submodule + +```bash +git -C drivers-evergreen-tools fetch --tags +git -C drivers-evergreen-tools checkout vX.Y.Z +git add drivers-evergreen-tools +``` + +### Evergreen patches that need tools changes + +Point the submodule at the needed commit and commit the new gitlink in the patch branch; +`configure-env.sh` checks out the recorded SHA on Evergreen hosts (`setup-dev-env.sh` does +the same for local checkouts with `just install`). + +### Using a local checkout instead + +Set `DRIVERS_TOOLS` to the path of a local clone — the environment variable wins over the +submodule default: + +```bash +export DRIVERS_TOOLS=/path/to/drivers-evergreen-tools +``` + +Alternatively, keep a local pin from being reset by `git submodule update`: + +```bash +git config submodule.drivers-evergreen-tools.update none +``` + ## Enable Debug Logs - Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest` to output all debug logs to the terminal. **Warning**: This will output a huge amount of logs. @@ -434,7 +475,10 @@ CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at - The uv binary version is pinned once in `[tool.uv] required-version` in `pyproject.toml`. `.evergreen/scripts/install-dependencies.sh` installs it with `uv tool install`, uv enforces it locally, and `astral-sh/setup-uv` reads it on GitHub. Bump it manually when a newer uv is needed. If uv cannot find the - requested Python, it installs it; if that fails, the task fails. + requested Python, it installs it; if that fails, the task fails. The pin is not enforced for scripts inside + the vendored `drivers-evergreen-tools` submodule: those scripts run uv versions this project does not pin, so + the setup scripts write a `uv.toml` boundary into the submodule checkout that stops uv's config discovery + (and with it the required-version check) at the submodule boundary. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. diff --git a/drivers-evergreen-tools b/drivers-evergreen-tools new file mode 160000 index 0000000000..ec0b1497b3 --- /dev/null +++ b/drivers-evergreen-tools @@ -0,0 +1 @@ +Subproject commit ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 diff --git a/pyproject.toml b/pyproject.toml index 2f7a68c837..cf824a72f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,11 @@ validate-bump = false [tool.hatch.build.targets.wheel] packages = ["bson","gridfs", "pymongo"] +[tool.hatch.build.targets.sdist] +# The drivers-evergreen-tools submodule must not ship in the sdist; hatchling's +# walk would otherwise pick it up when the submodule is populated locally. +exclude = ["drivers-evergreen-tools"] + [tool.hatch.metadata.hooks.requirements_txt] files = ["requirements.txt"] From 6c92519559f1ff360debc7583873cc0bd12e4cf8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 5 Oct 2026 06:59:11 -0500 Subject: [PATCH 16/24] PYTHON-5805 Add kms_connect_callback for KMS connections Add a kms_connect_callback option to AutoEncryptionOpts, ClientEncryption, and AsyncClientEncryption. The callable receives a KMSConnectContext (host, port, timeout) and returns a connected, unwrapped socket over which the driver performs the KMS TLS handshake, so verification still targets the KMS host rather than the peer reached. This enables routing KMS requests through custom channels such as proxies. Split TLS wrapping out of the configured socket helpers so the callback path reuses it, and shield the async wrap so a cancelled or timed-out wait still closes a late-produced socket. --- doc/changelog.rst | 8 + pymongo/asynchronous/encryption.py | 148 +++++++++- pymongo/encryption_options.py | 56 +++- pymongo/pool_shared.py | 77 ++++- pymongo/synchronous/encryption.py | 149 +++++++++- test/asynchronous/test_encryption.py | 28 +- test/asynchronous/test_kms_connect.py | 393 ++++++++++++++++++++++++++ test/asynchronous/test_pooling.py | 21 +- test/test_encryption.py | 28 +- test/test_kms_connect.py | 391 +++++++++++++++++++++++++ test/test_pooling.py | 21 +- tools/synchro.py | 14 + 12 files changed, 1287 insertions(+), 47 deletions(-) create mode 100644 test/asynchronous/test_kms_connect.py create mode 100644 test/test_kms_connect.py diff --git a/doc/changelog.rst b/doc/changelog.rst index cb3ded580a..9ea2f05c2e 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -16,6 +16,14 @@ PyMongo 4.19 brings a number of changes including: interpreter remain daemon threads and shutdown behavior is unchanged. Note that because these threads are non-daemon, a subinterpreter may block on teardown until any in-flight monitor work completes. +- Added the ``kms_connect_callback`` option on + :class:`~pymongo.encryption_options.AutoEncryptionOpts`, + :class:`~pymongo.encryption.ClientEncryption`, and + :class:`~pymongo.asynchronous.encryption.AsyncClientEncryption`: a callable + that opens the connection to a Key Management Service (KMS) host for + Client-Side Field Level Encryption and Queryable Encryption, e.g. to route + KMS requests through a proxy. The driver performs the KMS TLS handshake over + the callback's connection, so verification still targets the KMS host. - Added the ``srv_host_validator`` keyword argument to :class:`~pymongo.synchronous.mongo_client.MongoClient` and :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient`, an alternative to diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 9ba2758f78..0eea43d898 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -17,8 +17,11 @@ from __future__ import annotations import asyncio +import contextlib import functools +import inspect import socket +import ssl import time as time # noqa: PLC0414 # needed in sync version import uuid import weakref @@ -60,7 +63,9 @@ from pymongo.common import CONNECT_TIMEOUT from pymongo.daemon import _spawn_daemon from pymongo.encryption_options import ( + AsyncKMSConnectCallback, AutoEncryptionOpts, + KMSConnectContext, RangeOpts, StringOpts, # Re-exported for backwards compatibility: TextOpts is deprecated but must @@ -90,6 +95,8 @@ from pymongo.pool_options import PoolOptions from pymongo.pool_shared import ( _async_configured_socket, + _async_wrap_socket_tls, + _close_late_socket, _raise_connection_failure, ) from pymongo.read_concern import ReadConcern @@ -120,11 +127,108 @@ _KEY_VAULT_OPTS = CodecOptions(document_class=RawBSONDocument) -async def _connect_kms(address: _Address, opts: PoolOptions) -> Union[socket.socket, _sslConn]: +def _close_rejected_kms_socket(obj: Any) -> None: + """Close a rejected kms_connect_callback return value, best effort. + + Nothing else will close it: _connect_kms raises before the result reaches + the caller's ``finally``. + """ + close = getattr(obj, "close", None) + if callable(close): + with contextlib.suppress(Exception): + close() + + +async def _connect_kms( + address: _Address, + opts: PoolOptions, + kms_connect_callback: Optional[AsyncKMSConnectCallback], + timeout: float, +) -> Union[socket.socket, _sslConn]: + """Connect to a KMS host and perform the TLS handshake over the socket. + + Uses ``kms_connect_callback`` when one is provided, otherwise connects + directly, and always verifies against ``address`` (the KMS host). + """ + if kms_connect_callback is None: + try: + return await _async_configured_socket(address, opts) + except Exception as exc: + _raise_connection_failure(address, exc, timeout_details=_get_timeout_details(opts)) + + # TLS targets address, not the peer, so verification follows the KMS host. + # A plain callable would block the event loop before we could reject it, + # so check the callback first. + if not _IS_SYNC: + callback_any: Any = kms_connect_callback + is_coro = inspect.iscoroutinefunction(callback_any) + if not is_coro and callable(callback_any): + is_coro = inspect.iscoroutinefunction(callback_any.__call__) + if not is_coro: + raise ConfigurationError( + "kms_connect_callback must be a coroutine function for the async API." + ) + # Typed as Any so the generated synchronous flavor type-checks: the sync + # callback returns a plain socket, which is not awaitable. + result: Any = kms_connect_callback( + KMSConnectContext(host=address[0], port=cast(int, address[1]), timeout=timeout) + ) + remaining = _csot.remaining() + if remaining is None or _IS_SYNC: + # The synchronous API cannot interrupt a callback that has started + # running; honoring the deadline is the callback's contract there. + sock = await result + else: + # CSOT is cooperative: a callback that ignores the timeout could block + # past the deadline. Shield the task so stopping the wait does not + # cancel it mid-flight, and close any socket it yields later. + task = asyncio.ensure_future(result) + try: + sock = await asyncio.wait_for(asyncio.shield(task), remaining) + except asyncio.CancelledError: + task.add_done_callback(_close_late_socket) + raise + except asyncio.TimeoutError: + task.add_done_callback(_close_late_socket) + _raise_connection_failure( + address, + socket.timeout("timed out"), + timeout_details=_get_timeout_details(opts), + ) + if not isinstance(sock, socket.socket) or isinstance(sock, ssl.SSLSocket): + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return a connected, unwrapped " + f"socket.socket, not {type(sock)}." + ) + # wrap_socket refuses a non-blocking socket, so normalize the mode here. + try: + sock.getpeername() + except OSError: + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return an already connected socket." + ) from None + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE) != socket.SOCK_STREAM: + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return a stream socket, not a datagram one." + ) + # The callback may have consumed much of the CSOT budget, and wrapping + # resets the socket timeout, so recompute the remaining time here and for + # the KMS request that follows. + sock.settimeout(max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001)) try: - return await _async_configured_socket(address, opts) + conn = await _async_wrap_socket_tls(sock, address, opts) + except asyncio.CancelledError: + # The executor may still be wrapping the socket; close it so a TLS + # proxy's relay threads wind down instead of leaking. + sock.close() + raise except Exception as exc: _raise_connection_failure(address, exc, timeout_details=_get_timeout_details(opts)) + conn.settimeout(max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001)) + return conn class _EncryptionIO(AsyncMongoCryptCallback): # type: ignore[misc] @@ -179,13 +283,6 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None: False, # disable_ocsp_endpoint_check _IS_SYNC, ) - # CSOT: set timeout for socket creation. - connect_timeout = max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001) - opts = PoolOptions( - connect_timeout=connect_timeout, - socket_timeout=connect_timeout, - ssl_context=ctx, - ) address = parse_host(endpoint, _HTTPS_PORT) if address[0].endswith(".sock"): raise ConfigurationError(f"Invalid KMS endpoint {endpoint!r}") @@ -193,8 +290,21 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None: if sleep_u: sleep_sec = float(sleep_u) / 1e6 await asyncio.sleep(sleep_sec) + # Set the connect timeout after the retry backoff so the budget + # reflects the sleep. + connect_timeout = max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001) + opts = PoolOptions( + connect_timeout=connect_timeout, + socket_timeout=connect_timeout, + ssl_context=ctx, + ) try: - conn = await _connect_kms(address, opts) + conn = await _connect_kms( + address, + opts, + self.opts._kms_connect_callback, + connect_timeout, + ) try: await async_socket_sendall(conn, message) while kms_context.bytes_needed > 0: @@ -230,6 +340,8 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None: conn.close() except MongoCryptError: raise # Propagate MongoCryptError errors directly. + except ConfigurationError: + raise # A callback contract violation is not transient. except Exception as exc: remaining = _csot.remaining() if isinstance(exc, NetworkTimeout) or (remaining is not None and remaining <= 0): @@ -526,6 +638,7 @@ def __init__( codec_options: CodecOptions[_DocumentTypeArg], kms_tls_options: Optional[Mapping[str, Any]] = None, key_expiration_ms: Optional[int] = None, + kms_connect_callback: Optional[AsyncKMSConnectCallback] = None, ) -> None: """Explicit client-side field level encryption. @@ -595,7 +708,19 @@ def __init__( :param key_expiration_ms: The cache expiration time for data encryption keys. Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000. Set to 0 to disable key expiration. - + :param kms_connect_callback: A callable that opens the connection to a + KMS host, used to route KMS requests through an HTTP proxy. It + receives a :class:`~pymongo.encryption_options.KMSConnectContext` + and returns a connected, unwrapped :class:`socket.socket`, over + which the driver performs the KMS TLS handshake. The callback + must be a coroutine function for the asynchronous API; a plain callable is rejected before it can block the event loop. + When a CSOT timeout is active, the driver stops waiting at the + deadline and closes any socket the callback yields later. + Defaults to ``None``, meaning the driver connects to KMS hosts + directly. + + .. versionchanged:: 4.19 + Added the `kms_connect_callback` parameter. .. versionchanged:: 4.12 Added the `key_expiration_ms` parameter. .. versionchanged:: 4.0 @@ -639,6 +764,7 @@ def __init__( key_vault_namespace, kms_tls_options=kms_tls_options, key_expiration_ms=key_expiration_ms, + kms_connect_callback=kms_connect_callback, ) self._kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index 0d90fe3b19..f77e461704 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -19,9 +19,11 @@ from __future__ import annotations +import socket import warnings -from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Optional, TypedDict +from collections.abc import Awaitable, Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, Optional, TypedDict from pymongo.uri_parser_shared import _parse_kms_tls_options @@ -55,6 +57,37 @@ def check_min_pymongocrypt() -> None: ) +@dataclass(frozen=True) +class KMSConnectContext: + """Information about a pending KMS connection. + + Passed to ``kms_connect_callback``, which must return a plain, unwrapped + :class:`socket.socket`. The driver performs the KMS TLS handshake over it, + verifying against ``host`` rather than the peer actually reached. + + :param host: Hostname of the KMS server, and the TLS verification target. + :param port: Port of the KMS server. + :param timeout: Seconds left in the timeout budget, or the default KMS + connect timeout when no timeout is active. + + .. note:: ``timeoutMS`` does not constrain KMS requests for explicit + encryption, so ``timeout`` is always the default there. Automatic + encryption passes the remaining budget. This deviates from the Client + Side Operations Timeout specification; see PYTHON-6037. + + .. versionadded:: 4.19 + """ + + host: str + port: int + timeout: float + + +# A callback that opens a connection to a KMS host. +AsyncKMSConnectCallback = Callable[[KMSConnectContext], Awaitable[socket.socket]] +KMSConnectCallback = Callable[[KMSConnectContext], socket.socket] + + class AutoEncryptionOpts: """Options to configure automatic client-side field level encryption.""" @@ -75,6 +108,7 @@ def __init__( bypass_query_analysis: bool = False, encrypted_fields_map: Optional[Mapping[str, Any]] = None, key_expiration_ms: Optional[int] = None, + kms_connect_callback: Optional[Callable[[KMSConnectContext], Any]] = None, ) -> None: """Options to configure automatic client-side field level encryption. @@ -212,7 +246,18 @@ def __init__( :param key_expiration_ms: The cache expiration time for data encryption keys. Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000. Set to 0 to disable key expiration. - + :param kms_connect_callback: A callable that opens the connection to a + KMS host, used to route KMS requests through an HTTP proxy. It + receives a :class:`KMSConnectContext` and returns a connected, + unwrapped :class:`socket.socket`, over which the driver performs + the KMS TLS handshake. Must be a coroutine function for + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` and a + regular function for + :class:`~pymongo.synchronous.mongo_client.MongoClient`. Defaults + to ``None``, meaning the driver connects to KMS hosts directly. + + .. versionchanged:: 4.19 + Added the `kms_connect_callback` parameter. .. versionchanged:: 4.12 Added the `key_expiration_ms` parameter. .. versionchanged:: 4.2 @@ -259,6 +304,11 @@ def __init__( self._async_kms_ssl_contexts: Optional[dict[str, SSLContext]] = None self._bypass_query_analysis = bypass_query_analysis self._key_expiration_ms = key_expiration_ms + if kms_connect_callback is not None and not callable(kms_connect_callback): + raise TypeError( + f"kms_connect_callback must be callable, not {type(kms_connect_callback)}" + ) + self._kms_connect_callback = kms_connect_callback def _kms_ssl_contexts(self, is_sync: bool) -> dict[str, SSLContext]: if is_sync: diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index 8cd546bda6..04663f6fc9 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -18,6 +18,7 @@ import asyncio import collections +import contextlib import functools import socket import ssl @@ -304,16 +305,29 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s raise OSError("getaddrinfo failed") -async def _async_configured_socket( - address: _Address, options: PoolOptions +def _close_late_socket(future: asyncio.Future[Any]) -> None: + """Close a socket produced after its awaiting task was cancelled.""" + if not future.cancelled() and future.exception() is None: + # The callback may have returned a non-socket; close best effort. + close = getattr(future.result(), "close", None) + if callable(close): + with contextlib.suppress(Exception): + close() + + +async def _async_wrap_socket_tls( + sock: socket.socket, address: _Address, options: PoolOptions ) -> Union[socket.socket, _sslConn]: - """Given (host, port) and PoolOptions, return a raw configured socket. + """Given a connected socket, (host, port), and PoolOptions, apply TLS. + + The handshake, SNI, and certificate/hostname verification all target + ``address``, which may differ from the peer ``sock`` is connected to, e.g. + when ``sock`` tunnels through an HTTP proxy. Can raise socket.error, ConnectionFailure, or _CertificateError. - Sets socket's SSL and timeout options. + Sets the socket's SSL and timeout options. """ - sock = await _async_create_connection(address, options) ssl_context = options._ssl_context if ssl_context is None: @@ -326,13 +340,19 @@ async def _async_configured_socket( # to use SSLContext.check_hostname. if _has_sni(False): loop = asyncio.get_running_loop() - ssl_sock = await loop.run_in_executor( - None, - functools.partial(ssl_context.wrap_socket, sock, server_hostname=host), # type: ignore[assignment, misc, unused-ignore] - ) + wrap = functools.partial(ssl_context.wrap_socket, sock, server_hostname=host) # type: ignore[assignment, misc, unused-ignore] else: loop = asyncio.get_running_loop() - ssl_sock = await loop.run_in_executor(None, ssl_context.wrap_socket, sock) # type: ignore[assignment, misc, unused-ignore] + wrap = functools.partial(ssl_context.wrap_socket, sock) # type: ignore[assignment, misc, unused-ignore] + # Shield the executor future: wrap_socket hands the fd to a new + # SSLSocket, so cancellation must not orphan the result it produces. + future = loop.run_in_executor(None, wrap) + try: + ssl_sock = await asyncio.shield(future) + except asyncio.CancelledError: + future.add_done_callback(_close_late_socket) + sock.close() + raise except _CertificateError: sock.close() # Raise _CertificateError directly like we do after match_hostname @@ -360,6 +380,19 @@ async def _async_configured_socket( return ssl_sock +async def _async_configured_socket( + address: _Address, options: PoolOptions +) -> Union[socket.socket, _sslConn]: + """Given (host, port) and PoolOptions, return a raw configured socket. + + Can raise socket.error, ConnectionFailure, or _CertificateError. + + Sets socket's SSL and timeout options. + """ + sock = await _async_create_connection(address, options) + return await _async_wrap_socket_tls(sock, address, options) + + async def _configured_protocol_interface( address: _Address, options: PoolOptions, @@ -510,14 +543,19 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket raise OSError("getaddrinfo failed") -def _configured_socket(address: _Address, options: PoolOptions) -> Union[socket.socket, _sslConn]: - """Given (host, port) and PoolOptions, return a raw configured socket. +def _wrap_socket_tls( + sock: socket.socket, address: _Address, options: PoolOptions +) -> Union[socket.socket, _sslConn]: + """Given a connected socket, (host, port), and PoolOptions, apply TLS. + + The handshake, SNI, and certificate/hostname verification all target + ``address``, which may differ from the peer ``sock`` is connected to, e.g. + when ``sock`` tunnels through an HTTP proxy. Can raise socket.error, ConnectionFailure, or _CertificateError. - Sets socket's SSL and timeout options. + Sets the socket's SSL and timeout options. """ - sock = _create_connection(address, options) ssl_context = options._ssl_context if ssl_context is None: @@ -559,6 +597,17 @@ def _configured_socket(address: _Address, options: PoolOptions) -> Union[socket. return ssl_sock +def _configured_socket(address: _Address, options: PoolOptions) -> Union[socket.socket, _sslConn]: + """Given (host, port) and PoolOptions, return a raw configured socket. + + Can raise socket.error, ConnectionFailure, or _CertificateError. + + Sets socket's SSL and timeout options. + """ + sock = _create_connection(address, options) + return _wrap_socket_tls(sock, address, options) + + def _configured_socket_interface( address: _Address, options: PoolOptions, diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index e7d8a366ca..b89ad933f1 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -16,8 +16,12 @@ from __future__ import annotations +import asyncio +import contextlib import functools +import inspect import socket +import ssl import time as time # noqa: PLC0414 # needed in sync version import uuid import weakref @@ -56,6 +60,8 @@ from pymongo.daemon import _spawn_daemon from pymongo.encryption_options import ( AutoEncryptionOpts, + KMSConnectCallback, + KMSConnectContext, RangeOpts, StringOpts, # Re-exported for backwards compatibility: TextOpts is deprecated but must @@ -84,8 +90,10 @@ from pymongo.operations import UpdateOne from pymongo.pool_options import PoolOptions from pymongo.pool_shared import ( + _close_late_socket, _configured_socket, _raise_connection_failure, + _wrap_socket_tls, ) from pymongo.read_concern import ReadConcern from pymongo.results import DeleteResult @@ -119,11 +127,108 @@ _KEY_VAULT_OPTS = CodecOptions(document_class=RawBSONDocument) -def _connect_kms(address: _Address, opts: PoolOptions) -> Union[socket.socket, _sslConn]: +def _close_rejected_kms_socket(obj: Any) -> None: + """Close a rejected kms_connect_callback return value, best effort. + + Nothing else will close it: _connect_kms raises before the result reaches + the caller's ``finally``. + """ + close = getattr(obj, "close", None) + if callable(close): + with contextlib.suppress(Exception): + close() + + +def _connect_kms( + address: _Address, + opts: PoolOptions, + kms_connect_callback: Optional[KMSConnectCallback], + timeout: float, +) -> Union[socket.socket, _sslConn]: + """Connect to a KMS host and perform the TLS handshake over the socket. + + Uses ``kms_connect_callback`` when one is provided, otherwise connects + directly, and always verifies against ``address`` (the KMS host). + """ + if kms_connect_callback is None: + try: + return _configured_socket(address, opts) + except Exception as exc: + _raise_connection_failure(address, exc, timeout_details=_get_timeout_details(opts)) + + # TLS targets address, not the peer, so verification follows the KMS host. + # A plain callable would block the event loop before we could reject it, + # so check the callback first. + if not _IS_SYNC: + callback_any: Any = kms_connect_callback + is_coro = inspect.iscoroutinefunction(callback_any) + if not is_coro and callable(callback_any): + is_coro = inspect.iscoroutinefunction(callback_any.__call__) + if not is_coro: + raise ConfigurationError( + "kms_connect_callback must be a coroutine function for the async API." + ) + # Typed as Any so the generated synchronous flavor type-checks: the sync + # callback returns a plain socket, which is not awaitable. + result: Any = kms_connect_callback( + KMSConnectContext(host=address[0], port=cast(int, address[1]), timeout=timeout) + ) + remaining = _csot.remaining() + if remaining is None or _IS_SYNC: + # The synchronous API cannot interrupt a callback that has started + # running; honoring the deadline is the callback's contract there. + sock = result + else: + # CSOT is cooperative: a callback that ignores the timeout could block + # past the deadline. Shield the task so stopping the wait does not + # cancel it mid-flight, and close any socket it yields later. + task = asyncio.ensure_future(result) + try: + sock = asyncio.wait_for(asyncio.shield(task), remaining) + except asyncio.CancelledError: + task.add_done_callback(_close_late_socket) + raise + except asyncio.TimeoutError: + task.add_done_callback(_close_late_socket) + _raise_connection_failure( + address, + socket.timeout("timed out"), + timeout_details=_get_timeout_details(opts), + ) + if not isinstance(sock, socket.socket) or isinstance(sock, ssl.SSLSocket): + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return a connected, unwrapped " + f"socket.socket, not {type(sock)}." + ) + # wrap_socket refuses a non-blocking socket, so normalize the mode here. + try: + sock.getpeername() + except OSError: + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return an already connected socket." + ) from None + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE) != socket.SOCK_STREAM: + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return a stream socket, not a datagram one." + ) + # The callback may have consumed much of the CSOT budget, and wrapping + # resets the socket timeout, so recompute the remaining time here and for + # the KMS request that follows. + sock.settimeout(max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001)) try: - return _configured_socket(address, opts) + conn = _wrap_socket_tls(sock, address, opts) + except asyncio.CancelledError: + # The executor may still be wrapping the socket; close it so a TLS + # proxy's relay threads wind down instead of leaking. + sock.close() + raise except Exception as exc: _raise_connection_failure(address, exc, timeout_details=_get_timeout_details(opts)) + conn.settimeout(max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001)) + return conn class _EncryptionIO(MongoCryptCallback): # type: ignore[misc] @@ -178,13 +283,6 @@ def kms_request(self, kms_context: MongoCryptKmsContext) -> None: False, # disable_ocsp_endpoint_check _IS_SYNC, ) - # CSOT: set timeout for socket creation. - connect_timeout = max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001) - opts = PoolOptions( - connect_timeout=connect_timeout, - socket_timeout=connect_timeout, - ssl_context=ctx, - ) address = parse_host(endpoint, _HTTPS_PORT) if address[0].endswith(".sock"): raise ConfigurationError(f"Invalid KMS endpoint {endpoint!r}") @@ -192,8 +290,21 @@ def kms_request(self, kms_context: MongoCryptKmsContext) -> None: if sleep_u: sleep_sec = float(sleep_u) / 1e6 time.sleep(sleep_sec) + # Set the connect timeout after the retry backoff so the budget + # reflects the sleep. + connect_timeout = max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001) + opts = PoolOptions( + connect_timeout=connect_timeout, + socket_timeout=connect_timeout, + ssl_context=ctx, + ) try: - conn = _connect_kms(address, opts) + conn = _connect_kms( + address, + opts, + self.opts._kms_connect_callback, + connect_timeout, + ) try: sendall(conn, message) while kms_context.bytes_needed > 0: @@ -229,6 +340,8 @@ def kms_request(self, kms_context: MongoCryptKmsContext) -> None: conn.close() except MongoCryptError: raise # Propagate MongoCryptError errors directly. + except ConfigurationError: + raise # A callback contract violation is not transient. except Exception as exc: remaining = _csot.remaining() if isinstance(exc, NetworkTimeout) or (remaining is not None and remaining <= 0): @@ -523,6 +636,7 @@ def __init__( codec_options: CodecOptions[_DocumentTypeArg], kms_tls_options: Optional[Mapping[str, Any]] = None, key_expiration_ms: Optional[int] = None, + kms_connect_callback: Optional[KMSConnectCallback] = None, ) -> None: """Explicit client-side field level encryption. @@ -592,7 +706,19 @@ def __init__( :param key_expiration_ms: The cache expiration time for data encryption keys. Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000. Set to 0 to disable key expiration. - + :param kms_connect_callback: A callable that opens the connection to a + KMS host, used to route KMS requests through an HTTP proxy. It + receives a :class:`~pymongo.encryption_options.KMSConnectContext` + and returns a connected, unwrapped :class:`socket.socket`, over + which the driver performs the KMS TLS handshake. The callback + must be a regular function; the async API requires a coroutine function and rejects plain callables before they can block the event loop. + When a CSOT timeout is active, the driver stops waiting at the + deadline and closes any socket the callback yields later. + Defaults to ``None``, meaning the driver connects to KMS hosts + directly. + + .. versionchanged:: 4.19 + Added the `kms_connect_callback` parameter. .. versionchanged:: 4.12 Added the `key_expiration_ms` parameter. .. versionchanged:: 4.0 @@ -632,6 +758,7 @@ def __init__( key_vault_namespace, kms_tls_options=kms_tls_options, key_expiration_ms=key_expiration_ms, + kms_connect_callback=kms_connect_callback, ) self._kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 128f26feb4..dc5d0e31e0 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -60,7 +60,11 @@ from bson.son import SON from pymongo import ReadPreference from pymongo.asynchronous import encryption -from pymongo.asynchronous.encryption import Algorithm, AsyncClientEncryption, QueryType +from pymongo.asynchronous.encryption import ( + Algorithm, + AsyncClientEncryption, + QueryType, +) from pymongo.asynchronous.helpers import anext from pymongo.asynchronous.mongo_client import AsyncMongoClient from pymongo.cursor_shared import CursorType @@ -222,6 +226,9 @@ async def test_init_kms_tls_options(self): self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) +# KMS connect callback unit and prose tests live in test_kms_connect.py. + + class TestClientOptions(AsyncPyMongoTestCase): async def test_default(self): client = self.simple_client(connect=False) @@ -316,9 +323,15 @@ def create_client_encryption( key_vault_client: AsyncMongoClient, codec_options: CodecOptions, kms_tls_options: Optional[Mapping[str, Any]] = None, + kms_connect_callback: Optional[Any] = None, ): client_encryption = AsyncClientEncryption( - kms_providers, key_vault_namespace, key_vault_client, codec_options, kms_tls_options + kms_providers, + key_vault_namespace, + key_vault_client, + codec_options, + kms_tls_options, + kms_connect_callback=kms_connect_callback, ) self.addAsyncCleanup(client_encryption.close) return client_encryption @@ -331,9 +344,15 @@ def unmanaged_create_client_encryption( key_vault_client: AsyncMongoClient, codec_options: CodecOptions, kms_tls_options: Optional[Mapping[str, Any]] = None, + kms_connect_callback: Optional[Any] = None, ): client_encryption = AsyncClientEncryption( - kms_providers, key_vault_namespace, key_vault_client, codec_options, kms_tls_options + kms_providers, + key_vault_namespace, + key_vault_client, + codec_options, + kms_tls_options, + kms_connect_callback=kms_connect_callback, ) return client_encryption @@ -1988,6 +2007,9 @@ async def test_invalid_hostname_in_kms_certificate(self): await self.client_encrypted.create_data_key("aws", master_key=key) +# KMS connect callback unit and prose tests live in test_kms_connect.py. + + # https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/tests/README.md#kms-tls-options-tests class TestKmsTLSOptions(AsyncEncryptionIntegrationTest): @unittest.skipUnless(any(AWS_CREDS.values()), "AWS environment credentials are not set") diff --git a/test/asynchronous/test_kms_connect.py b/test/asynchronous/test_kms_connect.py new file mode 100644 index 0000000000..994ec5ca21 --- /dev/null +++ b/test/asynchronous/test_kms_connect.py @@ -0,0 +1,393 @@ +"""Tests for the KMS connect callback.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import socket +import ssl +import threading +import time +import unittest +from asyncio.trsock import TransportSocket +from unittest import mock + +import pytest + +import pymongo +from pymongo.asynchronous.encryption import ( + AsyncClientEncryption, + _connect_kms, + _EncryptionIO, + _wrap_encryption_errors, +) +from pymongo.encryption_options import ( + AutoEncryptionOpts, + KMSConnectContext, +) +from pymongo.errors import ConfigurationError, EncryptionError, NetworkTimeout +from pymongo.pool_options import PoolOptions +from pymongo.ssl_support import get_ssl_context +from test.asynchronous import AsyncPyMongoTestCase +from test.asynchronous.test_encryption import OPTS +from test.helpers_shared import CLIENT_PEM + +_IS_SYNC = False + +pytestmark = pytest.mark.encryption + + +class TestKmsConnectCallbackUnit(AsyncPyMongoTestCase): + """Contract checks for kms_connect_callback that need no KMS server.""" + + @staticmethod + def _pool_options(): + return PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=None) + + async def test_init_kms_connect_callback(self): + opts = AutoEncryptionOpts({}, "k.d") + self.assertIsNone(opts._kms_connect_callback) + + async def callback(context): + raise AssertionError("not called") + + opts = AutoEncryptionOpts({}, "k.d", kms_connect_callback=callback) + self.assertIs(opts._kms_connect_callback, callback) + + for bad in [1, "not-callable", object()]: + with self.assertRaisesRegex(TypeError, "kms_connect_callback must be callable"): + AutoEncryptionOpts({}, "k.d", kms_connect_callback=bad) # type: ignore[arg-type] + + context = KMSConnectContext(host="kms.example.com", port=443, timeout=9.5) + self.assertEqual(context.host, "kms.example.com") + self.assertEqual(context.port, 443) + self.assertEqual(context.timeout, 9.5) + with self.assertRaises(dataclasses.FrozenInstanceError): + context.host = "evil.example.com" # type: ignore[misc] + + async def test_non_socket_return_raises_configuration_error(self): + async def callback(context): + return "not-a-socket" + + with self.assertRaisesRegex(ConfigurationError, "must return a connected"): + await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + async def test_already_wrapped_socket_is_rejected(self): + # ssl.SSLSocket passes isinstance but cannot be TLS-wrapped again. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + left, right = socket.socketpair() + self.addCleanup(right.close) + # No peer needed to produce a genuine ssl.SSLSocket. + wrapped = ctx.wrap_socket(left, do_handshake_on_connect=False, server_hostname="x") + self.addCleanup(wrapped.close) + + async def callback(context): + return wrapped + + with self.assertRaisesRegex(ConfigurationError, "unwrapped"): + await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + async def test_context_receives_host_port_and_timeout(self): + received = [] + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + + async def callback(context): + received.append(context) + return left + + # ssl_context=None returns the socket unchanged, so a plain socket is accepted. + conn = await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 12.5) + self.assertIs(conn, left) + + self.assertEqual(len(received), 1) + self.assertEqual(received[0].host, "kms.example.com") + self.assertEqual(received[0].port, 443) + self.assertEqual(received[0].timeout, 12.5) + + async def test_non_blocking_socket_from_callback_is_accepted(self): + # Without the driver normalizing the mode, this raises ValueError. + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(CLIENT_PEM) + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + self.addCleanup(listener.close) + + def serve(): + try: + conn, _ = listener.accept() + server_ctx.wrap_socket(conn, server_side=True).close() + except OSError: + pass + + threading.Thread(target=serve, daemon=True).start() + + # Built as the driver does, for the flavor-correct type; the local cert won't verify. + client_ctx = get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) + options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) + sock.setblocking(False) + return sock + + async def callback(context): + if _IS_SYNC: + return connect() + return await asyncio.get_running_loop().run_in_executor(None, connect) + + conn = await _connect_kms(listener.getsockname(), options, callback, 10.0) + self.addCleanup(conn.close) + self.assertIsNotNone(conn.gettimeout()) + + async def test_asyncio_transport_socket_is_rejected(self): + # get_extra_info("socket") is a TransportSocket, not a socket.socket. + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + + async def callback(context): + return TransportSocket(left) + + with self.assertRaisesRegex(ConfigurationError, "TransportSocket"): + await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + async def test_cancelled_tls_wrap_closes_late_socket(self): + # A cancelled wrap can leave the executor producing an SSLSocket; the + # done callback must close it. + if _IS_SYNC: + raise unittest.SkipTest("the cancel-safe wrap is an async path") + from pymongo.pool_shared import _close_late_socket + + left, right = socket.socketpair() + future = asyncio.get_running_loop().create_future() + future.set_result(left) + self.assertNotEqual(left.fileno(), -1) + _close_late_socket(future) + self.assertEqual(left.fileno(), -1) + self.addCleanup(right.close) + + async def test_non_coroutine_callback_is_rejected(self): + # A plain def must be rejected before it blocks the event loop. + if _IS_SYNC: + raise unittest.SkipTest("a regular function is correct for the sync API") + + entered = [] + + def callback(context): + entered.append(context) + return None + + with self.assertRaisesRegex(ConfigurationError, "coroutine function"): + await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + self.assertEqual(entered, [], "invalid callback must not be entered") + + async def test_unconnected_socket_from_callback_is_rejected(self): + # An unconnected socket would fail later as a transient error and be retried. + bare = socket.socket() + self.addCleanup(bare.close) + + async def callback(context): + return bare + + with self.assertRaisesRegex(ConfigurationError, "already connected"): + await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + async def test_datagram_socket_from_callback_is_rejected(self): + # TLS on a connected UDP socket raises NotImplementedError, which would be retried. + left = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + right = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(left.close) + self.addCleanup(right.close) + right.bind(("127.0.0.1", 0)) + left.connect(right.getsockname()) + + async def callback(context): + return left + + with self.assertRaisesRegex(ConfigurationError, "stream socket"): + await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + async def test_kms_request_does_not_retry_a_contract_violation(self): + # _connect_kms has no retry loop; the no-retry guarantee is in + # kms_request, so exercise that instead. + calls = [] + + async def callback(context): + calls.append(context) + return "not-a-socket" + + opts = AutoEncryptionOpts({}, "k.d", kms_connect_callback=callback) + io = _EncryptionIO(None, mock.MagicMock(), None, opts) + + class StubKmsContext: + endpoint = "kms.example.com:443" + message = b"request" + kms_provider = "aws" + usleep = 0 + bytes_needed = 1 + + def feed(self, data): + raise AssertionError("should not reach the socket") + + def fail(self): + raise AssertionError("a contract violation must not be retried") + + with self.assertRaises(ConfigurationError): + await io.kms_request(StubKmsContext()) + self.assertEqual(len(calls), 1) + + async def test_contract_violation_surfaces_as_encryption_error(self): + # Callers see EncryptionError with ConfigurationError as its cause. + with self.assertRaises(EncryptionError) as caught: + with _wrap_encryption_errors(): + raise ConfigurationError("kms_connect_callback must return ...") + self.assertIsInstance(caught.exception.__cause__, ConfigurationError) + + async def test_network_error_from_callback_propagates(self): + async def callback(context): + raise OSError("proxy unreachable") + + # Not a ConfigurationError, so kms_request retries it. + with self.assertRaises(OSError): + await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + async def test_csot_deadline_stops_a_hung_callback(self): + # A callback that ignores the timeout cannot block past the CSOT + # deadline, and a socket it yields later must be closed. + if _IS_SYNC: + raise unittest.SkipTest("the sync API cannot interrupt a callback") + + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + + async def hung_callback(context): + await asyncio.sleep(0.5) + return left + + with self.assertRaises(NetworkTimeout): + with pymongo.timeout(0.1): + await _connect_kms( + ("kms.example.com", 443), self._pool_options(), hung_callback, 10.0 + ) + self.assertNotEqual(left.fileno(), -1) + # Let the shielded callback finish; the driver closes the late result. + await asyncio.sleep(0.75) + self.assertEqual(left.fileno(), -1) + + async def test_cancelling_kms_connect_closes_the_callback_socket(self): + # Cancelling during the TLS handshake must close the callback's socket, + # so a TLS proxy's relay threads wind down. + if _IS_SYNC: + raise unittest.SkipTest("cancellation is an async-only behavior") + + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(CLIENT_PEM) + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + self.addCleanup(listener.close) + gate = threading.Event() + eof = threading.Event() + + def stub_server(): + conn = None + try: + conn, _ = listener.accept() + # The cancel may land before or after the executor starts the + # handshake. Peek for the ClientHello without consuming it, or + # for EOF if the driver closed it, before wrap_socket detaches conn. + while True: + data = conn.recv(4096, socket.MSG_PEEK) + if not data: + eof.set() + return + if data[:1] == b"\x16": # TLS handshake record + break + # Hold the handshake open until the test has cancelled. + if not gate.wait(5): + return + tls = server_ctx.wrap_socket(conn, server_side=True, do_handshake_on_connect=False) + try: + tls.do_handshake() + # A discarded connection may end in a reset rather than a + # clean EOF; either proves the driver closed it. + while tls.recv(4096): + pass + except OSError: + pass + eof.set() + tls.close() + except OSError: + pass + finally: + if conn is not None: + conn.close() + + threading.Thread(target=stub_server, daemon=True).start() + + client_ctx = get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) + options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + socks = [] + + async def callback(context): + sock = await asyncio.get_running_loop().run_in_executor( + None, + lambda: socket.create_connection(listener.getsockname(), timeout=10), + ) + socks.append(sock) + return sock + + # The sync flavor returns a socket instead of a coroutine, so both + # error codes are needed depending on the flavor being checked. + connect = _connect_kms(listener.getsockname(), options, callback, 10.0) + task = asyncio.ensure_future(connect) # type: ignore[type-var,arg-type] + for _ in range(100): + if socks: + break + await asyncio.sleep(0.01) + self.assertTrue(socks, "callback was never invoked") + # Bias the cancel to land mid-handshake; the stub handles the earlier + # window too. + await asyncio.sleep(0.1) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + # The late SSLSocket (or raw socket) must be closed; the stub sees EOF. + gate.set() + for _ in range(50): + if eof.is_set(): + break + await asyncio.sleep(0.1) + self.assertTrue(eof.is_set(), "driver never closed the callback socket") + + async def test_client_encryption_accepts_callback(self): + async def callback(context): + raise AssertionError("not called") + + client = self.simple_client() + encryption = AsyncClientEncryption( + {"local": {"key": b"\x00" * 96}}, + "keyvault.datakeys", + client, + OPTS, + kms_connect_callback=callback, + ) + self.addAsyncCleanup(encryption.close) + self.assertIs(encryption._io_callbacks.opts._kms_connect_callback, callback) + + async def test_client_encryption_rejects_non_callable(self): + client = self.simple_client() + with self.assertRaisesRegex(TypeError, "kms_connect_callback must be callable"): + AsyncClientEncryption( + {"local": {"key": b"\x00" * 96}}, + "keyvault.datakeys", + client, + OPTS, + kms_connect_callback="not-callable", # type: ignore[arg-type] + ) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index 25da5134b9..9bd221f6e7 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -34,13 +34,19 @@ from pymongo.hello import HelloCompat from pymongo.lock import _async_create_lock from pymongo.monitoring import _EventListeners +from pymongo.pool_shared import _async_wrap_socket_tls from test.asynchronous.utils import async_get_pool, async_joinall, flaky sys.path[0:0] = [""] from pymongo.asynchronous.pool import Pool, PoolOptions from pymongo.socket_checker import SocketChecker -from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest +from test.asynchronous import ( + AsyncIntegrationTest, + AsyncPyMongoTestCase, + async_client_context, + unittest, +) from test.asynchronous.helpers import ConcurrentRunner from test.utils_shared import CMAPListener, delay @@ -919,5 +925,18 @@ def test_certificate_error_is_not_labeled_overloaded(self): self.assertFalse(err.has_error_label("SystemOverloadedError")) +class TestWrapSocketTLS(AsyncPyMongoTestCase): + async def test_wrap_socket_tls_without_ssl_context_returns_same_socket(self): + options = PoolOptions(socket_timeout=7.5) + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + + result = await _async_wrap_socket_tls(left, ("kms.example.com", 443), options) + + self.assertIs(result, left) + self.assertEqual(result.gettimeout(), 7.5) + + if __name__ == "__main__": unittest.main() diff --git a/test/test_encryption.py b/test/test_encryption.py index adb6005ea1..01cf8cfc83 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -82,7 +82,11 @@ ) from pymongo.operations import InsertOne, ReplaceOne, UpdateOne from pymongo.synchronous import encryption -from pymongo.synchronous.encryption import Algorithm, ClientEncryption, QueryType +from pymongo.synchronous.encryption import ( + Algorithm, + ClientEncryption, + QueryType, +) from pymongo.synchronous.helpers import next from pymongo.synchronous.mongo_client import MongoClient from pymongo.write_concern import WriteConcern @@ -222,6 +226,9 @@ def test_init_kms_tls_options(self): self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) +# KMS connect callback unit and prose tests live in test_kms_connect.py. + + class TestClientOptions(PyMongoTestCase): def test_default(self): client = self.simple_client(connect=False) @@ -316,9 +323,15 @@ def create_client_encryption( key_vault_client: MongoClient, codec_options: CodecOptions, kms_tls_options: Optional[Mapping[str, Any]] = None, + kms_connect_callback: Optional[Any] = None, ): client_encryption = ClientEncryption( - kms_providers, key_vault_namespace, key_vault_client, codec_options, kms_tls_options + kms_providers, + key_vault_namespace, + key_vault_client, + codec_options, + kms_tls_options, + kms_connect_callback=kms_connect_callback, ) self.addCleanup(client_encryption.close) return client_encryption @@ -331,9 +344,15 @@ def unmanaged_create_client_encryption( key_vault_client: MongoClient, codec_options: CodecOptions, kms_tls_options: Optional[Mapping[str, Any]] = None, + kms_connect_callback: Optional[Any] = None, ): client_encryption = ClientEncryption( - kms_providers, key_vault_namespace, key_vault_client, codec_options, kms_tls_options + kms_providers, + key_vault_namespace, + key_vault_client, + codec_options, + kms_tls_options, + kms_connect_callback=kms_connect_callback, ) return client_encryption @@ -1980,6 +1999,9 @@ def test_invalid_hostname_in_kms_certificate(self): self.client_encrypted.create_data_key("aws", master_key=key) +# KMS connect callback unit and prose tests live in test_kms_connect.py. + + # https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/tests/README.md#kms-tls-options-tests class TestKmsTLSOptions(EncryptionIntegrationTest): @unittest.skipUnless(any(AWS_CREDS.values()), "AWS environment credentials are not set") diff --git a/test/test_kms_connect.py b/test/test_kms_connect.py new file mode 100644 index 0000000000..25c96191ab --- /dev/null +++ b/test/test_kms_connect.py @@ -0,0 +1,391 @@ +"""Tests for the KMS connect callback.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import socket +import ssl +import threading +import time +import unittest +from asyncio.trsock import TransportSocket +from unittest import mock + +import pytest + +import pymongo +from pymongo.encryption_options import ( + AutoEncryptionOpts, + KMSConnectContext, +) +from pymongo.errors import ConfigurationError, EncryptionError, NetworkTimeout +from pymongo.pool_options import PoolOptions +from pymongo.ssl_support import get_ssl_context +from pymongo.synchronous.encryption import ( + ClientEncryption, + _connect_kms, + _EncryptionIO, + _wrap_encryption_errors, +) +from test import PyMongoTestCase +from test.helpers_shared import CLIENT_PEM +from test.test_encryption import OPTS + +_IS_SYNC = True + +pytestmark = pytest.mark.encryption + + +class TestKmsConnectCallbackUnit(PyMongoTestCase): + """Contract checks for kms_connect_callback that need no KMS server.""" + + @staticmethod + def _pool_options(): + return PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=None) + + def test_init_kms_connect_callback(self): + opts = AutoEncryptionOpts({}, "k.d") + self.assertIsNone(opts._kms_connect_callback) + + def callback(context): + raise AssertionError("not called") + + opts = AutoEncryptionOpts({}, "k.d", kms_connect_callback=callback) + self.assertIs(opts._kms_connect_callback, callback) + + for bad in [1, "not-callable", object()]: + with self.assertRaisesRegex(TypeError, "kms_connect_callback must be callable"): + AutoEncryptionOpts({}, "k.d", kms_connect_callback=bad) # type: ignore[arg-type] + + context = KMSConnectContext(host="kms.example.com", port=443, timeout=9.5) + self.assertEqual(context.host, "kms.example.com") + self.assertEqual(context.port, 443) + self.assertEqual(context.timeout, 9.5) + with self.assertRaises(dataclasses.FrozenInstanceError): + context.host = "evil.example.com" # type: ignore[misc] + + def test_non_socket_return_raises_configuration_error(self): + def callback(context): + return "not-a-socket" + + with self.assertRaisesRegex(ConfigurationError, "must return a connected"): + _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + def test_already_wrapped_socket_is_rejected(self): + # ssl.SSLSocket passes isinstance but cannot be TLS-wrapped again. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + left, right = socket.socketpair() + self.addCleanup(right.close) + # No peer needed to produce a genuine ssl.SSLSocket. + wrapped = ctx.wrap_socket(left, do_handshake_on_connect=False, server_hostname="x") + self.addCleanup(wrapped.close) + + def callback(context): + return wrapped + + with self.assertRaisesRegex(ConfigurationError, "unwrapped"): + _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + def test_context_receives_host_port_and_timeout(self): + received = [] + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + + def callback(context): + received.append(context) + return left + + # ssl_context=None returns the socket unchanged, so a plain socket is accepted. + conn = _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 12.5) + self.assertIs(conn, left) + + self.assertEqual(len(received), 1) + self.assertEqual(received[0].host, "kms.example.com") + self.assertEqual(received[0].port, 443) + self.assertEqual(received[0].timeout, 12.5) + + def test_non_blocking_socket_from_callback_is_accepted(self): + # Without the driver normalizing the mode, this raises ValueError. + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(CLIENT_PEM) + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + self.addCleanup(listener.close) + + def serve(): + try: + conn, _ = listener.accept() + server_ctx.wrap_socket(conn, server_side=True).close() + except OSError: + pass + + threading.Thread(target=serve, daemon=True).start() + + # Built as the driver does, for the flavor-correct type; the local cert won't verify. + client_ctx = get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) + options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) + sock.setblocking(False) + return sock + + def callback(context): + if _IS_SYNC: + return connect() + return asyncio.get_running_loop().run_in_executor(None, connect) + + conn = _connect_kms(listener.getsockname(), options, callback, 10.0) + self.addCleanup(conn.close) + self.assertIsNotNone(conn.gettimeout()) + + def test_asyncio_transport_socket_is_rejected(self): + # get_extra_info("socket") is a TransportSocket, not a socket.socket. + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + + def callback(context): + return TransportSocket(left) + + with self.assertRaisesRegex(ConfigurationError, "TransportSocket"): + _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + def test_cancelled_tls_wrap_closes_late_socket(self): + # A cancelled wrap can leave the executor producing an SSLSocket; the + # done callback must close it. + if _IS_SYNC: + raise unittest.SkipTest("the cancel-safe wrap is an async path") + from pymongo.pool_shared import _close_late_socket + + left, right = socket.socketpair() + future = asyncio.get_running_loop().create_future() + future.set_result(left) + self.assertNotEqual(left.fileno(), -1) + _close_late_socket(future) + self.assertEqual(left.fileno(), -1) + self.addCleanup(right.close) + + def test_non_coroutine_callback_is_rejected(self): + # A plain def must be rejected before it blocks the event loop. + if _IS_SYNC: + raise unittest.SkipTest("a regular function is correct for the sync API") + + entered = [] + + def callback(context): + entered.append(context) + return None + + with self.assertRaisesRegex(ConfigurationError, "coroutine function"): + _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + self.assertEqual(entered, [], "invalid callback must not be entered") + + def test_unconnected_socket_from_callback_is_rejected(self): + # An unconnected socket would fail later as a transient error and be retried. + bare = socket.socket() + self.addCleanup(bare.close) + + def callback(context): + return bare + + with self.assertRaisesRegex(ConfigurationError, "already connected"): + _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + def test_datagram_socket_from_callback_is_rejected(self): + # TLS on a connected UDP socket raises NotImplementedError, which would be retried. + left = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + right = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(left.close) + self.addCleanup(right.close) + right.bind(("127.0.0.1", 0)) + left.connect(right.getsockname()) + + def callback(context): + return left + + with self.assertRaisesRegex(ConfigurationError, "stream socket"): + _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + def test_kms_request_does_not_retry_a_contract_violation(self): + # _connect_kms has no retry loop; the no-retry guarantee is in + # kms_request, so exercise that instead. + calls = [] + + def callback(context): + calls.append(context) + return "not-a-socket" + + opts = AutoEncryptionOpts({}, "k.d", kms_connect_callback=callback) + io = _EncryptionIO(None, mock.MagicMock(), None, opts) + + class StubKmsContext: + endpoint = "kms.example.com:443" + message = b"request" + kms_provider = "aws" + usleep = 0 + bytes_needed = 1 + + def feed(self, data): + raise AssertionError("should not reach the socket") + + def fail(self): + raise AssertionError("a contract violation must not be retried") + + with self.assertRaises(ConfigurationError): + io.kms_request(StubKmsContext()) + self.assertEqual(len(calls), 1) + + def test_contract_violation_surfaces_as_encryption_error(self): + # Callers see EncryptionError with ConfigurationError as its cause. + with self.assertRaises(EncryptionError) as caught: + with _wrap_encryption_errors(): + raise ConfigurationError("kms_connect_callback must return ...") + self.assertIsInstance(caught.exception.__cause__, ConfigurationError) + + def test_network_error_from_callback_propagates(self): + def callback(context): + raise OSError("proxy unreachable") + + # Not a ConfigurationError, so kms_request retries it. + with self.assertRaises(OSError): + _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + + def test_csot_deadline_stops_a_hung_callback(self): + # A callback that ignores the timeout cannot block past the CSOT + # deadline, and a socket it yields later must be closed. + if _IS_SYNC: + raise unittest.SkipTest("the sync API cannot interrupt a callback") + + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + + def hung_callback(context): + time.sleep(0.5) + return left + + with self.assertRaises(NetworkTimeout): + with pymongo.timeout(0.1): + _connect_kms(("kms.example.com", 443), self._pool_options(), hung_callback, 10.0) + self.assertNotEqual(left.fileno(), -1) + # Let the shielded callback finish; the driver closes the late result. + time.sleep(0.75) + self.assertEqual(left.fileno(), -1) + + def test_cancelling_kms_connect_closes_the_callback_socket(self): + # Cancelling during the TLS handshake must close the callback's socket, + # so a TLS proxy's relay threads wind down. + if _IS_SYNC: + raise unittest.SkipTest("cancellation is an async-only behavior") + + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(CLIENT_PEM) + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + self.addCleanup(listener.close) + gate = threading.Event() + eof = threading.Event() + + def stub_server(): + conn = None + try: + conn, _ = listener.accept() + # The cancel may land before or after the executor starts the + # handshake. Peek for the ClientHello without consuming it, or + # for EOF if the driver closed it, before wrap_socket detaches conn. + while True: + data = conn.recv(4096, socket.MSG_PEEK) + if not data: + eof.set() + return + if data[:1] == b"\x16": # TLS handshake record + break + # Hold the handshake open until the test has cancelled. + if not gate.wait(5): + return + tls = server_ctx.wrap_socket(conn, server_side=True, do_handshake_on_connect=False) + try: + tls.do_handshake() + # A discarded connection may end in a reset rather than a + # clean EOF; either proves the driver closed it. + while tls.recv(4096): + pass + except OSError: + pass + eof.set() + tls.close() + except OSError: + pass + finally: + if conn is not None: + conn.close() + + threading.Thread(target=stub_server, daemon=True).start() + + client_ctx = get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) + options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + socks = [] + + def callback(context): + sock = asyncio.get_running_loop().run_in_executor( + None, + lambda: socket.create_connection(listener.getsockname(), timeout=10), + ) + socks.append(sock) + return sock + + # The sync flavor returns a socket instead of a coroutine, so both + # error codes are needed depending on the flavor being checked. + connect = _connect_kms(listener.getsockname(), options, callback, 10.0) + task = asyncio.ensure_future(connect) # type: ignore[type-var,arg-type] + for _ in range(100): + if socks: + break + time.sleep(0.01) + self.assertTrue(socks, "callback was never invoked") + # Bias the cancel to land mid-handshake; the stub handles the earlier + # window too. + time.sleep(0.1) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + task + # The late SSLSocket (or raw socket) must be closed; the stub sees EOF. + gate.set() + for _ in range(50): + if eof.is_set(): + break + time.sleep(0.1) + self.assertTrue(eof.is_set(), "driver never closed the callback socket") + + def test_client_encryption_accepts_callback(self): + def callback(context): + raise AssertionError("not called") + + client = self.simple_client() + encryption = ClientEncryption( + {"local": {"key": b"\x00" * 96}}, + "keyvault.datakeys", + client, + OPTS, + kms_connect_callback=callback, + ) + self.addCleanup(encryption.close) + self.assertIs(encryption._io_callbacks.opts._kms_connect_callback, callback) + + def test_client_encryption_rejects_non_callable(self): + client = self.simple_client() + with self.assertRaisesRegex(TypeError, "kms_connect_callback must be callable"): + ClientEncryption( + {"local": {"key": b"\x00" * 96}}, + "keyvault.datakeys", + client, + OPTS, + kms_connect_callback="not-callable", # type: ignore[arg-type] + ) diff --git a/test/test_pooling.py b/test/test_pooling.py index 3a81bc796f..d690de0ffb 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -34,13 +34,19 @@ from pymongo.hello import HelloCompat from pymongo.lock import _create_lock from pymongo.monitoring import _EventListeners +from pymongo.pool_shared import _wrap_socket_tls from test.utils import flaky, get_pool, joinall sys.path[0:0] = [""] from pymongo.socket_checker import SocketChecker from pymongo.synchronous.pool import Pool, PoolOptions -from test import IntegrationTest, client_context, unittest +from test import ( + IntegrationTest, + PyMongoTestCase, + client_context, + unittest, +) from test.helpers import ConcurrentRunner from test.utils_shared import CMAPListener, delay @@ -917,5 +923,18 @@ def test_certificate_error_is_not_labeled_overloaded(self): self.assertFalse(err.has_error_label("SystemOverloadedError")) +class TestWrapSocketTLS(PyMongoTestCase): + def test_wrap_socket_tls_without_ssl_context_returns_same_socket(self): + options = PoolOptions(socket_timeout=7.5) + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + + result = _wrap_socket_tls(left, ("kms.example.com", 443), options) + + self.assertIs(result, left) + self.assertEqual(result.gettimeout(), 7.5) + + if __name__ == "__main__": unittest.main() diff --git a/tools/synchro.py b/tools/synchro.py index bebf92c005..bd5edebda3 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -72,6 +72,7 @@ "_a_grid_out_property": "_grid_out_property", "AsyncClientEncryption": "ClientEncryption", "AsyncMongoCryptCallback": "MongoCryptCallback", + "AsyncKMSConnectCallback": "KMSConnectCallback", "AsyncExplicitEncrypter": "ExplicitEncrypter", "AsyncAutoEncrypter": "AutoEncrypter", "AsyncContextManager": "ContextManager", @@ -127,6 +128,7 @@ "AsyncNetworkingInterface": "NetworkingInterface", "_configured_protocol_interface": "_configured_socket_interface", "_async_configured_socket": "_configured_socket", + "_async_wrap_socket_tls": "_wrap_socket_tls", "SpecRunnerTask": "SpecRunnerThread", "AsyncMockConnection": "MockConnection", "AsyncMockPool": "MockPool", @@ -297,6 +299,18 @@ def translate_docstrings(lines: list[str]) -> list[str]: lines[i] = lines[i].replace("an asynchronous", "a") if "An asynchronous" in lines[i]: lines[i] = lines[i].replace("An asynchronous", "A") + # This sentence states the callback contract, whose meaning + # would invert under the async -> sync word replacements. + if ( + "must be a coroutine function for the asynchronous API; a plain callable is rejected before it can block the event loop" + in lines[i] + ): + lines[i] = lines[i].replace( + "must be a coroutine function for the asynchronous API; a plain callable is rejected before it can block the event loop", + "must be a regular function; the async API requires a " + "coroutine function and rejects plain callables before " + "they can block the event loop", + ) # This ensures docstring links are for `pymongo.X` instead of `pymongo.synchronous.X` if "pymongo.asynchronous" in lines[i] and "import" not in lines[i]: lines[i] = lines[i].replace("pymongo.asynchronous", "pymongo") From fd1b5a076396c5fa8955a5f6daa33fd0f08291d2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 5 Oct 2026 09:09:36 -0500 Subject: [PATCH 17/24] PYTHON-5805 Address follow-up review - Add a regression test that the KMS TLS handshake verifies against the KMS address, not the callback's connected peer. - Drop the stale prose-test references in test_encryption.py comments; prose tests moved with the PYTHON-6147 split. - Correct the KMSConnectContext.timeout docstring: it is the default KMS connect timeout capped by any active operation timeout. --- pymongo/encryption_options.py | 5 ++- test/asynchronous/test_encryption.py | 4 +- test/asynchronous/test_kms_connect.py | 55 ++++++++++++++++++++++++++- test/test_encryption.py | 4 +- test/test_kms_connect.py | 55 ++++++++++++++++++++++++++- 5 files changed, 113 insertions(+), 10 deletions(-) diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index f77e461704..b1d13700da 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -67,8 +67,9 @@ class KMSConnectContext: :param host: Hostname of the KMS server, and the TLS verification target. :param port: Port of the KMS server. - :param timeout: Seconds left in the timeout budget, or the default KMS - connect timeout when no timeout is active. + :param timeout: Seconds allowed for the connection: the default KMS + connect timeout, capped by the remaining time of an active operation + timeout (``timeoutMS``). .. note:: ``timeoutMS`` does not constrain KMS requests for explicit encryption, so ``timeout`` is always the default there. Automatic diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index dc5d0e31e0..e22d185f77 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -226,7 +226,7 @@ async def test_init_kms_tls_options(self): self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) -# KMS connect callback unit and prose tests live in test_kms_connect.py. +# KMS connect callback tests live in test_kms_connect.py. class TestClientOptions(AsyncPyMongoTestCase): @@ -2007,7 +2007,7 @@ async def test_invalid_hostname_in_kms_certificate(self): await self.client_encrypted.create_data_key("aws", master_key=key) -# KMS connect callback unit and prose tests live in test_kms_connect.py. +# KMS connect callback tests live in test_kms_connect.py. # https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/tests/README.md#kms-tls-options-tests diff --git a/test/asynchronous/test_kms_connect.py b/test/asynchronous/test_kms_connect.py index 994ec5ca21..99258cadda 100644 --- a/test/asynchronous/test_kms_connect.py +++ b/test/asynchronous/test_kms_connect.py @@ -4,6 +4,7 @@ import asyncio import dataclasses +import os import socket import ssl import threading @@ -25,12 +26,12 @@ AutoEncryptionOpts, KMSConnectContext, ) -from pymongo.errors import ConfigurationError, EncryptionError, NetworkTimeout +from pymongo.errors import ConfigurationError, ConnectionFailure, EncryptionError, NetworkTimeout from pymongo.pool_options import PoolOptions from pymongo.ssl_support import get_ssl_context from test.asynchronous import AsyncPyMongoTestCase from test.asynchronous.test_encryption import OPTS -from test.helpers_shared import CLIENT_PEM +from test.helpers_shared import CA_PEM, CERT_PATH, CLIENT_PEM _IS_SYNC = False @@ -144,6 +145,56 @@ async def callback(context): self.addCleanup(conn.close) self.assertIsNotNone(conn.gettimeout()) + async def test_tls_verification_targets_the_kms_host(self): + # The handshake must verify against the KMS address, not the peer the + # callback connected to. The server cert covers 127.0.0.1 (the peer) + # and localhost, but not the KMS hostname used below, so only + # address-based verification produces this outcome. + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(os.path.join(CERT_PATH, "server.pem")) + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(2) + self.addCleanup(listener.close) + + def serve(): + for _ in range(2): + try: + conn, _ = listener.accept() + server_ctx.wrap_socket(conn, server_side=True).close() + except (OSError, ssl.SSLError): + # The mismatched-name attempt fails mid-handshake. + pass + + threading.Thread(target=serve, daemon=True).start() + + # Full verification: trusted CA, invalid certs and hostnames rejected. + client_ctx = get_ssl_context(None, None, CA_PEM, None, False, False, False, _IS_SYNC) + options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + + created = [] + + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) + created.append(sock) + return sock + + async def callback(context): + if _IS_SYNC: + return connect() + return await asyncio.get_running_loop().run_in_executor(None, connect) + + port = listener.getsockname()[1] + # The cert covers localhost: verifying against the KMS address succeeds. + conn = await _connect_kms(("localhost", port), options, callback, 10.0) + self.addCleanup(conn.close) + # TLS-wrapped in either SSL flavor: a new object, not the plain socket. + self.assertIsNot(conn, created[0]) + # The cert does not cover this name: verification must fail even + # though the peer (127.0.0.1) presents a cert valid for itself. + with self.assertRaises(ConnectionFailure): + await _connect_kms(("kms.example.com", port), options, callback, 10.0) + async def test_asyncio_transport_socket_is_rejected(self): # get_extra_info("socket") is a TransportSocket, not a socket.socket. left, right = socket.socketpair() diff --git a/test/test_encryption.py b/test/test_encryption.py index 01cf8cfc83..2f9ed382ca 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -226,7 +226,7 @@ def test_init_kms_tls_options(self): self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) -# KMS connect callback unit and prose tests live in test_kms_connect.py. +# KMS connect callback tests live in test_kms_connect.py. class TestClientOptions(PyMongoTestCase): @@ -1999,7 +1999,7 @@ def test_invalid_hostname_in_kms_certificate(self): self.client_encrypted.create_data_key("aws", master_key=key) -# KMS connect callback unit and prose tests live in test_kms_connect.py. +# KMS connect callback tests live in test_kms_connect.py. # https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/tests/README.md#kms-tls-options-tests diff --git a/test/test_kms_connect.py b/test/test_kms_connect.py index 25c96191ab..4bbf6570d0 100644 --- a/test/test_kms_connect.py +++ b/test/test_kms_connect.py @@ -4,6 +4,7 @@ import asyncio import dataclasses +import os import socket import ssl import threading @@ -19,7 +20,7 @@ AutoEncryptionOpts, KMSConnectContext, ) -from pymongo.errors import ConfigurationError, EncryptionError, NetworkTimeout +from pymongo.errors import ConfigurationError, ConnectionFailure, EncryptionError, NetworkTimeout from pymongo.pool_options import PoolOptions from pymongo.ssl_support import get_ssl_context from pymongo.synchronous.encryption import ( @@ -29,7 +30,7 @@ _wrap_encryption_errors, ) from test import PyMongoTestCase -from test.helpers_shared import CLIENT_PEM +from test.helpers_shared import CA_PEM, CERT_PATH, CLIENT_PEM from test.test_encryption import OPTS _IS_SYNC = True @@ -144,6 +145,56 @@ def callback(context): self.addCleanup(conn.close) self.assertIsNotNone(conn.gettimeout()) + def test_tls_verification_targets_the_kms_host(self): + # The handshake must verify against the KMS address, not the peer the + # callback connected to. The server cert covers 127.0.0.1 (the peer) + # and localhost, but not the KMS hostname used below, so only + # address-based verification produces this outcome. + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(os.path.join(CERT_PATH, "server.pem")) + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(2) + self.addCleanup(listener.close) + + def serve(): + for _ in range(2): + try: + conn, _ = listener.accept() + server_ctx.wrap_socket(conn, server_side=True).close() + except (OSError, ssl.SSLError): + # The mismatched-name attempt fails mid-handshake. + pass + + threading.Thread(target=serve, daemon=True).start() + + # Full verification: trusted CA, invalid certs and hostnames rejected. + client_ctx = get_ssl_context(None, None, CA_PEM, None, False, False, False, _IS_SYNC) + options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + + created = [] + + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) + created.append(sock) + return sock + + def callback(context): + if _IS_SYNC: + return connect() + return asyncio.get_running_loop().run_in_executor(None, connect) + + port = listener.getsockname()[1] + # The cert covers localhost: verifying against the KMS address succeeds. + conn = _connect_kms(("localhost", port), options, callback, 10.0) + self.addCleanup(conn.close) + # TLS-wrapped in either SSL flavor: a new object, not the plain socket. + self.assertIsNot(conn, created[0]) + # The cert does not cover this name: verification must fail even + # though the peer (127.0.0.1) presents a cert valid for itself. + with self.assertRaises(ConnectionFailure): + _connect_kms(("kms.example.com", port), options, callback, 10.0) + def test_asyncio_transport_socket_is_rejected(self): # get_extra_info("socket") is a TransportSocket, not a socket.socket. left, right = socket.socketpair() From 5eee12d5e3afa47cf4a47610c65554085b272498 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 5 Oct 2026 09:20:32 -0500 Subject: [PATCH 18/24] PYTHON-5805 Factor shared helpers into test_kms_connect.py Extract the repeated listener/socketpair setup, TLS context construction, and the flavor-aware run-blocking wrapper into shared helpers, and collapse the fixed-value callbacks into a factory. --- test/asynchronous/test_kms_connect.py | 163 +++++++++++++------------- test/test_kms_connect.py | 159 +++++++++++++------------ 2 files changed, 166 insertions(+), 156 deletions(-) diff --git a/test/asynchronous/test_kms_connect.py b/test/asynchronous/test_kms_connect.py index 99258cadda..a96b0368d2 100644 --- a/test/asynchronous/test_kms_connect.py +++ b/test/asynchronous/test_kms_connect.py @@ -37,13 +37,57 @@ pytestmark = pytest.mark.encryption +_KMS_ADDRESS = ("kms.example.com", 443) + + +def _tls_server_context(cert=CLIENT_PEM): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(cert) + return ctx + + +def _client_tls_context(verify=False): + # verify=False matches the driver's test mode: the local certs don't verify. + if verify: + return get_ssl_context(None, None, CA_PEM, None, False, False, False, _IS_SYNC) + return get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) + + +async def _run_blocking(func, *args): + """Run a blocking callable off the event loop (inline in the sync flavor).""" + if _IS_SYNC: + return func(*args) + return await asyncio.get_running_loop().run_in_executor(None, func, *args) + + +def _callback_returning(value): + """A kms_connect_callback that always produces ``value``.""" + + async def callback(context): + return value + + return callback + class TestKmsConnectCallbackUnit(AsyncPyMongoTestCase): """Contract checks for kms_connect_callback that need no KMS server.""" @staticmethod - def _pool_options(): - return PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=None) + def _pool_options(ssl_context=None): + return PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=ssl_context) + + def _listen(self, backlog=1): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(backlog) + self.addCleanup(listener.close) + return listener + + def _socketpair(self): + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + return left, right async def test_init_kms_connect_callback(self): opts = AutoEncryptionOpts({}, "k.d") @@ -67,41 +111,36 @@ async def callback(context): context.host = "evil.example.com" # type: ignore[misc] async def test_non_socket_return_raises_configuration_error(self): - async def callback(context): - return "not-a-socket" - with self.assertRaisesRegex(ConfigurationError, "must return a connected"): - await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + await _connect_kms( + _KMS_ADDRESS, self._pool_options(), _callback_returning("not-a-socket"), 10.0 + ) async def test_already_wrapped_socket_is_rejected(self): # ssl.SSLSocket passes isinstance but cannot be TLS-wrapped again. ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE - left, right = socket.socketpair() - self.addCleanup(right.close) + left, _right = self._socketpair() # No peer needed to produce a genuine ssl.SSLSocket. wrapped = ctx.wrap_socket(left, do_handshake_on_connect=False, server_hostname="x") self.addCleanup(wrapped.close) - async def callback(context): - return wrapped - with self.assertRaisesRegex(ConfigurationError, "unwrapped"): - await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + await _connect_kms( + _KMS_ADDRESS, self._pool_options(), _callback_returning(wrapped), 10.0 + ) async def test_context_receives_host_port_and_timeout(self): received = [] - left, right = socket.socketpair() - self.addCleanup(left.close) - self.addCleanup(right.close) + left, _right = self._socketpair() async def callback(context): received.append(context) return left # ssl_context=None returns the socket unchanged, so a plain socket is accepted. - conn = await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 12.5) + conn = await _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 12.5) self.assertIs(conn, left) self.assertEqual(len(received), 1) @@ -111,12 +150,8 @@ async def callback(context): async def test_non_blocking_socket_from_callback_is_accepted(self): # Without the driver normalizing the mode, this raises ValueError. - server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - server_ctx.load_cert_chain(CLIENT_PEM) - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - self.addCleanup(listener.close) + server_ctx = _tls_server_context() + listener = self._listen() def serve(): try: @@ -127,9 +162,7 @@ def serve(): threading.Thread(target=serve, daemon=True).start() - # Built as the driver does, for the flavor-correct type; the local cert won't verify. - client_ctx = get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) - options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + options = self._pool_options(_client_tls_context()) def connect(): sock = socket.create_connection(listener.getsockname(), timeout=10) @@ -137,9 +170,7 @@ def connect(): return sock async def callback(context): - if _IS_SYNC: - return connect() - return await asyncio.get_running_loop().run_in_executor(None, connect) + return await _run_blocking(connect) conn = await _connect_kms(listener.getsockname(), options, callback, 10.0) self.addCleanup(conn.close) @@ -150,12 +181,8 @@ async def test_tls_verification_targets_the_kms_host(self): # callback connected to. The server cert covers 127.0.0.1 (the peer) # and localhost, but not the KMS hostname used below, so only # address-based verification produces this outcome. - server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - server_ctx.load_cert_chain(os.path.join(CERT_PATH, "server.pem")) - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(2) - self.addCleanup(listener.close) + server_ctx = _tls_server_context(os.path.join(CERT_PATH, "server.pem")) + listener = self._listen(2) def serve(): for _ in range(2): @@ -169,8 +196,7 @@ def serve(): threading.Thread(target=serve, daemon=True).start() # Full verification: trusted CA, invalid certs and hostnames rejected. - client_ctx = get_ssl_context(None, None, CA_PEM, None, False, False, False, _IS_SYNC) - options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + options = self._pool_options(_client_tls_context(verify=True)) created = [] @@ -180,9 +206,7 @@ def connect(): return sock async def callback(context): - if _IS_SYNC: - return connect() - return await asyncio.get_running_loop().run_in_executor(None, connect) + return await _run_blocking(connect) port = listener.getsockname()[1] # The cert covers localhost: verifying against the KMS address succeeds. @@ -197,15 +221,12 @@ async def callback(context): async def test_asyncio_transport_socket_is_rejected(self): # get_extra_info("socket") is a TransportSocket, not a socket.socket. - left, right = socket.socketpair() - self.addCleanup(left.close) - self.addCleanup(right.close) - - async def callback(context): - return TransportSocket(left) + left, _right = self._socketpair() with self.assertRaisesRegex(ConfigurationError, "TransportSocket"): - await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + await _connect_kms( + _KMS_ADDRESS, self._pool_options(), _callback_returning(TransportSocket(left)), 10.0 + ) async def test_cancelled_tls_wrap_closes_late_socket(self): # A cancelled wrap can leave the executor producing an SSLSocket; the @@ -214,13 +235,12 @@ async def test_cancelled_tls_wrap_closes_late_socket(self): raise unittest.SkipTest("the cancel-safe wrap is an async path") from pymongo.pool_shared import _close_late_socket - left, right = socket.socketpair() + left, _right = self._socketpair() future = asyncio.get_running_loop().create_future() future.set_result(left) self.assertNotEqual(left.fileno(), -1) _close_late_socket(future) self.assertEqual(left.fileno(), -1) - self.addCleanup(right.close) async def test_non_coroutine_callback_is_rejected(self): # A plain def must be rejected before it blocks the event loop. @@ -234,7 +254,7 @@ def callback(context): return None with self.assertRaisesRegex(ConfigurationError, "coroutine function"): - await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + await _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 10.0) self.assertEqual(entered, [], "invalid callback must not be entered") async def test_unconnected_socket_from_callback_is_rejected(self): @@ -242,11 +262,8 @@ async def test_unconnected_socket_from_callback_is_rejected(self): bare = socket.socket() self.addCleanup(bare.close) - async def callback(context): - return bare - with self.assertRaisesRegex(ConfigurationError, "already connected"): - await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + await _connect_kms(_KMS_ADDRESS, self._pool_options(), _callback_returning(bare), 10.0) async def test_datagram_socket_from_callback_is_rejected(self): # TLS on a connected UDP socket raises NotImplementedError, which would be retried. @@ -257,11 +274,8 @@ async def test_datagram_socket_from_callback_is_rejected(self): right.bind(("127.0.0.1", 0)) left.connect(right.getsockname()) - async def callback(context): - return left - with self.assertRaisesRegex(ConfigurationError, "stream socket"): - await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + await _connect_kms(_KMS_ADDRESS, self._pool_options(), _callback_returning(left), 10.0) async def test_kms_request_does_not_retry_a_contract_violation(self): # _connect_kms has no retry loop; the no-retry guarantee is in @@ -305,7 +319,7 @@ async def callback(context): # Not a ConfigurationError, so kms_request retries it. with self.assertRaises(OSError): - await _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + await _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 10.0) async def test_csot_deadline_stops_a_hung_callback(self): # A callback that ignores the timeout cannot block past the CSOT @@ -313,9 +327,7 @@ async def test_csot_deadline_stops_a_hung_callback(self): if _IS_SYNC: raise unittest.SkipTest("the sync API cannot interrupt a callback") - left, right = socket.socketpair() - self.addCleanup(left.close) - self.addCleanup(right.close) + left, _right = self._socketpair() async def hung_callback(context): await asyncio.sleep(0.5) @@ -323,9 +335,7 @@ async def hung_callback(context): with self.assertRaises(NetworkTimeout): with pymongo.timeout(0.1): - await _connect_kms( - ("kms.example.com", 443), self._pool_options(), hung_callback, 10.0 - ) + await _connect_kms(_KMS_ADDRESS, self._pool_options(), hung_callback, 10.0) self.assertNotEqual(left.fileno(), -1) # Let the shielded callback finish; the driver closes the late result. await asyncio.sleep(0.75) @@ -337,12 +347,8 @@ async def test_cancelling_kms_connect_closes_the_callback_socket(self): if _IS_SYNC: raise unittest.SkipTest("cancellation is an async-only behavior") - server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - server_ctx.load_cert_chain(CLIENT_PEM) - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - self.addCleanup(listener.close) + server_ctx = _tls_server_context() + listener = self._listen() gate = threading.Event() eof = threading.Event() @@ -382,22 +388,21 @@ def stub_server(): threading.Thread(target=stub_server, daemon=True).start() - client_ctx = get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) - options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + options = self._pool_options(_client_tls_context()) socks = [] - async def callback(context): - sock = await asyncio.get_running_loop().run_in_executor( - None, - lambda: socket.create_connection(listener.getsockname(), timeout=10), - ) + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) socks.append(sock) return sock + async def callback(context): + return await _run_blocking(connect) + # The sync flavor returns a socket instead of a coroutine, so both # error codes are needed depending on the flavor being checked. - connect = _connect_kms(listener.getsockname(), options, callback, 10.0) - task = asyncio.ensure_future(connect) # type: ignore[type-var,arg-type] + pending = _connect_kms(listener.getsockname(), options, callback, 10.0) + task = asyncio.ensure_future(pending) # type: ignore[type-var,arg-type] for _ in range(100): if socks: break diff --git a/test/test_kms_connect.py b/test/test_kms_connect.py index 4bbf6570d0..28d9bc6e1e 100644 --- a/test/test_kms_connect.py +++ b/test/test_kms_connect.py @@ -37,13 +37,57 @@ pytestmark = pytest.mark.encryption +_KMS_ADDRESS = ("kms.example.com", 443) + + +def _tls_server_context(cert=CLIENT_PEM): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(cert) + return ctx + + +def _client_tls_context(verify=False): + # verify=False matches the driver's test mode: the local certs don't verify. + if verify: + return get_ssl_context(None, None, CA_PEM, None, False, False, False, _IS_SYNC) + return get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) + + +def _run_blocking(func, *args): + """Run a blocking callable off the event loop (inline in the sync flavor).""" + if _IS_SYNC: + return func(*args) + return asyncio.get_running_loop().run_in_executor(None, func, *args) + + +def _callback_returning(value): + """A kms_connect_callback that always produces ``value``.""" + + def callback(context): + return value + + return callback + class TestKmsConnectCallbackUnit(PyMongoTestCase): """Contract checks for kms_connect_callback that need no KMS server.""" @staticmethod - def _pool_options(): - return PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=None) + def _pool_options(ssl_context=None): + return PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=ssl_context) + + def _listen(self, backlog=1): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(backlog) + self.addCleanup(listener.close) + return listener + + def _socketpair(self): + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + return left, right def test_init_kms_connect_callback(self): opts = AutoEncryptionOpts({}, "k.d") @@ -67,41 +111,34 @@ def callback(context): context.host = "evil.example.com" # type: ignore[misc] def test_non_socket_return_raises_configuration_error(self): - def callback(context): - return "not-a-socket" - with self.assertRaisesRegex(ConfigurationError, "must return a connected"): - _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + _connect_kms( + _KMS_ADDRESS, self._pool_options(), _callback_returning("not-a-socket"), 10.0 + ) def test_already_wrapped_socket_is_rejected(self): # ssl.SSLSocket passes isinstance but cannot be TLS-wrapped again. ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE - left, right = socket.socketpair() - self.addCleanup(right.close) + left, _right = self._socketpair() # No peer needed to produce a genuine ssl.SSLSocket. wrapped = ctx.wrap_socket(left, do_handshake_on_connect=False, server_hostname="x") self.addCleanup(wrapped.close) - def callback(context): - return wrapped - with self.assertRaisesRegex(ConfigurationError, "unwrapped"): - _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + _connect_kms(_KMS_ADDRESS, self._pool_options(), _callback_returning(wrapped), 10.0) def test_context_receives_host_port_and_timeout(self): received = [] - left, right = socket.socketpair() - self.addCleanup(left.close) - self.addCleanup(right.close) + left, _right = self._socketpair() def callback(context): received.append(context) return left # ssl_context=None returns the socket unchanged, so a plain socket is accepted. - conn = _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 12.5) + conn = _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 12.5) self.assertIs(conn, left) self.assertEqual(len(received), 1) @@ -111,12 +148,8 @@ def callback(context): def test_non_blocking_socket_from_callback_is_accepted(self): # Without the driver normalizing the mode, this raises ValueError. - server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - server_ctx.load_cert_chain(CLIENT_PEM) - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - self.addCleanup(listener.close) + server_ctx = _tls_server_context() + listener = self._listen() def serve(): try: @@ -127,9 +160,7 @@ def serve(): threading.Thread(target=serve, daemon=True).start() - # Built as the driver does, for the flavor-correct type; the local cert won't verify. - client_ctx = get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) - options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + options = self._pool_options(_client_tls_context()) def connect(): sock = socket.create_connection(listener.getsockname(), timeout=10) @@ -137,9 +168,7 @@ def connect(): return sock def callback(context): - if _IS_SYNC: - return connect() - return asyncio.get_running_loop().run_in_executor(None, connect) + return _run_blocking(connect) conn = _connect_kms(listener.getsockname(), options, callback, 10.0) self.addCleanup(conn.close) @@ -150,12 +179,8 @@ def test_tls_verification_targets_the_kms_host(self): # callback connected to. The server cert covers 127.0.0.1 (the peer) # and localhost, but not the KMS hostname used below, so only # address-based verification produces this outcome. - server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - server_ctx.load_cert_chain(os.path.join(CERT_PATH, "server.pem")) - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(2) - self.addCleanup(listener.close) + server_ctx = _tls_server_context(os.path.join(CERT_PATH, "server.pem")) + listener = self._listen(2) def serve(): for _ in range(2): @@ -169,8 +194,7 @@ def serve(): threading.Thread(target=serve, daemon=True).start() # Full verification: trusted CA, invalid certs and hostnames rejected. - client_ctx = get_ssl_context(None, None, CA_PEM, None, False, False, False, _IS_SYNC) - options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + options = self._pool_options(_client_tls_context(verify=True)) created = [] @@ -180,9 +204,7 @@ def connect(): return sock def callback(context): - if _IS_SYNC: - return connect() - return asyncio.get_running_loop().run_in_executor(None, connect) + return _run_blocking(connect) port = listener.getsockname()[1] # The cert covers localhost: verifying against the KMS address succeeds. @@ -197,15 +219,12 @@ def callback(context): def test_asyncio_transport_socket_is_rejected(self): # get_extra_info("socket") is a TransportSocket, not a socket.socket. - left, right = socket.socketpair() - self.addCleanup(left.close) - self.addCleanup(right.close) - - def callback(context): - return TransportSocket(left) + left, _right = self._socketpair() with self.assertRaisesRegex(ConfigurationError, "TransportSocket"): - _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + _connect_kms( + _KMS_ADDRESS, self._pool_options(), _callback_returning(TransportSocket(left)), 10.0 + ) def test_cancelled_tls_wrap_closes_late_socket(self): # A cancelled wrap can leave the executor producing an SSLSocket; the @@ -214,13 +233,12 @@ def test_cancelled_tls_wrap_closes_late_socket(self): raise unittest.SkipTest("the cancel-safe wrap is an async path") from pymongo.pool_shared import _close_late_socket - left, right = socket.socketpair() + left, _right = self._socketpair() future = asyncio.get_running_loop().create_future() future.set_result(left) self.assertNotEqual(left.fileno(), -1) _close_late_socket(future) self.assertEqual(left.fileno(), -1) - self.addCleanup(right.close) def test_non_coroutine_callback_is_rejected(self): # A plain def must be rejected before it blocks the event loop. @@ -234,7 +252,7 @@ def callback(context): return None with self.assertRaisesRegex(ConfigurationError, "coroutine function"): - _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 10.0) self.assertEqual(entered, [], "invalid callback must not be entered") def test_unconnected_socket_from_callback_is_rejected(self): @@ -242,11 +260,8 @@ def test_unconnected_socket_from_callback_is_rejected(self): bare = socket.socket() self.addCleanup(bare.close) - def callback(context): - return bare - with self.assertRaisesRegex(ConfigurationError, "already connected"): - _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + _connect_kms(_KMS_ADDRESS, self._pool_options(), _callback_returning(bare), 10.0) def test_datagram_socket_from_callback_is_rejected(self): # TLS on a connected UDP socket raises NotImplementedError, which would be retried. @@ -257,11 +272,8 @@ def test_datagram_socket_from_callback_is_rejected(self): right.bind(("127.0.0.1", 0)) left.connect(right.getsockname()) - def callback(context): - return left - with self.assertRaisesRegex(ConfigurationError, "stream socket"): - _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + _connect_kms(_KMS_ADDRESS, self._pool_options(), _callback_returning(left), 10.0) def test_kms_request_does_not_retry_a_contract_violation(self): # _connect_kms has no retry loop; the no-retry guarantee is in @@ -305,7 +317,7 @@ def callback(context): # Not a ConfigurationError, so kms_request retries it. with self.assertRaises(OSError): - _connect_kms(("kms.example.com", 443), self._pool_options(), callback, 10.0) + _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 10.0) def test_csot_deadline_stops_a_hung_callback(self): # A callback that ignores the timeout cannot block past the CSOT @@ -313,9 +325,7 @@ def test_csot_deadline_stops_a_hung_callback(self): if _IS_SYNC: raise unittest.SkipTest("the sync API cannot interrupt a callback") - left, right = socket.socketpair() - self.addCleanup(left.close) - self.addCleanup(right.close) + left, _right = self._socketpair() def hung_callback(context): time.sleep(0.5) @@ -323,7 +333,7 @@ def hung_callback(context): with self.assertRaises(NetworkTimeout): with pymongo.timeout(0.1): - _connect_kms(("kms.example.com", 443), self._pool_options(), hung_callback, 10.0) + _connect_kms(_KMS_ADDRESS, self._pool_options(), hung_callback, 10.0) self.assertNotEqual(left.fileno(), -1) # Let the shielded callback finish; the driver closes the late result. time.sleep(0.75) @@ -335,12 +345,8 @@ def test_cancelling_kms_connect_closes_the_callback_socket(self): if _IS_SYNC: raise unittest.SkipTest("cancellation is an async-only behavior") - server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - server_ctx.load_cert_chain(CLIENT_PEM) - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - self.addCleanup(listener.close) + server_ctx = _tls_server_context() + listener = self._listen() gate = threading.Event() eof = threading.Event() @@ -380,22 +386,21 @@ def stub_server(): threading.Thread(target=stub_server, daemon=True).start() - client_ctx = get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) - options = PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=client_ctx) + options = self._pool_options(_client_tls_context()) socks = [] - def callback(context): - sock = asyncio.get_running_loop().run_in_executor( - None, - lambda: socket.create_connection(listener.getsockname(), timeout=10), - ) + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) socks.append(sock) return sock + def callback(context): + return _run_blocking(connect) + # The sync flavor returns a socket instead of a coroutine, so both # error codes are needed depending on the flavor being checked. - connect = _connect_kms(listener.getsockname(), options, callback, 10.0) - task = asyncio.ensure_future(connect) # type: ignore[type-var,arg-type] + pending = _connect_kms(listener.getsockname(), options, callback, 10.0) + task = asyncio.ensure_future(pending) # type: ignore[type-var,arg-type] for _ in range(100): if socks: break From 0870b51be60e126e29e09611000551110b2c49d7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 5 Oct 2026 09:24:13 -0500 Subject: [PATCH 19/24] PYTHON-5805 Clarify _close_rejected_kms_socket docstring --- pymongo/asynchronous/encryption.py | 7 ++++--- pymongo/synchronous/encryption.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 0eea43d898..80cc292ffa 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -128,10 +128,11 @@ def _close_rejected_kms_socket(obj: Any) -> None: - """Close a rejected kms_connect_callback return value, best effort. + """Close a callback return value that failed validation, best effort. - Nothing else will close it: _connect_kms raises before the result reaches - the caller's ``finally``. + ``_connect_kms`` raises on a contract violation instead of returning the + value, so no caller ever takes ownership of it. Close it here, tolerating + non-socket values and ``close()`` failures. """ close = getattr(obj, "close", None) if callable(close): diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index b89ad933f1..0b2e68f663 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -128,10 +128,11 @@ def _close_rejected_kms_socket(obj: Any) -> None: - """Close a rejected kms_connect_callback return value, best effort. + """Close a callback return value that failed validation, best effort. - Nothing else will close it: _connect_kms raises before the result reaches - the caller's ``finally``. + ``_connect_kms`` raises on a contract violation instead of returning the + value, so no caller ever takes ownership of it. Close it here, tolerating + non-socket values and ``close()`` failures. """ close = getattr(obj, "close", None) if callable(close): From eadebcaeb05d9604a7ad923bdd1ba76ab8ad8462 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 5 Oct 2026 09:28:13 -0500 Subject: [PATCH 20/24] PYTHON-5805 Simplify the result: Any comment --- pymongo/asynchronous/encryption.py | 3 +-- pymongo/synchronous/encryption.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 80cc292ffa..a8891fc576 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -169,8 +169,7 @@ async def _connect_kms( raise ConfigurationError( "kms_connect_callback must be a coroutine function for the async API." ) - # Typed as Any so the generated synchronous flavor type-checks: the sync - # callback returns a plain socket, which is not awaitable. + # Any: the generated sync flavor's callback returns a plain socket. result: Any = kms_connect_callback( KMSConnectContext(host=address[0], port=cast(int, address[1]), timeout=timeout) ) diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 0b2e68f663..f2b114d3e6 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -169,8 +169,7 @@ def _connect_kms( raise ConfigurationError( "kms_connect_callback must be a coroutine function for the async API." ) - # Typed as Any so the generated synchronous flavor type-checks: the sync - # callback returns a plain socket, which is not awaitable. + # Any: the generated sync flavor's callback returns a plain socket. result: Any = kms_connect_callback( KMSConnectContext(host=address[0], port=cast(int, address[1]), timeout=timeout) ) From 07b62ad85b2a7a5372898c938f7918963b550dd0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 5 Oct 2026 09:30:25 -0500 Subject: [PATCH 21/24] PYTHON-5805 Tighten the kms_connect_callback docstring --- pymongo/asynchronous/encryption.py | 17 ++++++++--------- pymongo/synchronous/encryption.py | 17 ++++++++--------- tools/synchro.py | 11 +++-------- 3 files changed, 19 insertions(+), 26 deletions(-) diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index a8891fc576..4aab64978e 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -709,15 +709,14 @@ def __init__( Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000. Set to 0 to disable key expiration. :param kms_connect_callback: A callable that opens the connection to a - KMS host, used to route KMS requests through an HTTP proxy. It - receives a :class:`~pymongo.encryption_options.KMSConnectContext` - and returns a connected, unwrapped :class:`socket.socket`, over - which the driver performs the KMS TLS handshake. The callback - must be a coroutine function for the asynchronous API; a plain callable is rejected before it can block the event loop. - When a CSOT timeout is active, the driver stops waiting at the - deadline and closes any socket the callback yields later. - Defaults to ``None``, meaning the driver connects to KMS hosts - directly. + KMS host, e.g. to route KMS requests through a proxy. It receives + a :class:`~pymongo.encryption_options.KMSConnectContext` and + returns a connected, unwrapped :class:`socket.socket`, over which + the driver performs the KMS TLS handshake. + Must be a coroutine function for the asynchronous API. + On timeout the driver stops waiting and closes any late-yielded + socket. Defaults to ``None``, meaning the driver connects to KMS + hosts directly. .. versionchanged:: 4.19 Added the `kms_connect_callback` parameter. diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index f2b114d3e6..aa4641303d 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -707,15 +707,14 @@ def __init__( Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000. Set to 0 to disable key expiration. :param kms_connect_callback: A callable that opens the connection to a - KMS host, used to route KMS requests through an HTTP proxy. It - receives a :class:`~pymongo.encryption_options.KMSConnectContext` - and returns a connected, unwrapped :class:`socket.socket`, over - which the driver performs the KMS TLS handshake. The callback - must be a regular function; the async API requires a coroutine function and rejects plain callables before they can block the event loop. - When a CSOT timeout is active, the driver stops waiting at the - deadline and closes any socket the callback yields later. - Defaults to ``None``, meaning the driver connects to KMS hosts - directly. + KMS host, e.g. to route KMS requests through a proxy. It receives + a :class:`~pymongo.encryption_options.KMSConnectContext` and + returns a connected, unwrapped :class:`socket.socket`, over which + the driver performs the KMS TLS handshake. + Must be a regular function. + On timeout the driver stops waiting and closes any late-yielded + socket. Defaults to ``None``, meaning the driver connects to KMS + hosts directly. .. versionchanged:: 4.19 Added the `kms_connect_callback` parameter. diff --git a/tools/synchro.py b/tools/synchro.py index bd5edebda3..b5c6b2873a 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -301,15 +301,10 @@ def translate_docstrings(lines: list[str]) -> list[str]: lines[i] = lines[i].replace("An asynchronous", "A") # This sentence states the callback contract, whose meaning # would invert under the async -> sync word replacements. - if ( - "must be a coroutine function for the asynchronous API; a plain callable is rejected before it can block the event loop" - in lines[i] - ): + if "Must be a coroutine function for the asynchronous API." in lines[i]: lines[i] = lines[i].replace( - "must be a coroutine function for the asynchronous API; a plain callable is rejected before it can block the event loop", - "must be a regular function; the async API requires a " - "coroutine function and rejects plain callables before " - "they can block the event loop", + "Must be a coroutine function for the asynchronous API.", + "Must be a regular function.", ) # This ensures docstring links are for `pymongo.X` instead of `pymongo.synchronous.X` if "pymongo.asynchronous" in lines[i] and "import" not in lines[i]: From 089ffafa86267e0e3c3ca1c7e0551d16c1e09215 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 5 Oct 2026 09:33:23 -0500 Subject: [PATCH 22/24] PYTHON-5805 Fix sync docstring's false deadline-enforcement promise The synchronous API waits for the callback without enforcing the CSOT deadline; state the flavor-neutral contract instead (callback honors context.timeout). Addresses Copilot review discussion_r4185118746. --- pymongo/asynchronous/encryption.py | 6 +++--- pymongo/synchronous/encryption.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 4aab64978e..30a0e6d574 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -714,9 +714,9 @@ def __init__( returns a connected, unwrapped :class:`socket.socket`, over which the driver performs the KMS TLS handshake. Must be a coroutine function for the asynchronous API. - On timeout the driver stops waiting and closes any late-yielded - socket. Defaults to ``None``, meaning the driver connects to KMS - hosts directly. + The callback is responsible for honoring ``context.timeout``. + Defaults to ``None``, meaning the driver connects to KMS hosts + directly. .. versionchanged:: 4.19 Added the `kms_connect_callback` parameter. diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index aa4641303d..e22cc50405 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -712,9 +712,9 @@ def __init__( returns a connected, unwrapped :class:`socket.socket`, over which the driver performs the KMS TLS handshake. Must be a regular function. - On timeout the driver stops waiting and closes any late-yielded - socket. Defaults to ``None``, meaning the driver connects to KMS - hosts directly. + The callback is responsible for honoring ``context.timeout``. + Defaults to ``None``, meaning the driver connects to KMS hosts + directly. .. versionchanged:: 4.19 Added the `kms_connect_callback` parameter. From fa6adfec4c8703f1e533fc4faf48ca319d9b90e4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 5 Oct 2026 10:28:11 -0500 Subject: [PATCH 23/24] PYTHON-5805 Reword the result: Any comment --- pymongo/asynchronous/encryption.py | 2 +- pymongo/synchronous/encryption.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 30a0e6d574..4ae63716e3 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -169,7 +169,7 @@ async def _connect_kms( raise ConfigurationError( "kms_connect_callback must be a coroutine function for the async API." ) - # Any: the generated sync flavor's callback returns a plain socket. + # Any: the sync version's callback returns a plain socket. result: Any = kms_connect_callback( KMSConnectContext(host=address[0], port=cast(int, address[1]), timeout=timeout) ) diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index e22cc50405..ac2f710382 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -169,7 +169,7 @@ def _connect_kms( raise ConfigurationError( "kms_connect_callback must be a coroutine function for the async API." ) - # Any: the generated sync flavor's callback returns a plain socket. + # Any: the sync version's callback returns a plain socket. result: Any = kms_connect_callback( KMSConnectContext(host=address[0], port=cast(int, address[1]), timeout=timeout) ) From d76bf9938623f30a98ba323fcceb3842055a6046 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 5 Oct 2026 10:30:09 -0500 Subject: [PATCH 24/24] PYTHON-5805 Say sync version instead of sync flavor --- test/asynchronous/test_kms_connect.py | 6 +++--- test/test_kms_connect.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/asynchronous/test_kms_connect.py b/test/asynchronous/test_kms_connect.py index a96b0368d2..6a577b90ee 100644 --- a/test/asynchronous/test_kms_connect.py +++ b/test/asynchronous/test_kms_connect.py @@ -54,7 +54,7 @@ def _client_tls_context(verify=False): async def _run_blocking(func, *args): - """Run a blocking callable off the event loop (inline in the sync flavor).""" + """Run a blocking callable off the event loop (inline in the sync version).""" if _IS_SYNC: return func(*args) return await asyncio.get_running_loop().run_in_executor(None, func, *args) @@ -399,8 +399,8 @@ def connect(): async def callback(context): return await _run_blocking(connect) - # The sync flavor returns a socket instead of a coroutine, so both - # error codes are needed depending on the flavor being checked. + # The sync version returns a socket instead of a coroutine, so both + # error codes are needed depending on the version being checked. pending = _connect_kms(listener.getsockname(), options, callback, 10.0) task = asyncio.ensure_future(pending) # type: ignore[type-var,arg-type] for _ in range(100): diff --git a/test/test_kms_connect.py b/test/test_kms_connect.py index 28d9bc6e1e..8bd59ca689 100644 --- a/test/test_kms_connect.py +++ b/test/test_kms_connect.py @@ -54,7 +54,7 @@ def _client_tls_context(verify=False): def _run_blocking(func, *args): - """Run a blocking callable off the event loop (inline in the sync flavor).""" + """Run a blocking callable off the event loop (inline in the sync version).""" if _IS_SYNC: return func(*args) return asyncio.get_running_loop().run_in_executor(None, func, *args) @@ -397,8 +397,8 @@ def connect(): def callback(context): return _run_blocking(connect) - # The sync flavor returns a socket instead of a coroutine, so both - # error codes are needed depending on the flavor being checked. + # The sync version returns a socket instead of a coroutine, so both + # error codes are needed depending on the version being checked. pending = _connect_kms(listener.getsockname(), options, callback, 10.0) task = asyncio.ensure_future(pending) # type: ignore[type-var,arg-type] for _ in range(100):