From 5f89e78a6932fe71ad239f167629eeb57a9d59d7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 05:54:50 -0500 Subject: [PATCH 01/13] PYTHON-6112 Use a stable set of tasks and coverage in Evergreen --- .evergreen/generated_configs/tasks.yml | 424 ++++++++++++-------- .evergreen/scripts/generate_config.py | 66 ++- .evergreen/scripts/generate_config_utils.py | 5 + CONTRIBUTING.md | 6 + 4 files changed, 303 insertions(+), 198 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index aeb5782901..b1f31b0ce2 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -102,12 +102,40 @@ tasks: AWS_ROLE_SESSION_NAME: test UV_PYTHON: 3.15t tags: [auth-aws, auth-aws-web-identity, free-threaded] - - name: test-auth-aws-rapid-regular-python3.15-cov + - name: test-auth-aws-latest-assume-role-python3.11-min-deps commands: - func: run server vars: AUTH_AWS: "1" - VERSION: rapid + VERSION: latest + - func: assume ec2 role + - func: run tests + vars: + TEST_NAME: auth_aws + SUB_TEST_NAME: assume-role + UV_PYTHON: "3.11" + TEST_MIN_DEPS: "1" + tags: [auth-aws, auth-aws-assume-role] + - name: test-auth-aws-latest-regular-python3.11-min-deps + commands: + - func: run server + vars: + AUTH_AWS: "1" + VERSION: latest + - func: assume ec2 role + - func: run tests + vars: + TEST_NAME: auth_aws + SUB_TEST_NAME: regular + UV_PYTHON: "3.11" + TEST_MIN_DEPS: "1" + tags: [auth-aws, auth-aws-regular, pr] + - name: test-auth-aws-latest-regular-python3.15-cov + commands: + - func: run server + vars: + AUTH_AWS: "1" + VERSION: latest - func: assume ec2 role - func: run tests vars: @@ -116,20 +144,19 @@ tasks: UV_PYTHON: "3.15" COVERAGE: "1" tags: [auth-aws, auth-aws-regular, pr] - - name: test-auth-aws-latest-assume-role-python3.11-min-deps + - name: test-auth-aws-rapid-regular-python3.15 commands: - func: run server vars: AUTH_AWS: "1" - VERSION: latest + VERSION: rapid - func: assume ec2 role - func: run tests vars: TEST_NAME: auth_aws - SUB_TEST_NAME: assume-role - UV_PYTHON: "3.11" - TEST_MIN_DEPS: "1" - tags: [auth-aws, auth-aws-assume-role] + SUB_TEST_NAME: regular + UV_PYTHON: "3.15" + tags: [auth-aws, auth-aws-regular] - name: test-auth-aws-ecs commands: - func: assume ec2 role @@ -2872,7 +2899,7 @@ tasks: - python-3.12 - replica_set-auth-ssl - sync - - name: test-server-version-python3.12-async-noauth-nossl-replica-set-cov + - name: test-server-version-python3.15-async-noauth-nossl-replica-set-cov commands: - func: run server vars: @@ -2886,35 +2913,33 @@ tasks: SSL: nossl TOPOLOGY: replica_set COVERAGE: "1" - UV_PYTHON: "3.12" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - server-version - - python-3.12 + - python-3.15 - replica_set-noauth-nossl - async - pr - - name: test-server-version-python3.11-sync-noauth-nossl-replica-set-min-deps-cov + - name: test-server-version-python3.15-sync-noauth-nossl-replica-set-cov commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: replica_set - TEST_MIN_DEPS: "1" COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: replica_set - TEST_MIN_DEPS: "1" COVERAGE: "1" - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - server-version - - python-3.11 + - python-3.15 - replica_set-noauth-nossl - sync - pr @@ -2996,7 +3021,7 @@ tasks: - python-3.14 - sharded_cluster-auth-nossl - sync - - name: test-server-version-python3.11-async-auth-ssl-sharded-cluster-min-deps-cov + - name: test-server-version-python3.11-async-auth-ssl-sharded-cluster-min-deps commands: - func: run server vars: @@ -3004,14 +3029,12 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" UV_PYTHON: "3.11" TEST_NAME: default_async tags: @@ -3019,7 +3042,6 @@ tasks: - python-3.11 - sharded_cluster-auth-ssl - async - - pr - name: test-server-version-python3.12-async-auth-ssl-sharded-cluster commands: - func: run server @@ -3097,18 +3119,20 @@ tasks: - sharded_cluster-auth-ssl - async - free-threaded - - name: test-server-version-python3.15-async-auth-ssl-sharded-cluster + - name: test-server-version-python3.15-async-auth-ssl-sharded-cluster-cov commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster + COVERAGE: "1" UV_PYTHON: "3.15" TEST_NAME: default_async tags: @@ -3116,6 +3140,7 @@ tasks: - python-3.15 - sharded_cluster-auth-ssl - async + - pr - name: test-server-version-python3.15t-async-auth-ssl-sharded-cluster commands: - func: run server @@ -3136,7 +3161,7 @@ tasks: - sharded_cluster-auth-ssl - async - free-threaded - - name: test-server-version-python3.11-sync-auth-ssl-sharded-cluster-min-deps-cov + - name: test-server-version-python3.11-sync-auth-ssl-sharded-cluster-min-deps commands: - func: run server vars: @@ -3144,14 +3169,12 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" UV_PYTHON: "3.11" TEST_NAME: default_sync tags: @@ -3159,7 +3182,6 @@ tasks: - python-3.11 - sharded_cluster-auth-ssl - sync - - pr - name: test-server-version-python3.12-sync-auth-ssl-sharded-cluster commands: - func: run server @@ -3237,18 +3259,20 @@ tasks: - sharded_cluster-auth-ssl - sync - free-threaded - - name: test-server-version-python3.15-sync-auth-ssl-sharded-cluster + - name: test-server-version-python3.15-sync-auth-ssl-sharded-cluster-cov commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster + COVERAGE: "1" UV_PYTHON: "3.15" TEST_NAME: default_sync tags: @@ -3256,6 +3280,7 @@ tasks: - python-3.15 - sharded_cluster-auth-ssl - sync + - pr - name: test-server-version-python3.15t-sync-auth-ssl-sharded-cluster commands: - func: run server @@ -3457,25 +3482,27 @@ tasks: - standalone-noauth-nossl - async - pr - - name: test-server-version-python3.15-sync-noauth-nossl-standalone-cov + - name: test-server-version-python3.11-sync-noauth-nossl-standalone-min-deps-cov commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone + TEST_MIN_DEPS: "1" COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone + TEST_MIN_DEPS: "1" COVERAGE: "1" - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - server-version - - python-3.15 + - python-3.11 - standalone-noauth-nossl - sync - pr @@ -3722,52 +3749,6 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - async - - name: test-standard-v5.0-python3.15-async-auth-ssl-sharded-cluster - commands: - - func: run server - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: "5.0" - - func: run tests - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: "5.0" - UV_PYTHON: "3.15" - TEST_NAME: default_async - tags: - - test-standard - - server-5.0 - - python-3.15 - - sharded_cluster-auth-ssl - - async - - name: test-standard-v5.0-python3.11-async-noauth-nossl-standalone-min-deps - commands: - - func: run server - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: "5.0" - TEST_MIN_DEPS: "1" - - func: run tests - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: "5.0" - TEST_MIN_DEPS: "1" - UV_PYTHON: "3.11" - TEST_NAME: default_async - tags: - - test-standard - - server-5.0 - - python-3.11 - - standalone-noauth-nossl - - async - name: test-standard-v5.0-python3.14-async-noauth-nossl-standalone commands: - func: run server @@ -3858,29 +3839,6 @@ tasks: - python-3.12 - sharded_cluster-auth-ssl - sync - - name: test-standard-v6.0-python3.15t-sync-auth-ssl-sharded-cluster - commands: - - func: run server - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: "6.0" - - func: run tests - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: "6.0" - UV_PYTHON: 3.15t - TEST_NAME: default_sync - tags: - - test-standard - - server-6.0 - - python-3.15t - - sharded_cluster-auth-ssl - - sync - - free-threaded - name: test-standard-v6.0-python3.14t-sync-noauth-nossl-standalone commands: - func: run server @@ -3904,30 +3862,6 @@ tasks: - standalone-noauth-nossl - sync - free-threaded - - name: test-standard-v7.0-python3.11-async-noauth-ssl-replica-set-min-deps - commands: - - func: run server - vars: - AUTH: noauth - SSL: ssl - TOPOLOGY: replica_set - VERSION: "7.0" - TEST_MIN_DEPS: "1" - - func: run tests - vars: - AUTH: noauth - SSL: ssl - TOPOLOGY: replica_set - VERSION: "7.0" - TEST_MIN_DEPS: "1" - UV_PYTHON: "3.11" - TEST_NAME: default_async - tags: - - test-standard - - server-7.0 - - python-3.11 - - replica_set-noauth-ssl - - async - name: test-standard-v7.0-python3.14-async-noauth-ssl-replica-set commands: - func: run server @@ -4154,30 +4088,6 @@ tasks: - replica_set-noauth-ssl - async - free-threaded - - name: test-standard-v9.0-python3.11-async-auth-ssl-sharded-cluster-min-deps - commands: - - func: run server - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: "9.0" - TEST_MIN_DEPS: "1" - - func: run tests - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: "9.0" - TEST_MIN_DEPS: "1" - UV_PYTHON: "3.11" - TEST_NAME: default_async - tags: - - test-standard - - server-9.0 - - python-3.11 - - sharded_cluster-auth-ssl - - async - name: test-standard-v9.0-python3.14-async-auth-ssl-sharded-cluster commands: - func: run server @@ -4222,29 +4132,32 @@ tasks: - python-3.13 - standalone-noauth-nossl - async - - name: test-standard-v9.0-python3.15-async-noauth-nossl-standalone + - name: test-standard-latest-python3.11-async-noauth-ssl-replica-set-min-deps commands: - func: run server vars: AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: "9.0" + SSL: ssl + TOPOLOGY: replica_set + VERSION: latest + TEST_MIN_DEPS: "1" - func: run tests vars: AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: "9.0" - UV_PYTHON: "3.15" + SSL: ssl + TOPOLOGY: replica_set + VERSION: latest + TEST_MIN_DEPS: "1" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard - - server-9.0 - - python-3.15 - - standalone-noauth-nossl + - server-latest + - python-3.11 + - replica_set-noauth-ssl - async - - name: test-standard-latest-python3.13-async-noauth-ssl-replica-set-cov + - pr + - name: test-standard-latest-python3.13-async-noauth-ssl-replica-set commands: - func: run server vars: @@ -4252,14 +4165,12 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set VERSION: latest - COVERAGE: "1" UV_PYTHON: "3.13" TEST_NAME: default_async tags: @@ -4268,7 +4179,6 @@ tasks: - python-3.13 - replica_set-noauth-ssl - async - - pr - name: test-standard-latest-python3.15-async-noauth-ssl-replica-set-cov commands: - func: run server @@ -4294,7 +4204,32 @@ tasks: - replica_set-noauth-ssl - async - pr - - name: test-standard-latest-python3.14t-async-auth-ssl-sharded-cluster-cov + - name: test-standard-latest-python3.11-async-auth-ssl-sharded-cluster-min-deps + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + TEST_MIN_DEPS: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + TEST_MIN_DEPS: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_async + tags: + - test-standard + - server-latest + - python-3.11 + - sharded_cluster-auth-ssl + - async + - pr + - name: test-standard-latest-python3.14t-async-auth-ssl-sharded-cluster commands: - func: run server vars: @@ -4302,14 +4237,12 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest - COVERAGE: "1" UV_PYTHON: 3.14t TEST_NAME: default_async tags: @@ -4319,8 +4252,83 @@ tasks: - sharded_cluster-auth-ssl - async - free-threaded + - name: test-standard-latest-python3.15-async-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_async + tags: + - test-standard + - server-latest + - python-3.15 + - sharded_cluster-auth-ssl + - async + - pr + - name: test-standard-latest-python3.15t-sync-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + COVERAGE: "1" + UV_PYTHON: 3.15t + TEST_NAME: default_sync + tags: + - test-standard + - server-latest + - python-3.15t + - sharded_cluster-auth-ssl + - sync + - free-threaded - pr - - name: test-standard-latest-python3.12-async-noauth-nossl-standalone-cov + - name: test-standard-latest-python3.11-async-noauth-nossl-standalone-min-deps + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: latest + TEST_MIN_DEPS: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: latest + TEST_MIN_DEPS: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_async + tags: + - test-standard + - server-latest + - python-3.11 + - standalone-noauth-nossl + - async + - pr + - name: test-standard-latest-python3.12-async-noauth-nossl-standalone commands: - func: run server vars: @@ -4328,14 +4336,12 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone VERSION: latest - COVERAGE: "1" UV_PYTHON: "3.12" TEST_NAME: default_async tags: @@ -4344,8 +4350,7 @@ tasks: - python-3.12 - standalone-noauth-nossl - async - - pr - - name: test-standard-latest-python3.15t-async-noauth-nossl-standalone-cov + - name: test-standard-latest-python3.15-async-noauth-nossl-standalone-cov commands: - func: run server vars: @@ -4361,6 +4366,29 @@ tasks: TOPOLOGY: standalone VERSION: latest COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_async + tags: + - test-standard + - server-latest + - python-3.15 + - standalone-noauth-nossl + - async + - pr + - name: test-standard-latest-python3.15t-async-noauth-nossl-standalone + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: latest + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: latest UV_PYTHON: 3.15t TEST_NAME: default_async tags: @@ -4370,7 +4398,6 @@ tasks: - standalone-noauth-nossl - async - free-threaded - - pr - name: test-standard-rapid-python3.13-sync-noauth-ssl-replica-set commands: - func: run server @@ -4918,7 +4945,7 @@ tasks: - python-3.12 - replica_set-noauth-ssl - noauth - - name: test-non-standard-latest-python3.13-auth-ssl-sharded-cluster-cov + - name: test-non-standard-latest-python3.13-auth-ssl-sharded-cluster commands: - func: run server vars: @@ -4926,14 +4953,12 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest - COVERAGE: "1" UV_PYTHON: "3.13" tags: - test-non-standard @@ -4941,9 +4966,32 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - auth + - name: test-non-standard-latest-python3.15-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + COVERAGE: "1" + UV_PYTHON: "3.15" + tags: + - test-non-standard + - server-latest + - python-3.15 + - sharded_cluster-auth-ssl + - auth - pr - cov - - name: test-non-standard-latest-python3.13-auth-ssl-sharded-cluster + - name: test-non-standard-latest-python3.15-auth-ssl-sharded-cluster commands: - func: run server vars: @@ -4957,11 +5005,11 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest - UV_PYTHON: "3.13" + UV_PYTHON: "3.15" tags: - test-non-standard-no-cov - server-latest - - python-3.13 + - python-3.15 - sharded_cluster-auth-ssl - auth - pr @@ -5232,6 +5280,30 @@ tasks: - sharded_cluster-auth-ssl - auth - pr + - name: test-standard-auth-latest-python3.15-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + COVERAGE: "1" + UV_PYTHON: "3.15" + tags: + - test-standard-auth + - server-latest + - python-3.15 + - sharded_cluster-auth-ssl + - auth + - pr - name: test-standard-auth-rapid-python3.15-auth-ssl-sharded-cluster commands: - func: run server diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index c7c7cd3be5..335af57dbe 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -12,8 +12,11 @@ C_EXTS, CPYTHONS, DEFAULT_HOST, + FREE_THREADED_PYTHON, HOSTS, + MAX_PYTHON, MIN_MAX_PYTHON, + MIN_PYTHON, OTHER_HOSTS, SYNCS, TOPOLOGIES, @@ -647,10 +650,20 @@ def create_aws_lambda_variants(): def create_server_version_tasks(): tasks = [] task_combos = set() + # The canonical PR combos, pinned to the Python boundaries: the min + # Python on standalone (min deps), the max Python on replica set and + # sharded cluster (coverage). The pins re-bind instead of reshuffling + # when a CPython is added or removed. + pr_pythons = { + ("standalone", "noauth", "nossl"): MIN_PYTHON, + ("replica_set", "noauth", "nossl"): MAX_PYTHON, + ("sharded_cluster", "auth", "ssl"): MAX_PYTHON, + } # All combinations of topology, auth, ssl, and sync should be tested. - for (topology, auth, ssl, sync), python in zip_cycle( + for (topology, auth, ssl, sync), rotated in zip_cycle( list(product(TOPOLOGIES, ["auth", "noauth"], ["ssl", "nossl"], SYNCS)), ALL_PYTHONS ): + python = pr_pythons.get((topology, auth, ssl), rotated) task_combos.add((topology, auth, ssl, sync, python)) # Every python should be tested with sharded cluster, auth, ssl, with sync and async. @@ -658,19 +671,11 @@ def create_server_version_tasks(): task_combos.add(("sharded_cluster", "auth", "ssl", sync, python)) # Assemble the tasks. - seen = set() for topology, auth, ssl, sync, python in sorted(task_combos): combo = f"{topology}-{auth}-{ssl}" tags = ["server-version", f"python-{python}", combo, sync] - if combo in [ - "standalone-noauth-nossl", - "replica_set-noauth-nossl", - "sharded_cluster-auth-ssl", - ]: - combo = f"{combo}-{sync}" - if combo not in seen: - seen.add(combo) - tags.append("pr") + if python == pr_pythons.get((topology, auth, ssl)): + tags.append("pr") expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" @@ -719,9 +724,11 @@ def create_test_non_standard_tasks(): # For each version and topology, rotate through the CPythons. # Only the sharded_cluster topology runs on PRs to keep patch build size manageable. for (version, topology), python in zip_cycle(list(product(ALL_VERSIONS, TOPOLOGIES)), CPYTHONS): - pr = version == "latest" and topology == "sharded_cluster" - task_combos.add((version, topology, python, pr)) - for version, topology, python, pr in sorted(task_combos): + task_combos.add((version, topology, python)) + # The pinned PR task: the max Python on the latest server, sharded cluster. + task_combos.add(("latest", "sharded_cluster", MAX_PYTHON)) + for version, topology, python in sorted(task_combos): + pr = version == "latest" and topology == "sharded_cluster" and python == MAX_PYTHON auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-non-standard", @@ -805,9 +812,12 @@ def create_test_standard_auth_tasks(): for (version, topology), python in zip_cycle( list(product(ALL_VERSIONS, ["sharded_cluster"])), CPYTHONS ): - pr = version == "latest" - task_combos.add((version, topology, python, pr)) - for version, topology, python, pr in sorted(task_combos): + task_combos.add((version, topology, python)) + # The pinned PR tasks: the min and max Python on the latest server. + task_combos.add(("latest", "sharded_cluster", MIN_PYTHON)) + task_combos.add(("latest", "sharded_cluster", MAX_PYTHON)) + for version, topology, python in sorted(task_combos): + pr = version == "latest" and python in (MIN_PYTHON, MAX_PYTHON) auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-standard-auth", @@ -839,10 +849,15 @@ def create_standard_tasks(): tasks = [] task_combos = set() # For each python and topology and sync/async, rotate through the the versions. - for (python, topology, sync), version in zip_cycle( + for (python, topology, sync), rotated in zip_cycle( list(product(CPYTHONS, TOPOLOGIES, SYNCS)), ALL_VERSIONS ): - pr = version == "latest" + # PR tasks are pinned to the Python boundaries on the latest server: + # the min and max Python on async, plus one free-threaded task. + pr = (python in (MIN_PYTHON, MAX_PYTHON) and sync == "async") or ( + python == FREE_THREADED_PYTHON and topology == "sharded_cluster" and sync == "sync" + ) + version = "latest" if pr else rotated task_combos.add((version, topology, python, sync, pr)) for version, topology, python, sync, pr in sorted(task_combos): @@ -923,7 +938,10 @@ def create_aws_tasks(): "web-identity", ] assume_func = FunctionCall(func="assume ec2 role") - for version, test_type, python in zip_cycle(get_versions_from("4.4"), aws_test_types, CPYTHONS): + task_combos = set(zip_cycle(get_versions_from("4.4"), aws_test_types, CPYTHONS)) + # The pinned PR tasks: the min and max Python on the latest server. + task_combos |= {("latest", "regular", MIN_PYTHON), ("latest", "regular", MAX_PYTHON)} + for version, test_type, python in sorted(task_combos): base_name = f"test-auth-aws-{version}" base_tags = ["auth-aws"] server_vars = dict(AUTH_AWS="1", VERSION=version) @@ -932,11 +950,15 @@ def create_aws_tasks(): if "t" in python: tags.append("free-threaded") test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, UV_PYTHON=python) + # PR tasks are pinned to the Python boundaries on the latest server; + # the rotation never produces (latest, regular). + pr = version == "latest" and test_type == "regular" and python in (MIN_PYTHON, MAX_PYTHON) if python == MIN_MAX_PYTHON[0]: test_vars["TEST_MIN_DEPS"] = "1" - elif python == MIN_MAX_PYTHON[-1]: + if pr: tags.append("pr") - test_vars["COVERAGE"] = "1" + if "TEST_MIN_DEPS" not in test_vars: + test_vars["COVERAGE"] = "1" name = get_task_name(f"{base_name}-{test_type}", **test_vars) test_func = FunctionCall(func="run tests", vars=test_vars) funcs = [server_func, assume_func, test_func] diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index 18be347d56..b67602ef88 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -26,6 +26,11 @@ MIN_SUPPORT_VERSIONS = ["3.11"] ALL_PYTHONS = CPYTHONS MIN_MAX_PYTHON = [CPYTHONS[0], CPYTHONS[-1]] +# The Python boundaries that PR tasks are pinned to: adding or removing a +# CPython re-binds the same pins instead of reshuffling the PR set. +MIN_PYTHON = CPYTHONS[0] +MAX_PYTHON = CPYTHONS[-1] +FREE_THREADED_PYTHON = [p for p in CPYTHONS if "t" in p][-1] BATCHTIME_WEEK = 10080 BATCHTIME_DAY = 1440 AUTH_SSLS = [("auth", "ssl"), ("noauth", "ssl"), ("noauth", "nossl")] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1b6a12605..cc0958a180 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -479,6 +479,12 @@ CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at the setup scripts write a `uv.toml` boundary into the submodule checkout that stops uv's config discovery (and with it the required-version check) at the submodule boundary. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. +- PR tasks are pinned to the Python boundaries, not to the rotation: the min Python + (`MIN_PYTHON`, `TEST_MIN_DEPS=1`), the max Python (`MAX_PYTHON`, `COVERAGE=1`), and one + free-threaded task (`FREE_THREADED_PYTHON`), each against the `latest` server. The pins + derive from `CPYTHONS` in `generate_config_utils.py`, so adding or removing a CPython + re-binds the same PR tasks instead of reshuffling them. Review the regenerated diff of + `generated_configs/tasks.yml` to confirm the `pr` tags still land on the boundary Pythons. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. ## Handling flaky tests From f83f11960d31dd0dede3d150b8d86ad664c804fd Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 06:32:28 -0500 Subject: [PATCH 02/13] PYTHON-6112 Trim comments in Evergreen config generator --- .evergreen/scripts/generate_config.py | 11 +++-------- .evergreen/scripts/generate_config_utils.py | 4 ++-- CONTRIBUTING.md | 11 +++++------ 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 335af57dbe..ce1a342f75 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -650,10 +650,7 @@ def create_aws_lambda_variants(): def create_server_version_tasks(): tasks = [] task_combos = set() - # The canonical PR combos, pinned to the Python boundaries: the min - # Python on standalone (min deps), the max Python on replica set and - # sharded cluster (coverage). The pins re-bind instead of reshuffling - # when a CPython is added or removed. + # PR tasks pin to the Python boundaries. pr_pythons = { ("standalone", "noauth", "nossl"): MIN_PYTHON, ("replica_set", "noauth", "nossl"): MAX_PYTHON, @@ -852,8 +849,7 @@ def create_standard_tasks(): for (python, topology, sync), rotated in zip_cycle( list(product(CPYTHONS, TOPOLOGIES, SYNCS)), ALL_VERSIONS ): - # PR tasks are pinned to the Python boundaries on the latest server: - # the min and max Python on async, plus one free-threaded task. + # PR tasks pin to the Python boundaries on the latest server. pr = (python in (MIN_PYTHON, MAX_PYTHON) and sync == "async") or ( python == FREE_THREADED_PYTHON and topology == "sharded_cluster" and sync == "sync" ) @@ -950,8 +946,7 @@ def create_aws_tasks(): if "t" in python: tags.append("free-threaded") test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, UV_PYTHON=python) - # PR tasks are pinned to the Python boundaries on the latest server; - # the rotation never produces (latest, regular). + # The rotation never produces (latest, regular). pr = version == "latest" and test_type == "regular" and python in (MIN_PYTHON, MAX_PYTHON) if python == MIN_MAX_PYTHON[0]: test_vars["TEST_MIN_DEPS"] = "1" diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index b67602ef88..4883f8de66 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -26,8 +26,8 @@ MIN_SUPPORT_VERSIONS = ["3.11"] ALL_PYTHONS = CPYTHONS MIN_MAX_PYTHON = [CPYTHONS[0], CPYTHONS[-1]] -# The Python boundaries that PR tasks are pinned to: adding or removing a -# CPython re-binds the same pins instead of reshuffling the PR set. +# The Python boundaries PR tasks pin to: adding or removing a CPython +# re-binds the same pins instead of reshuffling the PR set. MIN_PYTHON = CPYTHONS[0] MAX_PYTHON = CPYTHONS[-1] FREE_THREADED_PYTHON = [p for p in CPYTHONS if "t" in p][-1] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cc0958a180..72b03226a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -479,12 +479,11 @@ CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at the setup scripts write a `uv.toml` boundary into the submodule checkout that stops uv's config discovery (and with it the required-version check) at the submodule boundary. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. -- PR tasks are pinned to the Python boundaries, not to the rotation: the min Python - (`MIN_PYTHON`, `TEST_MIN_DEPS=1`), the max Python (`MAX_PYTHON`, `COVERAGE=1`), and one - free-threaded task (`FREE_THREADED_PYTHON`), each against the `latest` server. The pins - derive from `CPYTHONS` in `generate_config_utils.py`, so adding or removing a CPython - re-binds the same PR tasks instead of reshuffling them. Review the regenerated diff of - `generated_configs/tasks.yml` to confirm the `pr` tags still land on the boundary Pythons. +- PR tasks pin to the Python boundaries: the min Python (`MIN_PYTHON`, `TEST_MIN_DEPS=1`), the max + Python (`MAX_PYTHON`, `COVERAGE=1`), and one free-threaded task (`FREE_THREADED_PYTHON`), each + against the `latest` server. The pins derive from `CPYTHONS` in `generate_config_utils.py`, so + adding or removing a CPython re-binds them instead of reshuffling the PR set. Review the + regenerated `generated_configs/tasks.yml` diff to confirm the `pr` tags land on the boundary Pythons. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. ## Handling flaky tests From 74cd4b5c8a8bcdc419078e60a2a827c845265478 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 06:41:55 -0500 Subject: [PATCH 03/13] PYTHON-6112 Pin PR tasks to latest-stable and cover min-deps --- .evergreen/generated_configs/tasks.yml | 321 +++++++++++--------- .evergreen/scripts/generate_config.py | 72 ++--- .evergreen/scripts/generate_config_utils.py | 2 +- .evergreen/scripts/test_generate_config.py | 50 +++ CONTRIBUTING.md | 5 +- 5 files changed, 267 insertions(+), 183 deletions(-) create mode 100644 .evergreen/scripts/test_generate_config.py diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index b1f31b0ce2..cdf8a72106 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -116,12 +116,12 @@ tasks: UV_PYTHON: "3.11" TEST_MIN_DEPS: "1" tags: [auth-aws, auth-aws-assume-role] - - name: test-auth-aws-latest-regular-python3.11-min-deps + - name: test-auth-aws-latest-stable-regular-python3.11-min-deps-cov commands: - func: run server vars: AUTH_AWS: "1" - VERSION: latest + VERSION: latest-stable - func: assume ec2 role - func: run tests vars: @@ -129,13 +129,14 @@ tasks: SUB_TEST_NAME: regular UV_PYTHON: "3.11" TEST_MIN_DEPS: "1" + COVERAGE: "1" tags: [auth-aws, auth-aws-regular, pr] - - name: test-auth-aws-latest-regular-python3.15-cov + - name: test-auth-aws-latest-stable-regular-python3.15-cov commands: - func: run server vars: AUTH_AWS: "1" - VERSION: latest + VERSION: latest-stable - func: assume ec2 role - func: run tests vars: @@ -781,7 +782,7 @@ tasks: UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - - name: test-ocsp-ecdsa-valid-cert-server-staples-v4.4-python3.11-min-deps + - name: test-ocsp-ecdsa-valid-cert-server-staples-v4.4-python3.11-min-deps-cov commands: - func: run tests vars: @@ -791,6 +792,7 @@ tasks: UV_PYTHON: "3.11" VERSION: "4.4" TEST_MIN_DEPS: "1" + COVERAGE: "1" tags: - ocsp - ocsp-ecdsa @@ -887,7 +889,7 @@ tasks: - ocsp-ecdsa - rapid - ocsp-staple - - name: test-ocsp-ecdsa-valid-cert-server-staples-latest-python3.15-cov + - name: test-ocsp-ecdsa-valid-cert-server-staples-latest-stable-python3.15-cov commands: - func: run tests vars: @@ -895,12 +897,12 @@ tasks: OCSP_SERVER_TYPE: valid TEST_NAME: ocsp UV_PYTHON: "3.15" - VERSION: latest + VERSION: latest-stable COVERAGE: "1" tags: - ocsp - ocsp-ecdsa - - latest + - latest-stable - ocsp-staple - pr - name: test-ocsp-ecdsa-invalid-cert-server-staples-v4.4-python3.11-min-deps @@ -4132,7 +4134,7 @@ tasks: - python-3.13 - standalone-noauth-nossl - async - - name: test-standard-latest-python3.11-async-noauth-ssl-replica-set-min-deps + - name: test-standard-latest-python3.13-async-noauth-ssl-replica-set commands: - func: run server vars: @@ -4140,264 +4142,270 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest - TEST_MIN_DEPS: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set VERSION: latest - TEST_MIN_DEPS: "1" - UV_PYTHON: "3.11" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - test-standard - server-latest - - python-3.11 + - python-3.13 - replica_set-noauth-ssl - async - - pr - - name: test-standard-latest-python3.13-async-noauth-ssl-replica-set + - name: test-standard-latest-python3.14t-async-auth-ssl-sharded-cluster commands: - func: run server vars: - AUTH: noauth + AUTH: auth SSL: ssl - TOPOLOGY: replica_set + TOPOLOGY: sharded_cluster VERSION: latest - func: run tests vars: - AUTH: noauth + AUTH: auth SSL: ssl - TOPOLOGY: replica_set + TOPOLOGY: sharded_cluster VERSION: latest - UV_PYTHON: "3.13" + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - test-standard - server-latest - - python-3.13 - - replica_set-noauth-ssl + - python-3.14t + - sharded_cluster-auth-ssl + - async + - free-threaded + - name: test-standard-latest-python3.12-async-noauth-nossl-standalone + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: latest + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: latest + UV_PYTHON: "3.12" + TEST_NAME: default_async + tags: + - test-standard + - server-latest + - python-3.12 + - standalone-noauth-nossl + - async + - name: test-standard-latest-python3.15t-async-noauth-nossl-standalone + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: latest + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: latest + UV_PYTHON: 3.15t + TEST_NAME: default_async + tags: + - test-standard + - server-latest + - python-3.15t + - standalone-noauth-nossl - async - - name: test-standard-latest-python3.15-async-noauth-ssl-replica-set-cov + - free-threaded + - name: test-standard-latest-stable-python3.11-async-noauth-ssl-replica-set-min-deps-cov commands: - func: run server vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - VERSION: latest + VERSION: latest-stable + TEST_MIN_DEPS: "1" COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - VERSION: latest + VERSION: latest-stable + TEST_MIN_DEPS: "1" COVERAGE: "1" - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard - - server-latest - - python-3.15 + - server-latest-stable + - python-3.11 - replica_set-noauth-ssl - async - pr - - name: test-standard-latest-python3.11-async-auth-ssl-sharded-cluster-min-deps + - name: test-standard-latest-stable-python3.15-async-noauth-ssl-replica-set-cov commands: - func: run server vars: - AUTH: auth + AUTH: noauth SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest - TEST_MIN_DEPS: "1" + TOPOLOGY: replica_set + VERSION: latest-stable + COVERAGE: "1" - func: run tests vars: - AUTH: auth + AUTH: noauth SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest - TEST_MIN_DEPS: "1" - UV_PYTHON: "3.11" + TOPOLOGY: replica_set + VERSION: latest-stable + COVERAGE: "1" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard - - server-latest - - python-3.11 - - sharded_cluster-auth-ssl + - server-latest-stable + - python-3.15 + - replica_set-noauth-ssl - async - pr - - name: test-standard-latest-python3.14t-async-auth-ssl-sharded-cluster + - name: test-standard-latest-stable-python3.11-async-auth-ssl-sharded-cluster-min-deps-cov commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest + VERSION: latest-stable + TEST_MIN_DEPS: "1" + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest - UV_PYTHON: 3.14t + VERSION: latest-stable + TEST_MIN_DEPS: "1" + COVERAGE: "1" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard - - server-latest - - python-3.14t + - server-latest-stable + - python-3.11 - sharded_cluster-auth-ssl - async - - free-threaded - - name: test-standard-latest-python3.15-async-auth-ssl-sharded-cluster-cov + - pr + - name: test-standard-latest-stable-python3.15-async-auth-ssl-sharded-cluster-cov commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest + VERSION: latest-stable COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest + VERSION: latest-stable COVERAGE: "1" UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard - - server-latest + - server-latest-stable - python-3.15 - sharded_cluster-auth-ssl - async - pr - - name: test-standard-latest-python3.15t-sync-auth-ssl-sharded-cluster-cov + - name: test-standard-latest-stable-python3.15t-sync-auth-ssl-sharded-cluster-cov commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest + VERSION: latest-stable COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest + VERSION: latest-stable COVERAGE: "1" UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - test-standard - - server-latest + - server-latest-stable - python-3.15t - sharded_cluster-auth-ssl - sync - free-threaded - pr - - name: test-standard-latest-python3.11-async-noauth-nossl-standalone-min-deps + - name: test-standard-latest-stable-python3.11-async-noauth-nossl-standalone-min-deps-cov commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone - VERSION: latest + VERSION: latest-stable TEST_MIN_DEPS: "1" + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone - VERSION: latest + VERSION: latest-stable TEST_MIN_DEPS: "1" + COVERAGE: "1" UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard - - server-latest + - server-latest-stable - python-3.11 - standalone-noauth-nossl - async - pr - - name: test-standard-latest-python3.12-async-noauth-nossl-standalone - commands: - - func: run server - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: latest - - func: run tests - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: latest - UV_PYTHON: "3.12" - TEST_NAME: default_async - tags: - - test-standard - - server-latest - - python-3.12 - - standalone-noauth-nossl - - async - - name: test-standard-latest-python3.15-async-noauth-nossl-standalone-cov + - name: test-standard-latest-stable-python3.15-async-noauth-nossl-standalone-cov commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone - VERSION: latest + VERSION: latest-stable COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone - VERSION: latest + VERSION: latest-stable COVERAGE: "1" UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard - - server-latest + - server-latest-stable - python-3.15 - standalone-noauth-nossl - async - pr - - name: test-standard-latest-python3.15t-async-noauth-nossl-standalone - commands: - - func: run server - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: latest - - func: run tests - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: latest - UV_PYTHON: 3.15t - TEST_NAME: default_async - tags: - - test-standard - - server-latest - - python-3.15t - - standalone-noauth-nossl - - async - - free-threaded - name: test-standard-rapid-python3.13-sync-noauth-ssl-replica-set commands: - func: run server @@ -4966,76 +4974,76 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - auth - - name: test-non-standard-latest-python3.15-auth-ssl-sharded-cluster-cov + - name: test-non-standard-latest-python3.11-noauth-nossl-standalone-min-deps + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: latest + TEST_MIN_DEPS: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: latest + TEST_MIN_DEPS: "1" + UV_PYTHON: "3.11" + tags: + - test-non-standard + - server-latest + - python-3.11 + - standalone-noauth-nossl + - noauth + - name: test-non-standard-latest-stable-python3.15-auth-ssl-sharded-cluster-cov commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest + VERSION: latest-stable COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest + VERSION: latest-stable COVERAGE: "1" UV_PYTHON: "3.15" tags: - test-non-standard - - server-latest + - server-latest-stable - python-3.15 - sharded_cluster-auth-ssl - auth - pr - cov - - name: test-non-standard-latest-python3.15-auth-ssl-sharded-cluster + - name: test-non-standard-latest-stable-python3.15-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest + VERSION: latest-stable - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest + VERSION: latest-stable UV_PYTHON: "3.15" tags: - test-non-standard-no-cov - - server-latest + - server-latest-stable - python-3.15 - sharded_cluster-auth-ssl - auth - pr - - name: test-non-standard-latest-python3.11-noauth-nossl-standalone-min-deps - commands: - - func: run server - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: latest - TEST_MIN_DEPS: "1" - - func: run tests - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: latest - TEST_MIN_DEPS: "1" - UV_PYTHON: "3.11" - tags: - - test-non-standard - - server-latest - - python-3.11 - - standalone-noauth-nossl - - noauth - name: test-non-standard-rapid-python3.15t-noauth-ssl-replica-set commands: - func: run server @@ -5279,27 +5287,52 @@ tasks: - python-3.11 - sharded_cluster-auth-ssl - auth + - name: test-standard-auth-latest-stable-python3.11-auth-ssl-sharded-cluster-min-deps-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest-stable + TEST_MIN_DEPS: "1" + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest-stable + TEST_MIN_DEPS: "1" + COVERAGE: "1" + UV_PYTHON: "3.11" + tags: + - test-standard-auth + - server-latest-stable + - python-3.11 + - sharded_cluster-auth-ssl + - auth - pr - - name: test-standard-auth-latest-python3.15-auth-ssl-sharded-cluster-cov + - name: test-standard-auth-latest-stable-python3.15-auth-ssl-sharded-cluster-cov commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest + VERSION: latest-stable COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest + VERSION: latest-stable COVERAGE: "1" UV_PYTHON: "3.15" tags: - test-standard-auth - - server-latest + - server-latest-stable - python-3.15 - sharded_cluster-auth-ssl - auth diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index ce1a342f75..a8feeec98d 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -722,10 +722,10 @@ def create_test_non_standard_tasks(): # Only the sharded_cluster topology runs on PRs to keep patch build size manageable. for (version, topology), python in zip_cycle(list(product(ALL_VERSIONS, TOPOLOGIES)), CPYTHONS): task_combos.add((version, topology, python)) - # The pinned PR task: the max Python on the latest server, sharded cluster. - task_combos.add(("latest", "sharded_cluster", MAX_PYTHON)) + # The pinned PR task: the max Python on the latest stable server, sharded cluster. + task_combos.add(("latest-stable", "sharded_cluster", MAX_PYTHON)) for version, topology, python in sorted(task_combos): - pr = version == "latest" and topology == "sharded_cluster" and python == MAX_PYTHON + pr = version == "latest-stable" auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-non-standard", @@ -810,11 +810,11 @@ def create_test_standard_auth_tasks(): list(product(ALL_VERSIONS, ["sharded_cluster"])), CPYTHONS ): task_combos.add((version, topology, python)) - # The pinned PR tasks: the min and max Python on the latest server. - task_combos.add(("latest", "sharded_cluster", MIN_PYTHON)) - task_combos.add(("latest", "sharded_cluster", MAX_PYTHON)) + # The pinned PR tasks: the min and max Python on the latest stable server. + task_combos.add(("latest-stable", "sharded_cluster", MIN_PYTHON)) + task_combos.add(("latest-stable", "sharded_cluster", MAX_PYTHON)) for version, topology, python in sorted(task_combos): - pr = version == "latest" and python in (MIN_PYTHON, MAX_PYTHON) + pr = version == "latest-stable" auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-standard-auth", @@ -830,7 +830,7 @@ def create_test_standard_auth_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" - elif pr: + if pr: expansions["COVERAGE"] = "1" name = get_task_name("test-standard-auth", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) @@ -849,11 +849,11 @@ def create_standard_tasks(): for (python, topology, sync), rotated in zip_cycle( list(product(CPYTHONS, TOPOLOGIES, SYNCS)), ALL_VERSIONS ): - # PR tasks pin to the Python boundaries on the latest server. + # PR tasks pin to the Python boundaries on the latest stable server. pr = (python in (MIN_PYTHON, MAX_PYTHON) and sync == "async") or ( python == FREE_THREADED_PYTHON and topology == "sharded_cluster" and sync == "sync" ) - version = "latest" if pr else rotated + version = "latest-stable" if pr else rotated task_combos.add((version, topology, python, sync, pr)) for version, topology, python, sync, pr in sorted(task_combos): @@ -872,7 +872,7 @@ def create_standard_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" - elif pr: + if pr: expansions["COVERAGE"] = "1" name = get_task_name("test-standard", python=python, sync=sync, **expansions) server_func = FunctionCall(func="run server", vars=expansions) @@ -935,8 +935,11 @@ def create_aws_tasks(): ] assume_func = FunctionCall(func="assume ec2 role") task_combos = set(zip_cycle(get_versions_from("4.4"), aws_test_types, CPYTHONS)) - # The pinned PR tasks: the min and max Python on the latest server. - task_combos |= {("latest", "regular", MIN_PYTHON), ("latest", "regular", MAX_PYTHON)} + # The pinned PR tasks: the min and max Python on the latest stable server. + task_combos |= { + ("latest-stable", "regular", MIN_PYTHON), + ("latest-stable", "regular", MAX_PYTHON), + } for version, test_type, python in sorted(task_combos): base_name = f"test-auth-aws-{version}" base_tags = ["auth-aws"] @@ -946,14 +949,13 @@ def create_aws_tasks(): if "t" in python: tags.append("free-threaded") test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, UV_PYTHON=python) - # The rotation never produces (latest, regular). - pr = version == "latest" and test_type == "regular" and python in (MIN_PYTHON, MAX_PYTHON) + # The rotation never produces (latest-stable, regular). + pr = version == "latest-stable" and test_type == "regular" if python == MIN_MAX_PYTHON[0]: test_vars["TEST_MIN_DEPS"] = "1" if pr: tags.append("pr") - if "TEST_MIN_DEPS" not in test_vars: - test_vars["COVERAGE"] = "1" + test_vars["COVERAGE"] = "1" name = get_task_name(f"{base_name}-{test_type}", **test_vars) test_func = FunctionCall(func="run tests", vars=test_vars) funcs = [server_func, assume_func, test_func] @@ -1006,7 +1008,19 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): file_name = f"{algo}-basic-tls-ocsp-{variant}.json" for version in get_versions_from("4.4"): - if version == "latest": + # Run exactly one min-deps and one latest-CPython stapled OCSP task on + # every PR (ecdsa only, to avoid doubling coverage across algorithms) + # so a cryptography-backend regression at either dependency extreme, + # like the min-deps ML-KEM breakage in PYTHON-6032, is caught before + # merge. The newest-server PR task downloads latest-stable; the + # "latest" nightly needs AWS credentials. + pr = ( + base_task_name == "valid-cert-server-staples" + and algo == "ecdsa" + and version in ("latest", "4.4") + ) + server_version = "latest-stable" if pr and version == "latest" else version + if server_version in ("latest", "latest-stable"): python = MIN_MAX_PYTHON[-1] else: python = MIN_MAX_PYTHON[0] @@ -1016,30 +1030,16 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): OCSP_SERVER_TYPE=server_type, TEST_NAME="ocsp", UV_PYTHON=python, - VERSION=version, + VERSION=server_version, ) if python == ALL_PYTHONS[0]: vars["TEST_MIN_DEPS"] = "1" - tags = ["ocsp", f"ocsp-{algo}", version] + tags = ["ocsp", f"ocsp-{algo}", server_version] if "disableStapling" not in variant: tags.append("ocsp-staple") - # Run exactly one min-deps and one latest-CPython stapled OCSP task on - # every PR (ecdsa only, to avoid doubling coverage across algorithms) - # so a cryptography-backend regression at either dependency extreme, - # like the min-deps ML-KEM breakage in PYTHON-6032, is caught before - # merge instead of on the next weekly mainline batch run. - if ( - base_task_name == "valid-cert-server-staples" - and algo == "ecdsa" - and version - in ( - "latest", - "4.4", - ) - ): + if pr: tags.append("pr") - if "TEST_MIN_DEPS" not in vars: - vars["COVERAGE"] = "1" + vars["COVERAGE"] = "1" test_func = FunctionCall(func="run tests", vars=vars) task_name = get_task_name(f"test-ocsp-{algo}-{base_task_name}", **vars) tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index 4883f8de66..0a47a4aafa 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -177,7 +177,7 @@ def get_common_name(base: str, sep: str, **kwargs) -> str: version = kwargs.pop("VERSION", None) version = version or kwargs.pop("version", None) if version: - if version not in ["rapid", "latest"]: + if version not in ["rapid", "latest", "latest-stable"]: version = f"v{version}" display_name = f"{display_name}{sep}{version}" for key, value in kwargs.items(): diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py new file mode 100644 index 0000000000..602db57662 --- /dev/null +++ b/.evergreen/scripts/test_generate_config.py @@ -0,0 +1,50 @@ +"""Tests for the Evergreen PR task rules (PYTHON-6112). + +Run: pytest .evergreen/scripts/test_generate_config.py -m "" +(-m "" overrides the repo-wide default marker filter.) +Requires the same dependencies as the generate-config hook: shrub.py, pyyaml. +""" + +from __future__ import annotations + +import sys +from inspect import getmembers, isfunction +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +import generate_config + + +def all_tasks(): + tasks = [] + for name, func in getmembers(generate_config, isfunction): + if name.startswith("create_") and name.endswith("_tasks"): + tasks.extend(func()) + return tasks + + +def task_vars(task): + vars = {} + for command in task.commands or []: + vars.update(getattr(command, "vars", None) or {}) + return vars + + +def pr_tasks(): + return [task for task in all_tasks() if "pr" in (task.tags or [])] + + +def test_pr_tasks_run_on_latest_stable(): + """PR tasks must not download the "latest" nightly: it needs AWS credentials.""" + offenders = [task.name for task in pr_tasks() if task_vars(task).get("VERSION") == "latest"] + assert not offenders, f"PR tasks pinned to 'latest': {offenders}" + + +def test_pr_min_deps_tasks_also_run_coverage(): + offenders = [ + task.name + for task in pr_tasks() + if task_vars(task).get("TEST_MIN_DEPS") == "1" and task_vars(task).get("COVERAGE") != "1" + ] + assert not offenders, f"min-deps PR tasks without COVERAGE: {offenders}" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 72b03226a2..722c5d8c29 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -480,8 +480,9 @@ CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at (and with it the required-version check) at the submodule boundary. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. - PR tasks pin to the Python boundaries: the min Python (`MIN_PYTHON`, `TEST_MIN_DEPS=1`), the max - Python (`MAX_PYTHON`, `COVERAGE=1`), and one free-threaded task (`FREE_THREADED_PYTHON`), each - against the `latest` server. The pins derive from `CPYTHONS` in `generate_config_utils.py`, so + Python (`MAX_PYTHON`), and one free-threaded task (`FREE_THREADED_PYTHON`), each against the + `latest-stable` server (the `latest` nightly needs AWS credentials to download). Every PR task + runs with `COVERAGE=1`. The pins derive from `CPYTHONS` in `generate_config_utils.py`, so adding or removing a CPython re-binds them instead of reshuffling the PR set. Review the regenerated `generated_configs/tasks.yml` diff to confirm the `pr` tags land on the boundary Pythons. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. From 38b8ad3788585409b7bbdf06c70173fa987e9570 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 08:05:10 -0500 Subject: [PATCH 04/13] PYTHON-6112 Run the generator unit tests in the generate-config hook Also corrects the comments' rationale for latest-stable: the "latest" nightly is unstable, not merely credentials-gated. --- .evergreen/scripts/generate-config.sh | 3 +++ .evergreen/scripts/generate_config.py | 2 +- .evergreen/scripts/test_generate_config.py | 2 +- .pre-commit-config.yaml | 2 +- CONTRIBUTING.md | 4 ++-- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.evergreen/scripts/generate-config.sh b/.evergreen/scripts/generate-config.sh index 70b4578cf9..3b85467d6f 100755 --- a/.evergreen/scripts/generate-config.sh +++ b/.evergreen/scripts/generate-config.sh @@ -3,4 +3,7 @@ set -eu +# Assert the generation rules before regenerating the config from them. +python -m pytest .evergreen/scripts/test_generate_config.py -m "" + python .evergreen/scripts/generate_config.py diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index a8feeec98d..dc515358ac 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -1013,7 +1013,7 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): # so a cryptography-backend regression at either dependency extreme, # like the min-deps ML-KEM breakage in PYTHON-6032, is caught before # merge. The newest-server PR task downloads latest-stable; the - # "latest" nightly needs AWS credentials. + # "latest" nightly is unstable. pr = ( base_task_name == "valid-cert-server-staples" and algo == "ecdsa" diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index 602db57662..95d898cf2a 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -36,7 +36,7 @@ def pr_tasks(): def test_pr_tasks_run_on_latest_stable(): - """PR tasks must not download the "latest" nightly: it needs AWS credentials.""" + """PR tasks must pin to latest-stable: the "latest" nightly is unstable.""" offenders = [task.name for task in pr_tasks() if task_vars(task).get("VERSION") == "latest"] assert not offenders, f"PR tasks pinned to 'latest': {offenders}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d15ce194d4..d4ad8c6317 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -134,4 +134,4 @@ repos: entry: .evergreen/scripts/generate-config.sh language: python require_serial: true - additional_dependencies: ["shrub.py>=3.10.0", "pyyaml>=6.0.2"] + additional_dependencies: ["pytest", "pytest-asyncio", "shrub.py>=3.10.0", "pyyaml>=6.0.2"] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 722c5d8c29..cee645aea9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -478,10 +478,10 @@ CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at the vendored `drivers-evergreen-tools` submodule: those scripts run uv versions this project does not pin, so the setup scripts write a `uv.toml` boundary into the submodule checkout that stops uv's config discovery (and with it the required-version check) at the submodule boundary. -- Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. +- Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`; the hook also runs the generation unit tests first. - PR tasks pin to the Python boundaries: the min Python (`MIN_PYTHON`, `TEST_MIN_DEPS=1`), the max Python (`MAX_PYTHON`), and one free-threaded task (`FREE_THREADED_PYTHON`), each against the - `latest-stable` server (the `latest` nightly needs AWS credentials to download). Every PR task + `latest-stable` server (the `latest` nightly is unstable). Every PR task runs with `COVERAGE=1`. The pins derive from `CPYTHONS` in `generate_config_utils.py`, so adding or removing a CPython re-binds them instead of reshuffling the PR set. Review the regenerated `generated_configs/tasks.yml` diff to confirm the `pr` tags land on the boundary Pythons. From cc2a5e9f39aa6250c286c50acd961fb60b0216a9 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 08:15:49 -0500 Subject: [PATCH 05/13] clean up contrib docs --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cee645aea9..fab838b9f0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -478,7 +478,7 @@ CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at the vendored `drivers-evergreen-tools` submodule: those scripts run uv versions this project does not pin, so the setup scripts write a `uv.toml` boundary into the submodule checkout that stops uv's config discovery (and with it the required-version check) at the submodule boundary. -- Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`; the hook also runs the generation unit tests first. +- Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. - PR tasks pin to the Python boundaries: the min Python (`MIN_PYTHON`, `TEST_MIN_DEPS=1`), the max Python (`MAX_PYTHON`), and one free-threaded task (`FREE_THREADED_PYTHON`), each against the `latest-stable` server (the `latest` nightly is unstable). Every PR task From f09263067b6b4b168684527fc621821bf0bb599a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 08:26:18 -0500 Subject: [PATCH 06/13] PYTHON-6112 Keep No C Ext and Compression PR runs at baseline --- .evergreen/generated_configs/tasks.yml | 196 ++++++++++----------- .evergreen/scripts/generate_config.py | 10 +- .evergreen/scripts/test_generate_config.py | 18 ++ 3 files changed, 117 insertions(+), 107 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index cdf8a72106..de8a3affb5 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -3751,6 +3751,28 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - async + - name: test-standard-v5.0-python3.15-async-auth-ssl-sharded-cluster + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "5.0" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "5.0" + UV_PYTHON: "3.15" + TEST_NAME: default_async + tags: + - test-standard + - server-5.0 + - python-3.15 + - sharded_cluster-auth-ssl + - async - name: test-standard-v5.0-python3.14-async-noauth-nossl-standalone commands: - func: run server @@ -3864,6 +3886,30 @@ tasks: - standalone-noauth-nossl - sync - free-threaded + - name: test-standard-v7.0-python3.11-async-noauth-ssl-replica-set-min-deps + commands: + - func: run server + vars: + AUTH: noauth + SSL: ssl + TOPOLOGY: replica_set + VERSION: "7.0" + TEST_MIN_DEPS: "1" + - func: run tests + vars: + AUTH: noauth + SSL: ssl + TOPOLOGY: replica_set + VERSION: "7.0" + TEST_MIN_DEPS: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_async + tags: + - test-standard + - server-7.0 + - python-3.11 + - replica_set-noauth-ssl + - async - name: test-standard-v7.0-python3.14-async-noauth-ssl-replica-set commands: - func: run server @@ -4090,6 +4136,30 @@ tasks: - replica_set-noauth-ssl - async - free-threaded + - name: test-standard-v9.0-python3.11-async-auth-ssl-sharded-cluster-min-deps + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + TEST_MIN_DEPS: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + TEST_MIN_DEPS: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_async + tags: + - test-standard + - server-9.0 + - python-3.11 + - sharded_cluster-auth-ssl + - async - name: test-standard-v9.0-python3.14-async-auth-ssl-sharded-cluster commands: - func: run server @@ -4134,6 +4204,28 @@ tasks: - python-3.13 - standalone-noauth-nossl - async + - name: test-standard-v9.0-python3.15-async-noauth-nossl-standalone + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: "9.0" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: "9.0" + UV_PYTHON: "3.15" + TEST_NAME: default_async + tags: + - test-standard + - server-9.0 + - python-3.15 + - standalone-noauth-nossl + - async - name: test-standard-latest-python3.13-async-noauth-ssl-replica-set commands: - func: run server @@ -4224,33 +4316,6 @@ tasks: - standalone-noauth-nossl - async - free-threaded - - name: test-standard-latest-stable-python3.11-async-noauth-ssl-replica-set-min-deps-cov - commands: - - func: run server - vars: - AUTH: noauth - SSL: ssl - TOPOLOGY: replica_set - VERSION: latest-stable - TEST_MIN_DEPS: "1" - COVERAGE: "1" - - func: run tests - vars: - AUTH: noauth - SSL: ssl - TOPOLOGY: replica_set - VERSION: latest-stable - TEST_MIN_DEPS: "1" - COVERAGE: "1" - UV_PYTHON: "3.11" - TEST_NAME: default_async - tags: - - test-standard - - server-latest-stable - - python-3.11 - - replica_set-noauth-ssl - - async - - pr - name: test-standard-latest-stable-python3.15-async-noauth-ssl-replica-set-cov commands: - func: run server @@ -4276,58 +4341,6 @@ tasks: - replica_set-noauth-ssl - async - pr - - name: test-standard-latest-stable-python3.11-async-auth-ssl-sharded-cluster-min-deps-cov - commands: - - func: run server - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest-stable - TEST_MIN_DEPS: "1" - COVERAGE: "1" - - func: run tests - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest-stable - TEST_MIN_DEPS: "1" - COVERAGE: "1" - UV_PYTHON: "3.11" - TEST_NAME: default_async - tags: - - test-standard - - server-latest-stable - - python-3.11 - - sharded_cluster-auth-ssl - - async - - pr - - name: test-standard-latest-stable-python3.15-async-auth-ssl-sharded-cluster-cov - commands: - - func: run server - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest-stable - COVERAGE: "1" - - func: run tests - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest-stable - COVERAGE: "1" - UV_PYTHON: "3.15" - TEST_NAME: default_async - tags: - - test-standard - - server-latest-stable - - python-3.15 - - sharded_cluster-auth-ssl - - async - - pr - name: test-standard-latest-stable-python3.15t-sync-auth-ssl-sharded-cluster-cov commands: - func: run server @@ -4381,31 +4394,6 @@ tasks: - standalone-noauth-nossl - async - pr - - name: test-standard-latest-stable-python3.15-async-noauth-nossl-standalone-cov - commands: - - func: run server - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: latest-stable - COVERAGE: "1" - - func: run tests - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: latest-stable - COVERAGE: "1" - UV_PYTHON: "3.15" - TEST_NAME: default_async - tags: - - test-standard - - server-latest-stable - - python-3.15 - - standalone-noauth-nossl - - async - - pr - name: test-standard-rapid-python3.13-sync-noauth-ssl-replica-set commands: - func: run server diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index dc515358ac..3eaf216b9b 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -849,9 +849,13 @@ def create_standard_tasks(): for (python, topology, sync), rotated in zip_cycle( list(product(CPYTHONS, TOPOLOGIES, SYNCS)), ALL_VERSIONS ): - # PR tasks pin to the Python boundaries on the latest stable server. - pr = (python in (MIN_PYTHON, MAX_PYTHON) and sync == "async") or ( - python == FREE_THREADED_PYTHON and topology == "sharded_cluster" and sync == "sync" + # PR tasks pin to the Python boundaries on the latest stable server: + # min Python on standalone, max Python on replica set, and the + # free-threaded Python on sharded cluster. + pr = ( + (python == MIN_PYTHON and topology == "standalone" and sync == "async") + or (python == MAX_PYTHON and topology == "replica_set" and sync == "async") + or (python == FREE_THREADED_PYTHON and topology == "sharded_cluster" and sync == "sync") ) version = "latest-stable" if pr else rotated task_combos.add((version, topology, python, sync, pr)) diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index 95d898cf2a..d166f1dac9 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -48,3 +48,21 @@ def test_pr_min_deps_tasks_also_run_coverage(): if task_vars(task).get("TEST_MIN_DEPS") == "1" and task_vars(task).get("COVERAGE") != "1" ] assert not offenders, f"min-deps PR tasks without COVERAGE: {offenders}" + + +def test_standard_family_pr_budget(): + """The No C Ext and Compression variants select .test-standard, so their + patch builds run one PR task per pr-tagged test-standard task. Keep the + count at or below the pre-pin baseline (5), and at or below 3 for the + ubuntu22 compression variant, which filters to Python 3.14 and newer.""" + tasks = [task for task in pr_tasks() if "test-standard" in (task.tags or [])] + assert len(tasks) <= 5, f"too many test-standard PR tasks: {[t.name for t in tasks]}" + newer = [ + task + for task in tasks + if any( + tag in (task.tags or []) + for tag in ("python-3.14", "python-3.14t", "python-3.15", "python-3.15t") + ) + ] + assert len(newer) <= 3, f"too many 3.14+ test-standard PR tasks: {[t.name for t in newer]}" From 17bc6d60265871a6fbee68984d702b6e6ed90564 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 08:54:07 -0500 Subject: [PATCH 07/13] PYTHON-6112 Pin PR tasks to the newest numbered server version --- .evergreen/generated_configs/tasks.yml | 496 ++++++++++---------- .evergreen/scripts/generate_config.py | 40 +- .evergreen/scripts/generate_config_utils.py | 4 +- .evergreen/scripts/test_generate_config.py | 18 +- CONTRIBUTING.md | 2 +- 5 files changed, 276 insertions(+), 284 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index de8a3affb5..24bb2bac05 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -75,7 +75,7 @@ tasks: SUB_TEST_NAME: session-creds UV_PYTHON: "3.14" tags: [auth-aws, auth-aws-session-creds] - - name: test-auth-aws-9.0-web-identity-python3.15t + - name: test-auth-aws-9.0-regular-python3.11-min-deps-cov commands: - func: run server vars: @@ -85,10 +85,12 @@ tasks: - func: run tests vars: TEST_NAME: auth_aws - SUB_TEST_NAME: web-identity - UV_PYTHON: 3.15t - tags: [auth-aws, auth-aws-web-identity, free-threaded] - - name: test-auth-aws-9.0-web-identity-session-name-python3.15t + SUB_TEST_NAME: regular + UV_PYTHON: "3.11" + TEST_MIN_DEPS: "1" + COVERAGE: "1" + tags: [auth-aws, auth-aws-regular, pr] + - name: test-auth-aws-9.0-regular-python3.15-cov commands: - func: run server vars: @@ -98,53 +100,51 @@ tasks: - func: run tests vars: TEST_NAME: auth_aws - SUB_TEST_NAME: web-identity - AWS_ROLE_SESSION_NAME: test - UV_PYTHON: 3.15t - tags: [auth-aws, auth-aws-web-identity, free-threaded] - - name: test-auth-aws-latest-assume-role-python3.11-min-deps + SUB_TEST_NAME: regular + UV_PYTHON: "3.15" + COVERAGE: "1" + tags: [auth-aws, auth-aws-regular, pr] + - name: test-auth-aws-9.0-web-identity-python3.15t commands: - func: run server vars: AUTH_AWS: "1" - VERSION: latest + VERSION: "9.0" - func: assume ec2 role - func: run tests vars: TEST_NAME: auth_aws - SUB_TEST_NAME: assume-role - UV_PYTHON: "3.11" - TEST_MIN_DEPS: "1" - tags: [auth-aws, auth-aws-assume-role] - - name: test-auth-aws-latest-stable-regular-python3.11-min-deps-cov + SUB_TEST_NAME: web-identity + UV_PYTHON: 3.15t + tags: [auth-aws, auth-aws-web-identity, free-threaded] + - name: test-auth-aws-9.0-web-identity-session-name-python3.15t commands: - func: run server vars: AUTH_AWS: "1" - VERSION: latest-stable + VERSION: "9.0" - func: assume ec2 role - func: run tests vars: TEST_NAME: auth_aws - SUB_TEST_NAME: regular - UV_PYTHON: "3.11" - TEST_MIN_DEPS: "1" - COVERAGE: "1" - tags: [auth-aws, auth-aws-regular, pr] - - name: test-auth-aws-latest-stable-regular-python3.15-cov + SUB_TEST_NAME: web-identity + AWS_ROLE_SESSION_NAME: test + UV_PYTHON: 3.15t + tags: [auth-aws, auth-aws-web-identity, free-threaded] + - name: test-auth-aws-latest-assume-role-python3.11-min-deps commands: - func: run server vars: AUTH_AWS: "1" - VERSION: latest-stable + VERSION: latest - func: assume ec2 role - func: run tests vars: TEST_NAME: auth_aws - SUB_TEST_NAME: regular - UV_PYTHON: "3.15" - COVERAGE: "1" - tags: [auth-aws, auth-aws-regular, pr] + SUB_TEST_NAME: assume-role + UV_PYTHON: "3.11" + TEST_MIN_DEPS: "1" + tags: [auth-aws, auth-aws-assume-role] - name: test-auth-aws-rapid-regular-python3.15 commands: - func: run server @@ -402,16 +402,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-valid-cert-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-valid-cert-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-valid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -489,16 +488,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-invalid-cert-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-invalid-cert-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-invalid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -576,16 +574,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-delegate-valid-cert-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-delegate-valid-cert-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-delegate-valid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -663,16 +660,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-delegate-invalid-cert-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-delegate-invalid-cert-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-delegate-invalid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -750,16 +746,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-soft-fail-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-soft-fail-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-soft-fail-rapid-python3.11-min-deps commands: @@ -859,16 +854,15 @@ tasks: - ocsp-ecdsa - "8.0" - ocsp-staple - - name: test-ocsp-ecdsa-valid-cert-server-staples-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-valid-cert-server-staples-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-ecdsa @@ -889,7 +883,7 @@ tasks: - ocsp-ecdsa - rapid - ocsp-staple - - name: test-ocsp-ecdsa-valid-cert-server-staples-latest-stable-python3.15-cov + - name: test-ocsp-ecdsa-valid-cert-server-staples-v9.0-python3.15-cov commands: - func: run tests vars: @@ -897,12 +891,12 @@ tasks: OCSP_SERVER_TYPE: valid TEST_NAME: ocsp UV_PYTHON: "3.15" - VERSION: latest-stable + VERSION: "9.0" COVERAGE: "1" tags: - ocsp - ocsp-ecdsa - - latest-stable + - "9.0" - ocsp-staple - pr - name: test-ocsp-ecdsa-invalid-cert-server-staples-v4.4-python3.11-min-deps @@ -980,16 +974,15 @@ tasks: - ocsp-ecdsa - "8.0" - ocsp-staple - - name: test-ocsp-ecdsa-invalid-cert-server-staples-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-invalid-cert-server-staples-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-ecdsa @@ -1099,16 +1092,15 @@ tasks: - ocsp-ecdsa - "8.0" - ocsp-staple - - name: test-ocsp-ecdsa-delegate-valid-cert-server-staples-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-delegate-valid-cert-server-staples-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-ecdsa @@ -1218,16 +1210,15 @@ tasks: - ocsp-ecdsa - "8.0" - ocsp-staple - - name: test-ocsp-ecdsa-delegate-invalid-cert-server-staples-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-delegate-invalid-cert-server-staples-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-ecdsa @@ -1317,16 +1308,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-malicious-invalid-cert-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1404,16 +1394,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1491,16 +1480,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-malicious-no-responder-muststaple-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-ecdsa-malicious-no-responder-muststaple-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-malicious-no-responder-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1578,16 +1566,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-valid-cert-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-valid-cert-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-valid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1665,16 +1652,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-invalid-cert-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-invalid-cert-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-invalid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1752,16 +1738,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-delegate-valid-cert-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-delegate-valid-cert-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-delegate-valid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1839,16 +1824,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-delegate-invalid-cert-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-delegate-invalid-cert-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-delegate-invalid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1926,16 +1910,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-soft-fail-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-soft-fail-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-soft-fail-rapid-python3.11-min-deps commands: @@ -2033,16 +2016,15 @@ tasks: - ocsp-rsa - "8.0" - ocsp-staple - - name: test-ocsp-rsa-valid-cert-server-staples-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-valid-cert-server-staples-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-rsa @@ -2152,16 +2134,15 @@ tasks: - ocsp-rsa - "8.0" - ocsp-staple - - name: test-ocsp-rsa-invalid-cert-server-staples-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-invalid-cert-server-staples-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-rsa @@ -2271,16 +2252,15 @@ tasks: - ocsp-rsa - "8.0" - ocsp-staple - - name: test-ocsp-rsa-delegate-valid-cert-server-staples-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-delegate-valid-cert-server-staples-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-rsa @@ -2390,16 +2370,15 @@ tasks: - ocsp-rsa - "8.0" - ocsp-staple - - name: test-ocsp-rsa-delegate-invalid-cert-server-staples-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-delegate-invalid-cert-server-staples-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-rsa @@ -2489,16 +2468,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-malicious-invalid-cert-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -2576,16 +2554,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -2663,16 +2640,15 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-malicious-no-responder-muststaple-server-does-not-staple-v9.0-python3.11-min-deps + - name: test-ocsp-rsa-malicious-no-responder-muststaple-server-does-not-staple-v9.0-python3.15 commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - UV_PYTHON: "3.11" + UV_PYTHON: "3.15" VERSION: "9.0" - TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-malicious-no-responder-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -4136,6 +4112,31 @@ tasks: - replica_set-noauth-ssl - async - free-threaded + - name: test-standard-v9.0-python3.15-async-noauth-ssl-replica-set-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: ssl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: ssl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_async + tags: + - test-standard + - server-9.0 + - python-3.15 + - replica_set-noauth-ssl + - async + - pr - name: test-standard-v9.0-python3.11-async-auth-ssl-sharded-cluster-min-deps commands: - func: run server @@ -4182,6 +4183,59 @@ tasks: - python-3.14 - sharded_cluster-auth-ssl - async + - name: test-standard-v9.0-python3.15t-sync-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: 3.15t + TEST_NAME: default_sync + tags: + - test-standard + - server-9.0 + - python-3.15t + - sharded_cluster-auth-ssl + - sync + - free-threaded + - pr + - name: test-standard-v9.0-python3.11-async-noauth-nossl-standalone-min-deps-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: "9.0" + TEST_MIN_DEPS: "1" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: "9.0" + TEST_MIN_DEPS: "1" + COVERAGE: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_async + tags: + - test-standard + - server-9.0 + - python-3.11 + - standalone-noauth-nossl + - async + - pr - name: test-standard-v9.0-python3.13-async-noauth-nossl-standalone commands: - func: run server @@ -4316,84 +4370,6 @@ tasks: - standalone-noauth-nossl - async - free-threaded - - name: test-standard-latest-stable-python3.15-async-noauth-ssl-replica-set-cov - commands: - - func: run server - vars: - AUTH: noauth - SSL: ssl - TOPOLOGY: replica_set - VERSION: latest-stable - COVERAGE: "1" - - func: run tests - vars: - AUTH: noauth - SSL: ssl - TOPOLOGY: replica_set - VERSION: latest-stable - COVERAGE: "1" - UV_PYTHON: "3.15" - TEST_NAME: default_async - tags: - - test-standard - - server-latest-stable - - python-3.15 - - replica_set-noauth-ssl - - async - - pr - - name: test-standard-latest-stable-python3.15t-sync-auth-ssl-sharded-cluster-cov - commands: - - func: run server - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest-stable - COVERAGE: "1" - - func: run tests - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest-stable - COVERAGE: "1" - UV_PYTHON: 3.15t - TEST_NAME: default_sync - tags: - - test-standard - - server-latest-stable - - python-3.15t - - sharded_cluster-auth-ssl - - sync - - free-threaded - - pr - - name: test-standard-latest-stable-python3.11-async-noauth-nossl-standalone-min-deps-cov - commands: - - func: run server - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: latest-stable - TEST_MIN_DEPS: "1" - COVERAGE: "1" - - func: run tests - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - VERSION: latest-stable - TEST_MIN_DEPS: "1" - COVERAGE: "1" - UV_PYTHON: "3.11" - TEST_NAME: default_async - tags: - - test-standard - - server-latest-stable - - python-3.11 - - standalone-noauth-nossl - - async - - pr - name: test-standard-rapid-python3.13-sync-noauth-ssl-replica-set commands: - func: run server @@ -4899,6 +4875,53 @@ tasks: - sharded_cluster-auth-ssl - auth - free-threaded + - name: test-non-standard-v9.0-python3.15-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + tags: + - test-non-standard + - server-9.0 + - python-3.15 + - sharded_cluster-auth-ssl + - auth + - pr + - cov + - name: test-non-standard-v9.0-python3.15-auth-ssl-sharded-cluster + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + UV_PYTHON: "3.15" + tags: + - test-non-standard-no-cov + - server-9.0 + - python-3.15 + - sharded_cluster-auth-ssl + - auth + - pr - name: test-non-standard-v9.0-python3.12-noauth-nossl-standalone commands: - func: run server @@ -4985,53 +5008,6 @@ tasks: - python-3.11 - standalone-noauth-nossl - noauth - - name: test-non-standard-latest-stable-python3.15-auth-ssl-sharded-cluster-cov - commands: - - func: run server - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest-stable - COVERAGE: "1" - - func: run tests - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest-stable - COVERAGE: "1" - UV_PYTHON: "3.15" - tags: - - test-non-standard - - server-latest-stable - - python-3.15 - - sharded_cluster-auth-ssl - - auth - - pr - - cov - - name: test-non-standard-latest-stable-python3.15-auth-ssl-sharded-cluster - commands: - - func: run server - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest-stable - - func: run tests - vars: - AUTH: auth - SSL: ssl - TOPOLOGY: sharded_cluster - VERSION: latest-stable - UV_PYTHON: "3.15" - tags: - - test-non-standard-no-cov - - server-latest-stable - - python-3.15 - - sharded_cluster-auth-ssl - - auth - - pr - name: test-non-standard-rapid-python3.15t-noauth-ssl-replica-set commands: - func: run server @@ -5230,7 +5206,7 @@ tasks: - python-3.14 - sharded_cluster-auth-ssl - auth - - name: test-standard-auth-v9.0-python3.15t-auth-ssl-sharded-cluster + - name: test-standard-auth-v9.0-python3.11-auth-ssl-sharded-cluster-min-deps-cov commands: - func: run server vars: @@ -5238,93 +5214,93 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" + TEST_MIN_DEPS: "1" + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - UV_PYTHON: 3.15t + TEST_MIN_DEPS: "1" + COVERAGE: "1" + UV_PYTHON: "3.11" tags: - test-standard-auth - server-9.0 - - python-3.15t + - python-3.11 - sharded_cluster-auth-ssl - auth - - free-threaded - - name: test-standard-auth-latest-python3.11-auth-ssl-sharded-cluster-min-deps + - pr + - name: test-standard-auth-v9.0-python3.15-auth-ssl-sharded-cluster-cov commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest - TEST_MIN_DEPS: "1" + VERSION: "9.0" + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest - TEST_MIN_DEPS: "1" - UV_PYTHON: "3.11" + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" tags: - test-standard-auth - - server-latest - - python-3.11 + - server-9.0 + - python-3.15 - sharded_cluster-auth-ssl - auth - - name: test-standard-auth-latest-stable-python3.11-auth-ssl-sharded-cluster-min-deps-cov + - pr + - name: test-standard-auth-v9.0-python3.15t-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest-stable - TEST_MIN_DEPS: "1" - COVERAGE: "1" + VERSION: "9.0" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest-stable - TEST_MIN_DEPS: "1" - COVERAGE: "1" - UV_PYTHON: "3.11" + VERSION: "9.0" + UV_PYTHON: 3.15t tags: - test-standard-auth - - server-latest-stable - - python-3.11 + - server-9.0 + - python-3.15t - sharded_cluster-auth-ssl - auth - - pr - - name: test-standard-auth-latest-stable-python3.15-auth-ssl-sharded-cluster-cov + - free-threaded + - name: test-standard-auth-latest-python3.11-auth-ssl-sharded-cluster-min-deps commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest-stable - COVERAGE: "1" + VERSION: latest + TEST_MIN_DEPS: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - VERSION: latest-stable - COVERAGE: "1" - UV_PYTHON: "3.15" + VERSION: latest + TEST_MIN_DEPS: "1" + UV_PYTHON: "3.11" tags: - test-standard-auth - - server-latest-stable - - python-3.15 + - server-latest + - python-3.11 - sharded_cluster-auth-ssl - auth - - pr - name: test-standard-auth-rapid-python3.15-auth-ssl-sharded-cluster commands: - func: run server diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 3eaf216b9b..763c9f2234 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -15,6 +15,7 @@ FREE_THREADED_PYTHON, HOSTS, MAX_PYTHON, + MAX_VERSION, MIN_MAX_PYTHON, MIN_PYTHON, OTHER_HOSTS, @@ -722,10 +723,11 @@ def create_test_non_standard_tasks(): # Only the sharded_cluster topology runs on PRs to keep patch build size manageable. for (version, topology), python in zip_cycle(list(product(ALL_VERSIONS, TOPOLOGIES)), CPYTHONS): task_combos.add((version, topology, python)) - # The pinned PR task: the max Python on the latest stable server, sharded cluster. - task_combos.add(("latest-stable", "sharded_cluster", MAX_PYTHON)) + # The pinned PR task: the max Python on the newest numbered server, + # sharded cluster. + task_combos.add((MAX_VERSION, "sharded_cluster", MAX_PYTHON)) for version, topology, python in sorted(task_combos): - pr = version == "latest-stable" + pr = version == MAX_VERSION and topology == "sharded_cluster" and python == MAX_PYTHON auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-non-standard", @@ -810,11 +812,11 @@ def create_test_standard_auth_tasks(): list(product(ALL_VERSIONS, ["sharded_cluster"])), CPYTHONS ): task_combos.add((version, topology, python)) - # The pinned PR tasks: the min and max Python on the latest stable server. - task_combos.add(("latest-stable", "sharded_cluster", MIN_PYTHON)) - task_combos.add(("latest-stable", "sharded_cluster", MAX_PYTHON)) + # The pinned PR tasks: the min and max Python on the newest numbered server. + task_combos.add((MAX_VERSION, "sharded_cluster", MIN_PYTHON)) + task_combos.add((MAX_VERSION, "sharded_cluster", MAX_PYTHON)) for version, topology, python in sorted(task_combos): - pr = version == "latest-stable" + pr = version == MAX_VERSION and python in (MIN_PYTHON, MAX_PYTHON) auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-standard-auth", @@ -849,7 +851,7 @@ def create_standard_tasks(): for (python, topology, sync), rotated in zip_cycle( list(product(CPYTHONS, TOPOLOGIES, SYNCS)), ALL_VERSIONS ): - # PR tasks pin to the Python boundaries on the latest stable server: + # PR tasks pin to the Python boundaries on the newest numbered server: # min Python on standalone, max Python on replica set, and the # free-threaded Python on sharded cluster. pr = ( @@ -857,7 +859,7 @@ def create_standard_tasks(): or (python == MAX_PYTHON and topology == "replica_set" and sync == "async") or (python == FREE_THREADED_PYTHON and topology == "sharded_cluster" and sync == "sync") ) - version = "latest-stable" if pr else rotated + version = MAX_VERSION if pr else rotated task_combos.add((version, topology, python, sync, pr)) for version, topology, python, sync, pr in sorted(task_combos): @@ -939,10 +941,10 @@ def create_aws_tasks(): ] assume_func = FunctionCall(func="assume ec2 role") task_combos = set(zip_cycle(get_versions_from("4.4"), aws_test_types, CPYTHONS)) - # The pinned PR tasks: the min and max Python on the latest stable server. + # The pinned PR tasks: the min and max Python on the newest numbered server. task_combos |= { - ("latest-stable", "regular", MIN_PYTHON), - ("latest-stable", "regular", MAX_PYTHON), + (MAX_VERSION, "regular", MIN_PYTHON), + (MAX_VERSION, "regular", MAX_PYTHON), } for version, test_type, python in sorted(task_combos): base_name = f"test-auth-aws-{version}" @@ -953,8 +955,10 @@ def create_aws_tasks(): if "t" in python: tags.append("free-threaded") test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, UV_PYTHON=python) - # The rotation never produces (latest-stable, regular). - pr = version == "latest-stable" and test_type == "regular" + # The rotation never produces (MAX_VERSION, regular). + pr = ( + version == MAX_VERSION and test_type == "regular" and python in (MIN_PYTHON, MAX_PYTHON) + ) if python == MIN_MAX_PYTHON[0]: test_vars["TEST_MIN_DEPS"] = "1" if pr: @@ -1016,15 +1020,15 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): # every PR (ecdsa only, to avoid doubling coverage across algorithms) # so a cryptography-backend regression at either dependency extreme, # like the min-deps ML-KEM breakage in PYTHON-6032, is caught before - # merge. The newest-server PR task downloads latest-stable; the - # "latest" nightly is unstable. + # merge. The newest-server PR task pins to the newest numbered server; + # the "latest" nightly is unstable. pr = ( base_task_name == "valid-cert-server-staples" and algo == "ecdsa" and version in ("latest", "4.4") ) - server_version = "latest-stable" if pr and version == "latest" else version - if server_version in ("latest", "latest-stable"): + server_version = MAX_VERSION if pr and version == "latest" else version + if server_version in ("latest", MAX_VERSION): python = MIN_MAX_PYTHON[-1] else: python = MIN_MAX_PYTHON[0] diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index 0a47a4aafa..1b49fbcb33 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -31,6 +31,8 @@ MIN_PYTHON = CPYTHONS[0] MAX_PYTHON = CPYTHONS[-1] FREE_THREADED_PYTHON = [p for p in CPYTHONS if "t" in p][-1] +# The newest numbered server version, which PR tasks pin to. +MAX_VERSION = max((v for v in ALL_VERSIONS if v[0].isdigit()), key=float) BATCHTIME_WEEK = 10080 BATCHTIME_DAY = 1440 AUTH_SSLS = [("auth", "ssl"), ("noauth", "ssl"), ("noauth", "nossl")] @@ -177,7 +179,7 @@ def get_common_name(base: str, sep: str, **kwargs) -> str: version = kwargs.pop("VERSION", None) version = version or kwargs.pop("version", None) if version: - if version not in ["rapid", "latest", "latest-stable"]: + if version not in ["rapid", "latest"]: version = f"v{version}" display_name = f"{display_name}{sep}{version}" for key, value in kwargs.items(): diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index d166f1dac9..0cfd7dc019 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).parent)) import generate_config +import generate_config_utils def all_tasks(): @@ -35,10 +36,19 @@ def pr_tasks(): return [task for task in all_tasks() if "pr" in (task.tags or [])] -def test_pr_tasks_run_on_latest_stable(): - """PR tasks must pin to latest-stable: the "latest" nightly is unstable.""" - offenders = [task.name for task in pr_tasks() if task_vars(task).get("VERSION") == "latest"] - assert not offenders, f"PR tasks pinned to 'latest': {offenders}" +def test_pr_tasks_pin_max_version(): + """PR tasks must pin to the newest numbered server version: the "latest" + nightly is unstable. The min-deps extreme is the exception: it pins to + the oldest numbered server (4.4) to match its oldest dependencies.""" + offenders = [ + (task.name, task_vars(task).get("VERSION")) + for task in pr_tasks() + if task_vars(task).get("VERSION") not in (None, generate_config_utils.MAX_VERSION) + and not ( + task_vars(task).get("VERSION") == "4.4" and task_vars(task).get("TEST_MIN_DEPS") == "1" + ) + ] + assert not offenders, f"PR tasks not pinned to MAX_VERSION: {offenders}" def test_pr_min_deps_tasks_also_run_coverage(): diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fab838b9f0..0f049eac22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -481,7 +481,7 @@ CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. - PR tasks pin to the Python boundaries: the min Python (`MIN_PYTHON`, `TEST_MIN_DEPS=1`), the max Python (`MAX_PYTHON`), and one free-threaded task (`FREE_THREADED_PYTHON`), each against the - `latest-stable` server (the `latest` nightly is unstable). Every PR task + newest numbered server (`MAX_VERSION`; the `latest` nightly is unstable). Every PR task runs with `COVERAGE=1`. The pins derive from `CPYTHONS` in `generate_config_utils.py`, so adding or removing a CPython re-binds them instead of reshuffling the PR set. Review the regenerated `generated_configs/tasks.yml` diff to confirm the `pr` tags land on the boundary Pythons. From c78d9cde535c763ec51285de37d9a53bcd86d33c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 09:59:34 -0500 Subject: [PATCH 08/13] PYTHON-6112 Preserve nightly batch coverage and rotation alongside the pinned PR tasks --- .evergreen/generated_configs/tasks.yml | 311 ++++++++++++++++---- .evergreen/generated_configs/variants.yml | 6 +- .evergreen/scripts/generate_config.py | 103 ++++--- .evergreen/scripts/generate_config_utils.py | 2 +- .evergreen/scripts/test_generate_config.py | 85 +++++- 5 files changed, 412 insertions(+), 95 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 24bb2bac05..455204ac14 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -145,7 +145,7 @@ tasks: UV_PYTHON: "3.11" TEST_MIN_DEPS: "1" tags: [auth-aws, auth-aws-assume-role] - - name: test-auth-aws-rapid-regular-python3.15 + - name: test-auth-aws-rapid-regular-python3.15-cov commands: - func: run server vars: @@ -157,6 +157,7 @@ tasks: TEST_NAME: auth_aws SUB_TEST_NAME: regular UV_PYTHON: "3.15" + COVERAGE: "1" tags: [auth-aws, auth-aws-regular] - name: test-auth-aws-ecs commands: @@ -402,15 +403,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-valid-cert-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-ecdsa-valid-cert-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-valid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -488,15 +490,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-invalid-cert-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-ecdsa-invalid-cert-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-invalid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -574,15 +577,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-delegate-valid-cert-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-ecdsa-delegate-valid-cert-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-delegate-valid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -660,15 +664,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-delegate-invalid-cert-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-ecdsa-delegate-invalid-cert-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-delegate-invalid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -746,15 +751,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-soft-fail-v9.0-python3.15 + - name: test-ocsp-ecdsa-soft-fail-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-soft-fail-rapid-python3.11-min-deps commands: @@ -854,15 +860,16 @@ tasks: - ocsp-ecdsa - "8.0" - ocsp-staple - - name: test-ocsp-ecdsa-valid-cert-server-staples-v9.0-python3.15 + - name: test-ocsp-ecdsa-valid-cert-server-staples-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-ecdsa @@ -883,6 +890,21 @@ tasks: - ocsp-ecdsa - rapid - ocsp-staple + - name: test-ocsp-ecdsa-valid-cert-server-staples-latest-python3.15-cov + commands: + - func: run tests + vars: + ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json + OCSP_SERVER_TYPE: valid + TEST_NAME: ocsp + UV_PYTHON: "3.15" + VERSION: latest + COVERAGE: "1" + tags: + - ocsp + - ocsp-ecdsa + - latest + - ocsp-staple - name: test-ocsp-ecdsa-valid-cert-server-staples-v9.0-python3.15-cov commands: - func: run tests @@ -974,15 +996,16 @@ tasks: - ocsp-ecdsa - "8.0" - ocsp-staple - - name: test-ocsp-ecdsa-invalid-cert-server-staples-v9.0-python3.15 + - name: test-ocsp-ecdsa-invalid-cert-server-staples-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-ecdsa @@ -1092,15 +1115,16 @@ tasks: - ocsp-ecdsa - "8.0" - ocsp-staple - - name: test-ocsp-ecdsa-delegate-valid-cert-server-staples-v9.0-python3.15 + - name: test-ocsp-ecdsa-delegate-valid-cert-server-staples-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-ecdsa @@ -1210,15 +1234,16 @@ tasks: - ocsp-ecdsa - "8.0" - ocsp-staple - - name: test-ocsp-ecdsa-delegate-invalid-cert-server-staples-v9.0-python3.15 + - name: test-ocsp-ecdsa-delegate-invalid-cert-server-staples-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-ecdsa @@ -1308,15 +1333,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-ecdsa-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-malicious-invalid-cert-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1394,15 +1420,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-ecdsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1480,15 +1507,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] - - name: test-ocsp-ecdsa-malicious-no-responder-muststaple-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-ecdsa-malicious-no-responder-muststaple-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] - name: test-ocsp-ecdsa-malicious-no-responder-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1566,15 +1594,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-valid-cert-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-rsa-valid-cert-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-valid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1652,15 +1681,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-invalid-cert-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-rsa-invalid-cert-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-invalid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1738,15 +1768,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-delegate-valid-cert-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-rsa-delegate-valid-cert-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-delegate-valid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1824,15 +1855,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-delegate-invalid-cert-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-rsa-delegate-invalid-cert-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-delegate-invalid-cert-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -1910,15 +1942,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-soft-fail-v9.0-python3.15 + - name: test-ocsp-rsa-soft-fail-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-soft-fail-rapid-python3.11-min-deps commands: @@ -2016,15 +2049,16 @@ tasks: - ocsp-rsa - "8.0" - ocsp-staple - - name: test-ocsp-rsa-valid-cert-server-staples-v9.0-python3.15 + - name: test-ocsp-rsa-valid-cert-server-staples-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-rsa @@ -2134,15 +2168,16 @@ tasks: - ocsp-rsa - "8.0" - ocsp-staple - - name: test-ocsp-rsa-invalid-cert-server-staples-v9.0-python3.15 + - name: test-ocsp-rsa-invalid-cert-server-staples-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-rsa @@ -2252,15 +2287,16 @@ tasks: - ocsp-rsa - "8.0" - ocsp-staple - - name: test-ocsp-rsa-delegate-valid-cert-server-staples-v9.0-python3.15 + - name: test-ocsp-rsa-delegate-valid-cert-server-staples-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-rsa @@ -2370,15 +2406,16 @@ tasks: - ocsp-rsa - "8.0" - ocsp-staple - - name: test-ocsp-rsa-delegate-invalid-cert-server-staples-v9.0-python3.15 + - name: test-ocsp-rsa-delegate-invalid-cert-server-staples-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: - ocsp - ocsp-rsa @@ -2468,15 +2505,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-rsa-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-malicious-invalid-cert-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -2554,15 +2592,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-rsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -2640,15 +2679,16 @@ tasks: VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] - - name: test-ocsp-rsa-malicious-no-responder-muststaple-server-does-not-staple-v9.0-python3.15 + - name: test-ocsp-rsa-malicious-no-responder-muststaple-server-does-not-staple-v9.0-python3.11-min-deps commands: - func: run tests vars: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - UV_PYTHON: "3.15" + UV_PYTHON: "3.11" VERSION: "9.0" + TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] - name: test-ocsp-rsa-malicious-no-responder-muststaple-server-does-not-staple-rapid-python3.11-min-deps commands: @@ -2877,6 +2917,25 @@ tasks: - python-3.12 - replica_set-auth-ssl - sync + - name: test-server-version-python3.12-async-noauth-nossl-replica-set + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + UV_PYTHON: "3.12" + TEST_NAME: default_async + tags: + - server-version + - python-3.12 + - replica_set-noauth-nossl + - async - name: test-server-version-python3.15-async-noauth-nossl-replica-set-cov commands: - func: run server @@ -2899,6 +2958,27 @@ tasks: - replica_set-noauth-nossl - async - pr + - name: test-server-version-python3.11-sync-noauth-nossl-replica-set-min-deps + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + TEST_MIN_DEPS: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + TEST_MIN_DEPS: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_sync + tags: + - server-version + - python-3.11 + - replica_set-noauth-nossl + - sync - name: test-server-version-python3.15-sync-noauth-nossl-replica-set-cov commands: - func: run server @@ -3484,6 +3564,25 @@ tasks: - standalone-noauth-nossl - sync - pr + - name: test-server-version-python3.15-sync-noauth-nossl-standalone + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + UV_PYTHON: "3.15" + TEST_NAME: default_sync + tags: + - server-version + - python-3.15 + - standalone-noauth-nossl + - sync - name: test-server-version-python3.15t-async-noauth-ssl-standalone commands: - func: run server @@ -3749,6 +3848,30 @@ tasks: - python-3.15 - sharded_cluster-auth-ssl - async + - name: test-standard-v5.0-python3.11-async-noauth-nossl-standalone-min-deps + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: "5.0" + TEST_MIN_DEPS: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: "5.0" + TEST_MIN_DEPS: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_async + tags: + - test-standard + - server-5.0 + - python-3.11 + - standalone-noauth-nossl + - async - name: test-standard-v5.0-python3.14-async-noauth-nossl-standalone commands: - func: run server @@ -3839,6 +3962,29 @@ tasks: - python-3.12 - sharded_cluster-auth-ssl - sync + - name: test-standard-v6.0-python3.15t-sync-auth-ssl-sharded-cluster + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "6.0" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "6.0" + UV_PYTHON: 3.15t + TEST_NAME: default_sync + tags: + - test-standard + - server-6.0 + - python-3.15t + - sharded_cluster-auth-ssl + - sync + - free-threaded - name: test-standard-v6.0-python3.14t-sync-noauth-nossl-standalone commands: - func: run server @@ -4280,7 +4426,7 @@ tasks: - python-3.15 - standalone-noauth-nossl - async - - name: test-standard-latest-python3.13-async-noauth-ssl-replica-set + - name: test-standard-latest-python3.13-async-noauth-ssl-replica-set-cov commands: - func: run server vars: @@ -4288,12 +4434,14 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set VERSION: latest + COVERAGE: "1" UV_PYTHON: "3.13" TEST_NAME: default_async tags: @@ -4302,7 +4450,31 @@ tasks: - python-3.13 - replica_set-noauth-ssl - async - - name: test-standard-latest-python3.14t-async-auth-ssl-sharded-cluster + - name: test-standard-latest-python3.15-async-noauth-ssl-replica-set-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: ssl + TOPOLOGY: replica_set + VERSION: latest + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: ssl + TOPOLOGY: replica_set + VERSION: latest + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_async + tags: + - test-standard + - server-latest + - python-3.15 + - replica_set-noauth-ssl + - async + - name: test-standard-latest-python3.14t-async-auth-ssl-sharded-cluster-cov commands: - func: run server vars: @@ -4310,12 +4482,14 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" UV_PYTHON: 3.14t TEST_NAME: default_async tags: @@ -4325,7 +4499,7 @@ tasks: - sharded_cluster-auth-ssl - async - free-threaded - - name: test-standard-latest-python3.12-async-noauth-nossl-standalone + - name: test-standard-latest-python3.12-async-noauth-nossl-standalone-cov commands: - func: run server vars: @@ -4333,12 +4507,14 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" UV_PYTHON: "3.12" TEST_NAME: default_async tags: @@ -4347,7 +4523,7 @@ tasks: - python-3.12 - standalone-noauth-nossl - async - - name: test-standard-latest-python3.15t-async-noauth-nossl-standalone + - name: test-standard-latest-python3.15t-async-noauth-nossl-standalone-cov commands: - func: run server vars: @@ -4355,12 +4531,14 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" UV_PYTHON: 3.15t TEST_NAME: default_async tags: @@ -4921,7 +5099,6 @@ tasks: - python-3.15 - sharded_cluster-auth-ssl - auth - - pr - name: test-non-standard-v9.0-python3.12-noauth-nossl-standalone commands: - func: run server @@ -4964,7 +5141,7 @@ tasks: - python-3.12 - replica_set-noauth-ssl - noauth - - name: test-non-standard-latest-python3.13-auth-ssl-sharded-cluster + - name: test-non-standard-latest-python3.13-auth-ssl-sharded-cluster-cov commands: - func: run server vars: @@ -4972,12 +5149,14 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" UV_PYTHON: "3.13" tags: - test-non-standard @@ -4985,6 +5164,28 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - auth + - cov + - name: test-non-standard-latest-python3.13-auth-ssl-sharded-cluster + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: latest + UV_PYTHON: "3.13" + tags: + - test-non-standard-no-cov + - server-latest + - python-3.13 + - sharded_cluster-auth-ssl + - auth - name: test-non-standard-latest-python3.11-noauth-nossl-standalone-min-deps commands: - func: run server diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 39cfcfe6d6..17b7db393c 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -91,13 +91,13 @@ buildvariants: run_on: - macos-14 tags: [pr] - - name: auth-aws-ecs-macos + - name: auth-aws-ecs-ubuntu-24 tasks: - name: .auth-aws-ecs - display_name: Auth AWS ECS macOS + display_name: Auth AWS ECS Ubuntu-24 run_on: - ubuntu2404-small - tags: [pr] + tags: [] # Aws lambda tests - name: faas-lambda diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 763c9f2234..4aeb7a07c0 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -586,12 +586,14 @@ def create_aws_auth_variants(): variants.append(variant) # The ECS test must be run on Ubuntu 24 to match the Fargate Config. + # Use fresh tags and host: the loop's macOS values must not leak into + # this variant. variant = create_variant( [".auth-aws-ecs"], - get_variant_name("Auth AWS ECS", host), + get_variant_name("Auth AWS ECS", HOSTS["ubuntu24"]), host=HOSTS["ubuntu24"], - tags=tags, - expansions=expansions, + tags=[], + expansions=dict(), ) variants.append(variant) return variants @@ -661,8 +663,11 @@ def create_server_version_tasks(): for (topology, auth, ssl, sync), rotated in zip_cycle( list(product(TOPOLOGIES, ["auth", "noauth"], ["ssl", "nossl"], SYNCS)), ALL_PYTHONS ): - python = pr_pythons.get((topology, auth, ssl), rotated) - task_combos.add((topology, auth, ssl, sync, python)) + task_combos.add((topology, auth, ssl, sync, rotated)) + # The pinned PR task re-binds to the boundary Python instead of + # reshuffling when a CPython is added or removed. + if pr_python := pr_pythons.get((topology, auth, ssl)): + task_combos.add((topology, auth, ssl, sync, pr_python)) # Every python should be tested with sharded cluster, auth, ssl, with sync and async. for python, sync in product(ALL_PYTHONS, SYNCS): @@ -723,8 +728,7 @@ def create_test_non_standard_tasks(): # Only the sharded_cluster topology runs on PRs to keep patch build size manageable. for (version, topology), python in zip_cycle(list(product(ALL_VERSIONS, TOPOLOGIES)), CPYTHONS): task_combos.add((version, topology, python)) - # The pinned PR task: the max Python on the newest numbered server, - # sharded cluster. + # The pinned PR task: the max Python on MAX_VERSION, sharded cluster. task_combos.add((MAX_VERSION, "sharded_cluster", MAX_PYTHON)) for version, topology, python in sorted(task_combos): pr = version == MAX_VERSION and topology == "sharded_cluster" and python == MAX_PYTHON @@ -743,7 +747,9 @@ def create_test_non_standard_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" - elif pr: + elif pr or (version == "latest" and topology == "sharded_cluster"): + # The pinned PR task and the rotated "latest" nightly sharded + # task both run coverage. expansions["COVERAGE"] = "1" tags.append("cov") name = get_task_name("test-non-standard", python=python, **expansions) @@ -755,14 +761,16 @@ def create_test_non_standard_tasks(): # For each coverage task, also emit a non-coverage companion so that # macOS/Win64 encryption variants (which filter out .cov due to timeout # constraints) still have a "latest" task to activate in patch builds. - if pr and "cov" in tags: + if "cov" in tags: nc_expansions = {k: v for k, v in expansions.items() if k != "COVERAGE"} # Use a distinct primary tag so companions are not selected by existing - # ".test-non-standard" selectors (e.g. load-balancer, PyOpenSSL variants). + # ".test-non-standard" selectors (e.g. load-balancer, PyOpenSSL variants), + # and drop the pr tag: the companion runs without coverage, so it is + # not a PR task. nc_tags = [ "test-non-standard-no-cov" if t == "test-non-standard" else t for t in tags - if t != "cov" + if t not in ("cov", "pr") ] nc_name = get_task_name("test-non-standard", python=python, **nc_expansions) nc_server_func = FunctionCall(func="run server", vars=nc_expansions) @@ -812,7 +820,7 @@ def create_test_standard_auth_tasks(): list(product(ALL_VERSIONS, ["sharded_cluster"])), CPYTHONS ): task_combos.add((version, topology, python)) - # The pinned PR tasks: the min and max Python on the newest numbered server. + # The pinned PR tasks: the min and max Python on MAX_VERSION. task_combos.add((MAX_VERSION, "sharded_cluster", MIN_PYTHON)) task_combos.add((MAX_VERSION, "sharded_cluster", MAX_PYTHON)) for version, topology, python in sorted(task_combos): @@ -851,18 +859,23 @@ def create_standard_tasks(): for (python, topology, sync), rotated in zip_cycle( list(product(CPYTHONS, TOPOLOGIES, SYNCS)), ALL_VERSIONS ): - # PR tasks pin to the Python boundaries on the newest numbered server: - # min Python on standalone, max Python on replica set, and the - # free-threaded Python on sharded cluster. + # PR tasks pin to the Python boundaries on MAX_VERSION: min Python on + # standalone, max Python on replica set, and the free-threaded Python + # on sharded cluster. They are added on top of the rotation, which + # keeps running every combination on its rotated server version. pr = ( (python == MIN_PYTHON and topology == "standalone" and sync == "async") or (python == MAX_PYTHON and topology == "replica_set" and sync == "async") or (python == FREE_THREADED_PYTHON and topology == "sharded_cluster" and sync == "sync") ) - version = MAX_VERSION if pr else rotated - task_combos.add((version, topology, python, sync, pr)) + task_combos.add((rotated, topology, python, sync, False)) + if pr: + task_combos.add((MAX_VERSION, topology, python, sync, True)) - for version, topology, python, sync, pr in sorted(task_combos): + # A pinned combo can collide with a rotated combo on MAX_VERSION; keep + # the pinned (pr) variant. + combos = {combo[:4]: combo for combo in sorted(task_combos)} + for version, topology, python, sync, pr in combos.values(): auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-standard", @@ -878,7 +891,9 @@ def create_standard_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" - if pr: + if pr or (version == "latest" and python != ALL_PYTHONS[0]): + # The pinned PR tasks and the rotated "latest" nightly tasks + # (min-deps excepted) run coverage. expansions["COVERAGE"] = "1" name = get_task_name("test-standard", python=python, sync=sync, **expansions) server_func = FunctionCall(func="run server", vars=expansions) @@ -941,7 +956,7 @@ def create_aws_tasks(): ] assume_func = FunctionCall(func="assume ec2 role") task_combos = set(zip_cycle(get_versions_from("4.4"), aws_test_types, CPYTHONS)) - # The pinned PR tasks: the min and max Python on the newest numbered server. + # The pinned PR tasks: the min and max Python on MAX_VERSION. task_combos |= { (MAX_VERSION, "regular", MIN_PYTHON), (MAX_VERSION, "regular", MAX_PYTHON), @@ -963,6 +978,9 @@ def create_aws_tasks(): test_vars["TEST_MIN_DEPS"] = "1" if pr: tags.append("pr") + if pr or python == MIN_MAX_PYTHON[-1]: + # The pinned PR tasks and the rotated max-Python tasks run + # coverage. test_vars["COVERAGE"] = "1" name = get_task_name(f"{base_name}-{test_type}", **test_vars) test_func = FunctionCall(func="run tests", vars=test_vars) @@ -1016,19 +1034,18 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): file_name = f"{algo}-basic-tls-ocsp-{variant}.json" for version in get_versions_from("4.4"): - # Run exactly one min-deps and one latest-CPython stapled OCSP task on - # every PR (ecdsa only, to avoid doubling coverage across algorithms) - # so a cryptography-backend regression at either dependency extreme, - # like the min-deps ML-KEM breakage in PYTHON-6032, is caught before - # merge. The newest-server PR task pins to the newest numbered server; - # the "latest" nightly is unstable. - pr = ( + # The rotated nightly tasks: the latest-CPython runs on the "latest" + # nightly, the min-deps Python runs everywhere else. + pr = base_task_name == "valid-cert-server-staples" and algo == "ecdsa" and version == "4.4" + # The rotated "latest" ECDSA staples task keeps its nightly coverage, + # but is no longer a PR task: the pinned MAX_VERSION task below + # supersedes it for PR selection. + nightly_cov = ( base_task_name == "valid-cert-server-staples" and algo == "ecdsa" - and version in ("latest", "4.4") + and version == "latest" ) - server_version = MAX_VERSION if pr and version == "latest" else version - if server_version in ("latest", MAX_VERSION): + if version == "latest": python = MIN_MAX_PYTHON[-1] else: python = MIN_MAX_PYTHON[0] @@ -1038,16 +1055,36 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): OCSP_SERVER_TYPE=server_type, TEST_NAME="ocsp", UV_PYTHON=python, - VERSION=server_version, + VERSION=version, ) if python == ALL_PYTHONS[0]: vars["TEST_MIN_DEPS"] = "1" - tags = ["ocsp", f"ocsp-{algo}", server_version] + tags = ["ocsp", f"ocsp-{algo}", version] if "disableStapling" not in variant: tags.append("ocsp-staple") + if pr or nightly_cov: + vars["COVERAGE"] = "1" if pr: tags.append("pr") - vars["COVERAGE"] = "1" + test_func = FunctionCall(func="run tests", vars=vars) + task_name = get_task_name(f"test-ocsp-{algo}-{base_task_name}", **vars) + tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) + + # The pinned PR task: exactly one latest-CPython stapled OCSP task on + # every PR (ecdsa only, to avoid doubling coverage across algorithms) so + # a cryptography-backend regression at either dependency extreme, like + # the min-deps ML-KEM breakage in PYTHON-6032, is caught before merge. + # It pins to MAX_VERSION; the "latest" nightly is unstable. + if base_task_name == "valid-cert-server-staples" and algo == "ecdsa": + vars = dict( + ORCHESTRATION_FILE=file_name, + OCSP_SERVER_TYPE=server_type, + TEST_NAME="ocsp", + UV_PYTHON=MIN_MAX_PYTHON[-1], + VERSION=MAX_VERSION, + COVERAGE="1", + ) + tags = ["ocsp", f"ocsp-{algo}", MAX_VERSION, "ocsp-staple", "pr"] test_func = FunctionCall(func="run tests", vars=vars) task_name = get_task_name(f"test-ocsp-{algo}-{base_task_name}", **vars) tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index 1b49fbcb33..e7a9d307db 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -31,7 +31,7 @@ MIN_PYTHON = CPYTHONS[0] MAX_PYTHON = CPYTHONS[-1] FREE_THREADED_PYTHON = [p for p in CPYTHONS if "t" in p][-1] -# The newest numbered server version, which PR tasks pin to. +# The version PR tasks pin to: the newest numbered entry in ALL_VERSIONS. MAX_VERSION = max((v for v in ALL_VERSIONS if v[0].isdigit()), key=float) BATCHTIME_WEEK = 10080 BATCHTIME_DAY = 1440 diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index 0cfd7dc019..52a43a58b0 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -37,9 +37,9 @@ def pr_tasks(): def test_pr_tasks_pin_max_version(): - """PR tasks must pin to the newest numbered server version: the "latest" - nightly is unstable. The min-deps extreme is the exception: it pins to - the oldest numbered server (4.4) to match its oldest dependencies.""" + """PR tasks must pin to MAX_VERSION: the "latest" nightly is unstable. + The min-deps extreme is the exception: it pins to 4.4 to match its + oldest dependencies.""" offenders = [ (task.name, task_vars(task).get("VERSION")) for task in pr_tasks() @@ -76,3 +76,82 @@ def test_standard_family_pr_budget(): ) ] assert len(newer) <= 3, f"too many 3.14+ test-standard PR tasks: {[t.name for t in newer]}" + + +def test_pr_tasks_run_coverage(): + """Every PR task that pins a server version runs with COVERAGE=1. The + no-cov companions are helpers for the macOS/Win64 encryption variants, + not PR tasks, and the non-test infrastructure tasks (coverage-report, + check-import-time, kms-fail) do not pin a server version.""" + offenders = [ + task.name + for task in pr_tasks() + if task_vars(task).get("VERSION") is not None + and "test-non-standard-no-cov" not in (task.tags or []) + and task_vars(task).get("COVERAGE") != "1" + ] + assert not offenders, f"PR tasks without COVERAGE: {offenders}" + + +def test_rotated_tasks_keep_nightly_coverage(): + """Mainline batch coverage rides the rotation, independently of the + pinned PR tasks: the rotated "latest" nightly tasks keep COVERAGE=1 + (min-deps excepted), as do the rotated max-Python auth-aws tasks.""" + offenders = [] + for task in all_tasks(): + tags = task.tags or [] + vars = task_vars(task) + if vars.get("TEST_MIN_DEPS") == "1" or "pr" in tags: + continue + rotated_latest = "server-latest" in tags or ("ocsp-ecdsa" in tags and "latest" in tags) + is_tested = ("test-standard" in tags and "server-latest" in tags) or ( + "test-non-standard" in tags + and "server-latest" in tags + and "sharded_cluster-auth-ssl" in tags + ) + is_aws_max_python = ( + "auth-aws" in tags + and "session-name" not in task.name + and vars.get("UV_PYTHON") == generate_config_utils.MAX_PYTHON + ) + if rotated_latest and is_tested and vars.get("COVERAGE") != "1": + offenders.append(task.name) + if is_aws_max_python and vars.get("COVERAGE") != "1": + offenders.append(task.name) + assert not offenders, f"rotated tasks without nightly COVERAGE: {offenders}" + # The rotated "latest" ECDSA staples task keeps its nightly coverage; + # the pinned PR task is emitted separately. + latest_staples = [ + task.name + for task in all_tasks() + if "ocsp-ecdsa" in (task.tags or []) + and "ocsp-staple" in (task.tags or []) + and "latest" in (task.tags or []) + and task_vars(task).get("COVERAGE") == "1" + ] + assert latest_staples, "no ECDSA staples task on the latest nightly with COVERAGE" + # The 9.0 nightly OCSP tasks stay on the min-deps Python; only the + # pinned PR task pins to MAX_VERSION. + offenders = [ + task.name + for task in all_tasks() + if "ocsp" in (task.tags or []) + and "9.0" in (task.tags or []) + and "pr" not in (task.tags or []) + and task_vars(task).get("TEST_MIN_DEPS") != "1" + ] + assert not offenders, f"nightly 9.0 OCSP tasks not on min-deps Python: {offenders}" + + +def test_non_standard_cov_tasks_have_no_cov_companions(): + """Each coverage task in the non-standard family also emits a + non-coverage companion so the macOS/Win64 encryption variants (which + filter out .cov due to timeout constraints) still have a task to + activate in patch builds.""" + no_cov = {task.name for task in all_tasks() if "test-non-standard-no-cov" in (task.tags or [])} + for task in all_tasks(): + if "test-non-standard-no-cov" in (task.tags or []): + assert "pr" not in (task.tags or []), f"no-cov companion tagged pr: {task.name}" + elif "cov" in (task.tags or []) and "test-non-standard" in (task.tags or []): + companion = task.name.removesuffix("-cov") + assert companion in no_cov, f"missing no-cov companion for {task.name}" From 39dae951c7d9c1d82ae8b46d1c99fb7137b70491 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 13:35:50 -0500 Subject: [PATCH 09/13] Pin the server-version PR tasks to MAX_VERSION in their own vars --- .evergreen/generated_configs/tasks.yml | 219 ++++++++++++++------- .evergreen/generated_configs/variants.yml | 14 +- .evergreen/scripts/generate_config.py | 88 ++++++--- .evergreen/scripts/test_generate_config.py | 115 ++++++++++- 4 files changed, 325 insertions(+), 111 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 455204ac14..0136edc4f4 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -2936,28 +2936,6 @@ tasks: - python-3.12 - replica_set-noauth-nossl - async - - name: test-server-version-python3.15-async-noauth-nossl-replica-set-cov - commands: - - func: run server - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: replica_set - COVERAGE: "1" - - func: run tests - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: replica_set - COVERAGE: "1" - UV_PYTHON: "3.15" - TEST_NAME: default_async - tags: - - server-version - - python-3.15 - - replica_set-noauth-nossl - - async - - pr - name: test-server-version-python3.11-sync-noauth-nossl-replica-set-min-deps commands: - func: run server @@ -2979,28 +2957,6 @@ tasks: - python-3.11 - replica_set-noauth-nossl - sync - - name: test-server-version-python3.15-sync-noauth-nossl-replica-set-cov - commands: - - func: run server - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: replica_set - COVERAGE: "1" - - func: run tests - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: replica_set - COVERAGE: "1" - UV_PYTHON: "3.15" - TEST_NAME: default_sync - tags: - - server-version - - python-3.15 - - replica_set-noauth-nossl - - sync - - pr - name: test-server-version-python3.15-async-noauth-ssl-replica-set commands: - func: run server @@ -3198,7 +3154,6 @@ tasks: - python-3.15 - sharded_cluster-auth-ssl - async - - pr - name: test-server-version-python3.15t-async-auth-ssl-sharded-cluster commands: - func: run server @@ -3338,7 +3293,6 @@ tasks: - python-3.15 - sharded_cluster-auth-ssl - sync - - pr - name: test-server-version-python3.15t-sync-auth-ssl-sharded-cluster commands: - func: run server @@ -3539,31 +3493,6 @@ tasks: - python-3.11 - standalone-noauth-nossl - async - - pr - - name: test-server-version-python3.11-sync-noauth-nossl-standalone-min-deps-cov - commands: - - func: run server - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - TEST_MIN_DEPS: "1" - COVERAGE: "1" - - func: run tests - vars: - AUTH: noauth - SSL: nossl - TOPOLOGY: standalone - TEST_MIN_DEPS: "1" - COVERAGE: "1" - UV_PYTHON: "3.11" - TEST_NAME: default_sync - tags: - - server-version - - python-3.11 - - standalone-noauth-nossl - - sync - - pr - name: test-server-version-python3.15-sync-noauth-nossl-standalone commands: - func: run server @@ -3622,6 +3551,154 @@ tasks: - python-3.14 - standalone-noauth-ssl - sync + - name: test-server-version-v9.0-python3.15-sync-noauth-nossl-replica-set-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_sync + tags: + - server-version + - python-3.15 + - replica_set-noauth-nossl + - sync + - pr + - name: test-server-version-v9.0-python3.15-async-noauth-nossl-replica-set-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_async + tags: + - server-version + - python-3.15 + - replica_set-noauth-nossl + - async + - pr + - name: test-server-version-v9.0-python3.15-sync-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_sync + tags: + - server-version + - python-3.15 + - sharded_cluster-auth-ssl + - sync + - pr + - name: test-server-version-v9.0-python3.15-async-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_async + tags: + - server-version + - python-3.15 + - sharded_cluster-auth-ssl + - async + - pr + - name: test-server-version-v9.0-python3.11-sync-noauth-nossl-standalone-min-deps-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_sync + tags: + - server-version + - python-3.11 + - standalone-noauth-nossl + - sync + - pr + - name: test-server-version-v9.0-python3.11-async-noauth-nossl-standalone-min-deps-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_async + tags: + - server-version + - python-3.11 + - standalone-noauth-nossl + - async + - pr # Standard tests - name: test-standard-v4.4-python3.12-sync-noauth-ssl-replica-set diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 17b7db393c..b9f6c0ae91 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -531,7 +531,7 @@ buildvariants: # Server version tests - name: mongodb-v4.4 tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB v4.4" run_on: - rhel8.10-small @@ -540,7 +540,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-v5.0 tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB v5.0" run_on: - rhel8.10-small @@ -549,7 +549,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-v6.0 tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB v6.0" run_on: - rhel8.10-small @@ -558,7 +558,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-v7.0 tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB v7.0" run_on: - rhel8.10-small @@ -567,7 +567,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-v8.0 tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB v8.0" run_on: - rhel8.10-small @@ -585,7 +585,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-rapid tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB rapid" run_on: - rhel8.10-small @@ -594,7 +594,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-latest tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB latest" run_on: - rhel8.10-small diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 4aeb7a07c0..68e03a360b 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -90,8 +90,12 @@ def create_server_version_variants() -> list[BuildVariant]: variants = [] for version in ALL_VERSIONS: display_name = get_variant_name("* MongoDB", version=version) + # The pr-tagged server-version tasks pin MAX_VERSION in their own vars + # and are selected only by the MAX_VERSION variant, so a patch build + # runs each of them exactly once. + tasks = [".server-version"] if version == MAX_VERSION else [".server-version !.pr"] variant = create_variant( - [".server-version"], + tasks, display_name, version=version, host=DEFAULT_HOST, @@ -650,42 +654,25 @@ def create_aws_lambda_variants(): ############## -def create_server_version_tasks(): - tasks = [] - task_combos = set() - # PR tasks pin to the Python boundaries. - pr_pythons = { - ("standalone", "noauth", "nossl"): MIN_PYTHON, - ("replica_set", "noauth", "nossl"): MAX_PYTHON, - ("sharded_cluster", "auth", "ssl"): MAX_PYTHON, - } - # All combinations of topology, auth, ssl, and sync should be tested. - for (topology, auth, ssl, sync), rotated in zip_cycle( - list(product(TOPOLOGIES, ["auth", "noauth"], ["ssl", "nossl"], SYNCS)), ALL_PYTHONS - ): - task_combos.add((topology, auth, ssl, sync, rotated)) - # The pinned PR task re-binds to the boundary Python instead of - # reshuffling when a CPython is added or removed. - if pr_python := pr_pythons.get((topology, auth, ssl)): - task_combos.add((topology, auth, ssl, sync, pr_python)) +# The server-version PR tasks pin to the Python boundaries: the min Python on +# standalone, the max Python on replica set and sharded cluster. +SERVER_VERSION_PR_PYTHONS = { + ("standalone", "noauth", "nossl"): MIN_PYTHON, + ("replica_set", "noauth", "nossl"): MAX_PYTHON, + ("sharded_cluster", "auth", "ssl"): MAX_PYTHON, +} - # Every python should be tested with sharded cluster, auth, ssl, with sync and async. - for python, sync in product(ALL_PYTHONS, SYNCS): - task_combos.add(("sharded_cluster", "auth", "ssl", sync, python)) - # Assemble the tasks. - for topology, auth, ssl, sync, python in sorted(task_combos): +def create_server_version_tasks(): + def make_task(topology, auth, ssl, sync, python, extra_tags=(), **extra_vars): combo = f"{topology}-{auth}-{ssl}" - tags = ["server-version", f"python-{python}", combo, sync] - if python == pr_pythons.get((topology, auth, ssl)): - tags.append("pr") + tags = ["server-version", f"python-{python}", combo, sync, *extra_tags] expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" + expansions.update(extra_vars) if "t" in python: tags.append("free-threaded") - if "pr" in tags: - expansions["COVERAGE"] = "1" name = get_task_name( "test-server-version", python=python, @@ -697,7 +684,48 @@ def create_server_version_tasks(): test_vars["UV_PYTHON"] = python test_vars["TEST_NAME"] = f"default_{sync}" test_func = FunctionCall(func="run tests", vars=test_vars) - tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) + return EvgTask(name=name, tags=tags, commands=[server_func, test_func]) + + tasks = [] + task_combos = set() + # All combinations of topology, auth, ssl, and sync should be tested, with + # the Python rotating so every CPython is tested on every server version. + for (topology, auth, ssl, sync), rotated in zip_cycle( + list(product(TOPOLOGIES, ["auth", "noauth"], ["ssl", "nossl"], SYNCS)), ALL_PYTHONS + ): + task_combos.add((topology, auth, ssl, sync, rotated)) + + # Every python should be tested with sharded cluster, auth, ssl, with sync and async. + for python, sync in product(ALL_PYTHONS, SYNCS): + task_combos.add(("sharded_cluster", "auth", "ssl", sync, python)) + + # Rotated batch tasks inherit the selecting variant's server version (one + # variant per MongoDB version). The boundary-Python tasks keep per-commit + # coverage without the pr tag: the pinned PR tasks below are the only pr + # tasks, and they are selected only by the MAX_VERSION variant. + for topology, auth, ssl, sync, python in sorted(task_combos): + extra_vars = {} + if python == SERVER_VERSION_PR_PYTHONS.get((topology, auth, ssl)): + extra_vars["COVERAGE"] = "1" + tasks.append(make_task(topology, auth, ssl, sync, python, **extra_vars)) + + # The pinned PR tasks re-bind to the boundary Pythons on MAX_VERSION + # instead of reshuffling when a CPython is added or removed. Their own + # VERSION var pins them to MAX_VERSION wherever they run. + for (topology, auth, ssl), python in sorted(SERVER_VERSION_PR_PYTHONS.items()): + for sync in SYNCS: + tasks.append( + make_task( + topology, + auth, + ssl, + sync, + python, + extra_tags=("pr",), + VERSION=MAX_VERSION, + COVERAGE="1", + ) + ) return tasks diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index 52a43a58b0..72c8dcb06b 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -36,10 +36,49 @@ def pr_tasks(): return [task for task in all_tasks() if "pr" in (task.tags or [])] +def all_variants(): + variants = [] + for name, func in getmembers(generate_config, isfunction): + if name.startswith("create_") and name.endswith("_variants"): + variants.extend(func()) + return variants + + +def variant_selects(variant, task): + """Whether the variant's task selectors select the task. Selector tokens + are split on whitespace: a "." prefix matches a tag, "!" negates the + following token, and a bare token matches a task name.""" + tags = task.tags or [] + for ref in variant.tasks: + includes, excludes, names = [], [], [] + for token in ref.name.split(): + if token.startswith("!."): + excludes.append(token[2:]) + elif token.startswith("!"): + excludes.append(token[1:]) + elif token.startswith("."): + includes.append(token[1:]) + else: + names.append(token) + if any(tag in tags for tag in excludes): + continue + if not includes and not names: + # A negation-only selector matches every remaining task. + return True + if names and task.name not in names: + continue + if includes and not all(tag in tags for tag in includes): + continue + return True + return False + + def test_pr_tasks_pin_max_version(): - """PR tasks must pin to MAX_VERSION: the "latest" nightly is unstable. - The min-deps extreme is the exception: it pins to 4.4 to match its - oldest dependencies.""" + """PR tasks that set a server version must pin MAX_VERSION: the "latest" + nightly is unstable. The min-deps extreme is the exception: it pins to + 4.4 to match its oldest dependencies. Tasks without a VERSION var inherit + the selecting variant's version instead; the merged result is checked by + test_pr_tasks_effectively_pin_max_version.""" offenders = [ (task.name, task_vars(task).get("VERSION")) for task in pr_tasks() @@ -51,6 +90,76 @@ def test_pr_tasks_pin_max_version(): assert not offenders, f"PR tasks not pinned to MAX_VERSION: {offenders}" +def test_pr_tasks_effectively_pin_max_version(): + """A patch build runs pr tasks in every selecting variant with that + variant's expansions, so a task without its own VERSION var inherits the + variant's server version (4.4 through the unstable latest nightly). Every + pr task that runs a server must resolve to MAX_VERSION once the variant's + VERSION expansion is merged in. The only exception is an explicit 4.4 + min-deps pin on the task itself.""" + tasks = [task for task in all_tasks() if "pr" in (task.tags or [])] + offenders = [] + for variant in all_variants(): + variant_version = (variant.expansions or {}).get("VERSION") + for task in tasks: + if not variant_selects(variant, task): + continue + task_vars_ = task_vars(task) + if task_vars_.get("VERSION") == "4.4" and task_vars_.get("TEST_MIN_DEPS") == "1": + continue + effective = task_vars_.get("VERSION") or variant_version + if effective is None: + # Helpers that run the stable default server, like kms-fail. + continue + if effective != generate_config_utils.MAX_VERSION: + offenders.append((variant.name, task.name, effective)) + assert not offenders, f"pr tasks not effectively pinned to MAX_VERSION: {offenders}" + + +def test_server_version_pr_tasks_selected_once(): + """The mongodb-* variants differ only in their server version, so a + pr-tagged server-version task selected by several of them would run + identically once per variant per patch build. Only the MAX_VERSION + variant selects them.""" + tasks = [ + task + for task in all_tasks() + if "server-version" in (task.tags or []) and "pr" in (task.tags or []) + ] + offenders = {} + for task in tasks: + selecting = [variant.name for variant in all_variants() if variant_selects(variant, task)] + if len(selecting) != 1: + offenders[task.name] = selecting + assert not offenders, ( + f"server-version pr tasks not selected by exactly one variant: {offenders}" + ) + + +def test_server_version_rotated_tasks_keep_coverage(): + """The rotated server-version tasks inherit the selecting variant's + version (one variant per MongoDB version), so the family's per-commit + coverage rides the rotated boundary-Python tasks: the pinned pr tasks run + only in the MAX_VERSION variant, and coverage keyed to pr alone would + collapse to 9.0-only.""" + family = [task for task in all_tasks() if "server-version" in (task.tags or [])] + cov = [task for task in family if task_vars(task).get("COVERAGE") == "1"] + assert any("pr" not in (task.tags or []) for task in cov), ( + "the server-version family's only COVERAGE tasks are pr tasks" + ) + offenders = [] + for task in family: + if "pr" in (task.tags or []): + continue + vars = task_vars(task) + boundary = generate_config.SERVER_VERSION_PR_PYTHONS.get( + (vars["TOPOLOGY"], vars["AUTH"], vars["SSL"]) + ) + if vars.get("UV_PYTHON") == boundary and vars.get("COVERAGE") != "1": + offenders.append(task.name) + assert not offenders, f"rotated server-version boundary tasks without COVERAGE: {offenders}" + + def test_pr_min_deps_tasks_also_run_coverage(): offenders = [ task.name From d5b7fa954342483473096caec282cd0592975c0e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 14:00:20 -0500 Subject: [PATCH 10/13] Document pytest and pytest-asyncio test dependencies --- .evergreen/scripts/test_generate_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index 72c8dcb06b..e4e3e90293 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -2,7 +2,8 @@ Run: pytest .evergreen/scripts/test_generate_config.py -m "" (-m "" overrides the repo-wide default marker filter.) -Requires the same dependencies as the generate-config hook: shrub.py, pyyaml. +Requires the same dependencies as the generate-config hook: pytest, +pytest-asyncio, shrub.py, pyyaml. """ from __future__ import annotations From e3dcaff1ab7a0d9c1faae0986e3275da00123294 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 15:08:53 -0500 Subject: [PATCH 11/13] Trim docstrings and comments in the Evergreen config scripts --- .evergreen/scripts/generate-config.sh | 2 +- .evergreen/scripts/generate_config.py | 58 ++++++----------- .evergreen/scripts/generate_config_utils.py | 4 +- .evergreen/scripts/test_generate_config.py | 72 ++++++++------------- CONTRIBUTING.md | 11 ++-- 5 files changed, 53 insertions(+), 94 deletions(-) diff --git a/.evergreen/scripts/generate-config.sh b/.evergreen/scripts/generate-config.sh index 3b85467d6f..4f46605901 100755 --- a/.evergreen/scripts/generate-config.sh +++ b/.evergreen/scripts/generate-config.sh @@ -3,7 +3,7 @@ set -eu -# Assert the generation rules before regenerating the config from them. +# Assert the generation rules, then regenerate the config. python -m pytest .evergreen/scripts/test_generate_config.py -m "" python .evergreen/scripts/generate_config.py diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 68e03a360b..751795f627 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -90,9 +90,8 @@ def create_server_version_variants() -> list[BuildVariant]: variants = [] for version in ALL_VERSIONS: display_name = get_variant_name("* MongoDB", version=version) - # The pr-tagged server-version tasks pin MAX_VERSION in their own vars - # and are selected only by the MAX_VERSION variant, so a patch build - # runs each of them exactly once. + # The pr tasks pin their own VERSION; only the MAX_VERSION variant + # selects them. tasks = [".server-version"] if version == MAX_VERSION else [".server-version !.pr"] variant = create_variant( tasks, @@ -590,8 +589,7 @@ def create_aws_auth_variants(): variants.append(variant) # The ECS test must be run on Ubuntu 24 to match the Fargate Config. - # Use fresh tags and host: the loop's macOS values must not leak into - # this variant. + # Fresh tags and expansions: the loop's macOS values must not leak here. variant = create_variant( [".auth-aws-ecs"], get_variant_name("Auth AWS ECS", HOSTS["ubuntu24"]), @@ -654,8 +652,7 @@ def create_aws_lambda_variants(): ############## -# The server-version PR tasks pin to the Python boundaries: the min Python on -# standalone, the max Python on replica set and sharded cluster. +# The Python boundaries the server-version PR tasks pin to. SERVER_VERSION_PR_PYTHONS = { ("standalone", "noauth", "nossl"): MIN_PYTHON, ("replica_set", "noauth", "nossl"): MAX_PYTHON, @@ -688,8 +685,7 @@ def make_task(topology, auth, ssl, sync, python, extra_tags=(), **extra_vars): tasks = [] task_combos = set() - # All combinations of topology, auth, ssl, and sync should be tested, with - # the Python rotating so every CPython is tested on every server version. + # All combinations of topology, auth, ssl, and sync should be tested. for (topology, auth, ssl, sync), rotated in zip_cycle( list(product(TOPOLOGIES, ["auth", "noauth"], ["ssl", "nossl"], SYNCS)), ALL_PYTHONS ): @@ -699,19 +695,16 @@ def make_task(topology, auth, ssl, sync, python, extra_tags=(), **extra_vars): for python, sync in product(ALL_PYTHONS, SYNCS): task_combos.add(("sharded_cluster", "auth", "ssl", sync, python)) - # Rotated batch tasks inherit the selecting variant's server version (one - # variant per MongoDB version). The boundary-Python tasks keep per-commit - # coverage without the pr tag: the pinned PR tasks below are the only pr - # tasks, and they are selected only by the MAX_VERSION variant. + # Rotated tasks inherit the variant's VERSION; boundary-Python tasks keep + # COVERAGE without pr so per-commit coverage is not pr-only. for topology, auth, ssl, sync, python in sorted(task_combos): extra_vars = {} if python == SERVER_VERSION_PR_PYTHONS.get((topology, auth, ssl)): extra_vars["COVERAGE"] = "1" tasks.append(make_task(topology, auth, ssl, sync, python, **extra_vars)) - # The pinned PR tasks re-bind to the boundary Pythons on MAX_VERSION - # instead of reshuffling when a CPython is added or removed. Their own - # VERSION var pins them to MAX_VERSION wherever they run. + # The pinned PR tasks: the boundary Pythons on MAX_VERSION, stable as + # CPYTHONS changes. for (topology, auth, ssl), python in sorted(SERVER_VERSION_PR_PYTHONS.items()): for sync in SYNCS: tasks.append( @@ -776,8 +769,6 @@ def create_test_non_standard_tasks(): if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" elif pr or (version == "latest" and topology == "sharded_cluster"): - # The pinned PR task and the rotated "latest" nightly sharded - # task both run coverage. expansions["COVERAGE"] = "1" tags.append("cov") name = get_task_name("test-non-standard", python=python, **expansions) @@ -792,9 +783,7 @@ def create_test_non_standard_tasks(): if "cov" in tags: nc_expansions = {k: v for k, v in expansions.items() if k != "COVERAGE"} # Use a distinct primary tag so companions are not selected by existing - # ".test-non-standard" selectors (e.g. load-balancer, PyOpenSSL variants), - # and drop the pr tag: the companion runs without coverage, so it is - # not a PR task. + # ".test-non-standard" selectors (e.g. load-balancer, PyOpenSSL variants). nc_tags = [ "test-non-standard-no-cov" if t == "test-non-standard" else t for t in tags @@ -887,9 +876,7 @@ def create_standard_tasks(): for (python, topology, sync), rotated in zip_cycle( list(product(CPYTHONS, TOPOLOGIES, SYNCS)), ALL_VERSIONS ): - # PR tasks pin to the Python boundaries on MAX_VERSION: min Python on - # standalone, max Python on replica set, and the free-threaded Python - # on sharded cluster. They are added on top of the rotation, which + # PR tasks pin to the Python boundaries on MAX_VERSION; the rotation # keeps running every combination on its rotated server version. pr = ( (python == MIN_PYTHON and topology == "standalone" and sync == "async") @@ -900,8 +887,7 @@ def create_standard_tasks(): if pr: task_combos.add((MAX_VERSION, topology, python, sync, True)) - # A pinned combo can collide with a rotated combo on MAX_VERSION; keep - # the pinned (pr) variant. + # A pinned combo can collide with a rotated one on MAX_VERSION; keep the pinned (pr) task. combos = {combo[:4]: combo for combo in sorted(task_combos)} for version, topology, python, sync, pr in combos.values(): auth, ssl = get_standard_auth_ssl(topology) @@ -920,8 +906,6 @@ def create_standard_tasks(): if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" if pr or (version == "latest" and python != ALL_PYTHONS[0]): - # The pinned PR tasks and the rotated "latest" nightly tasks - # (min-deps excepted) run coverage. expansions["COVERAGE"] = "1" name = get_task_name("test-standard", python=python, sync=sync, **expansions) server_func = FunctionCall(func="run server", vars=expansions) @@ -1007,8 +991,6 @@ def create_aws_tasks(): if pr: tags.append("pr") if pr or python == MIN_MAX_PYTHON[-1]: - # The pinned PR tasks and the rotated max-Python tasks run - # coverage. test_vars["COVERAGE"] = "1" name = get_task_name(f"{base_name}-{test_type}", **test_vars) test_func = FunctionCall(func="run tests", vars=test_vars) @@ -1062,12 +1044,10 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): file_name = f"{algo}-basic-tls-ocsp-{variant}.json" for version in get_versions_from("4.4"): - # The rotated nightly tasks: the latest-CPython runs on the "latest" - # nightly, the min-deps Python runs everywhere else. + # Rotated nightlies: latest-CPython on "latest", min-deps Python elsewhere. pr = base_task_name == "valid-cert-server-staples" and algo == "ecdsa" and version == "4.4" - # The rotated "latest" ECDSA staples task keeps its nightly coverage, - # but is no longer a PR task: the pinned MAX_VERSION task below - # supersedes it for PR selection. + # The "latest" ECDSA staples task keeps nightly coverage; the pinned + # MAX_VERSION task below is the pr task. nightly_cov = ( base_task_name == "valid-cert-server-staples" and algo == "ecdsa" @@ -1098,11 +1078,9 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): task_name = get_task_name(f"test-ocsp-{algo}-{base_task_name}", **vars) tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) - # The pinned PR task: exactly one latest-CPython stapled OCSP task on - # every PR (ecdsa only, to avoid doubling coverage across algorithms) so - # a cryptography-backend regression at either dependency extreme, like - # the min-deps ML-KEM breakage in PYTHON-6032, is caught before merge. - # It pins to MAX_VERSION; the "latest" nightly is unstable. + # One latest-CPython ECDSA staples task on every PR, pinned to MAX_VERSION + # (the "latest" nightly is unstable): catches cryptography-backend + # regressions at the dependency extremes before merge (PYTHON-6032). if base_task_name == "valid-cert-server-staples" and algo == "ecdsa": vars = dict( ORCHESTRATION_FILE=file_name, diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index e7a9d307db..9880dc10df 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -26,12 +26,10 @@ MIN_SUPPORT_VERSIONS = ["3.11"] ALL_PYTHONS = CPYTHONS MIN_MAX_PYTHON = [CPYTHONS[0], CPYTHONS[-1]] -# The Python boundaries PR tasks pin to: adding or removing a CPython -# re-binds the same pins instead of reshuffling the PR set. +# The Python boundaries and newest numbered server version that PR tasks pin to. MIN_PYTHON = CPYTHONS[0] MAX_PYTHON = CPYTHONS[-1] FREE_THREADED_PYTHON = [p for p in CPYTHONS if "t" in p][-1] -# The version PR tasks pin to: the newest numbered entry in ALL_VERSIONS. MAX_VERSION = max((v for v in ALL_VERSIONS if v[0].isdigit()), key=float) BATCHTIME_WEEK = 10080 BATCHTIME_DAY = 1440 diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index e4e3e90293..4ecc9b0030 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -1,9 +1,8 @@ """Tests for the Evergreen PR task rules (PYTHON-6112). Run: pytest .evergreen/scripts/test_generate_config.py -m "" -(-m "" overrides the repo-wide default marker filter.) -Requires the same dependencies as the generate-config hook: pytest, -pytest-asyncio, shrub.py, pyyaml. +(-m "" clears the repo-wide marker filter.) +Requires the generate-config hook's dependencies: pytest, pytest-asyncio, shrub.py, pyyaml. """ from __future__ import annotations @@ -46,9 +45,8 @@ def all_variants(): def variant_selects(variant, task): - """Whether the variant's task selectors select the task. Selector tokens - are split on whitespace: a "." prefix matches a tag, "!" negates the - following token, and a bare token matches a task name.""" + """Whether the variant's task selectors match the task: "." selects a + tag, "!" negates the token that follows, a bare token names a task.""" tags = task.tags or [] for ref in variant.tasks: includes, excludes, names = [], [], [] @@ -75,11 +73,9 @@ def variant_selects(variant, task): def test_pr_tasks_pin_max_version(): - """PR tasks that set a server version must pin MAX_VERSION: the "latest" - nightly is unstable. The min-deps extreme is the exception: it pins to - 4.4 to match its oldest dependencies. Tasks without a VERSION var inherit - the selecting variant's version instead; the merged result is checked by - test_pr_tasks_effectively_pin_max_version.""" + """Task-level pins: pr tasks pin MAX_VERSION (the "latest" nightly is + unstable); the explicit 4.4 min-deps pin is the exception. Variant-merged + pins are checked by test_pr_tasks_effectively_pin_max_version.""" offenders = [ (task.name, task_vars(task).get("VERSION")) for task in pr_tasks() @@ -92,12 +88,10 @@ def test_pr_tasks_pin_max_version(): def test_pr_tasks_effectively_pin_max_version(): - """A patch build runs pr tasks in every selecting variant with that - variant's expansions, so a task without its own VERSION var inherits the - variant's server version (4.4 through the unstable latest nightly). Every - pr task that runs a server must resolve to MAX_VERSION once the variant's - VERSION expansion is merged in. The only exception is an explicit 4.4 - min-deps pin on the task itself.""" + """Effective pins: a pr task without its own VERSION var inherits the + selecting variant's version, so every pr task that runs a server must + resolve to MAX_VERSION once variant expansions merge in. The explicit + 4.4 min-deps pin is the exception.""" tasks = [task for task in all_tasks() if "pr" in (task.tags or [])] offenders = [] for variant in all_variants(): @@ -118,10 +112,9 @@ def test_pr_tasks_effectively_pin_max_version(): def test_server_version_pr_tasks_selected_once(): - """The mongodb-* variants differ only in their server version, so a - pr-tagged server-version task selected by several of them would run - identically once per variant per patch build. Only the MAX_VERSION - variant selects them.""" + """The mongodb-* variants differ only in server version, so a pr task + they all select would run identically per variant. Only MAX_VERSION + selects them.""" tasks = [ task for task in all_tasks() @@ -138,11 +131,8 @@ def test_server_version_pr_tasks_selected_once(): def test_server_version_rotated_tasks_keep_coverage(): - """The rotated server-version tasks inherit the selecting variant's - version (one variant per MongoDB version), so the family's per-commit - coverage rides the rotated boundary-Python tasks: the pinned pr tasks run - only in the MAX_VERSION variant, and coverage keyed to pr alone would - collapse to 9.0-only.""" + """Per-commit coverage for this family rides the rotated boundary-Python + tasks, not the pr tasks (which run only on MAX_VERSION).""" family = [task for task in all_tasks() if "server-version" in (task.tags or [])] cov = [task for task in family if task_vars(task).get("COVERAGE") == "1"] assert any("pr" not in (task.tags or []) for task in cov), ( @@ -171,10 +161,9 @@ def test_pr_min_deps_tasks_also_run_coverage(): def test_standard_family_pr_budget(): - """The No C Ext and Compression variants select .test-standard, so their - patch builds run one PR task per pr-tagged test-standard task. Keep the - count at or below the pre-pin baseline (5), and at or below 3 for the - ubuntu22 compression variant, which filters to Python 3.14 and newer.""" + """Patch-build PR budget: at most 5 pr-tagged test-standard tasks (the + pre-pin baseline), at most 3 on Python 3.14+ (the ubuntu22 compression + filter).""" tasks = [task for task in pr_tasks() if "test-standard" in (task.tags or [])] assert len(tasks) <= 5, f"too many test-standard PR tasks: {[t.name for t in tasks]}" newer = [ @@ -189,10 +178,8 @@ def test_standard_family_pr_budget(): def test_pr_tasks_run_coverage(): - """Every PR task that pins a server version runs with COVERAGE=1. The - no-cov companions are helpers for the macOS/Win64 encryption variants, - not PR tasks, and the non-test infrastructure tasks (coverage-report, - check-import-time, kms-fail) do not pin a server version.""" + """Pr tasks that pin a server version run with COVERAGE=1; no-cov + companions and infrastructure tasks (no VERSION) are exempt.""" offenders = [ task.name for task in pr_tasks() @@ -204,9 +191,9 @@ def test_pr_tasks_run_coverage(): def test_rotated_tasks_keep_nightly_coverage(): - """Mainline batch coverage rides the rotation, independently of the - pinned PR tasks: the rotated "latest" nightly tasks keep COVERAGE=1 - (min-deps excepted), as do the rotated max-Python auth-aws tasks.""" + """Batch coverage rides the rotation independently of the pr tasks: + rotated "latest" tasks and max-Python auth-aws tasks keep COVERAGE=1 + (min-deps excepted).""" offenders = [] for task in all_tasks(): tags = task.tags or [] @@ -229,8 +216,7 @@ def test_rotated_tasks_keep_nightly_coverage(): if is_aws_max_python and vars.get("COVERAGE") != "1": offenders.append(task.name) assert not offenders, f"rotated tasks without nightly COVERAGE: {offenders}" - # The rotated "latest" ECDSA staples task keeps its nightly coverage; - # the pinned PR task is emitted separately. + # The "latest" ECDSA staples task keeps its nightly coverage. latest_staples = [ task.name for task in all_tasks() @@ -240,8 +226,7 @@ def test_rotated_tasks_keep_nightly_coverage(): and task_vars(task).get("COVERAGE") == "1" ] assert latest_staples, "no ECDSA staples task on the latest nightly with COVERAGE" - # The 9.0 nightly OCSP tasks stay on the min-deps Python; only the - # pinned PR task pins to MAX_VERSION. + # The 9.0 nightly OCSP tasks stay on the min-deps Python. offenders = [ task.name for task in all_tasks() @@ -254,9 +239,8 @@ def test_rotated_tasks_keep_nightly_coverage(): def test_non_standard_cov_tasks_have_no_cov_companions(): - """Each coverage task in the non-standard family also emits a - non-coverage companion so the macOS/Win64 encryption variants (which - filter out .cov due to timeout constraints) still have a task to + """Each non-standard coverage task also emits a no-cov companion, so the + macOS/Win64 encryption variants (which filter .cov) have a task to activate in patch builds.""" no_cov = {task.name for task in all_tasks() if "test-non-standard-no-cov" in (task.tags or [])} for task in all_tasks(): diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f049eac22..11ddd4b59d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -479,12 +479,11 @@ CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at the setup scripts write a `uv.toml` boundary into the submodule checkout that stops uv's config discovery (and with it the required-version check) at the submodule boundary. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. -- PR tasks pin to the Python boundaries: the min Python (`MIN_PYTHON`, `TEST_MIN_DEPS=1`), the max - Python (`MAX_PYTHON`), and one free-threaded task (`FREE_THREADED_PYTHON`), each against the - newest numbered server (`MAX_VERSION`; the `latest` nightly is unstable). Every PR task - runs with `COVERAGE=1`. The pins derive from `CPYTHONS` in `generate_config_utils.py`, so - adding or removing a CPython re-binds them instead of reshuffling the PR set. Review the - regenerated `generated_configs/tasks.yml` diff to confirm the `pr` tags land on the boundary Pythons. +- PR tasks pin to the Python boundaries: `MIN_PYTHON` with `TEST_MIN_DEPS=1`, `MAX_PYTHON`, and one + `FREE_THREADED_PYTHON`, each against the newest numbered server (`MAX_VERSION`; the `latest` + nightly is unstable), each with `COVERAGE=1`. The pins derive from `CPYTHONS` in + `generate_config_utils.py`, so CPython changes re-bind them rather than reshuffle the set. + Check the regenerated `generated_configs/tasks.yml` diff to confirm the `pr` tags. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. ## Handling flaky tests From fbab1f030fb6b193a5ee528835cdcda7dc6ccdb2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 15:30:02 -0500 Subject: [PATCH 12/13] Test the min server version on PRs --- .evergreen/generated_configs/tasks.yml | 60 ++++++++++++++++++++ .evergreen/generated_configs/variants.yml | 4 +- .evergreen/scripts/generate_config.py | 62 +++++++++++++-------- .evergreen/scripts/generate_config_utils.py | 3 +- .evergreen/scripts/test_generate_config.py | 55 +++++++++++++----- CONTRIBUTING.md | 9 +-- 6 files changed, 150 insertions(+), 43 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 0136edc4f4..4924f05ffb 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -3551,6 +3551,60 @@ tasks: - python-3.14 - standalone-noauth-ssl - sync + - name: test-server-version-v4.4-python3.11-sync-noauth-nossl-standalone-min-deps-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "4.4" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "4.4" + COVERAGE: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_sync + tags: + - server-version + - python-3.11 + - standalone-noauth-nossl + - sync + - pr + - "4.4" + - name: test-server-version-v4.4-python3.11-async-noauth-nossl-standalone-min-deps-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "4.4" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "4.4" + COVERAGE: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_async + tags: + - server-version + - python-3.11 + - standalone-noauth-nossl + - async + - pr + - "4.4" - name: test-server-version-v9.0-python3.15-sync-noauth-nossl-replica-set-cov commands: - func: run server @@ -3575,6 +3629,7 @@ tasks: - replica_set-noauth-nossl - sync - pr + - "9.0" - name: test-server-version-v9.0-python3.15-async-noauth-nossl-replica-set-cov commands: - func: run server @@ -3599,6 +3654,7 @@ tasks: - replica_set-noauth-nossl - async - pr + - "9.0" - name: test-server-version-v9.0-python3.15-sync-auth-ssl-sharded-cluster-cov commands: - func: run server @@ -3623,6 +3679,7 @@ tasks: - sharded_cluster-auth-ssl - sync - pr + - "9.0" - name: test-server-version-v9.0-python3.15-async-auth-ssl-sharded-cluster-cov commands: - func: run server @@ -3647,6 +3704,7 @@ tasks: - sharded_cluster-auth-ssl - async - pr + - "9.0" - name: test-server-version-v9.0-python3.11-sync-noauth-nossl-standalone-min-deps-cov commands: - func: run server @@ -3673,6 +3731,7 @@ tasks: - standalone-noauth-nossl - sync - pr + - "9.0" - name: test-server-version-v9.0-python3.11-async-noauth-nossl-standalone-min-deps-cov commands: - func: run server @@ -3699,6 +3758,7 @@ tasks: - standalone-noauth-nossl - async - pr + - "9.0" # Standard tests - name: test-standard-v4.4-python3.12-sync-noauth-ssl-replica-set diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index b9f6c0ae91..b1d34bce8d 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -532,6 +532,7 @@ buildvariants: - name: mongodb-v4.4 tasks: - name: .server-version !.pr + - name: .server-version .4.4 display_name: "* MongoDB v4.4" run_on: - rhel8.10-small @@ -576,7 +577,8 @@ buildvariants: tags: [coverage_tag] - name: mongodb-v9.0 tasks: - - name: .server-version + - name: .server-version !.pr + - name: .server-version .9.0 display_name: "* MongoDB v9.0" run_on: - rhel8.10-small diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 751795f627..7544d31a6a 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -18,6 +18,7 @@ MAX_VERSION, MIN_MAX_PYTHON, MIN_PYTHON, + MIN_VERSION, OTHER_HOSTS, SYNCS, TOPOLOGIES, @@ -90,9 +91,12 @@ def create_server_version_variants() -> list[BuildVariant]: variants = [] for version in ALL_VERSIONS: display_name = get_variant_name("* MongoDB", version=version) - # The pr tasks pin their own VERSION; only the MAX_VERSION variant - # selects them. - tasks = [".server-version"] if version == MAX_VERSION else [".server-version !.pr"] + # The pr tasks pin their own VERSION; each version-boundary variant + # selects only the pr tasks pinned to that version. + if version in SERVER_VERSION_PR_PYTHONS: + tasks = [".server-version !.pr", f".server-version .{version}"] + else: + tasks = [".server-version !.pr"] variant = create_variant( tasks, display_name, @@ -652,11 +656,16 @@ def create_aws_lambda_variants(): ############## -# The Python boundaries the server-version PR tasks pin to. +# The server-version PR tasks: the Python boundaries to pin per server +# version. The min server tests its min-Python floor; the newest numbered +# server tests every boundary. SERVER_VERSION_PR_PYTHONS = { - ("standalone", "noauth", "nossl"): MIN_PYTHON, - ("replica_set", "noauth", "nossl"): MAX_PYTHON, - ("sharded_cluster", "auth", "ssl"): MAX_PYTHON, + MIN_VERSION: {("standalone", "noauth", "nossl"): MIN_PYTHON}, + MAX_VERSION: { + ("standalone", "noauth", "nossl"): MIN_PYTHON, + ("replica_set", "noauth", "nossl"): MAX_PYTHON, + ("sharded_cluster", "auth", "ssl"): MAX_PYTHON, + }, } @@ -697,28 +706,35 @@ def make_task(topology, auth, ssl, sync, python, extra_tags=(), **extra_vars): # Rotated tasks inherit the variant's VERSION; boundary-Python tasks keep # COVERAGE without pr so per-commit coverage is not pr-only. + pr_pythons = { + combo: python + for combos in SERVER_VERSION_PR_PYTHONS.values() + for combo, python in combos.items() + } for topology, auth, ssl, sync, python in sorted(task_combos): extra_vars = {} - if python == SERVER_VERSION_PR_PYTHONS.get((topology, auth, ssl)): + if python == pr_pythons.get((topology, auth, ssl)): extra_vars["COVERAGE"] = "1" tasks.append(make_task(topology, auth, ssl, sync, python, **extra_vars)) - # The pinned PR tasks: the boundary Pythons on MAX_VERSION, stable as - # CPYTHONS changes. - for (topology, auth, ssl), python in sorted(SERVER_VERSION_PR_PYTHONS.items()): - for sync in SYNCS: - tasks.append( - make_task( - topology, - auth, - ssl, - sync, - python, - extra_tags=("pr",), - VERSION=MAX_VERSION, - COVERAGE="1", + # The pinned PR tasks: the boundary Pythons pinned per server version, + # stable as CPYTHONS changes. Each carries its version as a tag so only + # the variant running that version selects it. + for version, combos in sorted(SERVER_VERSION_PR_PYTHONS.items()): + for (topology, auth, ssl), python in sorted(combos.items()): + for sync in SYNCS: + tasks.append( + make_task( + topology, + auth, + ssl, + sync, + python, + extra_tags=("pr", version), + VERSION=version, + COVERAGE="1", + ) ) - ) return tasks diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index 9880dc10df..0ba291ab5b 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -26,10 +26,11 @@ MIN_SUPPORT_VERSIONS = ["3.11"] ALL_PYTHONS = CPYTHONS MIN_MAX_PYTHON = [CPYTHONS[0], CPYTHONS[-1]] -# The Python boundaries and newest numbered server version that PR tasks pin to. +# The Python and server version boundaries that PR tasks pin to. MIN_PYTHON = CPYTHONS[0] MAX_PYTHON = CPYTHONS[-1] FREE_THREADED_PYTHON = [p for p in CPYTHONS if "t" in p][-1] +MIN_VERSION = min((v for v in ALL_VERSIONS if v[0].isdigit()), key=float) MAX_VERSION = max((v for v in ALL_VERSIONS if v[0].isdigit()), key=float) BATCHTIME_WEEK = 10080 BATCHTIME_DAY = 1440 diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py index 4ecc9b0030..f078f2f305 100644 --- a/.evergreen/scripts/test_generate_config.py +++ b/.evergreen/scripts/test_generate_config.py @@ -113,23 +113,47 @@ def test_pr_tasks_effectively_pin_max_version(): def test_server_version_pr_tasks_selected_once(): """The mongodb-* variants differ only in server version, so a pr task - they all select would run identically per variant. Only MAX_VERSION - selects them.""" - tasks = [ - task - for task in all_tasks() - if "server-version" in (task.tags or []) and "pr" in (task.tags or []) - ] + must be selected only by the variant running the server version it pins.""" offenders = {} - for task in tasks: + for task in pr_tasks(): + if "server-version" not in (task.tags or []): + continue + own = task_vars(task).get("VERSION") selecting = [variant.name for variant in all_variants() if variant_selects(variant, task)] - if len(selecting) != 1: - offenders[task.name] = selecting + matching = [ + variant.name + for variant in all_variants() + if (variant.expansions or {}).get("VERSION") == own + ] + if selecting != matching: + offenders[task.name] = {"selected by": selecting, "pins": own} assert not offenders, ( - f"server-version pr tasks not selected by exactly one variant: {offenders}" + f"server-version pr tasks not selected by the variant matching their pin: {offenders}" ) +def test_pr_tasks_cover_version_boundaries(): + """PRs test the server version boundaries: the newest numbered server + with every Python boundary, the min server with its min-Python floor.""" + for version, pythons in ( + ( + generate_config_utils.MAX_VERSION, + (generate_config_utils.MIN_PYTHON, generate_config_utils.MAX_PYTHON), + ), + (generate_config_utils.MIN_VERSION, (generate_config_utils.MIN_PYTHON,)), + ): + tasks = [ + task + for task in pr_tasks() + if "server-version" in (task.tags or []) and task_vars(task).get("VERSION") == version + ] + seen = {task_vars(task).get("UV_PYTHON") for task in tasks} + assert seen == set(pythons), ( + f"{version} pr tasks pin {sorted(seen)}, want {sorted(pythons)}" + ) + assert len(tasks) >= 2, f"{version} has too few server-version pr tasks" + + def test_server_version_rotated_tasks_keep_coverage(): """Per-commit coverage for this family rides the rotated boundary-Python tasks, not the pr tasks (which run only on MAX_VERSION).""" @@ -138,14 +162,17 @@ def test_server_version_rotated_tasks_keep_coverage(): assert any("pr" not in (task.tags or []) for task in cov), ( "the server-version family's only COVERAGE tasks are pr tasks" ) + pr_pythons = { + combo: python + for combos in generate_config.SERVER_VERSION_PR_PYTHONS.values() + for combo, python in combos.items() + } offenders = [] for task in family: if "pr" in (task.tags or []): continue vars = task_vars(task) - boundary = generate_config.SERVER_VERSION_PR_PYTHONS.get( - (vars["TOPOLOGY"], vars["AUTH"], vars["SSL"]) - ) + boundary = pr_pythons.get((vars["TOPOLOGY"], vars["AUTH"], vars["SSL"])) if vars.get("UV_PYTHON") == boundary and vars.get("COVERAGE") != "1": offenders.append(task.name) assert not offenders, f"rotated server-version boundary tasks without COVERAGE: {offenders}" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 11ddd4b59d..1c20f99b1f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -480,10 +480,11 @@ CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at (and with it the required-version check) at the submodule boundary. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. - PR tasks pin to the Python boundaries: `MIN_PYTHON` with `TEST_MIN_DEPS=1`, `MAX_PYTHON`, and one - `FREE_THREADED_PYTHON`, each against the newest numbered server (`MAX_VERSION`; the `latest` - nightly is unstable), each with `COVERAGE=1`. The pins derive from `CPYTHONS` in - `generate_config_utils.py`, so CPython changes re-bind them rather than reshuffle the set. - Check the regenerated `generated_configs/tasks.yml` diff to confirm the `pr` tags. + `FREE_THREADED_PYTHON`, against the newest numbered server (`MAX_VERSION`; the `latest` nightly + is unstable), plus the min server (`MIN_VERSION`) through its `MIN_PYTHON` floor. Each runs with + `COVERAGE=1`. The pins derive from `CPYTHONS` in `generate_config_utils.py`, so CPython changes + re-bind them rather than reshuffle the set. Check the regenerated `generated_configs/tasks.yml` + diff to confirm the `pr` tags. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. ## Handling flaky tests From ce26198f3f5f557c2bc31ae0901ca96cecd95129 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 15 Oct 2026 15:31:49 -0500 Subject: [PATCH 13/13] Trim the min-server comments --- .evergreen/scripts/generate_config.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 7544d31a6a..4652d33d57 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -91,8 +91,7 @@ def create_server_version_variants() -> list[BuildVariant]: variants = [] for version in ALL_VERSIONS: display_name = get_variant_name("* MongoDB", version=version) - # The pr tasks pin their own VERSION; each version-boundary variant - # selects only the pr tasks pinned to that version. + # Each version-boundary variant selects only the pr tasks pinned to its version. if version in SERVER_VERSION_PR_PYTHONS: tasks = [".server-version !.pr", f".server-version .{version}"] else: @@ -656,9 +655,7 @@ def create_aws_lambda_variants(): ############## -# The server-version PR tasks: the Python boundaries to pin per server -# version. The min server tests its min-Python floor; the newest numbered -# server tests every boundary. +# The server-version pr tasks: the Python boundaries pinned per server version. SERVER_VERSION_PR_PYTHONS = { MIN_VERSION: {("standalone", "noauth", "nossl"): MIN_PYTHON}, MAX_VERSION: { @@ -717,9 +714,7 @@ def make_task(topology, auth, ssl, sync, python, extra_tags=(), **extra_vars): extra_vars["COVERAGE"] = "1" tasks.append(make_task(topology, auth, ssl, sync, python, **extra_vars)) - # The pinned PR tasks: the boundary Pythons pinned per server version, - # stable as CPYTHONS changes. Each carries its version as a tag so only - # the variant running that version selects it. + # The pinned pr tasks: the boundary Pythons per server version, stable as CPYTHONS changes. for version, combos in sorted(SERVER_VERSION_PR_PYTHONS.items()): for (topology, auth, ssl), python in sorted(combos.items()): for sync in SYNCS: