diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index aeb5782901..4924f05ffb 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -75,7 +75,7 @@ tasks: SUB_TEST_NAME: session-creds UV_PYTHON: "3.14" tags: [auth-aws, auth-aws-session-creds] - - name: test-auth-aws-9.0-web-identity-python3.15t + - name: test-auth-aws-9.0-regular-python3.11-min-deps-cov commands: - func: run server vars: @@ -85,10 +85,26 @@ tasks: - func: run tests vars: TEST_NAME: auth_aws - SUB_TEST_NAME: web-identity - UV_PYTHON: 3.15t - tags: [auth-aws, auth-aws-web-identity, free-threaded] - - name: test-auth-aws-9.0-web-identity-session-name-python3.15t + SUB_TEST_NAME: regular + UV_PYTHON: "3.11" + TEST_MIN_DEPS: "1" + COVERAGE: "1" + tags: [auth-aws, auth-aws-regular, pr] + - name: test-auth-aws-9.0-regular-python3.15-cov + commands: + - func: run server + vars: + AUTH_AWS: "1" + VERSION: "9.0" + - func: assume ec2 role + - func: run tests + vars: + TEST_NAME: auth_aws + SUB_TEST_NAME: regular + UV_PYTHON: "3.15" + COVERAGE: "1" + tags: [auth-aws, auth-aws-regular, pr] + - name: test-auth-aws-9.0-web-identity-python3.15t commands: - func: run server vars: @@ -99,23 +115,22 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: web-identity - AWS_ROLE_SESSION_NAME: test UV_PYTHON: 3.15t tags: [auth-aws, auth-aws-web-identity, free-threaded] - - name: test-auth-aws-rapid-regular-python3.15-cov + - name: test-auth-aws-9.0-web-identity-session-name-python3.15t commands: - func: run server vars: AUTH_AWS: "1" - VERSION: rapid + VERSION: "9.0" - func: assume ec2 role - func: run tests vars: TEST_NAME: auth_aws - SUB_TEST_NAME: regular - UV_PYTHON: "3.15" - COVERAGE: "1" - tags: [auth-aws, auth-aws-regular, pr] + SUB_TEST_NAME: web-identity + AWS_ROLE_SESSION_NAME: test + UV_PYTHON: 3.15t + tags: [auth-aws, auth-aws-web-identity, free-threaded] - name: test-auth-aws-latest-assume-role-python3.11-min-deps commands: - func: run server @@ -130,6 +145,20 @@ tasks: UV_PYTHON: "3.11" TEST_MIN_DEPS: "1" tags: [auth-aws, auth-aws-assume-role] + - name: test-auth-aws-rapid-regular-python3.15-cov + commands: + - func: run server + vars: + AUTH_AWS: "1" + VERSION: rapid + - func: assume ec2 role + - func: run tests + vars: + TEST_NAME: auth_aws + SUB_TEST_NAME: regular + UV_PYTHON: "3.15" + COVERAGE: "1" + tags: [auth-aws, auth-aws-regular] - name: test-auth-aws-ecs commands: - func: assume ec2 role @@ -754,7 +783,7 @@ tasks: UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - - name: test-ocsp-ecdsa-valid-cert-server-staples-v4.4-python3.11-min-deps + - name: test-ocsp-ecdsa-valid-cert-server-staples-v4.4-python3.11-min-deps-cov commands: - func: run tests vars: @@ -764,6 +793,7 @@ tasks: UV_PYTHON: "3.11" VERSION: "4.4" TEST_MIN_DEPS: "1" + COVERAGE: "1" tags: - ocsp - ocsp-ecdsa @@ -875,6 +905,21 @@ tasks: - ocsp-ecdsa - latest - ocsp-staple + - name: test-ocsp-ecdsa-valid-cert-server-staples-v9.0-python3.15-cov + commands: + - func: run tests + vars: + ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json + OCSP_SERVER_TYPE: valid + TEST_NAME: ocsp + UV_PYTHON: "3.15" + VERSION: "9.0" + COVERAGE: "1" + tags: + - ocsp + - ocsp-ecdsa + - "9.0" + - ocsp-staple - pr - name: test-ocsp-ecdsa-invalid-cert-server-staples-v4.4-python3.11-min-deps commands: @@ -2872,20 +2917,18 @@ tasks: - python-3.12 - replica_set-auth-ssl - sync - - name: test-server-version-python3.12-async-noauth-nossl-replica-set-cov + - name: test-server-version-python3.12-async-noauth-nossl-replica-set commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: replica_set - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: replica_set - COVERAGE: "1" UV_PYTHON: "3.12" TEST_NAME: default_async tags: @@ -2893,8 +2936,7 @@ tasks: - python-3.12 - replica_set-noauth-nossl - async - - pr - - name: test-server-version-python3.11-sync-noauth-nossl-replica-set-min-deps-cov + - name: test-server-version-python3.11-sync-noauth-nossl-replica-set-min-deps commands: - func: run server vars: @@ -2902,14 +2944,12 @@ tasks: SSL: nossl TOPOLOGY: replica_set TEST_MIN_DEPS: "1" - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: replica_set TEST_MIN_DEPS: "1" - COVERAGE: "1" UV_PYTHON: "3.11" TEST_NAME: default_sync tags: @@ -2917,7 +2957,6 @@ tasks: - python-3.11 - replica_set-noauth-nossl - sync - - pr - name: test-server-version-python3.15-async-noauth-ssl-replica-set commands: - func: run server @@ -2996,7 +3035,7 @@ tasks: - python-3.14 - sharded_cluster-auth-nossl - sync - - name: test-server-version-python3.11-async-auth-ssl-sharded-cluster-min-deps-cov + - name: test-server-version-python3.11-async-auth-ssl-sharded-cluster-min-deps commands: - func: run server vars: @@ -3004,14 +3043,12 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" UV_PYTHON: "3.11" TEST_NAME: default_async tags: @@ -3019,7 +3056,6 @@ tasks: - python-3.11 - sharded_cluster-auth-ssl - async - - pr - name: test-server-version-python3.12-async-auth-ssl-sharded-cluster commands: - func: run server @@ -3097,18 +3133,20 @@ tasks: - sharded_cluster-auth-ssl - async - free-threaded - - name: test-server-version-python3.15-async-auth-ssl-sharded-cluster + - name: test-server-version-python3.15-async-auth-ssl-sharded-cluster-cov commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster + COVERAGE: "1" UV_PYTHON: "3.15" TEST_NAME: default_async tags: @@ -3136,7 +3174,7 @@ tasks: - sharded_cluster-auth-ssl - async - free-threaded - - name: test-server-version-python3.11-sync-auth-ssl-sharded-cluster-min-deps-cov + - name: test-server-version-python3.11-sync-auth-ssl-sharded-cluster-min-deps commands: - func: run server vars: @@ -3144,14 +3182,12 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" UV_PYTHON: "3.11" TEST_NAME: default_sync tags: @@ -3159,7 +3195,6 @@ tasks: - python-3.11 - sharded_cluster-auth-ssl - sync - - pr - name: test-server-version-python3.12-sync-auth-ssl-sharded-cluster commands: - func: run server @@ -3237,18 +3272,20 @@ tasks: - sharded_cluster-auth-ssl - sync - free-threaded - - name: test-server-version-python3.15-sync-auth-ssl-sharded-cluster + - name: test-server-version-python3.15-sync-auth-ssl-sharded-cluster-cov commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster + COVERAGE: "1" UV_PYTHON: "3.15" TEST_NAME: default_sync tags: @@ -3456,21 +3493,18 @@ tasks: - python-3.11 - standalone-noauth-nossl - async - - pr - - name: test-server-version-python3.15-sync-noauth-nossl-standalone-cov + - name: test-server-version-python3.15-sync-noauth-nossl-standalone commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone - COVERAGE: "1" UV_PYTHON: "3.15" TEST_NAME: default_sync tags: @@ -3478,7 +3512,6 @@ tasks: - python-3.15 - standalone-noauth-nossl - sync - - pr - name: test-server-version-python3.15t-async-noauth-ssl-standalone commands: - func: run server @@ -3518,6 +3551,214 @@ tasks: - python-3.14 - standalone-noauth-ssl - sync + - name: test-server-version-v4.4-python3.11-sync-noauth-nossl-standalone-min-deps-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "4.4" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "4.4" + COVERAGE: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_sync + tags: + - server-version + - python-3.11 + - standalone-noauth-nossl + - sync + - pr + - "4.4" + - name: test-server-version-v4.4-python3.11-async-noauth-nossl-standalone-min-deps-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "4.4" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "4.4" + COVERAGE: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_async + tags: + - server-version + - python-3.11 + - standalone-noauth-nossl + - async + - pr + - "4.4" + - name: test-server-version-v9.0-python3.15-sync-noauth-nossl-replica-set-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_sync + tags: + - server-version + - python-3.15 + - replica_set-noauth-nossl + - sync + - pr + - "9.0" + - name: test-server-version-v9.0-python3.15-async-noauth-nossl-replica-set-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_async + tags: + - server-version + - python-3.15 + - replica_set-noauth-nossl + - async + - pr + - "9.0" + - name: test-server-version-v9.0-python3.15-sync-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_sync + tags: + - server-version + - python-3.15 + - sharded_cluster-auth-ssl + - sync + - pr + - "9.0" + - name: test-server-version-v9.0-python3.15-async-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_async + tags: + - server-version + - python-3.15 + - sharded_cluster-auth-ssl + - async + - pr + - "9.0" + - name: test-server-version-v9.0-python3.11-sync-noauth-nossl-standalone-min-deps-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_sync + tags: + - server-version + - python-3.11 + - standalone-noauth-nossl + - sync + - pr + - "9.0" + - name: test-server-version-v9.0-python3.11-async-noauth-nossl-standalone-min-deps-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + TEST_MIN_DEPS: "1" + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_async + tags: + - server-version + - python-3.11 + - standalone-noauth-nossl + - async + - pr + - "9.0" # Standard tests - name: test-standard-v4.4-python3.12-sync-noauth-ssl-replica-set @@ -4154,6 +4395,31 @@ tasks: - replica_set-noauth-ssl - async - free-threaded + - name: test-standard-v9.0-python3.15-async-noauth-ssl-replica-set-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: ssl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: ssl + TOPOLOGY: replica_set + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + TEST_NAME: default_async + tags: + - test-standard + - server-9.0 + - python-3.15 + - replica_set-noauth-ssl + - async + - pr - name: test-standard-v9.0-python3.11-async-auth-ssl-sharded-cluster-min-deps commands: - func: run server @@ -4200,6 +4466,59 @@ tasks: - python-3.14 - sharded_cluster-auth-ssl - async + - name: test-standard-v9.0-python3.15t-sync-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: 3.15t + TEST_NAME: default_sync + tags: + - test-standard + - server-9.0 + - python-3.15t + - sharded_cluster-auth-ssl + - sync + - free-threaded + - pr + - name: test-standard-v9.0-python3.11-async-noauth-nossl-standalone-min-deps-cov + commands: + - func: run server + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: "9.0" + TEST_MIN_DEPS: "1" + COVERAGE: "1" + - func: run tests + vars: + AUTH: noauth + SSL: nossl + TOPOLOGY: standalone + VERSION: "9.0" + TEST_MIN_DEPS: "1" + COVERAGE: "1" + UV_PYTHON: "3.11" + TEST_NAME: default_async + tags: + - test-standard + - server-9.0 + - python-3.11 + - standalone-noauth-nossl + - async + - pr - name: test-standard-v9.0-python3.13-async-noauth-nossl-standalone commands: - func: run server @@ -4268,7 +4587,6 @@ tasks: - python-3.13 - replica_set-noauth-ssl - async - - pr - name: test-standard-latest-python3.15-async-noauth-ssl-replica-set-cov commands: - func: run server @@ -4293,7 +4611,6 @@ tasks: - python-3.15 - replica_set-noauth-ssl - async - - pr - name: test-standard-latest-python3.14t-async-auth-ssl-sharded-cluster-cov commands: - func: run server @@ -4319,7 +4636,6 @@ tasks: - sharded_cluster-auth-ssl - async - free-threaded - - pr - name: test-standard-latest-python3.12-async-noauth-nossl-standalone-cov commands: - func: run server @@ -4344,7 +4660,6 @@ tasks: - python-3.12 - standalone-noauth-nossl - async - - pr - name: test-standard-latest-python3.15t-async-noauth-nossl-standalone-cov commands: - func: run server @@ -4370,7 +4685,6 @@ tasks: - standalone-noauth-nossl - async - free-threaded - - pr - name: test-standard-rapid-python3.13-sync-noauth-ssl-replica-set commands: - func: run server @@ -4876,6 +5190,52 @@ tasks: - sharded_cluster-auth-ssl - auth - free-threaded + - name: test-non-standard-v9.0-python3.15-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + tags: + - test-non-standard + - server-9.0 + - python-3.15 + - sharded_cluster-auth-ssl + - auth + - pr + - cov + - name: test-non-standard-v9.0-python3.15-auth-ssl-sharded-cluster + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + UV_PYTHON: "3.15" + tags: + - test-non-standard-no-cov + - server-9.0 + - python-3.15 + - sharded_cluster-auth-ssl + - auth - name: test-non-standard-v9.0-python3.12-noauth-nossl-standalone commands: - func: run server @@ -4941,7 +5301,6 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - auth - - pr - cov - name: test-non-standard-latest-python3.13-auth-ssl-sharded-cluster commands: @@ -4964,7 +5323,6 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - auth - - pr - name: test-non-standard-latest-python3.11-noauth-nossl-standalone-min-deps commands: - func: run server @@ -5186,6 +5544,56 @@ tasks: - python-3.14 - sharded_cluster-auth-ssl - auth + - name: test-standard-auth-v9.0-python3.11-auth-ssl-sharded-cluster-min-deps-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + TEST_MIN_DEPS: "1" + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + TEST_MIN_DEPS: "1" + COVERAGE: "1" + UV_PYTHON: "3.11" + tags: + - test-standard-auth + - server-9.0 + - python-3.11 + - sharded_cluster-auth-ssl + - auth + - pr + - name: test-standard-auth-v9.0-python3.15-auth-ssl-sharded-cluster-cov + commands: + - func: run server + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + - func: run tests + vars: + AUTH: auth + SSL: ssl + TOPOLOGY: sharded_cluster + VERSION: "9.0" + COVERAGE: "1" + UV_PYTHON: "3.15" + tags: + - test-standard-auth + - server-9.0 + - python-3.15 + - sharded_cluster-auth-ssl + - auth + - pr - name: test-standard-auth-v9.0-python3.15t-auth-ssl-sharded-cluster commands: - func: run server @@ -5231,7 +5639,6 @@ tasks: - python-3.11 - sharded_cluster-auth-ssl - auth - - pr - name: test-standard-auth-rapid-python3.15-auth-ssl-sharded-cluster commands: - func: run server diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 39cfcfe6d6..b1d34bce8d 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -91,13 +91,13 @@ buildvariants: run_on: - macos-14 tags: [pr] - - name: auth-aws-ecs-macos + - name: auth-aws-ecs-ubuntu-24 tasks: - name: .auth-aws-ecs - display_name: Auth AWS ECS macOS + display_name: Auth AWS ECS Ubuntu-24 run_on: - ubuntu2404-small - tags: [pr] + tags: [] # Aws lambda tests - name: faas-lambda @@ -531,7 +531,8 @@ buildvariants: # Server version tests - name: mongodb-v4.4 tasks: - - name: .server-version + - name: .server-version !.pr + - name: .server-version .4.4 display_name: "* MongoDB v4.4" run_on: - rhel8.10-small @@ -540,7 +541,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-v5.0 tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB v5.0" run_on: - rhel8.10-small @@ -549,7 +550,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-v6.0 tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB v6.0" run_on: - rhel8.10-small @@ -558,7 +559,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-v7.0 tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB v7.0" run_on: - rhel8.10-small @@ -567,7 +568,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-v8.0 tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB v8.0" run_on: - rhel8.10-small @@ -576,7 +577,8 @@ buildvariants: tags: [coverage_tag] - name: mongodb-v9.0 tasks: - - name: .server-version + - name: .server-version !.pr + - name: .server-version .9.0 display_name: "* MongoDB v9.0" run_on: - rhel8.10-small @@ -585,7 +587,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-rapid tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB rapid" run_on: - rhel8.10-small @@ -594,7 +596,7 @@ buildvariants: tags: [coverage_tag] - name: mongodb-latest tasks: - - name: .server-version + - name: .server-version !.pr display_name: "* MongoDB latest" run_on: - rhel8.10-small diff --git a/.evergreen/scripts/generate-config.sh b/.evergreen/scripts/generate-config.sh index 70b4578cf9..4f46605901 100755 --- a/.evergreen/scripts/generate-config.sh +++ b/.evergreen/scripts/generate-config.sh @@ -3,4 +3,7 @@ set -eu +# Assert the generation rules, then regenerate the config. +python -m pytest .evergreen/scripts/test_generate_config.py -m "" + python .evergreen/scripts/generate_config.py diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index c7c7cd3be5..4652d33d57 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -12,8 +12,13 @@ C_EXTS, CPYTHONS, DEFAULT_HOST, + FREE_THREADED_PYTHON, HOSTS, + MAX_PYTHON, + MAX_VERSION, MIN_MAX_PYTHON, + MIN_PYTHON, + MIN_VERSION, OTHER_HOSTS, SYNCS, TOPOLOGIES, @@ -86,8 +91,13 @@ def create_server_version_variants() -> list[BuildVariant]: variants = [] for version in ALL_VERSIONS: display_name = get_variant_name("* MongoDB", version=version) + # Each version-boundary variant selects only the pr tasks pinned to its version. + if version in SERVER_VERSION_PR_PYTHONS: + tasks = [".server-version !.pr", f".server-version .{version}"] + else: + tasks = [".server-version !.pr"] variant = create_variant( - [".server-version"], + tasks, display_name, version=version, host=DEFAULT_HOST, @@ -582,12 +592,13 @@ def create_aws_auth_variants(): variants.append(variant) # The ECS test must be run on Ubuntu 24 to match the Fargate Config. + # Fresh tags and expansions: the loop's macOS values must not leak here. variant = create_variant( [".auth-aws-ecs"], - get_variant_name("Auth AWS ECS", host), + get_variant_name("Auth AWS ECS", HOSTS["ubuntu24"]), host=HOSTS["ubuntu24"], - tags=tags, - expansions=expansions, + tags=[], + expansions=dict(), ) variants.append(variant) return variants @@ -644,40 +655,27 @@ def create_aws_lambda_variants(): ############## -def create_server_version_tasks(): - tasks = [] - task_combos = set() - # All combinations of topology, auth, ssl, and sync should be tested. - for (topology, auth, ssl, sync), python in zip_cycle( - list(product(TOPOLOGIES, ["auth", "noauth"], ["ssl", "nossl"], SYNCS)), ALL_PYTHONS - ): - task_combos.add((topology, auth, ssl, sync, python)) +# The server-version pr tasks: the Python boundaries pinned per server version. +SERVER_VERSION_PR_PYTHONS = { + MIN_VERSION: {("standalone", "noauth", "nossl"): MIN_PYTHON}, + MAX_VERSION: { + ("standalone", "noauth", "nossl"): MIN_PYTHON, + ("replica_set", "noauth", "nossl"): MAX_PYTHON, + ("sharded_cluster", "auth", "ssl"): MAX_PYTHON, + }, +} - # Every python should be tested with sharded cluster, auth, ssl, with sync and async. - for python, sync in product(ALL_PYTHONS, SYNCS): - task_combos.add(("sharded_cluster", "auth", "ssl", sync, python)) - # Assemble the tasks. - seen = set() - for topology, auth, ssl, sync, python in sorted(task_combos): +def create_server_version_tasks(): + def make_task(topology, auth, ssl, sync, python, extra_tags=(), **extra_vars): combo = f"{topology}-{auth}-{ssl}" - tags = ["server-version", f"python-{python}", combo, sync] - if combo in [ - "standalone-noauth-nossl", - "replica_set-noauth-nossl", - "sharded_cluster-auth-ssl", - ]: - combo = f"{combo}-{sync}" - if combo not in seen: - seen.add(combo) - tags.append("pr") + tags = ["server-version", f"python-{python}", combo, sync, *extra_tags] expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" + expansions.update(extra_vars) if "t" in python: tags.append("free-threaded") - if "pr" in tags: - expansions["COVERAGE"] = "1" name = get_task_name( "test-server-version", python=python, @@ -689,7 +687,49 @@ def create_server_version_tasks(): test_vars["UV_PYTHON"] = python test_vars["TEST_NAME"] = f"default_{sync}" test_func = FunctionCall(func="run tests", vars=test_vars) - tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) + return EvgTask(name=name, tags=tags, commands=[server_func, test_func]) + + tasks = [] + task_combos = set() + # All combinations of topology, auth, ssl, and sync should be tested. + for (topology, auth, ssl, sync), rotated in zip_cycle( + list(product(TOPOLOGIES, ["auth", "noauth"], ["ssl", "nossl"], SYNCS)), ALL_PYTHONS + ): + task_combos.add((topology, auth, ssl, sync, rotated)) + + # Every python should be tested with sharded cluster, auth, ssl, with sync and async. + for python, sync in product(ALL_PYTHONS, SYNCS): + task_combos.add(("sharded_cluster", "auth", "ssl", sync, python)) + + # Rotated tasks inherit the variant's VERSION; boundary-Python tasks keep + # COVERAGE without pr so per-commit coverage is not pr-only. + pr_pythons = { + combo: python + for combos in SERVER_VERSION_PR_PYTHONS.values() + for combo, python in combos.items() + } + for topology, auth, ssl, sync, python in sorted(task_combos): + extra_vars = {} + if python == pr_pythons.get((topology, auth, ssl)): + extra_vars["COVERAGE"] = "1" + tasks.append(make_task(topology, auth, ssl, sync, python, **extra_vars)) + + # The pinned pr tasks: the boundary Pythons per server version, stable as CPYTHONS changes. + for version, combos in sorted(SERVER_VERSION_PR_PYTHONS.items()): + for (topology, auth, ssl), python in sorted(combos.items()): + for sync in SYNCS: + tasks.append( + make_task( + topology, + auth, + ssl, + sync, + python, + extra_tags=("pr", version), + VERSION=version, + COVERAGE="1", + ) + ) return tasks @@ -719,9 +759,11 @@ def create_test_non_standard_tasks(): # For each version and topology, rotate through the CPythons. # Only the sharded_cluster topology runs on PRs to keep patch build size manageable. for (version, topology), python in zip_cycle(list(product(ALL_VERSIONS, TOPOLOGIES)), CPYTHONS): - pr = version == "latest" and topology == "sharded_cluster" - task_combos.add((version, topology, python, pr)) - for version, topology, python, pr in sorted(task_combos): + task_combos.add((version, topology, python)) + # The pinned PR task: the max Python on MAX_VERSION, sharded cluster. + task_combos.add((MAX_VERSION, "sharded_cluster", MAX_PYTHON)) + for version, topology, python in sorted(task_combos): + pr = version == MAX_VERSION and topology == "sharded_cluster" and python == MAX_PYTHON auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-non-standard", @@ -737,7 +779,7 @@ def create_test_non_standard_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" - elif pr: + elif pr or (version == "latest" and topology == "sharded_cluster"): expansions["COVERAGE"] = "1" tags.append("cov") name = get_task_name("test-non-standard", python=python, **expansions) @@ -749,14 +791,14 @@ def create_test_non_standard_tasks(): # For each coverage task, also emit a non-coverage companion so that # macOS/Win64 encryption variants (which filter out .cov due to timeout # constraints) still have a "latest" task to activate in patch builds. - if pr and "cov" in tags: + if "cov" in tags: nc_expansions = {k: v for k, v in expansions.items() if k != "COVERAGE"} # Use a distinct primary tag so companions are not selected by existing # ".test-non-standard" selectors (e.g. load-balancer, PyOpenSSL variants). nc_tags = [ "test-non-standard-no-cov" if t == "test-non-standard" else t for t in tags - if t != "cov" + if t not in ("cov", "pr") ] nc_name = get_task_name("test-non-standard", python=python, **nc_expansions) nc_server_func = FunctionCall(func="run server", vars=nc_expansions) @@ -805,9 +847,12 @@ def create_test_standard_auth_tasks(): for (version, topology), python in zip_cycle( list(product(ALL_VERSIONS, ["sharded_cluster"])), CPYTHONS ): - pr = version == "latest" - task_combos.add((version, topology, python, pr)) - for version, topology, python, pr in sorted(task_combos): + task_combos.add((version, topology, python)) + # The pinned PR tasks: the min and max Python on MAX_VERSION. + task_combos.add((MAX_VERSION, "sharded_cluster", MIN_PYTHON)) + task_combos.add((MAX_VERSION, "sharded_cluster", MAX_PYTHON)) + for version, topology, python in sorted(task_combos): + pr = version == MAX_VERSION and python in (MIN_PYTHON, MAX_PYTHON) auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-standard-auth", @@ -823,7 +868,7 @@ def create_test_standard_auth_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" - elif pr: + if pr: expansions["COVERAGE"] = "1" name = get_task_name("test-standard-auth", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) @@ -839,13 +884,23 @@ def create_standard_tasks(): tasks = [] task_combos = set() # For each python and topology and sync/async, rotate through the the versions. - for (python, topology, sync), version in zip_cycle( + for (python, topology, sync), rotated in zip_cycle( list(product(CPYTHONS, TOPOLOGIES, SYNCS)), ALL_VERSIONS ): - pr = version == "latest" - task_combos.add((version, topology, python, sync, pr)) + # PR tasks pin to the Python boundaries on MAX_VERSION; the rotation + # keeps running every combination on its rotated server version. + pr = ( + (python == MIN_PYTHON and topology == "standalone" and sync == "async") + or (python == MAX_PYTHON and topology == "replica_set" and sync == "async") + or (python == FREE_THREADED_PYTHON and topology == "sharded_cluster" and sync == "sync") + ) + task_combos.add((rotated, topology, python, sync, False)) + if pr: + task_combos.add((MAX_VERSION, topology, python, sync, True)) - for version, topology, python, sync, pr in sorted(task_combos): + # A pinned combo can collide with a rotated one on MAX_VERSION; keep the pinned (pr) task. + combos = {combo[:4]: combo for combo in sorted(task_combos)} + for version, topology, python, sync, pr in combos.values(): auth, ssl = get_standard_auth_ssl(topology) tags = [ "test-standard", @@ -861,7 +916,7 @@ def create_standard_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" - elif pr: + if pr or (version == "latest" and python != ALL_PYTHONS[0]): expansions["COVERAGE"] = "1" name = get_task_name("test-standard", python=python, sync=sync, **expansions) server_func = FunctionCall(func="run server", vars=expansions) @@ -923,7 +978,13 @@ def create_aws_tasks(): "web-identity", ] assume_func = FunctionCall(func="assume ec2 role") - for version, test_type, python in zip_cycle(get_versions_from("4.4"), aws_test_types, CPYTHONS): + task_combos = set(zip_cycle(get_versions_from("4.4"), aws_test_types, CPYTHONS)) + # The pinned PR tasks: the min and max Python on MAX_VERSION. + task_combos |= { + (MAX_VERSION, "regular", MIN_PYTHON), + (MAX_VERSION, "regular", MAX_PYTHON), + } + for version, test_type, python in sorted(task_combos): base_name = f"test-auth-aws-{version}" base_tags = ["auth-aws"] server_vars = dict(AUTH_AWS="1", VERSION=version) @@ -932,10 +993,15 @@ def create_aws_tasks(): if "t" in python: tags.append("free-threaded") test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, UV_PYTHON=python) + # The rotation never produces (MAX_VERSION, regular). + pr = ( + version == MAX_VERSION and test_type == "regular" and python in (MIN_PYTHON, MAX_PYTHON) + ) if python == MIN_MAX_PYTHON[0]: test_vars["TEST_MIN_DEPS"] = "1" - elif python == MIN_MAX_PYTHON[-1]: + if pr: tags.append("pr") + if pr or python == MIN_MAX_PYTHON[-1]: test_vars["COVERAGE"] = "1" name = get_task_name(f"{base_name}-{test_type}", **test_vars) test_func = FunctionCall(func="run tests", vars=test_vars) @@ -989,6 +1055,15 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): file_name = f"{algo}-basic-tls-ocsp-{variant}.json" for version in get_versions_from("4.4"): + # Rotated nightlies: latest-CPython on "latest", min-deps Python elsewhere. + pr = base_task_name == "valid-cert-server-staples" and algo == "ecdsa" and version == "4.4" + # The "latest" ECDSA staples task keeps nightly coverage; the pinned + # MAX_VERSION task below is the pr task. + nightly_cov = ( + base_task_name == "valid-cert-server-staples" + and algo == "ecdsa" + and version == "latest" + ) if version == "latest": python = MIN_MAX_PYTHON[-1] else: @@ -1006,23 +1081,27 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): tags = ["ocsp", f"ocsp-{algo}", version] if "disableStapling" not in variant: tags.append("ocsp-staple") - # Run exactly one min-deps and one latest-CPython stapled OCSP task on - # every PR (ecdsa only, to avoid doubling coverage across algorithms) - # so a cryptography-backend regression at either dependency extreme, - # like the min-deps ML-KEM breakage in PYTHON-6032, is caught before - # merge instead of on the next weekly mainline batch run. - if ( - base_task_name == "valid-cert-server-staples" - and algo == "ecdsa" - and version - in ( - "latest", - "4.4", - ) - ): + if pr or nightly_cov: + vars["COVERAGE"] = "1" + if pr: tags.append("pr") - if "TEST_MIN_DEPS" not in vars: - vars["COVERAGE"] = "1" + test_func = FunctionCall(func="run tests", vars=vars) + task_name = get_task_name(f"test-ocsp-{algo}-{base_task_name}", **vars) + tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) + + # One latest-CPython ECDSA staples task on every PR, pinned to MAX_VERSION + # (the "latest" nightly is unstable): catches cryptography-backend + # regressions at the dependency extremes before merge (PYTHON-6032). + if base_task_name == "valid-cert-server-staples" and algo == "ecdsa": + vars = dict( + ORCHESTRATION_FILE=file_name, + OCSP_SERVER_TYPE=server_type, + TEST_NAME="ocsp", + UV_PYTHON=MIN_MAX_PYTHON[-1], + VERSION=MAX_VERSION, + COVERAGE="1", + ) + tags = ["ocsp", f"ocsp-{algo}", MAX_VERSION, "ocsp-staple", "pr"] test_func = FunctionCall(func="run tests", vars=vars) task_name = get_task_name(f"test-ocsp-{algo}-{base_task_name}", **vars) tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index 18be347d56..0ba291ab5b 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -26,6 +26,12 @@ MIN_SUPPORT_VERSIONS = ["3.11"] ALL_PYTHONS = CPYTHONS MIN_MAX_PYTHON = [CPYTHONS[0], CPYTHONS[-1]] +# The Python and server version boundaries that PR tasks pin to. +MIN_PYTHON = CPYTHONS[0] +MAX_PYTHON = CPYTHONS[-1] +FREE_THREADED_PYTHON = [p for p in CPYTHONS if "t" in p][-1] +MIN_VERSION = min((v for v in ALL_VERSIONS if v[0].isdigit()), key=float) +MAX_VERSION = max((v for v in ALL_VERSIONS if v[0].isdigit()), key=float) BATCHTIME_WEEK = 10080 BATCHTIME_DAY = 1440 AUTH_SSLS = [("auth", "ssl"), ("noauth", "ssl"), ("noauth", "nossl")] diff --git a/.evergreen/scripts/test_generate_config.py b/.evergreen/scripts/test_generate_config.py new file mode 100644 index 0000000000..f078f2f305 --- /dev/null +++ b/.evergreen/scripts/test_generate_config.py @@ -0,0 +1,278 @@ +"""Tests for the Evergreen PR task rules (PYTHON-6112). + +Run: pytest .evergreen/scripts/test_generate_config.py -m "" +(-m "" clears the repo-wide marker filter.) +Requires the generate-config hook's dependencies: pytest, pytest-asyncio, shrub.py, pyyaml. +""" + +from __future__ import annotations + +import sys +from inspect import getmembers, isfunction +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +import generate_config +import generate_config_utils + + +def all_tasks(): + tasks = [] + for name, func in getmembers(generate_config, isfunction): + if name.startswith("create_") and name.endswith("_tasks"): + tasks.extend(func()) + return tasks + + +def task_vars(task): + vars = {} + for command in task.commands or []: + vars.update(getattr(command, "vars", None) or {}) + return vars + + +def pr_tasks(): + return [task for task in all_tasks() if "pr" in (task.tags or [])] + + +def all_variants(): + variants = [] + for name, func in getmembers(generate_config, isfunction): + if name.startswith("create_") and name.endswith("_variants"): + variants.extend(func()) + return variants + + +def variant_selects(variant, task): + """Whether the variant's task selectors match the task: "." selects a + tag, "!" negates the token that follows, a bare token names a task.""" + tags = task.tags or [] + for ref in variant.tasks: + includes, excludes, names = [], [], [] + for token in ref.name.split(): + if token.startswith("!."): + excludes.append(token[2:]) + elif token.startswith("!"): + excludes.append(token[1:]) + elif token.startswith("."): + includes.append(token[1:]) + else: + names.append(token) + if any(tag in tags for tag in excludes): + continue + if not includes and not names: + # A negation-only selector matches every remaining task. + return True + if names and task.name not in names: + continue + if includes and not all(tag in tags for tag in includes): + continue + return True + return False + + +def test_pr_tasks_pin_max_version(): + """Task-level pins: pr tasks pin MAX_VERSION (the "latest" nightly is + unstable); the explicit 4.4 min-deps pin is the exception. Variant-merged + pins are checked by test_pr_tasks_effectively_pin_max_version.""" + offenders = [ + (task.name, task_vars(task).get("VERSION")) + for task in pr_tasks() + if task_vars(task).get("VERSION") not in (None, generate_config_utils.MAX_VERSION) + and not ( + task_vars(task).get("VERSION") == "4.4" and task_vars(task).get("TEST_MIN_DEPS") == "1" + ) + ] + assert not offenders, f"PR tasks not pinned to MAX_VERSION: {offenders}" + + +def test_pr_tasks_effectively_pin_max_version(): + """Effective pins: a pr task without its own VERSION var inherits the + selecting variant's version, so every pr task that runs a server must + resolve to MAX_VERSION once variant expansions merge in. The explicit + 4.4 min-deps pin is the exception.""" + tasks = [task for task in all_tasks() if "pr" in (task.tags or [])] + offenders = [] + for variant in all_variants(): + variant_version = (variant.expansions or {}).get("VERSION") + for task in tasks: + if not variant_selects(variant, task): + continue + task_vars_ = task_vars(task) + if task_vars_.get("VERSION") == "4.4" and task_vars_.get("TEST_MIN_DEPS") == "1": + continue + effective = task_vars_.get("VERSION") or variant_version + if effective is None: + # Helpers that run the stable default server, like kms-fail. + continue + if effective != generate_config_utils.MAX_VERSION: + offenders.append((variant.name, task.name, effective)) + assert not offenders, f"pr tasks not effectively pinned to MAX_VERSION: {offenders}" + + +def test_server_version_pr_tasks_selected_once(): + """The mongodb-* variants differ only in server version, so a pr task + must be selected only by the variant running the server version it pins.""" + offenders = {} + for task in pr_tasks(): + if "server-version" not in (task.tags or []): + continue + own = task_vars(task).get("VERSION") + selecting = [variant.name for variant in all_variants() if variant_selects(variant, task)] + matching = [ + variant.name + for variant in all_variants() + if (variant.expansions or {}).get("VERSION") == own + ] + if selecting != matching: + offenders[task.name] = {"selected by": selecting, "pins": own} + assert not offenders, ( + f"server-version pr tasks not selected by the variant matching their pin: {offenders}" + ) + + +def test_pr_tasks_cover_version_boundaries(): + """PRs test the server version boundaries: the newest numbered server + with every Python boundary, the min server with its min-Python floor.""" + for version, pythons in ( + ( + generate_config_utils.MAX_VERSION, + (generate_config_utils.MIN_PYTHON, generate_config_utils.MAX_PYTHON), + ), + (generate_config_utils.MIN_VERSION, (generate_config_utils.MIN_PYTHON,)), + ): + tasks = [ + task + for task in pr_tasks() + if "server-version" in (task.tags or []) and task_vars(task).get("VERSION") == version + ] + seen = {task_vars(task).get("UV_PYTHON") for task in tasks} + assert seen == set(pythons), ( + f"{version} pr tasks pin {sorted(seen)}, want {sorted(pythons)}" + ) + assert len(tasks) >= 2, f"{version} has too few server-version pr tasks" + + +def test_server_version_rotated_tasks_keep_coverage(): + """Per-commit coverage for this family rides the rotated boundary-Python + tasks, not the pr tasks (which run only on MAX_VERSION).""" + family = [task for task in all_tasks() if "server-version" in (task.tags or [])] + cov = [task for task in family if task_vars(task).get("COVERAGE") == "1"] + assert any("pr" not in (task.tags or []) for task in cov), ( + "the server-version family's only COVERAGE tasks are pr tasks" + ) + pr_pythons = { + combo: python + for combos in generate_config.SERVER_VERSION_PR_PYTHONS.values() + for combo, python in combos.items() + } + offenders = [] + for task in family: + if "pr" in (task.tags or []): + continue + vars = task_vars(task) + boundary = pr_pythons.get((vars["TOPOLOGY"], vars["AUTH"], vars["SSL"])) + if vars.get("UV_PYTHON") == boundary and vars.get("COVERAGE") != "1": + offenders.append(task.name) + assert not offenders, f"rotated server-version boundary tasks without COVERAGE: {offenders}" + + +def test_pr_min_deps_tasks_also_run_coverage(): + offenders = [ + task.name + for task in pr_tasks() + if task_vars(task).get("TEST_MIN_DEPS") == "1" and task_vars(task).get("COVERAGE") != "1" + ] + assert not offenders, f"min-deps PR tasks without COVERAGE: {offenders}" + + +def test_standard_family_pr_budget(): + """Patch-build PR budget: at most 5 pr-tagged test-standard tasks (the + pre-pin baseline), at most 3 on Python 3.14+ (the ubuntu22 compression + filter).""" + tasks = [task for task in pr_tasks() if "test-standard" in (task.tags or [])] + assert len(tasks) <= 5, f"too many test-standard PR tasks: {[t.name for t in tasks]}" + newer = [ + task + for task in tasks + if any( + tag in (task.tags or []) + for tag in ("python-3.14", "python-3.14t", "python-3.15", "python-3.15t") + ) + ] + assert len(newer) <= 3, f"too many 3.14+ test-standard PR tasks: {[t.name for t in newer]}" + + +def test_pr_tasks_run_coverage(): + """Pr tasks that pin a server version run with COVERAGE=1; no-cov + companions and infrastructure tasks (no VERSION) are exempt.""" + offenders = [ + task.name + for task in pr_tasks() + if task_vars(task).get("VERSION") is not None + and "test-non-standard-no-cov" not in (task.tags or []) + and task_vars(task).get("COVERAGE") != "1" + ] + assert not offenders, f"PR tasks without COVERAGE: {offenders}" + + +def test_rotated_tasks_keep_nightly_coverage(): + """Batch coverage rides the rotation independently of the pr tasks: + rotated "latest" tasks and max-Python auth-aws tasks keep COVERAGE=1 + (min-deps excepted).""" + offenders = [] + for task in all_tasks(): + tags = task.tags or [] + vars = task_vars(task) + if vars.get("TEST_MIN_DEPS") == "1" or "pr" in tags: + continue + rotated_latest = "server-latest" in tags or ("ocsp-ecdsa" in tags and "latest" in tags) + is_tested = ("test-standard" in tags and "server-latest" in tags) or ( + "test-non-standard" in tags + and "server-latest" in tags + and "sharded_cluster-auth-ssl" in tags + ) + is_aws_max_python = ( + "auth-aws" in tags + and "session-name" not in task.name + and vars.get("UV_PYTHON") == generate_config_utils.MAX_PYTHON + ) + if rotated_latest and is_tested and vars.get("COVERAGE") != "1": + offenders.append(task.name) + if is_aws_max_python and vars.get("COVERAGE") != "1": + offenders.append(task.name) + assert not offenders, f"rotated tasks without nightly COVERAGE: {offenders}" + # The "latest" ECDSA staples task keeps its nightly coverage. + latest_staples = [ + task.name + for task in all_tasks() + if "ocsp-ecdsa" in (task.tags or []) + and "ocsp-staple" in (task.tags or []) + and "latest" in (task.tags or []) + and task_vars(task).get("COVERAGE") == "1" + ] + assert latest_staples, "no ECDSA staples task on the latest nightly with COVERAGE" + # The 9.0 nightly OCSP tasks stay on the min-deps Python. + offenders = [ + task.name + for task in all_tasks() + if "ocsp" in (task.tags or []) + and "9.0" in (task.tags or []) + and "pr" not in (task.tags or []) + and task_vars(task).get("TEST_MIN_DEPS") != "1" + ] + assert not offenders, f"nightly 9.0 OCSP tasks not on min-deps Python: {offenders}" + + +def test_non_standard_cov_tasks_have_no_cov_companions(): + """Each non-standard coverage task also emits a no-cov companion, so the + macOS/Win64 encryption variants (which filter .cov) have a task to + activate in patch builds.""" + no_cov = {task.name for task in all_tasks() if "test-non-standard-no-cov" in (task.tags or [])} + for task in all_tasks(): + if "test-non-standard-no-cov" in (task.tags or []): + assert "pr" not in (task.tags or []), f"no-cov companion tagged pr: {task.name}" + elif "cov" in (task.tags or []) and "test-non-standard" in (task.tags or []): + companion = task.name.removesuffix("-cov") + assert companion in no_cov, f"missing no-cov companion for {task.name}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c1053779ce..55c1ed6add 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -135,4 +135,4 @@ repos: entry: .evergreen/scripts/generate-config.sh language: python require_serial: true - additional_dependencies: ["shrub.py>=3.10.0", "pyyaml>=6.0.2"] + additional_dependencies: ["pytest", "pytest-asyncio", "shrub.py>=3.10.0", "pyyaml>=6.0.2"] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1b6a12605..1c20f99b1f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -479,6 +479,12 @@ CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at the setup scripts write a `uv.toml` boundary into the submodule checkout that stops uv's config discovery (and with it the required-version check) at the submodule boundary. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. +- PR tasks pin to the Python boundaries: `MIN_PYTHON` with `TEST_MIN_DEPS=1`, `MAX_PYTHON`, and one + `FREE_THREADED_PYTHON`, against the newest numbered server (`MAX_VERSION`; the `latest` nightly + is unstable), plus the min server (`MIN_VERSION`) through its `MIN_PYTHON` floor. Each runs with + `COVERAGE=1`. The pins derive from `CPYTHONS` in `generate_config_utils.py`, so CPython changes + re-bind them rather than reshuffle the set. Check the regenerated `generated_configs/tasks.yml` + diff to confirm the `pr` tags. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. ## Handling flaky tests