diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 3fcb4eda49..34416bc183 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -339,6 +339,9 @@ jobs: # every group before it merges, so a passing fast core reports as Verify and each pull # request runs the heavy suites once, in the queue. Unsetting the variable restores the old # behaviour. + # GitHub takes the required Verify from the newest Verify check suite on the head. Re-running + # an older suite never repaints it: to recover a head whose newest suite reported another + # name, start a new suite (push, or remove and re-add full-verify), not a rerun. # The label and draft state come from the pull request as it is now, not the event payload: # `gh pr create --label` can deliver `labeled` before `opened`, and the later run cancels the # earlier one, so the surviving run must still see the label. @@ -372,26 +375,56 @@ jobs: if: ${{ !((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'full-verify') }} runs-on: ubuntu-24.04 steps: + # Release Please pushes its version bump before release-please.yml refreshes Cargo.lock and + # uv.lock in a commit of its own, so a bump head always disagrees with its lockfiles. That + # head waits for the refresh commit, whose run checks the lockfiles. The refresh step itself + # runs cargo metadata --locked and uv sync --locked before it commits, and the merge queue + # always checks the lockfiles. + - name: Wait for the release lockfile refresh + id: release + if: >- + github.event_name == 'pull_request' + && github.event.pull_request.head.repo.full_name == github.repository + && startsWith(github.event.pull_request.head.ref, 'release-please--branches--') + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + subject="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${HEAD_SHA}" \ + --jq '.commit.message | split("\n")[0]')" + if [[ "${subject}" != "chore: refresh release lockfiles" ]]; then + echo "pending=true" >> "${GITHUB_OUTPUT}" + echo "Version bump ${HEAD_SHA}: the lockfile refresh commit's run checks the lockfiles" \ + | tee -a "${GITHUB_STEP_SUMMARY}" + fi - uses: actions/checkout@v4 + if: steps.release.outputs.pending != 'true' - uses: ./.github/actions/setup-uv + if: steps.release.outputs.pending != 'true' with: version: "0.12.3" - uses: dtolnay/rust-toolchain@master + if: steps.release.outputs.pending != 'true' with: toolchain: stable - name: Install Python + if: steps.release.outputs.pending != 'true' run: uv python install 3.12 - name: Verify lockfile + if: steps.release.outputs.pending != 'true' run: uv lock --check --python 3.12 - name: Verify native workspace lockfile + if: steps.release.outputs.pending != 'true' run: cargo metadata --locked --format-version 1 > /dev/null - checks: - name: Static checks and unit + integration tests + # The extension and CLI build once here; the static job and every test shard install these + # wheels instead of compiling them again. + native_build: + name: Build the extension and CLI wheels needs: [changes, lockfile] if: needs.changes.outputs.checks == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-uv @@ -413,8 +446,58 @@ jobs: save-if: false - name: Install Python run: uv python install 3.12 - - name: Install dependencies - run: uv sync --locked --all-groups --all-extras + - name: Install the build tools without building the packages + run: uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli + - name: Build the extension and CLI wheels + run: | + uv run --no-sync maturin build --release --compatibility linux --out dist + uv run --no-sync maturin build --manifest-path crates/sqlbuild-cli/Cargo.toml --release --compatibility linux --out dist + - uses: actions/upload-artifact@v4 + with: + name: verify-native-wheels + path: dist/*.whl + if-no-files-found: error + compression-level: 0 + # Long enough for "Re-run failed jobs" on a shard, which reuses this run's wheels. + retention-days: 3 + overwrite: true + + checks: + name: Static checks + needs: [changes, lockfile, native_build] + if: needs.changes.outputs.checks == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + # A syncing `uv run` would replace the installed wheels with a fresh editable build. + UV_NO_SYNC: "1" + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-uv + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + # Clippy in check-ci builds its own check artifacts from the same cache as the native checks. + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - uses: Swatinem/rust-cache@v2 + with: + shared-key: verify-checks + save-if: false + - name: Install Python + run: uv python install 3.12 + - name: Install dependencies without building the packages + run: uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli + - uses: actions/download-artifact@v4 + with: + name: verify-native-wheels + path: dist + - name: Install the prebuilt extension and CLI + run: uv pip install --no-deps dist/*.whl - uses: actions/setup-node@v4 with: node-version: "22" @@ -428,6 +511,47 @@ jobs: run: make check-ci env: DUPES_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || 'origin/main' }} + + tests: + # check-test-shards, part of the static checks, proves that the TEST_SHARD_*_PATHS lists in + # the Makefile run every test once; each of those lists needs an entry here. + name: Unit + integration tests (${{ matrix.name }}) + needs: [changes, lockfile, native_build] + if: needs.changes.outputs.checks == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - name: core + target: test-shard-core + workers: "4" + - name: publication + target: test-shard-publication + workers: "2" + - name: warehouse history + target: test-shard-history + workers: "4" + env: + UV_NO_SYNC: "1" + PYTEST_XDIST_AUTO_NUM_WORKERS: ${{ matrix.workers }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-uv + with: + version: "0.12.3" + enable-cache: true + - name: Install Python + run: uv python install 3.12 + - name: Install dependencies without building the packages + run: uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli + - uses: actions/download-artifact@v4 + with: + name: verify-native-wheels + path: dist + - name: Install the prebuilt extension and CLI + run: uv pip install --no-deps dist/*.whl - name: Prepare Docker worker tests run: | docker info > /dev/null @@ -435,7 +559,7 @@ jobs: - name: Run unit and integration tests env: SQLBUILD_REQUIRE_DOCKER_TESTS: "1" - run: make test + run: make ${{ matrix.target }} native_checks: name: Native workspace checks @@ -1330,7 +1454,9 @@ jobs: [ changes, lockfile, + native_build, checks, + tests, native_checks, e2e_shards, compiler_differential, @@ -1358,7 +1484,9 @@ jobs: CHANGES_RESULT: ${{ needs.changes.result }} CHECKS_SELECTED: ${{ needs.changes.outputs.checks }} LOCKFILE_RESULT: ${{ needs.lockfile.result }} + NATIVE_BUILD_RESULT: ${{ needs.native_build.result }} CHECKS_RESULT: ${{ needs.checks.result }} + TESTS_RESULT: ${{ needs.tests.result }} NATIVE_CHECKS_RESULT: ${{ needs.native_checks.result }} E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }} COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }} @@ -1399,7 +1527,7 @@ jobs: echo "Required classification or lockfile checks did not pass" >&2 exit 1 fi - for result in "${CHECKS_RESULT}" "${NATIVE_CHECKS_RESULT}"; do + for result in "${NATIVE_BUILD_RESULT}" "${CHECKS_RESULT}" "${TESTS_RESULT}" "${NATIVE_CHECKS_RESULT}"; do if [[ "${result}" != "success" ]] && \ ! [[ "${CHECKS_SELECTED}" == "false" && "${result}" == "skipped" ]]; then echo "Required static or native checks did not pass: ${result}" >&2 diff --git a/Makefile b/Makefile index 6ed94078e8..b3dad765e5 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ SHELL := /bin/bash .PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-dupes-new check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client \ - check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \ + check-e2e-shards check-test-shards test-shard-core test-shard-publication test-shard-history \ + test-e2e-duckdb test-e2e-duckdb-build-core \ test-e2e-duckdb-build-incremental \ test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \ test-e2e-duckdb-integrations test-e2e-performance \ @@ -69,14 +70,46 @@ rust-lint: cargo clippy --workspace --all-targets --all-features -- -D warnings -test: +UNIT_INTEGRATION_MARKERS := not real_warehouse and not dbt +PUBLICATION_TESTS := tests/integration/src/sqlbuild/compiler/compiled_project/publication + +# CI runs the unit and integration tests in these shards; check-test-shards proves that together +# they run every test once. Each shard lists the pytest path arguments it runs. +TEST_SHARD_CORE_PATHS := tests/unit tests/integration --ignore=$(PUBLICATION_TESTS) +TEST_SHARD_HISTORY_PATHS := $(PUBLICATION_TESTS)/test_warehouse_metadata.py +TEST_SHARD_PUBLICATION_PATHS := $(PUBLICATION_TESTS) --ignore=$(TEST_SHARD_HISTORY_PATHS) + +# $(1) target, $(2) pytest path arguments, $(3) xdist distribution. +define run_unit_integration @mkdir -p /tmp/opencode - @log="/tmp/opencode/test-$$(date +%Y%m%d-%H%M%S).log"; \ + @log="/tmp/opencode/$(1)-$$(date +%Y%m%d-%H%M%S).log"; \ echo "Logging to $$log"; \ - env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=$(SQLBUILD_CONCURRENCY) uv run pytest tests/unit tests/integration -m "not real_warehouse and not dbt" -vv --color=yes -n auto --dist loadfile --max-worker-restart=0 2>&1 | tee "$$log"; \ + env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=$(SQLBUILD_CONCURRENCY) uv run pytest $(2) -m "$(UNIT_INTEGRATION_MARKERS)" -vv --color=yes -n auto --dist $(3) --max-worker-restart=0 2>&1 | tee "$$log"; \ status=$${PIPESTATUS[0]}; \ echo "TEST_EXIT=$$status (log: $$log)" | tee -a "$$log"; \ exit $$status +endef + +test: + $(call run_unit_integration,$@,tests/unit tests/integration,loadfile) + +test-shard-core: + $(call run_unit_integration,$@,$(TEST_SHARD_CORE_PATHS),loadfile) + +test-shard-publication: + $(call run_unit_integration,$@,$(TEST_SHARD_PUBLICATION_PATHS),loadfile) + +# Its location-backed cases wait on reconcile loops, and no test there shares module state, so +# they spread test by test instead of running as one file on one worker. +test-shard-history: + $(call run_unit_integration,$@,$(TEST_SHARD_HISTORY_PATHS),load) + +check-test-shards: + bash scripts/check_test_shards.sh --root tests/unit --root tests/integration \ + --markers "$(UNIT_INTEGRATION_MARKERS)" \ + --shard core $(TEST_SHARD_CORE_PATHS) \ + --shard publication $(TEST_SHARD_PUBLICATION_PATHS) \ + --shard history $(TEST_SHARD_HISTORY_PATHS) test-all: @@ -291,7 +324,7 @@ test-e2e-duckdb-integrations: $(call run_e2e_duckdb,$@,$(E2E_DUCKDB_INTEGRATIONS_PATHS)) check-e2e-shards: - bash scripts/check_e2e_shards.sh \ + bash scripts/check_test_shards.sh --root tests/e2e --markers "$(E2E_DUCKDB_MARKERS)" \ --shard build-core $(E2E_DUCKDB_BUILD_CORE_PATHS) \ --shard build-incremental $(E2E_DUCKDB_BUILD_INCREMENTAL_PATHS) \ --shard cli-data $(E2E_DUCKDB_CLI_DATA_PATHS) \ @@ -707,7 +740,7 @@ verify-quick: exit $$status -check-ci: check-dupes-new check-e2e-shards check-native-layering check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client rust-lint +check-ci: check-dupes-new check-e2e-shards check-test-shards check-native-layering check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client rust-lint uv run ruff format --check . uv run ruff check . uv run ty check src tests diff --git a/scripts/check_e2e_shards.sh b/scripts/check_e2e_shards.sh deleted file mode 100644 index 5c5966e160..0000000000 --- a/scripts/check_e2e_shards.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -readonly marker_expression="not real_warehouse and not dbt and not performance" -readonly e2e_root="tests/e2e" - -declare -a shard_names=() -declare -a root_names=() -declare -a root_shards=() -declare -A seen_shards=() -declare -A seen_roots=() - -while (( $# > 0 )); do - if [[ "$1" != "--shard" || $# -lt 3 ]]; then - echo "usage: $0 --shard NAME PATH [PATH ...] [--shard ...]" >&2 - exit 2 - fi - - shard="$2" - shift 2 - if [[ -n "${seen_shards[$shard]:-}" ]]; then - echo "duplicate shard name: $shard" >&2 - exit 1 - fi - seen_shards[$shard]=1 - shard_names+=("$shard") - - path_count=0 - while (( $# > 0 )) && [[ "$1" != "--shard" ]]; do - root="$1" - shift - ((path_count += 1)) - if [[ -n "${seen_roots[$root]:-}" ]]; then - echo "path is declared more than once: $root" >&2 - exit 1 - fi - if [[ "$root" != "$e2e_root/"* ]]; then - echo "path is outside $e2e_root: $root" >&2 - exit 1 - fi - if [[ ! -e "$root" ]]; then - echo "path does not exist: $root" >&2 - exit 1 - fi - seen_roots[$root]=1 - root_names+=("$root") - root_shards+=("$shard") - done - if (( path_count == 0 )); then - echo "shard has no paths: $shard" >&2 - exit 1 - fi -done - -mkdir -p /tmp/opencode -collection_file=$(mktemp /tmp/opencode/check-e2e-shards-XXXXXX.log) -trap 'rm -f "$collection_file"' EXIT - -if ! uv run pytest "$e2e_root" --collect-only -q --color=no \ - -m "$marker_expression" >"$collection_file"; then - cat "$collection_file" - echo "DuckDB E2E pytest collection failed" >&2 - exit 1 -fi - -declare -A counts=() -declare -A invalid_files=() -test_count=0 -while IFS= read -r node_id; do - if [[ "$node_id" != "$e2e_root/"*::* ]]; then - continue - fi - - ((test_count += 1)) - test_file="${node_id%%::*}" - owner_count=0 - owner="" - ownership="" - for index in "${!root_names[@]}"; do - root="${root_names[$index]}" - if [[ "$test_file" == "$root" || "$test_file" == "$root/"* ]]; then - ((owner_count += 1)) - owner="${root_shards[$index]}" - ownership+="${ownership:+, }$owner:$root" - fi - done - - if (( owner_count != 1 )); then - invalid_files[$test_file]="${ownership:-none}" - else - counts[$owner]=$(( ${counts[$owner]:-0} + 1 )) - fi -done <"$collection_file" - -if (( test_count == 0 )); then - echo "pytest collection returned no deterministic DuckDB E2E tests" >&2 - exit 1 -fi - -if (( ${#invalid_files[@]} > 0 )); then - echo "DuckDB E2E shard coverage failed:" >&2 - for test_file in "${!invalid_files[@]}"; do - echo " - $test_file owners: ${invalid_files[$test_file]}" >&2 - done - exit 1 -fi - -echo "DuckDB E2E shard coverage passed ($test_count tests):" -for shard in "${shard_names[@]}"; do - echo " $shard: ${counts[$shard]:-0}" -done diff --git a/scripts/check_test_shards.sh b/scripts/check_test_shards.sh new file mode 100644 index 0000000000..fd8e0cfdc2 --- /dev/null +++ b/scripts/check_test_shards.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# Check that every collected test runs in exactly one shard. +# +# usage: check_test_shards.sh --root DIR [--root DIR ...] --markers EXPR +# --shard NAME ARG [ARG ...] [--shard ...] +# +# A shard's ARGs are the pytest path arguments it runs: paths under a root, and +# --ignore=PATH for a path under one of them that another shard runs instead. + +set -euo pipefail + +declare -a collection_roots=() +markers="" +declare -a shard_names=() +declare -a root_names=() +declare -a root_shards=() +declare -A seen_shards=() +declare -A seen_roots=() +declare -A shard_ignores=() + +usage() { + echo "usage: $0 --root DIR [--root DIR ...] --markers EXPR --shard NAME ARG [ARG ...] [--shard ...]" >&2 + exit 2 +} + +under() { + [[ "$1" == "$2" || "$1" == "$2/"* ]] +} + +under_collection_root() { + local root + for root in "${collection_roots[@]}"; do + if under "$1" "$root"; then + return 0 + fi + done + return 1 +} + +while (( $# > 0 )) && [[ "$1" != "--shard" ]]; do + case "$1" in + --root) + (( $# >= 2 )) || usage + collection_roots+=("$2") + shift 2 + ;; + --markers) + (( $# >= 2 )) || usage + markers="$2" + shift 2 + ;; + *) + usage + ;; + esac +done +if (( ${#collection_roots[@]} == 0 )) || [[ -z "$markers" ]] || (( $# == 0 )); then + usage +fi + +while (( $# > 0 )); do + if [[ "$1" != "--shard" || $# -lt 3 ]]; then + usage + fi + + shard="$2" + shift 2 + if [[ -n "${seen_shards[$shard]:-}" ]]; then + echo "duplicate shard name: $shard" >&2 + exit 1 + fi + seen_shards[$shard]=1 + shard_names+=("$shard") + shard_ignores[$shard]="" + + path_count=0 + declare -a ignores=() + while (( $# > 0 )) && [[ "$1" != "--shard" ]]; do + argument="$1" + shift + if [[ "$argument" == --ignore=* ]]; then + ignores+=("${argument#--ignore=}") + continue + fi + root="$argument" + ((path_count += 1)) + if [[ -n "${seen_roots[$root]:-}" ]]; then + echo "path is declared more than once: $root" >&2 + exit 1 + fi + if ! under_collection_root "$root"; then + echo "path is outside ${collection_roots[*]}: $root" >&2 + exit 1 + fi + if [[ ! -e "$root" ]]; then + echo "path does not exist: $root" >&2 + exit 1 + fi + seen_roots[$root]=1 + root_names+=("$root") + root_shards+=("$shard") + done + if (( path_count == 0 )); then + echo "shard has no paths: $shard" >&2 + exit 1 + fi + for ignored in "${ignores[@]}"; do + if [[ ! -e "$ignored" ]]; then + echo "ignored path does not exist: $ignored" >&2 + exit 1 + fi + shard_ignores[$shard]+="$ignored"$'\n' + done + unset ignores +done + +mkdir -p /tmp/opencode +collection_file=$(mktemp /tmp/opencode/check-test-shards-XXXXXX.log) +trap 'rm -f "$collection_file"' EXIT + +if ! uv run pytest "${collection_roots[@]}" --collect-only -q --color=no \ + -m "$markers" >"$collection_file"; then + cat "$collection_file" + echo "pytest collection failed for ${collection_roots[*]}" >&2 + exit 1 +fi + +ignored_by() { + local shard="$1" test_file="$2" ignored + while IFS= read -r ignored; do + if [[ -n "$ignored" ]] && under "$test_file" "$ignored"; then + return 0 + fi + done <<<"${shard_ignores[$shard]}" + return 1 +} + +declare -A counts=() +declare -A invalid_files=() +test_count=0 +while IFS= read -r node_id; do + if [[ "$node_id" != *::* ]] || ! under_collection_root "${node_id%%::*}"; then + continue + fi + + ((test_count += 1)) + test_file="${node_id%%::*}" + owner_count=0 + owner="" + ownership="" + for index in "${!root_names[@]}"; do + root="${root_names[$index]}" + shard="${root_shards[$index]}" + if under "$test_file" "$root" && ! ignored_by "$shard" "$test_file"; then + ((owner_count += 1)) + owner="$shard" + ownership+="${ownership:+, }$owner:$root" + fi + done + + if (( owner_count != 1 )); then + invalid_files[$test_file]="${ownership:-none}" + else + counts[$owner]=$(( ${counts[$owner]:-0} + 1 )) + fi +done <"$collection_file" + +if (( test_count == 0 )); then + echo "pytest collection returned no tests under ${collection_roots[*]}" >&2 + exit 1 +fi + +if (( ${#invalid_files[@]} > 0 )); then + echo "Test shard coverage failed for ${collection_roots[*]}:" >&2 + for test_file in "${!invalid_files[@]}"; do + echo " - $test_file owners: ${invalid_files[$test_file]}" >&2 + done + exit 1 +fi + +echo "Test shard coverage passed for ${collection_roots[*]} ($test_count tests):" +for shard in "${shard_names[@]}"; do + echo " $shard: ${counts[$shard]:-0}" +done diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index 015c086250..1e5a97b70e 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -112,7 +112,7 @@ def analysis_error_codes(compiler_root: Path) -> frozenset[str]: path: Path = compiler_root / root for module in sorted(path.rglob("*.py")) if path.is_dir() else (path,): found.update(_module_codes(module)) - repo_root: Path = compiler_root.parents[2] + repo_root: Path = Path(__file__).resolve().parents[4] for native_module in ANALYSIS_NATIVE_CODE_SCAN_ROOTS: text: str = (repo_root / native_module).read_text(encoding="utf-8") found.update(NATIVE_CODE_CONSTANT_PATTERN.findall(text)) @@ -131,7 +131,7 @@ def inline_compile_codes(compile_root: Path) -> dict[str, tuple[str, ...]]: code: str | None = _code_literal(node) if code is not None: found.setdefault(code, []).append(f"{relative}:{getattr(node, 'lineno', 0)}") - repo_root: Path = compile_root.parents[3] + repo_root: Path = Path(__file__).resolve().parents[4] for directory in RENDER_NATIVE_CODE_SCAN_ROOTS: for path in sorted((repo_root / directory).rglob("*.rs")): if NATIVE_CODE_TEST_DIRECTORY in path.parts: