From 3a9b64a7fbfa2f90c018cad8e05292a0a78befa2 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Thu, 9 Jul 2026 06:02:15 +0000 Subject: [PATCH] =?UTF-8?q?ci:=20make=20fork=20offline=20CI=20robust=20?= =?UTF-8?q?=E2=80=94=203.9=20skip,=20mypy=20flags,=20warmer=20matrix=20(#8?= =?UTF-8?q?31)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from the fork verification run (#849) once the cache finally hit. The offline restore mechanism works (10 legs green offline incl. all kernel legs); these fix the remaining offline-path failures: 1. Python 3.9 can't be provisioned offline — the protected-runner image preinstalls 3.10-3.14 but not 3.9 (EOL), so setup-python tries to DOWNLOAD it and fails with no network on a fork. That hard-failed the 3.9 legs and, under fail-fast, cancelled the working 3.10-3.14 siblings. Move setup-python into restore-deps with continue-on-error, probe the restored interpreter, and export venv-usable; callers gate their offline run on it and skip 3.9 neutrally (3.9 fork coverage is in the merge queue) instead of failing. Keeps 3.9 in the matrix per maintainer preference without letting it sink the other legs. 2. check-types offline ran , which mypy rejects without --install-types ('--non-interactive is only supported with --install-types'). The warmer already primed the stubs into the venv, so run plain . 3. The warmer excluded 3.14-min but the PR matrix runs 3.14-min pyarrow (PR excludes only 3.12-min/3.13-min) — that leg missed the cache and fell back to JFrog. Align the warmer's excludes to the PR matrix exactly. Refs #831 Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/actions/restore-deps/action.yml | 55 ++++++++++++++++++----- .github/workflows/code-quality-checks.yml | 50 +++++++-------------- .github/workflows/warm-deps-cache.yml | 7 +-- 3 files changed, 63 insertions(+), 49 deletions(-) diff --git a/.github/actions/restore-deps/action.yml b/.github/actions/restore-deps/action.yml index 1acd03225..c90d56846 100644 --- a/.github/actions/restore-deps/action.yml +++ b/.github/actions/restore-deps/action.yml @@ -1,13 +1,22 @@ name: Restore Prewarmed Dependencies description: | Restores a fully-built in-project .venv from the "forkvenv-*" cache written by - warm-deps-cache.yml, so FORK PRs run offline without touching JFrog (#831). + warm-deps-cache.yml, provisions the matching Python interpreter, and probes + that the restored venv actually runs — so FORK PRs run offline without JFrog + (#831). - Outputs cache-hit. When true, the caller must SKIP setup-poetry (the .venv is - already complete) and only install Poetry itself. When false — a same-repo PR, - or a fork whose lockfiles changed and haven't been warmed — the caller falls - back to setup-poetry (JFrog OIDC), which works in every trusted context and on - cache-miss for same-repo PRs. + Outputs: + - cache-hit: "true" if a warmed .venv was restored. When false — a same-repo + PR, or a fork whose lockfiles changed and haven't been re-warmed — the + caller falls back to setup-poetry (JFrog OIDC), which works in every + trusted context and on cache-miss for same-repo PRs. + - venv-usable: "true" if the restored venv's interpreter runs offline. It is + false when the interpreter can't be provisioned without network — the + protected-runner image preinstalls 3.10–3.14 but not 3.9 (EOL), so on 3.9 + setup-python would try to download the interpreter and fail on a fork. + Callers gate their offline run on venv-usable=='true' and skip neutrally + otherwise (3.9 fork coverage comes from the merge queue), which also keeps + a 3.9 leg from cancelling its 3.10–3.14 siblings under fail-fast. inputs: python-version: @@ -26,6 +35,9 @@ outputs: cache-hit: description: "true if a warmed .venv was restored (install can be skipped)" value: ${{ steps.restore.outputs.cache-matched-key != '' }} + venv-usable: + description: "true if the restored venv interpreter runs offline (run tests) — false means skip neutrally" + value: ${{ steps.probe.outputs.usable }} runs: using: composite @@ -38,16 +50,37 @@ runs: # The warmer appends a timestamp to the key (caches are immutable), so # there is no fixed exact key to hit. restore-keys does a PREFIX match # and returns the most-recently-created entry for this lockfile + - # matrix leg — exactly the freshest warmed venv. + # matrix leg — exactly the freshest warmed venv. The lockfile hash MUST + # be computed here on the PRISTINE poetry.lock (before any `poetry + # lock` rewrites it) to match the warmer's key — see warm-deps-cache.yml. key: forkvenv-${{ runner.os }}-${{ inputs.python-version }}-${{ inputs.dependency-version }}-${{ inputs.extras || 'base' }}-${{ hashFiles('**/poetry.lock') }}-never restore-keys: | forkvenv-${{ runner.os }}-${{ inputs.python-version }}-${{ inputs.dependency-version }}-${{ inputs.extras || 'base' }}-${{ hashFiles('**/poetry.lock') }}- - - name: Report restore outcome + - name: Set up Python for the restored venv + # Only meaningful on a cache hit — the restored venv symlinks its + # interpreter to a hostedtoolcache path this step provisions. + # continue-on-error: 3.9 isn't preinstalled on the runner image, so this + # would try to DOWNLOAD it and fail offline on a fork. Tolerate that and + # let the probe decide, rather than hard-fail (and cancel siblings). + if: steps.restore.outputs.cache-matched-key != '' + continue-on-error: true + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Probe restored venv interpreter + id: probe shell: bash run: | - if [ -n "${{ steps.restore.outputs.cache-matched-key }}" ]; then - echo "Restored warmed .venv (${{ steps.restore.outputs.cache-matched-key }}) — install will be skipped." + if [ -n "${{ steps.restore.outputs.cache-matched-key }}" ] && .venv/bin/python --version >/dev/null 2>&1; then + echo "usable=true" >> "$GITHUB_OUTPUT" + echo "Restored venv is usable offline: $(.venv/bin/python --version 2>&1)" else - echo "No warmed .venv for this lockfile/leg — caller falls back to setup-poetry (JFrog)." + echo "usable=false" >> "$GITHUB_OUTPUT" + if [ -n "${{ steps.restore.outputs.cache-matched-key }}" ]; then + echo "::notice::Prewarmed venv interpreter (Python ${{ inputs.python-version }}) is unavailable offline on this runner; the caller will skip its offline run for this leg. Covered in the merge queue." + else + echo "No warmed .venv for this lockfile/leg — caller falls back to setup-poetry (JFrog)." + fi fi diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 9cb2cbcc1..42195e722 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -49,19 +49,17 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 # --- Fork path: restore a prewarmed .venv and run offline ------------- + # restore-deps also provisions the interpreter and probes it; venv-usable + # is false when the interpreter can't be provisioned offline (3.9), in + # which case we skip the offline run neutrally instead of failing. - name: Restore prewarmed dependencies id: deps uses: ./.github/actions/restore-deps with: python-version: ${{ matrix.python-version }} dependency-version: ${{ matrix.dependency-version }} - - name: Set up Python (cache hit — needed for the restored venv interpreter) - if: steps.deps.outputs.cache-hit == 'true' - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: ${{ matrix.python-version }} - name: Run tests (offline, from prewarmed venv) - if: steps.deps.outputs.cache-hit == 'true' + if: steps.deps.outputs.venv-usable == 'true' run: .venv/bin/python -m pytest tests/unit -m "not realkernel" # --- Fallback path: same-repo PRs / cache miss use JFrog (unchanged) -- @@ -126,13 +124,8 @@ jobs: python-version: ${{ matrix.python-version }} dependency-version: ${{ matrix.dependency-version }} extras: pyarrow - - name: Set up Python (cache hit — needed for the restored venv interpreter) - if: steps.deps.outputs.cache-hit == 'true' - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: ${{ matrix.python-version }} - name: Run tests (offline, from prewarmed venv) - if: steps.deps.outputs.cache-hit == 'true' + if: steps.deps.outputs.venv-usable == 'true' run: .venv/bin/python -m pytest tests/unit -m "not realkernel" # --- Fallback path: same-repo PRs / cache miss use JFrog (unchanged) --- @@ -200,19 +193,14 @@ jobs: with: python-version: ${{ matrix.python-version }} extras: kernel - - name: Set up Python (cache hit — needed for the restored venv interpreter) - if: steps.deps.outputs.cache-hit == 'true' - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: ${{ matrix.python-version }} - name: Assert the real kernel wheel is installed (offline) - if: steps.deps.outputs.cache-hit == 'true' + if: steps.deps.outputs.venv-usable == 'true' run: .venv/bin/python -c "import databricks_sql_kernel as k; assert k.__file__, 'kernel wheel missing __file__ — not the real wheel'; print('real kernel wheel:', k.__file__)" - name: Unit tests, realkernel deselected (offline) - if: steps.deps.outputs.cache-hit == 'true' + if: steps.deps.outputs.venv-usable == 'true' run: .venv/bin/python -m pytest tests/unit -m "not realkernel" - name: Drive use_kernel=True through the REAL wheel — routing (offline) - if: steps.deps.outputs.cache-hit == 'true' + if: steps.deps.outputs.venv-usable == 'true' run: .venv/bin/python -m pytest tests/unit/test_session.py -m realkernel -v # --- Fallback path: same-repo PRs / cache miss use JFrog (unchanged) -- @@ -267,13 +255,8 @@ jobs: uses: ./.github/actions/restore-deps with: python-version: ${{ matrix.python-version }} - - name: Set up Python (cache hit — needed for the restored venv interpreter) - if: steps.deps.outputs.cache-hit == 'true' - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: ${{ matrix.python-version }} - name: Black (offline, from prewarmed venv) - if: steps.deps.outputs.cache-hit == 'true' + if: steps.deps.outputs.venv-usable == 'true' run: .venv/bin/black --check src - name: Setup Poetry if: steps.deps.outputs.cache-hit != 'true' @@ -299,18 +282,15 @@ jobs: uses: ./.github/actions/restore-deps with: python-version: ${{ matrix.python-version }} - - name: Set up Python (cache hit — needed for the restored venv interpreter) - if: steps.deps.outputs.cache-hit == 'true' - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: ${{ matrix.python-version }} - name: Mypy (offline, from prewarmed venv) - if: steps.deps.outputs.cache-hit == 'true' - # --install-types would reach the index; the warmed venv already has - # the stub packages, so run mypy without it on the offline path. + if: steps.deps.outputs.venv-usable == 'true' + # The warmer already ran `mypy --install-types` to bake the stub + # packages into the venv, so just run mypy plainly here. (mypy rejects + # --non-interactive unless --install-types is also given, and + # --install-types would reach the index — neither is wanted offline.) run: | mkdir -p .mypy_cache - .venv/bin/mypy --non-interactive src + .venv/bin/mypy src - name: Setup Poetry if: steps.deps.outputs.cache-hit != 'true' uses: ./.github/actions/setup-poetry diff --git a/.github/workflows/warm-deps-cache.yml b/.github/workflows/warm-deps-cache.yml index 8c36b1c91..2871eb21d 100644 --- a/.github/workflows/warm-deps-cache.yml +++ b/.github/workflows/warm-deps-cache.yml @@ -90,13 +90,14 @@ jobs: dependency-version: ["default", "min"] extras: ["", "pyarrow", "kernel"] exclude: - # min deps are only exercised on 3.9/3.10/3.11 in the PR matrix. + # MUST match code-quality-checks.yml's excludes exactly, or a PR leg + # with no warmed cache entry misses and falls back to JFrog (which + # forks can't reach). The PR matrix excludes ONLY 3.12-min and + # 3.13-min — 3.14-min IS a real PR leg, so it must be warmed. - python-version: "3.12" dependency-version: "min" - python-version: "3.13" dependency-version: "min" - - python-version: "3.14" - dependency-version: "min" # The kernel wheel is cp310-abi3 (Requires-Python >=3.10); the # [kernel] extra is a no-op on 3.9, so there's no kernel leg to warm. - python-version: "3.9"