diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 959fbe5f..ca8e5eb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,11 +30,13 @@ jobs: test: runs-on: ubuntu-latest - # Serially, the test job on main took 29.5 minutes at b0c8f2e and hit - # this limit at ec5413c (2026-11-02). In parallel it took 12.3 minutes - # with -n logical (4 workers, run 41526298328), and 12.5 to 16.2 minutes - # in three attempts with -n auto (run 41490222217). Fail a hung job - # instead of holding a shared runner for GitHub's 6-hour default. + # Serially, the test job on main took 29.5 minutes at b0c8f2e, hit this + # limit at ec5413c and took 22.3 minutes at f4c3c38 (2026-11-02). In + # parallel it took 12.5 to 16.2 minutes in three attempts with -n auto on + # a merge with b0c8f2e (run 41490222217). With -n logical (4 workers) on + # merges with f4c3c38 it took 12.3 and 15.7 minutes (runs 41526298328 and + # 41529016470). Fail a hung job instead of holding a shared runner for + # GitHub's 6-hour default. timeout-minutes: 30 steps: - uses: actions/checkout@v6 @@ -49,8 +51,8 @@ jobs: - run: uv sync --locked --extra dev # One worker per logical CPU (-n auto would count physical cores, since # psutil is installed); the "created: N/N workers" line gives the count. - # --dist loadfile keeps each test file on one worker; on the 4-core VM it - # ran faster than --dist load (10.1 minutes). + # --dist loadfile keeps each test file on one worker; on a 4-vCPU VM it + # ran faster than --dist load (9.2 to 9.7 minutes, against 10.1). - run: uv run pytest -m "not slow" -n logical --dist loadfile --tb=short app: diff --git a/CLAUDE.md b/CLAUDE.md index e33cbcc5..364cf817 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,7 +3,7 @@ ## Quick start ```bash pip install -e ".[dev]" -pytest # Run tests (all external calls mocked) +pytest # Run tests (see "Testing" for network use) ruff check . # Lint ruff format . # Format ``` @@ -26,9 +26,11 @@ ruff format . # Format - `policybench/analysis.py` — Metrics and reporting ## Testing -- All tests mock external calls (LiteLLM, PE-US API) +- Tests mock LiteLLM's model calls, the provider batch adapters and `gh`, and the tests that need policyengine-us run it in-process. +- LiteLLM fetches its model cost map from GitHub in every process that imports it, and falls back to its bundled copy if the fetch fails. Each pytest process that runs tests imports it, and so do the `policybench.cli` runs that some slow tests start. Set `LITELLM_LOCAL_MODEL_COST_MAP=True` to skip the fetch. +- Some `tests/test_scenarios.py` tests download the certified US dataset from Hugging Face. - `pytest -m "not slow"` to skip slow tests -- `pytest -m "not slow" -n 4 --dist loadfile` to split them across four workers (CI starts one per CPU) +- `pytest -m "not slow" -n 4 --dist loadfile` to run the same tests on four workers (CI starts one per logical CPU) - Some tests read git history, so run the suite from a full clone (`git fetch --unshallow`) - Full benchmark runs are manual and expensive