diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 080fab50..4088cea1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -3,12 +3,6 @@ name: PR on: pull_request: branches: [main] - # Also grade every commit that lands on main. The release workflow's preflight - # reads the latest pr.yml conclusion for the SHA it's about to tag, so main - # HEAD must be validated in its own right (merge-queue or squash-merge can - # produce a SHA that no PR run ever graded). - push: - branches: [main] # Cancel in-progress runs when new commits are pushed to the same PR # so we never waste minutes on outdated code. On main (push), github.ref is @@ -165,7 +159,23 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: "npm" - run: make install - - run: uv run --project host playwright install --with-deps chromium + # Playwright's chromium download is ~150MB — cache it keyed on uv.lock so + # we only re-download when the pinned playwright version moves. OS deps + # (apt packages) aren't cacheable across runs but install-deps is fast on + # the GH runner image since most libs are preinstalled. + - name: Cache Playwright browsers + id: playwright-cache + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ hashFiles('uv.lock') }} + - name: Install Playwright chromium + run: | + if [ "${{ steps.playwright-cache.outputs.cache-hit }}" = "true" ]; then + uv run --project host playwright install-deps chromium + else + uv run --project host playwright install --with-deps chromium + fi - run: make gen-pages - run: uv run --project host alembic -c host/alembic.ini upgrade heads - name: Start API + Vite diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4c155746..7ed80e66 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,79 +3,70 @@ name: release on: workflow_dispatch: inputs: + bump: + description: "Semver bump (ignored if `version` is set)" + type: choice + options: [patch, minor, major] + default: patch version: - description: "Version to release (e.g., 0.0.1)" - required: true + description: "Explicit version override (e.g., 1.2.3)" + required: false type: string - target: - description: "Publish target" - type: choice - options: [pypi, testpypi] - default: testpypi permissions: contents: write id-token: write # required for PyPI Trusted Publishing (OIDC); npm uses NPM_TOKEN - actions: read # preflight's `gh run list` needs this; omitting it silently returns empty env: NODE_VERSION: "24" PYTHON_VERSION: "3.12" jobs: - # Fail fast on obvious problems before touching any packaging or git state: - # - malformed version string - # - tag already exists (re-dispatch would silently build on top of it) - # - pr.yml hasn't passed on the commit we're about to release - # The release workflow uses workflow_dispatch, so github.sha == main HEAD - # at dispatch time; pr.yml's push trigger must have already graded that SHA. - preflight: + # Resolve the release version once and share it with every downstream job. + # If `version` is set, use it verbatim; otherwise bump the current version in + # framework/core/pyproject.toml by the chosen level. + resolve: runs-on: ubuntu-latest + outputs: + version: ${{ steps.compute.outputs.version }} steps: - uses: actions/checkout@v6 with: fetch-depth: 0 - - name: Validate version string - run: | - echo "${{ inputs.version }}" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+([.-]?(a|b|rc|alpha|beta)[0-9]*)?$' - - name: Fail if tag already exists on origin + - id: compute + shell: bash run: | - git fetch --tags --quiet - if git rev-parse --verify --quiet "refs/tags/v${{ inputs.version }}" >/dev/null; then - echo "::error::tag v${{ inputs.version }} already exists — choose a new version" - exit 1 + if [ -n "${{ inputs.version }}" ]; then + ver="${{ inputs.version }}" + else + ver=$(BUMP="${{ inputs.bump }}" python3 <<'PY' + import os, pathlib, re, tomllib + cur = tomllib.loads(pathlib.Path("framework/core/pyproject.toml").read_text())["project"]["version"] + m = re.match(r"^(\d+)\.(\d+)\.(\d+)", cur) + if not m: + raise SystemExit(f"cannot parse current version: {cur!r}") + major, minor, patch = map(int, m.groups()) + bump = os.environ["BUMP"] + if bump == "major": + major, minor, patch = major + 1, 0, 0 + elif bump == "minor": + minor, patch = minor + 1, 0 + else: + patch += 1 + print(f"{major}.{minor}.{patch}") + PY + ) fi - - name: Verify pr.yml is green on ${{ github.sha }} - env: - GH_TOKEN: ${{ github.token }} - run: | - for i in $(seq 1 30); do - conclusion=$(gh run list \ - --commit "${{ github.sha }}" \ - --workflow pr.yml \ - --branch main \ - --json conclusion,status \ - --jq 'map(select(.status == "completed")) | .[0].conclusion' || true) - case "${conclusion:-}" in - success) - echo "pr.yml passed on ${{ github.sha }}" - exit 0 ;; - failure|cancelled|timed_out|action_required|startup_failure) - echo "::error::pr.yml concluded '${conclusion}' on ${{ github.sha }}" - exit 1 ;; - *) - echo "waiting for pr.yml on ${{ github.sha }} (attempt ${i}/30)" - sleep 10 ;; - esac - done - echo "::error::pr.yml did not complete on ${{ github.sha }} within timeout" - exit 1 + echo "${ver}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+([.-]?(a|b|rc|alpha|beta)[0-9]*)?$' \ + || { echo "::error::'${ver}' is not a valid version"; exit 1; } + echo "version=${ver}" >> "$GITHUB_OUTPUT" + echo "Resolved version: ${ver}" # Bump versions in the working tree (no commit yet), build artifacts, upload. # The commit + tag only hit origin in `finalize`, after publishes succeed, # so a failed build never leaves an orphan tag to clean up. build: - needs: preflight + needs: resolve runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -88,7 +79,7 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} - name: Bump all package versions (working tree only) - run: uv run python scripts/bump_version.py "${{ inputs.version }}" + run: uv run python scripts/bump_version.py "${{ needs.resolve.outputs.version }}" - name: Regenerate npm lockfile run: npm install --package-lock-only - name: Build Python wheels + sdists @@ -132,8 +123,8 @@ jobs: - simple_module_settings - simple_module_users environment: - name: ${{ inputs.target }} - url: https://${{ inputs.target == 'testpypi' && 'test.' || '' }}pypi.org/project/${{ matrix.package }}/ + name: pypi + url: https://pypi.org/project/${{ matrix.package }}/ steps: - uses: actions/download-artifact@v4 with: @@ -155,11 +146,9 @@ jobs: - uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: to-publish - repository-url: ${{ inputs.target == 'testpypi' && 'https://test.pypi.org/legacy/' || '' }} publish-npm: needs: build - if: inputs.target == 'pypi' runs-on: ubuntu-latest strategy: fail-fast: false @@ -194,13 +183,24 @@ jobs: ls -la dist-npm exit 1 fi - npm publish --access public "${files[0]}" + # Guard: the stripped-@ filename is ambiguous between scoped and + # unscoped packages (both would yield simple-module-py-ui-*.tgz), + # so verify the tarball's package.json actually names an + # @simple-module-py/* package before publishing. + name=$(tar -xzOf "${files[0]}" package/package.json | python3 -c 'import json,sys; print(json.load(sys.stdin)["name"])') + expected="@simple-module-py/${{ matrix.package }}" + if [ "$name" != "$expected" ]; then + echo "::error::refusing to publish ${files[0]}: package name is '$name', expected '$expected'" + exit 1 + fi + # Prefix with ./ so npm treats the arg as a file path, not a + # GitHub `user/repo` shorthand (which it does for any bare + # single-slash arg, even when the tarball exists on disk). + npm publish --access public "./${files[0]}" # Only after every publish succeeded do we commit the version bump + tag. - # Skipped for testpypi so we don't burn a version number on a dry-run. finalize: - needs: [publish-pypi, publish-npm] - if: inputs.target == 'pypi' + needs: [resolve, publish-pypi, publish-npm] runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -214,103 +214,21 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} - name: Re-apply version bump - run: uv run python scripts/bump_version.py "${{ inputs.version }}" + run: uv run python scripts/bump_version.py "${{ needs.resolve.outputs.version }}" - name: Regenerate npm lockfile run: npm install --package-lock-only - name: Commit, tag, push + env: + VERSION: ${{ needs.resolve.outputs.version }} run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add -A if ! git diff --cached --quiet; then - git commit -m "release: v${{ inputs.version }}" + git commit -m "release: v${VERSION}" git push origin HEAD:main else echo "working tree clean — packages already published at this version, tagging HEAD" fi - git tag "v${{ inputs.version }}" - git push origin "v${{ inputs.version }}" - - smoke: - needs: finalize - if: inputs.target == 'pypi' - runs-on: ubuntu-latest - steps: - - uses: actions/setup-node@v6 - with: - node-version: ${{ env.NODE_VERSION }} - - uses: astral-sh/setup-uv@v8.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install CLI from PyPI (retry for index propagation) - run: | - for i in $(seq 1 12); do - if uv tool install "simple_module_hosting==${{ inputs.version }}"; then - exit 0 - fi - echo "PyPI index not yet showing ${{ inputs.version }} — retry ${i}/12 in 15s" - sleep 15 - done - echo "::error::PyPI never served simple_module_hosting==${{ inputs.version }}" - exit 1 - - name: Generate smoke app - run: uv tool run simple-module new smoke-app --yes --db sqlite --no-install - - name: Install smoke app deps (Python + npm, with propagation retry) - working-directory: smoke-app - run: | - for i in $(seq 1 6); do - if uv sync; then break; fi - echo "uv sync failed — retry ${i}/6 in 15s" - sleep 15 - done - for i in $(seq 1 6); do - if npm install; then break; fi - echo "npm install failed — retry ${i}/6 in 15s" - sleep 15 - done - - name: Run smoke app tests - working-directory: smoke-app - run: uv run pytest -q - - # TestPyPI equivalent of `smoke`. Exercises the release path before real PyPI - # sees the version — the whole point of the testpypi target. Python-only: - # npm packages are only published on the pypi target, so the npm deps from - # the generated smoke app can't resolve here. - smoke-testpypi: - needs: publish-pypi - if: inputs.target == 'testpypi' - runs-on: ubuntu-latest - env: - UV_INDEX_URL: https://test.pypi.org/simple/ - UV_EXTRA_INDEX_URL: https://pypi.org/simple/ - steps: - - uses: astral-sh/setup-uv@v8.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install CLI from TestPyPI (retry for index propagation) - run: | - for i in $(seq 1 12); do - if uv tool install \ - --index-url "$UV_INDEX_URL" \ - --extra-index-url "$UV_EXTRA_INDEX_URL" \ - "simple_module_hosting==${{ inputs.version }}"; then - exit 0 - fi - echo "TestPyPI not yet showing ${{ inputs.version }} — retry ${i}/12 in 15s" - sleep 15 - done - echo "::error::TestPyPI never served simple_module_hosting==${{ inputs.version }}" - exit 1 - - name: Generate smoke app - run: uv tool run simple-module new smoke-app --yes --db sqlite --no-install - - name: Install Python deps from TestPyPI - working-directory: smoke-app - run: | - for i in $(seq 1 6); do - if uv sync; then break; fi - echo "uv sync failed — retry ${i}/6 in 15s" - sleep 15 - done - - name: Run smoke app tests (Python only) - working-directory: smoke-app - run: uv run pytest -q + git tag "v${VERSION}" + git push origin "v${VERSION}" diff --git a/docs/release.md b/docs/release.md index 7cf1313f..dfe57287 100644 --- a/docs/release.md +++ b/docs/release.md @@ -4,13 +4,13 @@ This repo publishes **14 Python packages** to PyPI and **3 JS packages** to npm - **Python packages** (`simple_module_*`) → [pypi.org](https://pypi.org) - **JS packages** (`@simple-module-py/*`) → [npmjs.com](https://www.npmjs.com) -- **Auth**: OIDC Trusted Publishing on both registries — no API tokens stored anywhere +- **Auth**: OIDC Trusted Publishing on PyPI; `NPM_TOKEN` on npm - **Entry point**: Actions → `release` → Run workflow ## TL;DR — already set up? Cut a release in 3 clicks 1. Ensure `main` is green (`make lint && make test`). -2. [Actions → release → Run workflow](https://github.com/antosubash/simple_module_python/actions/workflows/release.yml) → version `X.Y.Z`, target `pypi` → **Run**. +2. [Actions → release → Run workflow](https://github.com/antosubash/simple_module_python/actions/workflows/release.yml) → pick **bump** (`patch` / `minor` / `major`), leave **version** blank → **Run**. 3. After it finishes, write release notes on the auto-created `vX.Y.Z` tag on GitHub. For the very first time, or if any of the above is unfamiliar, keep reading. @@ -19,11 +19,11 @@ For the very first time, or if any of the above is unfamiliar, keep reading. ## First-time setup (once per registry account) -You need to set up Trusted Publisher entries on PyPI, TestPyPI, and npm *before* running the workflow. These entries tell each registry: "trust OIDC tokens minted by this exact GitHub Actions workflow." No tokens are exchanged — the registry validates the token's GitHub-issued claims at publish time. +You need to set up Trusted Publisher entries on PyPI and npm *before* running the workflow. These entries tell each registry: "trust OIDC tokens minted by this exact GitHub Actions workflow." No tokens are exchanged — the registry validates the token's GitHub-issued claims at publish time. -### 1. PyPI (and TestPyPI) +### 1. PyPI -For *each* of the 14 Python project names, on *both* [pypi.org](https://pypi.org/manage/account/publishing/) and [test.pypi.org](https://test.pypi.org/manage/account/publishing/): +For *each* of the 14 Python project names, on [pypi.org](https://pypi.org/manage/account/publishing/): 1. Log in as the owner account (`antosubash`). 2. Go to **Your account → Publishing** (or click "Add a new pending publisher" if the project doesn't exist yet). @@ -32,7 +32,7 @@ For *each* of the 14 Python project names, on *both* [pypi.org](https://pypi.org - **Owner**: `antosubash` - **Repository name**: `simple_module_python` - **Workflow name**: `release.yml` - - **Environment name**: `pypi` on pypi.org, `testpypi` on test.pypi.org + - **Environment name**: `pypi` 4. Save. Repeat for every project in this list: @@ -60,23 +60,17 @@ simple_module_users 1. On [npmjs.com](https://www.npmjs.com), sign in as the owner. 2. Create the `@simple-module-py` organization (Settings → "Create a new organization"). This is a one-time step. -3. For each of `@simple-module-py/ui`, `@simple-module-py/i18n`, `@simple-module-py/tsconfig`: - - Go to the package settings (or "Add a pending publisher" if unpublished). - - Under **Trusted Publishers**, add a GitHub Actions publisher: - - **Repository**: `antosubash/simple_module_python` - - **Workflow name**: `release.yml` - - **Environment name**: `npm` -4. Save. +3. Generate an automation-type access token with publish rights for `@simple-module-py/*`. +4. In this repo: **Settings → Secrets and variables → Actions → New repository secret**, name `NPM_TOKEN`, paste the token. ### 3. GitHub Environments -In the repo's **Settings → Environments → New environment**, create three environments — they match the `environment:` fields used by the release workflow jobs: +In the repo's **Settings → Environments → New environment**, create two environments — they match the `environment:` fields used by the release workflow jobs: - `pypi` -- `testpypi` - `npm` -No secrets or variables are needed. Optionally, add a **deployment-protection rule** requiring a manual approval on `pypi` and `npm` so every release gets a human click before it goes live. +No secrets or variables are needed on the `pypi` environment (OIDC). Optionally, add a **deployment-protection rule** requiring a manual approval on both so every release gets a human click before it goes live. ### 4. Branch protection @@ -96,53 +90,38 @@ make lint make test ``` -Both should pass locally. CI should be green on `main` too. +Both should pass locally. CI should be green on every PR into `main`. ### Step 2. Pick a version All 17 packages bump in lockstep to the same version. We follow a relaxed SemVer during the 0.x phase: -| Situation | Bump | +| Situation | Input | |---|---| -| Bug fix, docs, internal refactor | `0.0.N` → `0.0.N+1` | -| New feature, no breaking changes | `0.0.N` → `0.1.0` | -| Breaking change (post-1.0) | `X.Y.Z` → `X+1.0.0` | -| Pre-release rehearsal | append `a0`, `b1`, `rc1` (PEP 440) | +| Bug fix, docs, internal refactor | **bump**: `patch` | +| New feature, no breaking changes | **bump**: `minor` | +| Breaking change (post-1.0) | **bump**: `major` | +| Pre-release rehearsal or specific number | **version**: e.g. `0.2.0rc1` | -Version strings must match `^[0-9]+\.[0-9]+\.[0-9]+([.-]?(a|b|rc|alpha|beta)[0-9]*)?$` — the workflow validates this up front. +The workflow derives the next version from the current `framework/core/pyproject.toml` version when you pick a bump level. If you set **version** explicitly, it overrides the bump choice. Version strings must match `^[0-9]+\.[0-9]+\.[0-9]+([.-]?(a|b|rc|alpha|beta)[0-9]*)?$`. -### Step 3. Rehearse on TestPyPI (recommended for anything bigger than a patch) +### Step 3. Run the release 1. Go to [Actions → release → Run workflow](https://github.com/antosubash/simple_module_python/actions/workflows/release.yml). -2. **Version**: e.g. `0.0.2a0` (PEP 440 alpha — doesn't collide with the real release). -3. **Target**: `testpypi`. -4. Click **Run workflow**. - -The rehearsal: -- Bumps all 17 packages to the alpha version, commits, and pushes a tag. -- Publishes Python wheels to [test.pypi.org](https://test.pypi.org). -- Skips npm publishes (npm has no equivalent test registry — we rely on the dry-run tarballs uploaded as workflow artifacts). -- Skips the smoke test (TestPyPI can't resolve the full dep tree). - -Download the `dist-npm` artifact from the workflow run and `tar tf` a tarball to confirm the JS package contents look right. If anything's off, fix on a PR, merge, and rehearse again with `0.0.2a1`. - -### Step 4. Real release - -Same form, two fields changed: - -1. **Version**: `0.0.2` (the real one). -2. **Target**: `pypi`. -3. **Run workflow**. +2. **bump**: `patch` (default), `minor`, or `major`. Leave **version** blank to use it. +3. **version**: optional — set to an explicit number (e.g. `0.2.0rc1`) to override the bump. +4. **Run workflow**. What happens: -- `bump-and-build` rewrites every version, commits `release: v0.0.2`, tags `v0.0.2`, pushes both, builds 14 wheels + 14 sdists + 3 npm tarballs. -- `publish-pypi` fans out 14 parallel jobs, each publishing one wheel+sdist pair via OIDC. -- `publish-npm` fans out 3 parallel jobs publishing via OIDC with `--provenance`. -- `smoke` installs `simple_module_hosting==0.0.2` from PyPI, runs `simple-module new smoke-app`, and runs the scaffolded app's tests against the just-published registries. +- `resolve` computes the final version and shares it with every downstream job. +- `build` rewrites every version, builds 14 wheels + 14 sdists + 3 npm tarballs, and uploads them as artifacts. +- `publish-pypi` fans out 14 parallel jobs, each publishing one wheel+sdist pair via OIDC Trusted Publishing. +- `publish-npm` fans out 3 parallel jobs publishing `@simple-module-py/*` tarballs with `NPM_TOKEN`. Each job verifies the tarball's `package.json` is actually scoped to `@simple-module-py/*` before invoking `npm publish`. +- `finalize` re-applies the bump, commits `release: vX.Y.Z`, tags it, and pushes both. Expected wall time: 5–8 minutes. -### Step 5. GitHub Release notes +### Step 4. GitHub Release notes The workflow creates the `vX.Y.Z` tag but not a GitHub Release. Do that manually: @@ -177,7 +156,7 @@ rm -rf dist-py dist-npm uv build --all-packages --out-dir dist-py mkdir -p dist-npm && for p in packages/*/; do npm pack "$p" --pack-destination dist-npm; done -# 6. Sanity-check wheel contents +# 6. Sanity-check contents ls dist-py/ | wc -l # expect 28 (14 wheels + 14 sdists) ls dist-npm/ | wc -l # expect 3 @@ -193,14 +172,19 @@ If step 5 fails for any package, the workflow will fail the same way — fix it ### Workflow fails at "Trusted publisher not configured" -The project's Trusted Publisher entry is missing or mismatched. Common causes: +The PyPI project's Trusted Publisher entry is missing or mismatched. Common causes: - Wrong workflow filename (must be exactly `release.yml`, not the full path) -- Wrong environment name (must be exactly `pypi` / `testpypi` / `npm`) +- Wrong environment name (must be exactly `pypi`) - Repository owner typo Fix the entry on the registry, re-run the failed job. -### `git push` fails in "Commit, tag, and push" step +### npm publish fails with `401 Unauthorized` or a weird git error + +- `401` → `NPM_TOKEN` secret is missing, expired, or lacks publish rights on `@simple-module-py/*`. Rotate and re-run. +- `git ls-remote` / `Permission denied (publickey)` → npm is mis-parsing the tarball path. This is guarded for in the workflow (the path is prefixed with `./` and the package name is verified against `@simple-module-py/`). If you see it again, the guard has regressed. + +### `git push` fails in "Commit, tag, push" step Branch protection is blocking `github-actions[bot]`. Add the `RELEASE_PUSH_TOKEN` secret (see First-time setup → Branch protection) and re-dispatch the workflow. The token is consumed automatically when present. @@ -210,27 +194,21 @@ PyPI is immutable — you cannot re-upload `0.0.2` under any circumstance. Optio 1. **Yank** the bad PyPI versions via the PyPI project UI (doesn't delete, but hides them from `pip install`). 2. **Unpublish** the good npm versions within 72 hours: `npm unpublish @simple-module-py/@0.0.2` for each. -3. Fix the root cause (almost always a Trusted Publisher misconfiguration). +3. Fix the root cause. 4. Bump to `0.0.3` and re-run. -The TestPyPI rehearsal in Step 3 is the mitigation — do it for anything you're unsure about. - -### Smoke job fails with "package not found" - -PyPI has a short CDN propagation delay (typically <1 min, occasionally up to 10). The smoke job can race against it. Re-run just the smoke job from the Actions UI after a minute or two. - ### A new module was added — how do I include it in releases? 1. Add its distribution name to `scripts/bump_version.py`'s package list (should be automatic if it lives under `modules/*/pyproject.toml`). 2. Add it to `.github/workflows/release.yml` under `publish-pypi` → `strategy.matrix.package`. -3. Create the PyPI (and TestPyPI) Trusted Publisher entry for its project name. +3. Create the PyPI Trusted Publisher entry for its project name. 4. Add a substantive README (`check_readmes.py` will fail otherwise). `scripts/check_metadata.py` and `scripts/check_readmes.py` run in `make lint` and will tell you what's missing. ### I need to rotate or recover the owner account -Trusted Publishing is tied to the GitHub repo, not any personal account — so a PyPI/npm account handover is the usual account-transfer flow at the registry, not a code change. Just update the "Project names" section of this doc afterward. +Trusted Publishing is tied to the GitHub repo, not any personal account — so a PyPI account handover is the usual account-transfer flow at the registry, not a code change. For npm, rotate `NPM_TOKEN` under the new owner. Update the "Project names" section of this doc afterward. ---