From 3f798c5ef4de45abdbddb4e134c9217a967a3065 Mon Sep 17 00:00:00 2001 From: jnasbyupgrade Date: Wed, 29 Jul 2026 16:43:10 -0500 Subject: [PATCH 1/2] Squashed 'pgxntool/' changes from bed3604..b67064b b67064b Release 2.2.0 (#77) d949d6d Fix stale README/CLAUDE.md docs; rename internal-only test shim (#76) 6d7822c ci: grant actions:write so bun cache save succeeds (#75) 2738f59 README.asc: fix stale docs, document undocumented API surface (#74) d1d3bee Add make pgxntool-version to print the embedded pgxntool version (#73) 493a0f8 Fix parallel-build SQL corruption, remote hardcoding, and add safety checks (#61) 3b0e3de Fix make pgtle PGTLE_VERSION=X being silently ignored (#66) 21b35ef ci: skip paired-test-PR requirement for doc-only PRs (#71) 55efd03 Rename EXTENSION_VERSION_FILES to EXTENSION__CURRENT_VERSION__FILES (#69) 9d0a856 Fix pgtle.sh rejecting non-numeric version aliases like 'stable' (#57) (#63) 9c2f75f Docs: versioned SQL file tracking tradeoffs and update-testing pattern (#60) c38cf2a ci: never check out fork PR head in claude-code-review pull_request_target job (#70) aab8f2b pgtle_versions.md: fix wrong uninstall_extension() version claim (#59) e2c9174 README.asc: make tag creates a git tag, not a branch (#58) eb84bc6 Stamp 2.1.0 9fbe6f4 Fix results ordering, control file whitespace, ENABLE_* override, debug levels (#31) 88bb4f2 Add Claude Code GitHub Actions workflows (#41) af5bbbb ci: pass repo owners to run-tests.yml for fork-account matching (#40) c7928af Fix repo-root guard to work inside a git worktree (#39) b062fca ci: point reusable test workflow at @master b6cdbfd Add CI workflows and multi-session PR guard (#33) 1ba0987 Stamp 2.0.3 1931cbe Fix pgxntool-sync remote and make it runnable without make (#37) 8176304 Stamp 2.0.2 3e142ab Fix parse_control_file: remove comments before stripping quotes (#27) cacc301 Stamp 2.0.1 bf1db6b Fix bash 3.2 / Linux compatibility issues (#26) 62d0fcb Fix broken ifeq for --load-language=plpgsql on PG < 13 (#24) 121f0b3 Stamp 2.0.0 ad3ca7e Remove .source support; add test/install, test/build, and verify-results (#18) c010cf8 Fix bash 3.2 compatibility (#23) abeb9d3 Remove .source file support from pg_regress integration (#22) 08c1879 Stamp 1.1.2 6e0dad2 Fix double --dbname bug that defeated unique test database names 639756c Stamp 1.1.1 6ba3176 Fix pg_tle exception handler and empty upgrade files (#15) 3b8cb2a Stamp 1.1.0 550a901 Remove commit.md (maintained in pgxntool-test) d73ca93 Add unique test database names to prevent conflicts (#13) 9b344be Add update-setup-files.sh for 3-way merging after pgxntool-sync (#12) ab7f6e2 Stamp 1.0.0 3a571ba Add pg_tle support and modernize test infrastructure (#11) b96ea6d Add support for Claude code; build and doc improvements (#9) e9c24de Fix pg_regress on versions > 12 (#5) REVERT: bed3604 Fix pg_regress on versions > 12 (#5) (#6) git-subtree-dir: pgxntool git-subtree-split: b67064b16ca2eb30761c078511dc8c974938480f --- .claude/CLAUDE.md | 28 + .claude/development.md | 47 + .gitattributes | 4 + .github/workflows/CLAUDE.md | 71 + .github/workflows/ci.yml | 336 +++++ .github/workflows/claude-code-review.yml | 115 ++ .github/workflows/claude.yml | 51 + .github/workflows/protect-label.yml | 146 ++ .gitignore | 1 + CLAUDE.md | 285 ++++ HISTORY.asc | 169 ++- LICENSE | 2 +- README.asc | 603 +++++++- README.html | 1757 ++++++++++++++++++---- _.gitignore | 18 +- base.mk | 508 ++++++- bin/version | 37 + build_meta.sh | 32 +- control.mk.sh | 93 ++ lib.sh | 110 ++ meta.mk.sh | 96 +- pgtle.sh | 866 +++++++++++ pgtle_versions.md | 46 + pgxntool-sync.sh | 50 + run-test-build.sh | 47 + setup.sh | 51 +- test/bin/check-stale-expected.sh | 107 ++ update-setup-files.sh | 181 +++ verify-results-pgtap.sh | 47 + 29 files changed, 5455 insertions(+), 449 deletions(-) create mode 100644 .claude/CLAUDE.md create mode 100644 .claude/development.md create mode 100644 .github/workflows/CLAUDE.md create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/claude-code-review.yml create mode 100644 .github/workflows/claude.yml create mode 100644 .github/workflows/protect-label.yml create mode 100644 CLAUDE.md create mode 100755 bin/version create mode 100755 control.mk.sh create mode 100644 lib.sh create mode 100755 pgtle.sh create mode 100644 pgtle_versions.md create mode 100755 pgxntool-sync.sh create mode 100755 run-test-build.sh create mode 100755 test/bin/check-stale-expected.sh create mode 100755 update-setup-files.sh create mode 100755 verify-results-pgtap.sh diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md new file mode 100644 index 0000000..e2e74fe --- /dev/null +++ b/.claude/CLAUDE.md @@ -0,0 +1,28 @@ +# Claude Development Notes + +This file contains guidance for Claude Code when working in this repository. +It is excluded from distributions via `.gitattributes export-ignore`. + +## CI Monitoring After Every Push + +**REQUIRED**: After every `git push`, immediately start a background task to +monitor the CI run for that push. If you pushed to both pgxntool and +pgxntool-test, start a background task for each repo — do not monitor them +sequentially. + +Use `gh run watch` or poll with `gh run list` / `gh pr checks` in the +background task. Report failures to the user as soon as they are detected; +do not wait for all jobs to finish before reporting. + +## Multiple Concurrent Sessions + +It is common to have multiple Claude Code sessions open simultaneously across +pgxntool and pgxntool-test. To avoid cross-session interference: + +**If you are asked to do something on an existing PR that you did not open or +are not already working on in this session, immediately ask for confirmation +before proceeding.** For example: "I see PR #32 exists. Were you asking me to +work on that, or did you mean to send this to a different session?" + +This applies to: editing PR branches, pushing to them, closing/reopening them, +adding commits, modifying PR descriptions, or any other PR-level action. diff --git a/.claude/development.md b/.claude/development.md new file mode 100644 index 0000000..886484a --- /dev/null +++ b/.claude/development.md @@ -0,0 +1,47 @@ +# pgxntool Development Guidelines + +**THIS FILE IS FOR PGXNTOOL DEVELOPERS ONLY.** + +If you are an extension developer using pgxntool in your project, this file does not +apply to you. See the top-level `CLAUDE.md` instead. + +## Critical: Work from pgxntool-test, Not Here + +**NEVER make changes to pgxntool directly from this repository.** + +pgxntool development must be done from a checkout of **pgxntool-test**, which contains +the full test infrastructure. Working here directly means you cannot run tests, and +any changes you commit cannot be validated before merging. + +**Correct workflow:** +1. Clone or use an existing checkout of `pgxntool-test` +2. Work in a worktree: both `pgxntool/` and `pgxntool-test/` will be siblings +3. Make changes to `pgxntool/` from within that pgxntool-test context +4. Run the test suite via `make test` in pgxntool-test before committing + +**See:** https://github.com/Postgres-Extensions/pgxntool-test for the full development +workflow. + +--- + +## Makefile Variable Assignment Rules + +**RULE: Do not use `:=` (simply expanded) unless you have a specific need for immediate evaluation.** + +Use `=` (recursively expanded) for standard variable assignments. Reserve `:=` for cases where the right-hand side must be evaluated exactly once at assignment time — for example, when assigning the result of a `$(call ...)` function that references the variable being set (which would cause infinite recursion with `=`). + +When a variable must also override command-line values, combine `override` with `:=` — but only where `override` is genuinely needed. + +## Debug Level Rules (lib.sh `debug` function) + +`debug LEVEL "msg"` prints when `DEBUG >= LEVEL`. LEVEL encodes how noisy/esoteric a message is — how far you'd crank `DEBUG` before you'd want to see it — **not** code nesting depth. A top-level line can warrant a high level if it's esoteric, and loop-body detail is usually high precisely because it's noisy. Judge by signal-to-noise. + +The tiers are anchors, not strict multiples of 10 — any value in range is fine, leaving room to fine-tune between existing calls without renumbering: + +- **10**: Critical errors, important warnings +- **20**: Warnings, significant state changes +- **30**: General debugging, function entry/exit, array operations +- **40**: Verbose details, loop iterations +- **50+**: Maximum verbosity (per-iteration innards) + +Note: The BATS test helper `debug` function (in `tests/lib/helpers.bash` in pgxntool-test) uses a separate 1–5 scale controlled by `$TESTDEBUG`. The two systems are independent. diff --git a/.gitattributes b/.gitattributes index c602ea0..8dc1599 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,8 @@ .gitattributes export-ignore +.claude/ export-ignore +.github/ export-ignore +*.md export-ignore +.DS_Store export-ignore *.asc export-ignore *.adoc export-ignore *.html export-ignore diff --git a/.github/workflows/CLAUDE.md b/.github/workflows/CLAUDE.md new file mode 100644 index 0000000..1d5a6f0 --- /dev/null +++ b/.github/workflows/CLAUDE.md @@ -0,0 +1,71 @@ +# .github/workflows — CI Architecture + +## Workflow files + +- **`ci.yml`** — main CI for pgxntool pull requests. Runs `check-test-pr` (verifies + the paired pgxntool-test PR's CI passed), then optionally runs `test` (only for the + commit-with-no-tests path — see below). +- **`protect-label.yml`** — enforces that only maintainers with write access can apply + or remove the `commit-with-no-tests` label. + +## Normal CI flow (paired test PR exists) + +When a pgxntool PR has a corresponding open PR in pgxntool-test with the same branch +name, the `check-test-pr` job polls (up to 20 minutes) for that test PR's CI to +complete and pass. If it passes, pgxntool CI passes — **no tests run here**. Tests run +exactly once, in pgxntool-test's own CI. + +## commit-with-no-tests path + +When a maintainer applies the `commit-with-no-tests` label (and no paired test PR +exists), the `test` job runs tests directly in pgxntool CI against pgxntool-test/master. +This is the rare exception, not the norm. + +## Doc-only bypass + +`check-test-pr` checks this first, before the paired-test-PR lookup or the +`commit-with-no-tests` label: if every changed file in the PR is pure +documentation (`*.md`, `*.asc`, `*.adoc`, `*.asciidoc`, anywhere including +under `.claude/`, but never under `.github/` — workflow definitions carry +real behavioral weight regardless of extension), it skips both the paired +branch requirement and the `test` job entirely. `claude-code-review.yml` is +a separate workflow gated by its own `if:` and always still runs. + +## Cross-repo reusable workflow — tradeoffs and constraints + +The `test` job calls a reusable workflow from pgxntool-test: +```yaml +uses: Postgres-Extensions/pgxntool-test/.github/workflows/run-tests.yml@ +``` + +GitHub Actions requires the `uses:` ref to be a **static string** — expressions like +`${{ }}` are not supported in the repo/path portion or the `@ref` suffix in practice. + +### The @branch → @master ref + +While developing on a feature branch where pgxntool-test also has changes, this ref +is set to `@` so CI can find `run-tests.yml` before it lands on master. + +**IMPORTANT**: This ref must be updated to `@master` before pgxntool merges. The +correct merge order is: **pgxntool-test merges first**, then update this ref to +`@master`, then pgxntool merges. + +**For Claude**: Do NOT leave a `@` ref without explicit user approval. The +user merges directly from the PR page — there are no manual steps between merges. +See `.github/workflows/CLAUDE.md` in pgxntool-test for the full picture. + +### Changes to run-tests.yml + +`run-tests.yml` lives in pgxntool-test and is the single source of truth for all test +steps. If it changes, pgxntool's CI uses `@master` — so it won't see the new version +until pgxntool-test merges. This is acceptable because: +- Changes to `run-tests.yml` require a paired test PR (not commit-with-no-tests) +- When a paired test PR exists, pgxntool's `test` job is skipped anyway +- The two scenarios are mutually exclusive in practice + +## Label name + +The label `commit-with-no-tests` is defined as a const (`NO_TEST_LABEL`) in `ci.yml` +and as `LABEL` in `protect-label.yml`. The job-level `if:` condition in +`protect-label.yml` must also use the literal string (YAML can't reference JS consts) +— keep these in sync if the label name ever changes. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2bfcbaa --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,336 @@ +name: CI + +on: + pull_request: + # We use 'pull_request' (not 'pull_request_target') deliberately. + # 'pull_request_target' runs with write access to the base repo, which is + # a security risk for untrusted fork code. Since this workflow only reads + # from other public repos (no secrets needed), 'pull_request' is correct + # and safe even for fork PRs. + +permissions: + contents: read # required by actions/checkout in the reusable test workflow + pull-requests: read + checks: read + +concurrency: + group: ci-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check-test-pr: + name: Check for paired pgxntool-test PR + runs-on: ubuntu-latest + # This check polls until the paired pgxntool-test CI run completes + # (up to 20 minutes). The job timeout gives a few minutes of headroom. + timeout-minutes: 25 + outputs: + run-tests: ${{ steps.check.outputs.run_tests }} + test-ref: ${{ steps.check.outputs.test_ref }} + + steps: + - name: Find paired pgxntool-test PR or check commit-with-no-tests label + id: check + # Pinned to an immutable SHA (supply-chain hardening); comment tracks the tag. + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + # GITHUB_TOKEN is sufficient for reading public repos. If these repos + # are ever made private, replace with a PAT stored as a secret with + # 'repo' scope on both repos. Note: PAT expiration causes silent + # failures here — the API returns 401 and the job errors out instead + # of failing gracefully with a useful message. + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const branch = context.payload.pull_request.head.ref; + const prNumber = context.payload.pull_request.number; + // Single source of truth for the label name. Must also match the + // literal string in the protect-label.yml job-level `if:` condition + // (YAML expressions can't reference JS constants). + const NO_TEST_LABEL = 'commit-with-no-tests'; + + // DOC-ONLY BYPASS: skip both the paired-test-PR requirement and + // the actual Postgres test run when every changed file is pure + // documentation. This is independent of, and takes priority + // over, everything below — a doc-only PR needs neither a paired + // branch nor the NO_TEST_LABEL override. + // + // Files under .github/ are never doc-only even if their + // extension matches (they're workflow definitions with real + // behavioral weight, some running with pull_request_target + // privileges). Everything else — including .claude/*.md prompt + // and command docs, and README.html (a generated rendering of + // README.asc, no execution weight of its own) — counts. + // + // This does NOT skip claude-review: that's a separate workflow + // gated by its own `if:`, unaffected by this check's outputs. + const DOC_EXTENSIONS = /\.(md|asc|adoc|asciidoc|html)$/i; + const changedFiles = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + per_page: 100 + }); + // Check both filename and previous_filename: a rename like + // src/foo.sql -> docs/foo.md must not read as doc-only just + // because the new name matches — the old path is a real change. + const changedPaths = changedFiles.flatMap(f => + f.previous_filename ? [f.filename, f.previous_filename] : [f.filename] + ); + const isDocOnly = changedPaths.length > 0 && changedPaths.every(p => + DOC_EXTENSIONS.test(p) && !p.startsWith('.github/') + ); + if (isDocOnly) { + core.info( + `All ${changedFiles.length} changed file(s) are documentation-only ` + + `(matched ${DOC_EXTENSIONS}, none under .github/); skipping the ` + + `paired-test-PR requirement and the Postgres test matrix.` + ); + core.setOutput('run_tests', 'false'); + core.setOutput('test_ref', ''); + return; + } + + // master-to-master PRs have no paired test PR by convention. + // Run tests against pgxntool-test/master directly. + // + // If a fork PR's branch is named 'master', that's almost certainly + // a mistake (contributors should use a feature branch), but we + // don't block it — just warn visibly as an annotation on the run. + // Note: pull_request gives a read-only token for fork PRs, so we + // can't post a PR comment back to the upstream repo from here. + // Gate on the BASE branch too: this shortcut is only for + // master-to-master PRs. A PR from master into some other base must + // still go through the normal paired-test lookup below. + if (branch === 'master' && context.payload.pull_request.base.ref === 'master') { + const headRepo = context.payload.pull_request.head.repo; + const isBaseRepo = + headRepo?.owner?.login === context.repo.owner && + headRepo?.name === context.repo.repo; + if (!isBaseRepo) { + core.warning( + `PR head branch is named 'master' but comes from a fork ` + + `(${headRepo?.full_name ?? 'unknown'}). Contributors should ` + + `use a feature branch, not master. Proceeding with tests ` + + `against pgxntool-test/master.` + ); + } + core.setOutput('run_tests', 'true'); + core.setOutput('test_ref', 'master'); + return; + } + + // The owner of this PR's head repo — the contributor's fork owner + // for fork PRs, or the base repo owner for maintainer PRs. + // The paired pgxntool-test PR must come from the SAME owner. + // We never cross-match PRs across different contributors' forks. + const prOwner = context.payload.pull_request.head.repo?.owner?.login; + + // Look for open pgxntool-test PRs with the SAME branch name AND + // the same fork owner. Branch names must match exactly. + // + // The GitHub API's 'head' filter requires "owner:branch" format. + // We list all open PRs and filter locally — safe for repos with + // few open PRs, and avoids needing to know the fork repo name. + // paginate() fetches all pages automatically, so this is correct + // even if pgxntool-test ever exceeds 100 open PRs (the per_page cap). + const prs = await github.paginate(github.rest.pulls.list, { + owner: context.repo.owner, + repo: 'pgxntool-test', + state: 'open', + per_page: 100 + }); + + const matching = prs.filter(pr => + pr.head.ref === branch && + pr.head.repo?.owner?.login === prOwner + ); + if (matching.length > 1) { + core.setFailed( + `Multiple open pgxntool-test PRs from ${prOwner} match branch ` + + `'${branch}'. Cannot determine which one to use.\n\n` + + `Close all but one, then re-run this check.` + ); + return; + } + + const testPR = matching.length === 1 ? matching[0] : null; + + if (testPR) { + // Error if the no-test label is also set — that's contradictory. + // Re-fetch the PR live (not from payload) in case the label was + // added after this workflow was triggered. + const { data: currentPR } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }); + if (currentPR.labels.some(l => l.name === NO_TEST_LABEL)) { + core.setFailed( + `PR has the '${NO_TEST_LABEL}' label, but a paired ` + + `pgxntool-test PR #${testPR.number} exists on branch '${branch}'.\n\n` + + `Remove the '${NO_TEST_LABEL}' label — it should only be used ` + + `when there is genuinely no paired test PR.` + ); + return; + } + + // A paired test PR exists. Verify its CI passed for the exact + // current HEAD SHA and that the run is recent enough to be valid. + const sha = testPR.head.sha; + const testPRUrl = + `https://github.com/${context.repo.owner}/pgxntool-test/pull/${testPR.number}`; + const recheckUrl = + `https://github.com/${context.repo.owner}/${context.repo.repo}/pull/${prNumber}/checks`; + + core.info(`Found pgxntool-test PR #${testPR.number} (${sha.slice(0, 7)})`); + + // Poll until all check runs for the exact HEAD SHA complete. + // Using 'ref: sha' (not branch name) ensures we only see runs for + // this commit — never stale runs from an older push on the same branch. + // + // We poll rather than fail immediately because both repos are often + // pushed close together. When that happens, pgxntool CI starts while + // pgxntool-test CI may not have queued yet. We wait up to 20 minutes. + const POLL_INTERVAL_MS = 30 * 1000; + const MAX_WAIT_MS = 20 * 60 * 1000; + const waitStart = Date.now(); + let runs; + + while (true) { + // per_page: 100 is intentional here — a single commit will + // not realistically have 100+ CI check runs, so pagination + // is unnecessary. (pulls.list uses paginate() above because + // an active repo could have many open PRs.) + const { data: checks } = await github.rest.checks.listForRef({ + owner: context.repo.owner, + repo: 'pgxntool-test', + ref: sha, + per_page: 100 + }); + runs = checks.check_runs; + + const incomplete = runs.filter(r => r.status !== 'completed'); + if (runs.length > 0 && incomplete.length === 0) break; + + const elapsed = Date.now() - waitStart; + if (elapsed >= MAX_WAIT_MS) { + const mins = Math.round(elapsed / 60000); + if (runs.length === 0) { + core.setFailed( + `pgxntool-test PR #${testPR.number} has no CI runs for ` + + `SHA ${sha.slice(0, 7)} after waiting ${mins} min.\n\n` + + `Push a commit (or manually re-run CI) on the test PR:\n` + + ` Test PR: ${testPRUrl}\n` + + ` Re-run this check: ${recheckUrl}` + ); + } else { + const names = incomplete.map(r => r.name).join(', '); + core.setFailed( + `pgxntool-test PR #${testPR.number} CI did not finish within ` + + `${mins} min for SHA ${sha.slice(0, 7)}: ${names}\n\n` + + ` Test PR: ${testPRUrl}\n` + + ` Re-run this check: ${recheckUrl}` + ); + } + return; + } + + if (runs.length === 0) { + core.info(`No CI runs yet for pgxntool-test PR #${testPR.number} (${sha.slice(0, 7)}); waiting 30s...`); + } else { + const names = incomplete.map(r => r.name).join(', '); + core.info(`pgxntool-test CI still running (${names}); waiting 30s...`); + } + await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL_MS)); + } + + // All checks complete — look for failures. + // 'success', 'skipped', 'neutral' are non-blocking. + const failed = runs.filter( + r => !['success', 'skipped', 'neutral'].includes(r.conclusion) + ); + if (failed.length > 0) { + const names = failed.map(r => `${r.name} (${r.conclusion})`).join(', '); + core.setFailed( + `pgxntool-test PR #${testPR.number} CI failed for ` + + `SHA ${sha.slice(0, 7)}: ${names}\n\n` + + `Fix the test PR CI, then re-run this check:\n` + + ` Test PR: ${testPRUrl}\n` + + ` Re-run this check: ${recheckUrl}` + ); + return; + } + + core.info( + `pgxntool-test PR #${testPR.number} CI passed for ` + + `SHA ${sha.slice(0, 7)} — tests run there, not here.` + ); + core.setOutput('run_tests', 'false'); + core.setOutput('test_ref', sha); + return; + } + + // No paired test PR found. Check for the NO_TEST_LABEL label, + // which a maintainer can apply when a pgxntool change genuinely + // needs no test changes (unusual). + // + // We make a live API call rather than reading from the event + // payload. The payload is a snapshot from when this workflow was + // triggered — a maintainer may have added the label after that. + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }); + + if (pr.labels.some(l => l.name === NO_TEST_LABEL)) { + core.info( + `'${NO_TEST_LABEL}' label is present; running tests ` + + "against pgxntool-test/master. The protect-label workflow " + + "ensures only maintainers can apply this label." + ); + core.setOutput('run_tests', 'true'); + core.setOutput('test_ref', 'master'); + return; + } + + // Neither a paired test PR nor the override label was found. + // Fail with a clear, actionable message. + core.setFailed( + `No paired pgxntool-test PR found for branch '${branch}', ` + + `and no '${NO_TEST_LABEL}' label on this PR.\n\n` + + `pgxntool changes should always be paired with matching test\n` + + `changes in pgxntool-test. This check enforces that pairing.\n\n` + + `To resolve:\n` + + ` 1. Open a PR in pgxntool-test from the SAME account (${prOwner}),\n` + + ` on a branch ALSO named '${branch}'. Both the branch name and\n` + + ` the head owner must match exactly for the pairing to work.\n\n` + + ` 2. If this pgxntool change truly needs no test updates (unusual),\n` + + ` ask a maintainer to apply the '${NO_TEST_LABEL}' label.\n` + + ` Only maintainers can apply this label. It is not a normal\n` + + ` shortcut — most pgxntool changes require test updates.\n\n` + + `See: https://github.com/Postgres-Extensions/pgxntool-test#ci-and-contributing` + ); + + test: + needs: check-test-pr + if: needs.check-test-pr.outputs.run-tests == 'true' + # ----------------------------------------------------------------------- + # CROSS-REPO REUSABLE WORKFLOW — READ BEFORE CHANGING THIS REF + # See: .github/workflows/CLAUDE.md for full architecture notes. + # + # The ref must be a static string — GitHub Actions does not support + # expressions in uses:. It points at pgxntool-test's run-tests.yml on + # master. (During feature-branch development this is temporarily set to + # @ so CI can find run-tests.yml before it lands on master, and + # flipped back to @master once pgxntool-test/ has merged.) + # ----------------------------------------------------------------------- + uses: Postgres-Extensions/pgxntool-test/.github/workflows/run-tests.yml@master + with: + # pgxntool: this PR's own branch, on its own account (a fork for fork PRs). + pgxntool-owner: ${{ github.event.pull_request.head.repo.owner.login }} + pgxntool-branch: ${{ github.event.pull_request.head.ref }} + # pgxntool-test: no paired test PR in this path, so use canonical master + # from Postgres-Extensions only (never a fork's master). + pgxntool-test-owner: Postgres-Extensions + pgxntool-test-ref: master diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 0000000..afa5099 --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,115 @@ +name: Claude Code Review + +# Runs on PRs INTO this repo. We use pull_request_target (not pull_request) so +# that PRs from a fork can access CLAUDE_CODE_OAUTH_TOKEN — GitHub withholds +# secrets from `pull_request` runs triggered by forks, which is why the plain +# `pull_request` version never worked for fork PRs. +# +# SECURITY: pull_request_target runs in the BASE repo with secrets and a +# write-capable token. The job is gated to PRs from the trusted `jnasbyupgrade` +# fork only — an arbitrary external fork can never trigger this secret-bearing +# job. The workflow file always comes from the base branch (master), so a PR +# cannot modify the reviewer that runs on it. We never check out the fork's PR +# head: GitHub Actions refuses that combination by default (the "pwn request" +# guard — see actions/checkout's allow-unsafe-pr-checkout), and +# anthropics/claude-code-action's own docs (docs/security.md) recommend +# checking out the base ref and letting the action read PR content via the +# GitHub API instead. The code-review prompt passes the PR number; the action +# has a GitHub token and pull-requests read/write, so it fetches the diff +# itself (e.g. `gh pr diff`) without ever writing fork code to disk. +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: claude-review-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + claude-review: + # Trusted fork only, and skip drafts (don't spend API/CI on unfinished PRs). + # To add more trusted owners, extend the head-owner check. + if: >- + github.event.pull_request.draft == false && + github.event.pull_request.head.repo.owner.login == 'jnasbyupgrade' + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + pull-requests: write # post the review comments + checks: read # read sibling check-runs for the cost gate + # write (not just read) needed so claude-code-action's internal + # bun-setup step can save its cache; read-only causes a harmless but + # noisy "Cache reservation failed: cache write denied: token has no + # writable scopes" warning. + actions: write + steps: + # COST GATE: the paid Claude review is the last thing to run. Wait for the + # PR head's OTHER check-runs to finish and only proceed if they are clean. + # If any sibling check failed we skip the review to avoid spending money + # reviewing a PR that is already known-broken. Uniform across all repos: + # it discovers sibling checks dynamically (no per-repo workflow names). + # - decision=run : all sibling checks completed with a good conclusion, + # OR no sibling checks exist after a short grace window + # (nothing to gate on), OR the poll timed out is treated + # as skip (see below). + # - decision=skip : at least one sibling check failed/cancelled/etc, or + # we timed out waiting for still-pending checks. + # We exclude this workflow's own check-run (job name `claude-review`) so the + # gate never waits on or fails because of itself. + - name: Wait for CI; skip the paid review if any check failed + id: gate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha }} + run: | + decision=skip + for i in $(seq 1 72); do # ~24 min max + json=$(gh api "repos/$REPO/commits/$SHA/check-runs" --paginate \ + --jq '[.check_runs[] | select(.name != "claude-review")]' 2>/dev/null) || json='' + [ -z "$json" ] && { sleep 20; continue; } + total=$(jq 'length' <<<"$json") + if [ "$total" -eq 0 ]; then + [ "$i" -ge 9 ] && { decision=run; break; } # ~3 min grace: nothing to gate on + sleep 20; continue + fi + pending=$(jq '[.[]|select(.status!="completed")]|length' <<<"$json") + if [ "$pending" -eq 0 ]; then + bad=$(jq '[.[]|select((.conclusion//"")|test("^(failure|cancelled|timed_out|action_required|stale)$"))]|length' <<<"$json") + [ "$bad" -eq 0 ] && decision=run || decision=skip + break + fi + sleep 20 + done + echo "decision=$decision" >> "$GITHUB_OUTPUT" + echo "gate decision: $decision" + + - name: Check out base branch + if: steps.gate.outputs.decision == 'run' + # Intentionally tracks the major-version tag (not a pinned SHA) so + # upstream fixes are picked up automatically. + # + # No `repository:`/`ref:` here on purpose — this checks out the base + # branch (master), never the fork's PR head. See the SECURITY note + # above. + uses: actions/checkout@v6 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Run Claude Code Review + if: steps.gate.outputs.decision == 'run' + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # Provide github_token so the action uses it directly for GitHub API + # calls instead of the OIDC->GitHub-App-token exchange, which 401s under + # pull_request_target. GITHUB_TOKEN is repo/workflow-scoped (independent + # of the actor's role) and has pull-requests: write here. + github_token: ${{ secrets.GITHUB_TOKEN }} + # NOTE: plugin_marketplaces can't be pinned — it tracks the + # marketplace repo's default branch (upstream anthropics/claude-code). + plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' + plugins: 'code-review@claude-code-plugins' + prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 0000000..f5b9ce9 --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,51 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +# No concurrency limit: @claude mentions are independent, read-only requests; +# serializing would only delay responses and cancelling would drop them. +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + # Required for Claude to read CI results on PRs. write (not just read) + # is also needed so claude-code-action's internal bun-setup step can + # save its cache; read-only causes a harmless but noisy + # "Cache reservation failed: cache write denied: token has no writable + # scopes" warning. + actions: write + steps: + - name: Checkout repository + # Intentionally tracks the major-version tag (not a pinned SHA) so + # upstream fixes are picked up automatically. + uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # Allows Claude to read CI results on PRs + additional_permissions: | + actions: read diff --git a/.github/workflows/protect-label.yml b/.github/workflows/protect-label.yml new file mode 100644 index 0000000..de71ea3 --- /dev/null +++ b/.github/workflows/protect-label.yml @@ -0,0 +1,146 @@ +name: Protect 'commit-with-no-tests' label + +on: + # IMPORTANT: Must use pull_request_target, NOT pull_request. + # + # 'pull_request' from a fork runs with a read-only GITHUB_TOKEN scoped to + # the fork. It cannot add or remove labels on the upstream repo (write + # operation), and cannot call getCollaboratorPermissionLevel (requires write + # permission to the target repo). + # + # 'pull_request_target' runs in the base repo's context with a token that + # has write access — exactly what we need here. + # + # Security: because pull_request_target has write access, never check out + # or execute code from the PR head in this workflow. This workflow only calls + # the GitHub API via actions/github-script and is safe. + pull_request_target: + types: [labeled, unlabeled] + +jobs: + protect: + # Only fire for the label we care about. All other label changes are + # unaffected by this workflow. + # Note: this literal must match the LABEL const defined in the script below. + if: github.event.label.name == 'commit-with-no-tests' + runs-on: ubuntu-latest + permissions: + pull-requests: write # To add/remove labels + issues: write # GitHub label API goes through the issues endpoint + + steps: + - name: Enforce write-access-only on 'commit-with-no-tests' label + # Pinned to an immutable SHA: this workflow runs as pull_request_target + # with write access, so a moved upstream tag must not change what runs. + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + const actor = context.actor; + const prNumber = context.payload.pull_request.number; + const action = context.payload.action; // 'labeled' or 'unlabeled' + // Single source of truth for the label name within this script. + // Must also match the literal in the job-level `if:` condition above + // (YAML expressions can't reference JS constants). + const LABEL = 'commit-with-no-tests'; + + // When this workflow re-adds or removes the label itself, that fires + // this event again with actor = 'github-actions[bot]'. Without this + // guard the job loops forever. We match any '[bot]' suffix to also + // cover other automation (Dependabot, Renovate, etc.). + if (actor.endsWith('[bot]')) { + core.info(`Actor is a bot (${actor}); skipping permission check`); + return; + } + + // Check the actor's effective permission level in this repo. + // + // EDGE CASE — 404 for non-collaborators: This API returns 404 when + // the user is not an explicit collaborator. This is the normal case + // for contributors who forked and opened a PR. If we don't catch + // this error, the job crashes with an unhandled exception and the + // label stays in whatever state the contributor put it in — + // defeating the entire protection. + // + // EDGE CASE — org team members: Users with write access via org + // team membership (not a direct collaborator invite) correctly show + // as 'write' here because the API returns effective permission. + // Exception: if the org has "private member visibility" set and the + // token can't enumerate team membership, they may get a 404 instead. + // If that becomes an issue, add a fallback to + // github.rest.orgs.getMembershipForUser(). + // + // EDGE CASE — other errors: Network blips, API outages, and rate + // limiting all throw here. We fail safe by treating any unexpected + // error as "no write access" and logging for debugging. + let hasWrite = false; + try { + const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: actor + }); + hasWrite = ['admin', 'write'].includes(perm.permission); + } catch (e) { + if (e.status === 404) { + // Not a collaborator — no write access. Expected and normal. + hasWrite = false; + } else { + core.warning( + `Unexpected error checking permissions for ${actor} ` + + `(HTTP ${e.status}): ${e.message}. Treating as no write access.` + ); + hasWrite = false; + } + } + + if (action === 'labeled' && !hasWrite) { + core.info(`${actor} lacks write access; removing '${LABEL}' label`); + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + name: LABEL + }); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: + `@${actor} The \`commit-with-no-tests\` label can only be applied by ` + + `maintainers with write access to this repository.\n\n` + + `If you believe no test changes are needed for this PR, please ask a ` + + `maintainer to apply the label after reviewing. Note that most pgxntool ` + + `changes do require paired test updates — this label should be used sparingly.` + }); + + } else if (action === 'unlabeled' && !hasWrite) { + // Non-writer removed the label. Put it back. + // + // EDGE CASE — brief label-absent window: There is a short window + // between removal and this workflow re-adding the label. During + // that window the label genuinely does not exist. This is harmless + // in practice: the ci.yml workflow reads labels via a live API + // call (not from its cached payload), so a re-run after the label + // is restored will pick it up correctly. + core.info(`${actor} lacks write access; re-adding '${LABEL}' label`); + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + labels: [LABEL] + }); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: + `@${actor} The \`commit-with-no-tests\` label can only be removed by ` + + `maintainers with write access to this repository.\n\n` + + `Contact a maintainer if you believe this label was applied in error.` + }); + + } else if (hasWrite) { + core.info( + `${actor} has write access; '${action}' on '${LABEL}' label is approved` + ); + } diff --git a/.gitignore b/.gitignore index a01ee28..5ffb236 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .*.swp +.claude/*.local.json diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..5db8c95 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,285 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## CI Monitoring After Every Push + +**REQUIRED**: After every `git push`, immediately start a background task to +monitor the CI run for that push. If you pushed to both pgxntool and +pgxntool-test, start a background task for each repo — do not monitor them +sequentially. + +The CI monitor lives in the pgxntool-test checkout: run +`bash ../pgxntool-test/.claude/skills/ci/scripts/monitor-ci.sh` (the `/ci` +skill). It monitors both repos and derives the owner from the current repo. +Pass the exact push SHA when available — `gh run list --branch` has a race +condition: if two pushes land close together on the same branch, `--branch` +may pick up the wrong run. `--commit SHA` targets the exact push and avoids it. + +## Scope of This File + +**CLAUDE.md is for people USING pgxntool** — extension developers who have embedded +pgxntool into their project via `git subtree`. It documents the build system, available +commands, and how pgxntool works. + +**If you are making changes to pgxntool itself**, stop — you are in the wrong place. +See `.claude/` in this directory for developer guidelines. More importantly, pgxntool +development must be done from the **pgxntool-test** repository, not from here. See the +`Development Workflow` section below. + +Any agent working in an extension project should always defer to that project's own +CLAUDE.md and instructions over anything stated here. + +## Git Commit Guidelines + +**IMPORTANT**: When creating commit messages, do not attribute commits to yourself (Claude). Commit messages should reflect the work being done without AI attribution in the message body. The standard Co-Authored-By trailer is acceptable. + +## Critical: What This Repo Actually Is + +**pgxntool is NOT a standalone project.** It is a meta-framework that exists ONLY to be embedded into PostgreSQL extension projects via `git subtree`. This repo cannot be built, tested, or run directly. + +**Think of it like this**: pgxntool is to PostgreSQL extensions what a Makefile template library is to C projects - it's infrastructure code that gets copied into other projects, not a project itself. + +## Critical: Directory Purity - NO Temporary Files + +**This directory contains ONLY files that get embedded into extension projects.** When extension developers run `git subtree add`, they pull the entire pgxntool directory into their project. + +**ABSOLUTE RULE**: NO temporary files, scratch work, or development tools may be added to this directory. + +**Examples of what NEVER belongs here:** +- Temporary files (scratch notes, test output, debugging artifacts) +- Development scripts or tools (these go in pgxntool-test/) +- Planning documents (PLAN-*.md files go in pgxntool-test/) +- Any file you wouldn't want in every extension project that uses pgxntool + +**CLAUDE.md exception**: CLAUDE.md exists here for AI assistant guidance, but is excluded from distributions via `.gitattributes export-ignore`. Same with `.claude/` directory. + +**Why this matters**: Any file you add here will be pulled into hundreds of extension projects via git subtree. Keep this directory lean and clean. + +## Development Workflow: Work from pgxntool-test + +**CRITICAL**: All development work on pgxntool should be done from the pgxntool-test repository, NOT from this repository. + +**For complete development workflow documentation, see:** +https://github.com/Postgres-Extensions/pgxntool-test + +## Two-Repository Development Pattern + +This codebase uses a two-repository pattern: + +1. **pgxntool/** (this repo) - The framework code that gets embedded into extension projects +2. **pgxntool-test** - The test harness that validates pgxntool functionality + +**For development and testing workflow, see:** +https://github.com/Postgres-Extensions/pgxntool-test + +## How Extension Developers Use pgxntool + +Extension projects include pgxntool via git subtree: + +```bash +git subtree add -P pgxntool --squash git@github.com:decibel/pgxntool.git release +pgxntool/setup.sh +``` + +After setup, their Makefile typically contains just: +```makefile +include pgxntool/base.mk +``` + +## Architecture: Two-Phase Build System + +### Phase 1: Meta Generation (`build_meta.sh`) +- Processes `META.in.json` (template with placeholders/empty values) +- Strips out X_comment fields and empty values +- Produces clean `META.json` + +### Phase 2: Variable Extraction (`meta.mk.sh`) +- Parses `META.json` using `JSON.sh` (a bash-based JSON parser) +- Generates `meta.mk` with Make variables: + - `PGXN` - distribution name + - `PGXNVERSION` - version number + - `EXTENSIONS` - list of extensions provided + - `EXTENSION_*_VERSION` - per-extension versions + - `EXTENSION__CURRENT_VERSION__FILES` - the current/most-recent auto-generated versioned SQL file for each extension (not all version files) +- `base.mk` includes `meta.mk` via `-include` + +### The Magic of base.mk + +`base.mk` provides a complete PGXS-based build system: +- Auto-detects extension SQL files in `sql/` +- Auto-detects C modules in `src/*.c` +- Auto-detects tests in `test/sql/*.sql` +- Auto-generates versioned extension files (`extension--version.sql`) +- Handles Asciidoc → HTML conversion +- Integrates with PGXN distribution format +- Manages git tagging and release packaging + +## File Structure for Consumer Projects + +Projects using pgxntool follow this layout: +``` +project/ +├── Makefile # include pgxntool/base.mk +├── META.in.json # Template metadata (customize for your extension) +├── META.json # Auto-generated from META.in.json +├── extension.control # Standard PostgreSQL control file +├── pgxntool/ # This repo, embedded via git subtree +├── sql/ +│ └── extension.sql # Base extension SQL +├── src/ # Optional C code (*.c files) +├── test/ +│ ├── deps.sql # Load extension and test dependencies +│ ├── sql/*.sql # Test SQL files +│ └── expected/*.out # Expected test outputs +└── doc/ # Optional docs (*.adoc, *.asciidoc) +``` + +## Commands for Extension Developers (End Users) + +These are the commands extension developers use (documented for context): + +```bash +make # Build extension (generates versioned SQL, docs) +make test # Full test: testdeps → install → installcheck → show diffs +make results # Run tests and update expected output files +make html # Generate HTML from Asciidoc sources +make tag # Create git tag for current META.json version +make dist # Create PGXN .zip (auto-tags, places in ../) +make pgtle # Generate pg_tle registration SQL (see pg_tle Support below) +make check-pgtle # Check pg_tle installation and report version +make run-pgtle # Register extensions with pg_tle in the database +make pgxntool-sync # Update to latest pgxntool via git subtree pull +``` + +## Testing with pgxntool + +### Critical Testing Rules + +**NEVER use `make installcheck` directly**. Always use `make test` instead. The `make test` target ensures: +- Clean builds before testing +- Proper test isolation +- Correct test dependency installation +- Proper cleanup and result comparison + +**Database Connection Requirement**: PostgreSQL must be running before executing `make test`. If you get connection errors (e.g., "could not connect to server"), stop and ask the user to start PostgreSQL. + +**Claude Code MUST NEVER run `make results`**. This target updates test expected output files and requires manual human verification of test changes before execution. + +**Claude Code MUST NEVER modify files in `test/expected/`**. These are expected test outputs that define correct behavior and must only be updated through the `make results` workflow. + +The workflow is: +1. Human runs `make test` and examines diffs +2. Human manually verifies changes are correct +3. Human manually runs `make results` to update expected files + +### Test Output Mechanics + +pgxntool uses PostgreSQL's pg_regress test framework: +- **Actual test output**: Written to `test/results/` directory +- **Expected output**: Stored in `test/expected/` directory +- **Test comparison**: pg_regress compares actual vs expected and generates diffs; `make test` displays them +- **Updating expectations**: `make results` copies `test/results/` → `test/expected/` + +When tests fail, examine the diff output carefully. The actual test output in `test/results/` shows what your code produced, while `test/expected/` shows what was expected. + +## Key Implementation Details + +### PostgreSQL Version Handling +- `MAJORVER` = version × 10 (e.g., 9.6 → 96, 13 → 130) +- Tests use `--load-language=plpgsql` for versions < 13 +- Version detection via `pg_config --version` + +### Test System (pg_regress based) +- Tests in `test/sql/*.sql`, outputs compared to `test/expected/*.out` +- Setup via `test/pgxntool/setup.sql` (loads pgTap and deps.sql) +- `.IGNORE: installcheck` allows `make test` to handle errors (show diffs, then exit with error status) +- `make results` updates expected outputs after test runs + +### Document Generation +- Auto-detects `asciidoctor` or `asciidoc` +- Generates HTML from `*.adoc` and `*.asciidoc` in `$(DOC_DIRS)` +- HTML required for `make dist`, optional for `make install` +- Template-based rules via `ASCIIDOC_template` + +### Distribution Packaging +- `make dist` creates `../PGXN-VERSION.zip` +- Always creates git tag matching version +- Uses `git archive` to package +- Validates repo is clean before tagging + +### Subtree Sync Support +- `make pgxntool-sync` pulls the latest release (the `release` tag) from the canonical repo +- `pgxntool/pgxntool-sync.sh [ []]` does the work and can be run without make +- `make pgxntool-sync-` pulls from the `pgxntool-sync-` variable (` `) +- Uses `git subtree pull --squash`, then `update-setup-files.sh` for a 3-way merge of copied files +- Requires clean repo (no uncommitted changes) + +### pg_tle Support + +pgxntool can generate pg_tle (Trusted Language Extensions) registration SQL for deploying extensions in AWS RDS/Aurora without filesystem access. + +**Usage:** `make pgtle` or `make pgtle PGTLE_VERSION=1.5.0+` + +**Output:** `pg_tle/{version_range}/{extension}.sql` + +**For version range details and API compatibility boundaries, see:** `pgtle_versions.md` + +**Installation targets:** + +- `make check-pgtle` - Checks if pg_tle is installed and reports the version. Reports the version from `pg_extension` if `CREATE EXTENSION pg_tle` has been run in the database. Errors if pg_tle is not available in the cluster. Assumes `PG*` environment variables are configured. + +- `make run-pgtle` - Registers all extensions with pg_tle by executing the generated pg_tle registration SQL files. Requires pg_tle to already be installed in the target database (`CREATE EXTENSION pg_tle;`) -- it does not create the extension itself, and errors out telling you to run `make check-pgtle` if it's missing. Depends on `pgtle`, so it generates the SQL files first if needed. Assumes `PG*` environment variables are configured. + +**Version notation:** +- `X.Y.Z+` means >= X.Y.Z +- `X.Y.Z-A.B.C` means >= X.Y.Z and < A.B.C (note boundary) + +**Key implementation details:** +- Script: `pgxntool/pgtle.sh` (bash) +- Parses `.control` files for metadata (NOT META.json) +- Fixed delimiter: `$_pgtle_wrap_delimiter_$` (validated not in source) +- Each output file contains ALL versions and ALL upgrade paths +- Multi-extension support (multiple .control files) +- Output directory `pg_tle/` excluded from git +- Depends on `make all` to ensure versioned SQL files exist first +- Only processes versioned files (`sql/{ext}--{version}.sql`), not base files + +**SQL file handling:** +- **Version files** (`sql/{ext}--{version}.sql`): Generated automatically by `make all` from base `sql/{ext}.sql` file +- **Upgrade scripts** (`sql/{ext}--{v1}--{v2}.sql`): Created manually by users when adding new extension versions +- The script ensures the default_version file exists if the base file exists (creates it from base file if missing) +- All version files and upgrade scripts are discovered and included in the generated pg_tle registration SQL + +**Dependencies:** +Generated files depend on: +- Control file (metadata source) +- All SQL files (sql/{ext}--*.sql) - must run `make all` first +- Generator script itself + +**Limitations:** +- No C code support (pg_tle requires trusted languages only) +- PostgreSQL 14.5+ required (pg_tle not available on earlier versions) + +## Critical Gotchas + +1. **Empty Variables**: If `DOCS` or `MODULES` is empty, base.mk sets to empty to prevent PGXS errors +2. **testdeps Pattern**: Never add recipes to `testdeps` - create separate target and make it a prerequisite +3. **META.json is Generated**: Always edit `META.in.json`, never `META.json` directly +4. **Control File Versions**: No automatic validation that `.control` matches `META.json` version +5. **PGXNTOOL_NO_PGXS_INCLUDE**: Setting this skips PGXS inclusion (for special scenarios) +6. **Distribution Placement**: `.zip` files go in parent directory (`../`) to avoid repo clutter +7. **Never hand-edit an old versioned SQL file**: `sql/{ext}--{version}.sql` is only auto-regenerated by `make` for the extension's *current* `default_version` (from its `.control` file — not `META.json`; see "PGXN Distributions vs. Extensions" in README.asc). Every other `sql/{ext}--{version}.sql` (any version that is no longer current) is a frozen historical record used to test update paths and PostgreSQL-version compatibility — editing one directly silently corrupts that record. If you need to change something after a version has shipped, bump the version and add a `sql/{ext}--{old}--{new}.sql` upgrade script instead; never edit `sql/{ext}--{old}.sql` in place. This applies even though the file carries a generic `DO NOT EDIT - AUTO-GENERATED FILE` header — that header doesn't distinguish "regenerated every build" (current version) from "generated once, now frozen" (every other version). See "Version-Specific SQL Files" in README.asc for the full reasoning, including when it's acceptable to not commit a given version's file at all, and why that should be a plain `rm` rather than a `.gitignore` entry. + +## Scripts + +- **setup.sh** - Initializes pgxntool in a new extension project (copies templates, creates directories) +- **build_meta.sh** - Strips empty fields from META.in.json to create META.json +- **meta.mk.sh** - Parses META.json via JSON.sh and generates meta.mk with Make variables +- **JSON.sh** - Third-party bash JSON parser (MIT licensed) +- **safesed** - Utility for safe sed operations + +## Related Repositories + +- **pgxntool-test** - Test harness for validating pgxntool functionality: https://github.com/Postgres-Extensions/pgxntool-test +- Never produce any kind of metrics or estimates unless you have data to back them up. If you do have data you MUST reference it. \ No newline at end of file diff --git a/HISTORY.asc b/HISTORY.asc index 9cb793b..231809f 100644 --- a/HISTORY.asc +++ b/HISTORY.asc @@ -1,5 +1,164 @@ -STABLE ------- +2.2.0 +----- +== Add `check-stale-expected` to catch orphaned test/expected files +`make test` now fails if `test/expected/*.out` (or `test/build/expected/*.out`) +has no corresponding `test/sql/*.sql` (or `test/build/*.sql`). If your project +has genuinely stale expected-output files sitting around, `make test` will +newly fail until they're removed. pg_regress's alternate expected-output +files (`test_0.out` .. `test_9.out`) are recognized and not flagged as stale. + +== Rename `EXTENSION_VERSION_FILES` to `EXTENSION__CURRENT_VERSION__FILES` +The old name implied it listed every version file for an extension; it +actually only ever holds the current/most-recent one per extension. If your +Makefile references `EXTENSION_VERSION_FILES` directly (e.g. to add extra +dependencies to `DATA`), update it to `EXTENSION__CURRENT_VERSION__FILES`. + +== Add `make pgxntool-version` to print the embedded pgxntool version +There was no way to tell which pgxntool version a project had embedded via +`git subtree` short of digging through git history. `make pgxntool-version` +(and `pgxntool/bin/version` directly) now prints it, read from the +first line of `pgxntool/HISTORY.asc` -- the same line the release process +stamps with the version number. The line is rejected unless it's `STABLE` +or a plain `X.Y.Z` version (digits only, no pre-release suffixes). + +== Add `PGXN_REMOTE` to override the remote used by tag/rmtag/forcetag/dist +These targets hardcoded the `origin` remote, silently re-tagging the wrong +repository for maintainers whose `origin` is a personal fork. Defaults to +`origin`, so existing behavior is unchanged unless you set it. + +== Fix `make pgtle PGTLE_VERSION=X` generating all versions instead of one +`base.mk`'s `pgtle` target never passed the `PGTLE_VERSION` make variable +through to `pgtle.sh --pgtle-version`, so it silently generated all pg_tle +version ranges regardless of the value given on the command line. + +== Fix `pgtle.sh` rejecting non-numeric version aliases like `stable` +`extract_version_from_filename()` required every version segment to start +with a digit, so extensions using a persistent `stable` pseudo-version +(`default_version = 'stable'`) failed with "Cannot parse version from +filename". Widened to accept any non-empty, `--`-free version string, +matching how pg_tle itself treats extension versions. + +== Fix versioned SQL file corruption under overlapping make invocations +The rule generating `sql/--.sql` used a `>` truncate followed +by a separate `>>` append, so a rule that fired more than once (e.g. +overlapping `make` processes) doubled the file's content. Now uses a single +atomic redirect. + +== Add include guard to base.mk +`base.mk` could end up included twice in one `make` run (for example, an +extension's own `.mk` module includes it in addition to the Makefile's own +include), which redefined every target and printed overriding-recipe +warnings. A second inclusion is now a harmless no-op. + +2.1.0 +----- +== Fix setup.sh / pgxntool-sync.sh / update-setup-files.sh inside a git worktree +These scripts guarded the project root with `[ -d .git ]`. In a linked worktree +`.git` is a file, not a directory, so the check failed: setup.sh would wrongly +re-run `git init`, and the sync scripts aborted with "Not in a git repository." +They now detect the repo with `git rev-parse --git-dir`, which works in both a +normal clone and a worktree. + +== Fix `verify-results` checking stale results in `make results` +`make results` ran `verify-results` before `make test`, so it checked stale +`regression.diffs` from a prior run. Reordered so `verify-results` always +checks the fresh results. + +== Fix `PGXNTOOL_ENABLE_TEST_BUILD`/`PGXNTOOL_ENABLE_TEST_INSTALL` ignoring command-line values +Without `override`, the `pgxntool_validate_yesno` normalization was silently +skipped when these variables were set on the command line. + +2.0.3 +----- +== Fix pgxntool-sync remote, and make it runnable without make +`make pgxntool-sync` pointed at the old SSH URL, which fails without +GitHub SSH keys; it now uses `https://github.com/Postgres-Extensions/pgxntool.git` +over HTTPS (still the `release` tag). The subtree pull + `update-setup-files.sh` +logic moved into `pgxntool/pgxntool-sync.sh`, which the make targets now wrap, so +you can also sync without `make` by running it directly. + +2.0.2 +----- +== Fix parse_control_file corrupting values with trailing comments +Control file values like `default_version = '1.0.0' # comment` were parsed +incorrectly — the trailing quote was left in the value due to comment removal +happening after quote stripping. Fixed by removing comments first. + +2.0.1 +----- +== Improve bash compatibility (specifically for Mac OS) +Mac OS uses the (very old) bash 3.2 as the default shell; fix a few compatibility bugs. + +2.0.0 +----- +== Remove .source file support +PostgreSQL removed `.source` file processing from `pg_regress` in PG15. +The `input/*.source` → `sql/*.sql` and `output/*.source` → +`expected/*.out` conversion mechanism no longer functions. All related +variables (`TEST__SOURCE__*`), the `make_results.sh` helper script, and +the special-case logic in `make results` have been removed. Extensions +that used `.source` files should convert them to regular `test/sql/*.sql` +and `test/expected/*.out` files using relative paths. + +== Add test-build for pre-test validation +When CREATE EXTENSION fails due to a SQL syntax error, PostgreSQL reports only a cryptic error with limited context. test-build runs your extension SQL directly through pg_regress first, so syntax errors show the exact file, line, and position — cutting debugging time significantly. Place SQL files in `test/build/` to enable; auto-detects based on file presence. + +== Add test/install for one-time test setup +Extensions that install dependencies or run expensive setup in every test file pay that cost once per test. test/install runs setup SQL once before the entire test suite, and all regular tests share the resulting database state. This can dramatically speed up test suites that install extensions or load fixtures. Place SQL files in `test/install/` to enable; auto-detects based on file presence. + +== Add verify-results safeguard for make results +`make results` now refuses to run when tests are failing (detected via `regression.diffs`). Prevents accidentally blessing incorrect output as the new expected results. Enabled by default; disable with `PGXNTOOL_ENABLE_VERIFY_RESULTS=no`. + +== Fix bash 3.2 compatibility and shebang portability +`${#ARRAY[@]:-0}` is a syntax error in bash 3.2; replaced with `${#ARRAY[@]}`. +Shell scripts now use `#!/usr/bin/env bash`. + +1.1.2 +----- +== Fix double --dbname bug that defeated unique test database names +The unique database naming introduced in 1.1.0 was ineffective because +base.mk added --dbname=$(REGRESS_DBNAME) to REGRESS_OPTS while PGXS +also appends --dbname=$(CONTRIB_TESTDB). The second --dbname caused +pg_regress to create a contrib_regression database that collided across +projects. Fixed by overriding CONTRIB_TESTDB after include $(PGXS) instead. + +1.1.1 +----- +== Fix pg_tle exception handler and empty upgrade files +The exception handler for `uninstall_extension()` now correctly catches +`no_data_found` (P0002) instead of `undefined_object` (42704). Empty upgrade +files are now treated as valid no-op upgrades for version bumps. Added +`ON_ERROR_STOP=1` to `run_pgtle_sql()` so psql errors propagate correctly. + +1.1.0 +----- +== Use unique database names for tests +Tests now use a unique database name based on the project name and a hash of the +current directory. This prevents test conflicts when running tests for multiple +projects in parallel. + +== Add 3-way merge support for setup files after pgxntool-sync +New `update-setup-files.sh` script handles merging changes to files initially +copied by `setup.sh` (`.gitignore`, `test/deps.sql`). After running `make +pgxntool-sync`, the script performs a 3-way merge if both you and pgxntool have +modified the same file, using git's native conflict markers for resolution. + +1.0.0 +----- +== Fix broken multi-extension support +Prior to this fix, distributions with multiple extensions or extensions with +versions different from the PGXN distribution version were completely broken. +Extension versions are now correctly read from each `.control` file's +`default_version` instead of using META.json's distribution version. + +== Add pg_tle support +New `make pgtle` target generates pg_tle registration SQL for extensions. +Supports pg_tle version ranges (1.0.0-1.4.0, 1.4.0-1.5.0, 1.5.0+) with +appropriate API calls for each range. See README for usage. + +== Use git tags for distribution versioning +The `tag` and `rmtag` targets now create/delete git tags instead of branches. + == Support 13+ The `--load-language` option was removed from `pg_regress` in 13. @@ -7,7 +166,9 @@ The `--load-language` option was removed from `pg_regress` in 13. As part of this change, you will want to review the changes to test/deps.sql. === Support asciidoc documentation targets -By default, if asciidoctor or asciidoc exists on the system, any files in doc/ that end in .adoc or .asciidoc will be processed to html. +By default, if asciidoctor or asciidoc exists on the system, any files in doc/ +that end in .adoc or .asciidoc will be processed to html. + See the README for full details. === Support 9.2 @@ -35,7 +196,7 @@ VERSION is defined by PGXS itself, so trying to use it causes problems. Old code didn't deal with the lack of a . that can appear in a 10+ version. 0.1.10 ------- +----- ### Remove invalid `git subtree pull` options 0.1.9 diff --git a/LICENSE b/LICENSE index 5a20925..4a507f7 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015, Jim Nasby, Blue Treble Solutions +Copyright (c) 2015-2026, Jim Nasby, Blue Treble Solutions All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.asc b/README.asc index c2c6683..a799c7c 100644 --- a/README.asc +++ b/README.asc @@ -7,9 +7,9 @@ Easier PGXN development PGXNtool is meant to make developing new Postgres extensions for http://pgxn.org[PGXN] easier. -Currently, it consists a base Makefile that you can include instead of writing your own, a template META.json, and some test framework. More features will be added over time. +Currently, it consists a base Makefile that you can include instead of writing your own, a template `META.in.json` (from which `META.json` is generated — you edit the former, never the latter directly), and some test framework. More features will be added over time. -If you find any bugs or have ideas for improvements, please https://github.com/decibel/pgxntool/issues[*open an issue*]. +If you find any bugs or have ideas for improvements, please https://github.com/Postgres-Extensions/pgxntool/issues[*open an issue*]. == Install This assumes that you've already initialized your extension in git. @@ -17,12 +17,18 @@ This assumes that you've already initialized your extension in git. NOTE: The `--squash` is important! Otherwise you'll clutter your repo with a bunch of commits you probably don't want. ---- -git subtree add -P pgxntool --squash git@github.com:decibel/pgxntool.git release +git subtree add -P pgxntool --squash git@github.com:Postgres-Extensions/pgxntool.git release pgxntool/setup.sh ---- TODO: Create a nice script that will init a new project for you. +== Development + +If you want to contribute to pgxntool development, work from the https://github.com/Postgres-Extensions/pgxntool-test[pgxntool-test] repository, not from this repository. That repository contains the test infrastructure and development tools needed to validate changes to pgxntool. This repository contains only the framework files that get embedded into extension projects via `git subtree`. + +Changes are normally paired across both repos: a `pgxntool` change should come with a matching branch (same name, on the same account) and PR in `pgxntool-test`, and CI enforces this pairing. See the https://github.com/Postgres-Extensions/pgxntool-test#ci-and-contributing[CI and Contributing] section in pgxntool-test for the full workflow. + == Usage Typically, you can just create a simple Makefile that does nothing but include base.mk: @@ -33,13 +39,165 @@ include pgxntool/base.mk == make targets These are the make targets that are provided by base.mk -NOTE: all the targets normally provided by Postgres http://www.postgresql.org/docs/current/static/extend-pgxs.html[PGXS] still work. +NOTE: all the targets normally provided by Postgres http://www.postgresql.org/docs/current/static/extend-pgxs.html[PGXS] still work. If you need to skip PGXS being included entirely (for advanced/non-standard setups), set `PGXNTOOL_NO_PGXS_INCLUDE` in your Makefile before `include pgxntool/base.mk`. === html This will build any .html files that can be created. See <<_Document_Handling>>. === test -Runs unit tests via the PGXS `installcheck` target. Unlike a simple `make installcheck` though, the `test` rule has the following prerequisites: clean testdeps install installcheck. All of those are PGXS rules, except for `testdeps`. +Runs your extension's test suite: installs the extension and runs it through PGXS's `installcheck`, first pulling in anything you've hooked into <<_testdeps>> and, if enabled, sanity-checking your test SQL via <<_test_build>>. + +Whether `test-build` runs is controlled by the `PGXNTOOL_ENABLE_TEST_BUILD` variable — see <<_test_build>> for what it does and how to turn it on/off. + +NOTE: `test` intentionally does *not* depend on `clean` — that caused problems with incremental/watch-based builds. If your tests need a clean build to pass, that's a sign of a missing dependency elsewhere rather than something to fix by adding `clean` back. + +NOTE: While you can still run `make installcheck` or any other valid PGXS make target directly, it's recommended to use `make test` when using pgxntool. The `test` target ensures proper test isolation and correct dependency installation. + +=== test-build +Validates that extension SQL files are syntactically correct before running the full test suite. This feature runs SQL files from `test/build/` through `pg_regress`, providing better error messages than `CREATE EXTENSION` failures when there are syntax errors in your extension code. + +**How it works:** + +1. Place SQL files in `test/build/*.sql` +2. Place expected output in `test/build/expected/*.out` +3. These files run through `pg_regress` before `make test` runs the main test suite +4. If any build test fails, the test run stops immediately with clear error messages + +**Directory structure:** + +---- +test/build/ +├── *.sql # SQL test files (checked in) +├── expected/ # Expected output files (checked in) +│ └── *.out +└── sql/ # GENERATED - do not edit or check in + └── *.sql # Synced from *.sql above +---- + +The `sql/` subdirectory is generated automatically by `make test-build`. It is listed in `.gitignore` and removed by `make clean`. Do not place files directly in `test/build/sql/`. + +**Configuration:** + +The feature auto-detects based on whether `test/build/*.sql` files exist: + +- Files present → feature enabled automatically +- No files → feature disabled (no impact on existing projects) + +You can override auto-detection by setting `PGXNTOOL_ENABLE_TEST_BUILD`: +---- +# In your Makefile +PGXNTOOL_ENABLE_TEST_BUILD = yes # or no +---- + +**Example: Validate extension SQL compiles** + +Create `test/build/build.sql` to run your extension's SQL directly: + +---- +\set ECHO none +-- Sets ON_ERROR_STOP, VERBOSITY verbose, and ON_ERROR_ROLLBACK +\i test/pgxntool/psql.sql +-- Suppress column headers and row counts for cleaner expected output +\t + +BEGIN; +SET client_min_messages = WARNING; + +-- Install dependencies your extension requires +CREATE EXTENSION IF NOT EXISTS pgtap CASCADE; + +-- Clean slate +DROP EXTENSION IF EXISTS myext; +DROP SCHEMA IF EXISTS myext; +CREATE SCHEMA myext; + +-- Run the actual extension SQL (not CREATE EXTENSION) +-- psql.sql above ensures errors abort immediately with clear messages +\i sql/myext.sql + +-- If we get here, the build succeeded; ON_ERROR_STOP would have aborted on any error above +\echo # BUILD TEST SUCCEEDED +ROLLBACK; +---- + +This approach catches SQL syntax errors *before* running `CREATE EXTENSION`, giving clearer error messages with line numbers. The `ROLLBACK` ensures nothing persists—this is purely validation. + +**Why use `\i` instead of `CREATE EXTENSION`?** + +When `CREATE EXTENSION` fails, PostgreSQL shows only "syntax error" with limited context. Running the SQL directly via `\i` shows the exact line and position of errors, making debugging much faster. + +=== test/install +Runs setup files before the main test suite within the same `pg_regress` invocation. This allows expensive one-time operations (like extension installation) to set up state that persists into the regular test files. + +**How it works:** + +1. Place SQL files in `test/install/*.sql` +2. Place expected output alongside as `test/install/*.out` +3. A schedule file is auto-generated that lists install files with `../install/` relative paths +4. `pg_regress` processes the install schedule first, then runs regular test files — all in one invocation, so database state persists + +**Directory structure:** + +---- +test/install/ +├── *.sql # SQL setup files (checked in) +├── *.out # Expected output (checked in, alongside .sql) +├── .gitignore # Ignores pg_regress artifacts (*.out.diff) +└── schedule # GENERATED - auto-created by make +---- + +The `schedule` file is generated automatically and listed in `.gitignore`. Do not edit it. + +**Configuration:** + +The feature auto-detects based on whether `test/install/*.sql` files exist: + +- Files present → feature enabled automatically +- No files → feature disabled (no impact on existing projects) + +You can override auto-detection by setting `PGXNTOOL_ENABLE_TEST_INSTALL`: +---- +# In your Makefile +PGXNTOOL_ENABLE_TEST_INSTALL = yes # or no +---- + +**Why this is useful:** + +Without `test/install`, each test file typically needs to run `CREATE EXTENSION` in its setup, which adds overhead and doesn't allow validating the installation step separately. With `test/install`, setup runs once before all tests, and any state it creates (tables, extensions, etc.) is available to every subsequent test file. + +**Key detail:** Install files and regular tests run in a single `pg_regress` invocation. This means the database is NOT dropped between install and test phases — state created by install files persists into the main test suite. + +==== Update & Upgrade (U&U) Testing + +Beyond validating a plain install, it's worth testing that your extension behaves correctly across the two transitions every extension with more than one release eventually goes through: + +- **Update**: `ALTER EXTENSION ext UPDATE`, moving to a newer extension version on the same PostgreSQL version. +- **Upgrade**: `pg_upgrade`, moving to a newer PostgreSQL major version while carrying the extension's on-disk catalog state along with it. + +Both catch bugs that a simple "did the script run without error" check misses — wrong mappings, missing `REVOKE`s, objects that drifted from the base SQL, or catalog state that doesn't survive a binary upgrade cleanly. This isn't a niche concern that only applies if your extension does something unusual (adds enum types, custom operators, etc.) — it applies to any extension that will ever be updated or upgraded in place, which in practice is every extension. + +`test/install` is the recommended place to build this, because its "load once, committed, before the suite" mechanic (described above) is exactly what update/upgrade testing needs. + +**The pattern**: load the extension exactly once, committed, from a `test/install/*.sql` file, in one of several modes selected by a custom backend setting that your `Makefile` sets via `PGOPTIONS` (e.g. `-c myext.test_load_mode=$(TEST_LOAD_SOURCE)`) and that `test/install` reads with `current_setting('myext.test_load_mode')`: + +- *fresh*: `CREATE EXTENSION ext;` +- *update*: `CREATE EXTENSION ext VERSION 'oldest_supported';` then `ALTER EXTENSION ext UPDATE;` (to the current `default_version`, or an explicit intermediate version) +- *existing*: the extension is already installed — by a real `pg_upgrade`, or an update performed outside the suite — and `test/install` only asserts it's present at the expected version, without dropping or touching it + +The *same* `test/sql/*` suite and the *same* `test/expected/*` output then run unchanged against every mode. Identical expected output passing in *update* mode *is* the update-equivalence assertion — if the updated database behaves any differently than a fresh install, some test will diff. Running that same suite in *existing* mode against a database that just went through a real `pg_upgrade` gives you upgrade testing using the same test files, no separate suite required. + +If you adopt this pattern, `test/deps.sql` should no longer run `CREATE EXTENSION` itself for the per-test setup — the committed install performed by `test/install` persists into every (rolled-back) test file, exactly as described above. + +**Why the install step must be committed, not run per-test in `test/deps.sql`:** every `test/sql/` file runs inside `BEGIN; ... ROLLBACK;` — a pgxntool convention (`test/pgxntool/setup.sql` opens the transaction; since it's never committed, it rolls back when the session ends). Running the update inside that transaction means the suite never actually exercises a *committed* update — which doesn't match production, where `ALTER EXTENSION UPDATE` commits before anything else touches the database, and it hides bugs that only show up once the update is durably committed. (Modifying an `enum` is one example of how this can turn into an outright failure rather than just a hidden bug — but it's just an example; the reason to commit first holds regardless of what the update script does.) + +As a bonus, sharing one committed install across every test file also removes the per-test re-install cost. + +NOTE: This pattern isn't (yet) native to pgxntool — you wire up the mode selection and `PGOPTIONS` plumbing yourself in `test/install` and your `Makefile`. First-class support (a standard mode-switching toggle, and scaffolding for real `pg_upgrade` testing) is planned; see https://github.com/Postgres-Extensions/pgxntool/issues/42[issue #42]. + +**Working examples** (specific files, not full PRs — both extensions' histories include plenty of unrelated changes): + +- https://github.com/Postgres-Extensions/cat_tools/blob/d01c933a3fcbf28d3bdd2bf6f2bd7b116e95e604/test/install/load.sql[cat_tools `test/install/load.sql`] and the `TEST_LOAD_SOURCE` block in its https://github.com/Postgres-Extensions/cat_tools/blob/d01c933a3fcbf28d3bdd2bf6f2bd7b116e95e604/Makefile[`Makefile`] — the fresh/update/existing pattern above, including the `PGOPTIONS` wiring. This is a reasonably complete example, and somewhat more elaborate than the minimum needed to get started. +- https://github.com/Postgres-Extensions/pg_count_nulls/blob/9f088ac36f6345023bb5b3d20ad0e4aa6ade0d69/test/sql/extension_tests.sql[pg_count_nulls `test/sql/extension_tests.sql`] additionally demonstrates testing an extension installed into a non-default schema. It's a useful reference for that specific problem, but it also defines its assertions as plpgsql functions rather than plain pgTAP calls — a pattern that adds complexity of its own and isn't recommended as a model for U&U testing itself. === testdeps This rule allows you to ensure certain actions have taken place before running tests. By default it has a single prerequisite, `pgtap`, which will attempt to install http://pgtap.org[pgtap] from PGXN. This depneds on having the pgxn client installed. @@ -55,33 +213,249 @@ check_control: grep -q "requires = 'pgtap, test_factory'" test_factory_pgtap.control ---- -If you want to over-ride the default dependency on `pgtap` you should be able to do that with a makefile override. If you need help with that, please https://github.com/decibel/pgxntool/issues[open an issue]. +If you want to over-ride the default dependency on `pgtap` you should be able to do that with a makefile override. If you need help with that, please https://github.com/Postgres-Extensions/pgxntool/issues[open an issue]. WARNING: It will probably cause problems if you try to create a `testdeps` rule that has a recipe. Instead of doing that, put the recipe in a separate rule and make that rule a prerequisite of `testdeps` as show in the example. === results -Because `make test` ultimately runs `installcheck`, it's using the Postgres test suite. Unfortunately, that suite is based on running `diff` between a raw output file and expected results. I *STRONGLY* recommend you use http://pgtap.org[pgTap] instead! The extra effort of learning pgTap will quickly pay for itself. https://github.com/decibel/trunklet-format/blob/master/test/sql/base.sql[This example] might help get you started. +Because `make test` ultimately runs `installcheck`, it's using the Postgres test suite. Unfortunately, that suite is based on running `diff` between a raw output file and expected results. I *STRONGLY* recommend you use http://pgtap.org[pgTap] instead! With pgTap, it's MUCH easier to determine whether a test is passing or not - tests explicitly pass or fail rather than requiring you to examine diff output. The extra effort of learning pgTap will quickly pay for itself. https://github.com/decibel/trunklet-format/blob/master/test/sql/base.sql[This example] might help get you started. No matter what method you use, once you know that all your tests are passing correctly, you need to create or update the test output expected files. `make results` does that for you. +IMPORTANT: *`make results` requires manual verification first*. The correct workflow is: + +1. Run `make test` and examine the diff output +2. Manually verify that the differences are correct and expected +3. Only then run `make results` to update the expected output files in `test/expected/` + +Never run `make results` without first verifying the test changes are correct. The `results` target copies files from `test/results/` to `test/expected/`, so running it blindly will make incorrect output become the new expected behavior. + +==== verify-results safeguard +By default, `make results` will refuse to run if your tests are failing, so you can't accidentally promote incorrect output into the new expected results. Which failures it looks for is controlled by `PGXNTOOL_VERIFY_RESULTS_MODE`: + +`pgtap` (the default):: Scans `test/results/*.out` for pgTAP `not ok` lines (ignoring `# TODO` items) and plan-count mismatches, then also checks `test/results/regression.diffs` for any output mismatch pgTAP itself wouldn't catch. +`diffs`:: Only checks whether `test/results/regression.diffs` exists. + +If tests are failing, `make results` prints an explanation of what failed and exits without touching `test/expected/`; fix the failures, then re-run `make results`. + +To select a mode explicitly: +---- +# In your Makefile +PGXNTOOL_VERIFY_RESULTS_MODE = diffs # or pgtap (the default) +---- + +To disable this safeguard entirely (not recommended): +---- +# In your Makefile +PGXNTOOL_ENABLE_VERIFY_RESULTS = no + +# Or on the command line +make PGXNTOOL_ENABLE_VERIFY_RESULTS=no results +---- + === tag -`make tag` will create a git branch for the current version of your extension, as determined by the META.json file. The reason to do this is so you can always refer to the exact code that went into a released version. +`make tag` will create a git tag for the current version of your extension, as determined by the META.json file (generated from `META.in.json` — see <<_pgxn_distributions_vs_extensions>>), and push it to `origin`. The reason to do this is so you can always refer to the exact code that went into a released version. -If there's already a tag for the current version that probably means you forgot to update META.json, so you'll get an error. If you're certain you want to over-write the tag, you can do `make forcetag`, which removes the existing tag (via `make rmtag`) and creates a new one. +If a tag for the current version already exists and points at your current commit, `make tag` does nothing — this makes it safe for other targets (like `dist`) to depend on `tag` without worrying about re-running it. If the existing tag points at a *different* commit — meaning you likely forgot to bump the version — you'll get an error. If you're certain you want to over-write the tag, you can do `make forcetag`, which removes the existing tag (via `make rmtag`) and creates a new one. -WARNING: You will be very unhappy if you forget to update the .control file for your extension! There is an https://github.com/decibel/pgxntool/issues/1[open issue] to improve this. +WARNING: You will be very unhappy if you forget to update the .control file for your extension! There is an https://github.com/Postgres-Extensions/pgxntool/issues/1[open issue] to improve this. === dist -`make dist` will create a .zip file for your current version that you can upload to PGXN. The file is named after the PGXN name and version (the top-level "name" and "version" attributes in META.json). The .zip file is placed in the *parent* directory so as not to clutter up your git repo. +`make dist` will create a .zip file for your current version that you can upload to PGXN. It first runs `tag` (creating/pushing the version's git tag as described above if needed), then uses `git archive` at that tag to build the .zip file, so the archive always matches the exact tagged commit. The file is named after the PGXN name and version (the top-level "name" and "version" attributes in META.json, generated from `META.in.json`). The .zip file is placed in the *parent* directory so as not to clutter up your git repo. NOTE: Part of the `clean` recipe is cleaning up these .zip files. If you accidentally clean before uploading, just run `make dist-only`. +WARNING: If your project has a `.gitattributes` file, `make dist`/`make dist-only` will refuse to run unless it's committed to git, and will tell you so. This is because `git archive` (which builds the .zip) only honors `export-ignore` attributes on *committed* files — an uncommitted `.gitattributes` would silently have no effect on the archive contents, which could leak files into your PGXN distribution that you meant to exclude. + +`make forcedist` is a shortcut for `forcetag dist`: it force-recreates the tag (see `forcetag` above) before rebuilding the distribution .zip. + === pgxntool-sync -This rule will pull down the latest released version of PGXNtool via `git subtree pull`. +This rule will pull down the latest released version of PGXNtool via `git subtree pull` and then reconcile the files `setup.sh` copied into your project (`.gitignore`, `test/deps.sql`) with a 3-way merge. NOTE: Your repository must be clean (no modified files) in order to run this. Running this command will produce a git commit of the merge. -TIP: There is also a `pgxntool-sync-%` rule if you need to do more advanced things. +NOTE: `git subtree pull` copies pgxntool's entire tree into your project, including dev-only directories (`pgxntool/.github`, `pgxntool/.claude`) that don't belong in a project that merely embeds pgxntool. Every sync automatically removes these two directories from `pgxntool/` afterward — this is expected, not a bug, and doesn't affect anything else in your project. + +TIP: The actual work is done by `pgxntool/pgxntool-sync.sh`, so you can run it directly (`pgxntool/pgxntool-sync.sh`) if you'd rather not go through `make`. It optionally takes `` and `` arguments to pull from somewhere other than the default. + +TIP: `pgxntool-sync.sh` itself delegates the 3-way merge of `setup.sh`-copied files to `pgxntool/update-setup-files.sh `, which you can also run directly -- for example, to redo or debug just the merge step without pulling again. `` is the pgxntool subtree commit that was current *before* the sync you want to re-merge. + +TIP: There is also a `pgxntool-sync-%` rule if you need to do more advanced things. `make pgxntool-sync-` pulls from the ` ` defined by the `pgxntool-sync-` make variable. + +=== pgxntool-version +`make pgxntool-version` prints the version of the embedded pgxntool copy, read from the first line of `pgxntool/HISTORY.asc`. That line is "STABLE" instead of a version number if this copy was synced from an unreleased commit rather than a tagged release. + +TIP: The actual work is done by `pgxntool/bin/version`, so you can run it directly (`pgxntool/bin/version`) if you'd rather not go through `make`. + +=== list +`make list` prints every make target defined in your project — including ones provided by PGXS and pgxntool, not just ones you wrote — one per line. Useful for discovering what's available without reading through the Makefile. + +=== print-% +`make print-VARNAME` prints the current value (and origin) of any make variable, e.g. `make print-PGXNVERSION`. Useful for debugging why a variable isn't set to what you expect. + +=== distclean +`make distclean` removes generated configuration files (`META.json` — generated from `META.in.json` — plus `meta.mk` and `control.mk`) that survive a normal `make clean`. + +NOTE: PGXS doesn't provide any special support for `distclean` — its built-in `distclean` target simply depends on `clean`. PGXNtool uses its own `PGXNTOOL_distclean` variable to track files that should only be removed by `distclean`, not `clean`. + +If your extension generates additional files that should be removed by `distclean` but not `clean`, you can add them: +---- +PGXNTOOL_distclean += my_generated_config.mk +---- + +=== pgtle +Generates pg_tle (Trusted Language Extensions) registration SQL files for deploying extensions in managed environments like AWS RDS/Aurora. See <<_pg_tle_Support>> for complete documentation. + +`make pgtle` generates SQL files in `pg_tle/` subdirectories organized by pg_tle version ranges. For version range details, see `pgtle_versions.md`. + +Set `PGTLE_VERSION` on the command line to limit generation to the single version range that value falls into, instead of every known range: + +---- +make pgtle PGTLE_VERSION=1.5.0 +---- + +=== check-pgtle +Checks if pg_tle is installed and reports the version. This target: +- Reports the version from `pg_extension` if `CREATE EXTENSION pg_tle` has been run in the database +- Errors if pg_tle is not available in the cluster + +This target assumes `PG*` environment variables are configured for `psql` connectivity. + +---- +make check-pgtle +---- + +=== run-pgtle +Registers all extensions with pg_tle by executing the generated pg_tle registration SQL files in a PostgreSQL database. This target: +- Requires pg_tle to be installed in the target database. This isn't a make-level dependency on `check-pgtle` — `pgtle.sh` checks it itself when it runs, and will tell you to run `make check-pgtle` if pg_tle isn't there +- Uses `pgtle.sh` to determine which version range directory to use based on the installed pg_tle version +- Runs all generated SQL files via `psql` to register your extensions with pg_tle + +This target assumes that running `psql` without any arguments will connect to the desired database. You can control this by setting the various PG* environment variables (and possibly using the `.pgpassword` file). See the PostgreSQL documentation for more details. + +NOTE: The `pgtle` target is a dependency, so `make run-pgtle` will automatically generate the SQL files if needed. + +---- +make run-pgtle +---- + +After running `make run-pgtle`, you can create your extension in the database: +---- +CREATE EXTENSION "your-extension-name"; +---- + +== Version-Specific SQL Files + +PGXNtool automatically generates version-specific SQL files from your base SQL file. These files follow the pattern `sql/{extension}--{version}.sql` and are used by PostgreSQL's extension system to install specific versions of your extension. + +=== PGXN Distributions vs. Extensions + +PGXN distinguishes two things it's easy to conflate: a *distribution* and an *extension*. Per https://pgxn.org/spec/[the PGXN meta spec], a distribution is "a collection of extensions, source code, utilities, tests, and/or documents that are distributed together" — the thing you release and upload to PGXN, identified by the top-level `name`/`version` in `META.json` (generated from `META.in.json` — you edit the latter, never `META.json` directly). An extension is the individual thing installed via `CREATE EXTENSION`, described by its own `.control` file. A single distribution routinely provides *more than one* extension — `META.in.json`'s `provides` map lists each one; the pgTAP distribution, for example, provides both the `pgtap` and `schematap` extensions from one release. + +This split matters for versioning, and it's where pgxntool has a real gap: `META.json`'s top-level `version` is the *distribution's* release version (used by `make tag` and `make dist` — it's the git tag name and the `.zip` filename). Each extension's *own* version, though, ends up tracked in *two* separate places that pgxntool does not keep in sync for you: + +- the extension's `.control` file (`default_version`) — what PostgreSQL and pgxntool's own build actually use; see <<_what_controls_the_version_number>> below +- that extension's entry under `META.in.json`'s `provides` map (`provides.{extension}.version`) — which the PGXN meta spec defines as "a Version for the extension" in its own right, *not* the distribution's version, and which is what PGXN's public index reads + +Nothing generates one from the other or checks that they still agree, so it's easy to bump one and forget the other; https://github.com/Postgres-Extensions/pgxntool/issues/47[issue #47] tracks fixing that. + +=== How Version Files Are Generated + +When you run `make` (or `make all`), PGXNtool: + +1. Reads each extension's own `.control` file to determine its version from `default_version` (via `control.mk.sh`) +2. Generates a Makefile rule that copies your base SQL file (`sql/{extension}.sql`) to the version-specific file (`sql/{extension}--{version}.sql`) +3. Executes this rule, creating the version-specific file with a header comment indicating it's auto-generated + +For example, if your `myext.control` contains: +---- +default_version = '1.2.3' +---- + +Running `make` will create `sql/myext--1.2.3.sql` by copying `sql/myext.sql`. + +=== What Controls the Version Number + +The version number comes from each extension's *own* `.control` file → `default_version`, *not* from `META.json`. PostgreSQL itself uses `default_version` to pick which versioned SQL file to load, so `control.mk.sh` parses the `.control` file(s) directly to keep the generated filename in sync with what PostgreSQL will actually use. `META.in.json` has its own, separate `provides.{extension}.version` for that same extension (see <<_pgxn_distributions_vs_extensions>> above) — pgxntool never reads it when generating SQL files, and nothing keeps it in sync with the `.control` file's `default_version`. + +To change the version of one of your extensions: +1. Update `default_version` in that extension's `.control` file +2. Run `make` to regenerate the version-specific file +3. Update that extension's `provides.{extension}.version` in `META.in.json` to match by hand, and regenerate `META.json` (`make META.json`) — pgxntool won't do this for you +4. Update `META.in.json`'s top-level `version` too if you're also cutting a new distribution release + +=== Committing Version Files + +Version-specific SQL files are treated as permanent files that should be committed to your repository by default. This makes it much easier to test updates to extensions, as you can see exactly what SQL was included in each version. + +IMPORTANT: These files are auto-generated and include a header comment warning not to edit them. Any manual changes will be overwritten the next time you run `make`. To modify the extension, edit the base SQL file (`sql/{extension}.sql`) instead. + +==== Why Commit Them: Update Testing + +The primary value of committing a version's install script is *update testing*: with the file in place, you can install that exact version (`CREATE EXTENSION ext VERSION 'x.y.z'`), run `ALTER EXTENSION ext UPDATE`, and verify the update path actually works. In principle you would only ever need the very first committed version's install script — every later version is reachable by chaining upgrade scripts from it. See <<_testinstall>> for how to wire this into your test suite. + +In practice, you should keep *most* old versions tracked rather than relying purely on that chain, because you cannot predict when a new *major PostgreSQL version* will break the ability to install an *older* extension version — a system catalog column changes type, a `SELECT *` over a catalog starts failing, and so on. Committing the old version's install script lets CI catch that regression directly, by attempting to install that exact historical version against the new PostgreSQL release. For any non-trivial extension, most old versions should stay tracked for this reason alone. + +==== When It's OK to Skip a Version + +For a *minor version change that doesn't meaningfully alter the extension* (e.g. a small bug fix), it's unlikely to straddle a PostgreSQL supported-version boundary, so there's much less test-coverage value in committing that specific version's generated install script. For large extensions, deliberately not committing some of these individual version files is worth doing to keep repository size down — the base `sql/{extension}.sql` still fully captures the change in git history, and the install script itself is trivially regenerated from that base file at build time. + +==== Don't `.gitignore` a Skipped Version — `rm` It Once + +When you decide not to track a particular version's generated install script, do *not* add a `.gitignore` entry for it. + +`make` only ever generates the *current* version's file, stamped from the base source according to that extension's `.control` file `default_version`. The moment you bump the version, `make` starts generating the *new* current version's file and stops touching the old one — the old file left behind in your working tree is a stale, one-off artifact, not something `make` keeps recreating on every build. + +Because it's only ever transiently present, the correct cleanup is a single `rm sql/{extension}--{old-version}.sql` right after the version bump — not a permanent `.gitignore` rule. A per-version `.gitignore` line would be perpetual clutter added on every release, for a file that's only ever present for one build cycle. A broad glob (e.g. `sql/*--*.sql`) is wrong for the same reason it's wrong below in <<_alternative_ignoring_all_version_files>>: it would also hide the prior-version install scripts and upgrade scripts (`sql/{extension}--{a}--{b}.sql`) that you *do* want tracked and visible in `git status`. + +==== Never Hand-Edit a Version File That's No Longer Current + +`control.mk.sh` generates a Make rule along these lines for the current version's file: + +---- +$(EXTENSION_ext_VERSION_FILE): sql/ext.sql extension.control + @(echo '/* DO NOT EDIT - AUTO-GENERATED FILE */'; cat sql/ext.sql) > $(EXTENSION_ext_VERSION_FILE) +---- + +That rule only ever targets the file matching the extension's *current* `default_version`, so editing the base `sql/{extension}.sql` and running `make` is the correct, safe way to change that one file. + +Every *other* versioned file — any `sql/{extension}--{version}.sql` where `{version}` is no longer current — is a frozen historical record, not something `make` will ever regenerate or overwrite again. Its entire purpose is to preserve exactly what shipped in that version (see <<_why_commit_them_update_testing>> above), so it can keep being used to test update paths and PostgreSQL-version compatibility against exactly what your users actually installed. + +**Hand-editing an old versioned file directly is never correct** — it silently corrupts that historical record. If you need to change behavior after a version has already shipped, bump the version and add a proper `sql/{extension}--{old}--{new}.sql` upgrade script instead. Never edit `sql/{extension}--{old}.sql` in place. + +CAUTION: This is especially relevant for AI coding agents, which won't know this convention exists unless it's spelled out. The generic `DO NOT EDIT - AUTO-GENERATED FILE` header on every version file doesn't distinguish "regenerated on every build" (the current version) from "was generated once, now frozen" (every other version). An agent without this context may reasonably — but incorrectly — treat any auto-generated file as fair game to patch directly. Old versioned SQL files must instead be treated as append-only history: the fix is always a new version plus an upgrade script, never an in-place edit. + +[[_alternative_ignoring_all_version_files]] +=== Alternative: Ignoring All Version Files + +NOTE: This is a different, all-or-nothing choice from <<_when_its_ok_to_skip_a_version>> above, where the recommendation is to keep tracking most versions and selectively skip only a few low-value minor ones. This section instead covers not tracking *any* version-specific files at all. + +If you prefer not to commit version-specific SQL files, you must add them to your `.gitignore` to prevent `make dist` from failing due to untracked files. Add the following to your `.gitignore`: + +---- +# Auto-generated version-specific SQL files (if not committing them) +sql/*--*.sql +!sql/*--*--*.sql +---- + +The second line (`!sql/*--*--*.sql`) ensures that upgrade scripts (which contain two version numbers and should be manually written) are still tracked. + +WARNING: If you ignore version files instead of committing them, they will NOT be included in your PGXN distribution (`make dist` uses `git archive`, which only includes tracked files). This means users installing your extension from PGXN will need `make` and PGXS available to build the extension - they cannot simply copy the SQL files into their PostgreSQL installation. For maximum compatibility, we recommend committing version files. + +=== Distribution Inclusion + +Version-specific files are included in distributions created by `make dist` only if they are committed to git. Since `make dist` uses `git archive`, only tracked files are included in the distribution archive. + +=== Multiple Versions + +If you need to support multiple versions of your extension: + +1. Create additional version-specific files manually (e.g., `sql/myext--1.0.0.sql`, `sql/myext--1.1.0.sql`) +2. Create upgrade scripts for version transitions (e.g., `sql/myext--1.0.0--1.1.0.sql`) +3. Update `default_version` in the extension's `.control` file to reflect the current version you're working on +4. Commit all version files and upgrade scripts to your repository + +The version file for the current version (specified in the `.control` file's `default_version`) will be automatically regenerated when you run `make`, but other version files you create manually will be preserved. == Document Handling PGXNtool supports generation and installation of document files. There are several variables and rules that control this behavior. @@ -106,7 +480,7 @@ Location of `asciidoc` or equivalent executable. If not set PGXNtool will search for first `asciidoctor`, then `asciidoc`. ASCIIDOC_EXTS:: File extensions to consider as Asciidoc. -Defined as `+= adoc asciidoc`. +Defined as `+= adoc asciidoc asc`. ASCIIDOC_FILES:: Asciidoc input files. PGXNtool searches each `$(DOC_DIRS)` directory, looking for files with any `$(ASCIIDOC_EXTS)` extension. @@ -118,9 +492,8 @@ PGXNtool replaces each `$(ASCIIDOC_EXTS)` in `$(ASCIIDOC_FILES)` with `html`. The result is appended to `ASCIIDOC_HTML` using `+=`. === Document Rules -If Asciidoc is found (or `$(ASCIIDOC)` is set), the `html` rule will be added as a prerequisite to the `install` and `installchec` rules. -That will ensure that docs are generated for install and test, but only if Asciidoc is available. -The `dist` rule will always depend on `html` though, to ensure html files are up-to-date before creating a distribution. +If Asciidoc is found (or `$(ASCIIDOC)` is set), `html` is added as a dependency of the `all` target, which both `install` and `installcheck` depend on — so docs get built for install and test, but only if Asciidoc is available. +The `dist` rule always depends on `html` directly, regardless of whether Asciidoc was found, to ensure html files are up-to-date before creating a distribution. The `html` rule simply depends on `$(ASCIIDOC_HTML). This rule is always present. @@ -133,11 +506,11 @@ These rules are generated from `ASCIIDOC_template`: ---- define ASCIIDOC_template %.html: %.$(1) # <1> -ifndef ASCIIDOC +ifeq (,$(strip $(ASCIIDOC))) $$(warning Could not find "asciidoc" or "asciidoctor". Add one of them to your PATH,) $$(warning or set ASCIIDOC to the correct location.) $$(error Could not build %$$@) -endif # ifndef ASCIIDOC +endif # ifeq ASCIIDOC $$(ASCIIDOC) $$(ASCIIDOC_FLAGS) $$< endef # define ASCIIDOC_template ---- @@ -147,6 +520,8 @@ These rules will *always* exist, even if `$(ASCIIDOC)` isn't set (ie: if Asciido These rules will throw an error if they are run if `$(ASCIIDOC)` isn't defined. On a normal user system that should never happen, because the `html` rule won't be included in `install` or `installcheck`. +`make docclean` removes all generated HTML files (`$(DOCS_HTML)`). It's separate from `make clean`/`make distclean`, so you can clear out generated docs without touching your build artifacts. + === The DOCS variable This variable has special meaning to PGXS. See the Postgres documentation for full details. @@ -158,7 +533,193 @@ Because of this, `base.mk` will forcibly define it to be NULL if it's empty. PGXNtool appends *all* files found in all `$(DOC_DIRS)` to `DOCS`. +== pg_tle Support +[[_pg_tle_Support]] +pgxntool can generate link:https://github.com/aws/pg_tle[pg_tle (Trusted Language Extensions)] registration SQL for deploying PostgreSQL extensions in managed environments like AWS RDS and Aurora where filesystem access is not available. + +For make targets, see: <<_pgtle>>, <<_check_pgtle>>, <<_run_pgtle>>. + +=== What is pg_tle? + +pg_tle is an AWS open-source framework that enables developers to create and deploy PostgreSQL extensions without filesystem access. Traditional PostgreSQL extensions require `.control` and `.sql` files on the filesystem, which isn't possible in managed services like RDS and Aurora. + +pg_tle solves this by: +- Storing extension metadata and SQL in database tables +- Using the `pgtle_admin` role for administrative operations +- Enabling `CREATE EXTENSION` to work in managed environments + +=== Quick Start + +Generate pg_tle registration SQL for your extension: + +---- +make pgtle +---- + +This creates files in `pg_tle/` subdirectories organized by pg_tle version ranges. See `pgtle_versions.md` for complete version range details and API compatibility boundaries. + +=== Version Groupings + +pgxntool creates different sets of files for different pg_tle versions to handle backward-incompatible API changes. Each version boundary represents a change to pg_tle's API functions that we use. + +For details on version boundaries and API changes, see `pgtle_versions.md`. + +=== Installation Example + +IMPORTANT: This is only a basic example. Always refer to the link:https://github.com/aws/pg_tle[main pg_tle documentation] for complete installation instructions and best practices. + +Basic installation steps: + +. Ensure pg_tle is installed and grant the `pgtle_admin` role to your user +. Generate and run the pg_tle registration SQL files: ++ +---- +make run-pgtle +---- ++ +This automatically detects your pg_tle version and runs the appropriate SQL files. See `pgtle_versions.md` for version range details. +. Create your extension: `CREATE EXTENSION myextension;` + +=== Advanced Usage + +==== Multi-Extension Projects + +If your project has multiple extensions (multiple `.control` files), `make pgtle` generates files for all of them: + +---- +myproject/ +├── ext1.control +├── ext2.control +└── pg_tle/ + ├── 1.0.0-1.5.0/ + │ ├── ext1.sql + │ └── ext2.sql + └── 1.5.0+/ + ├── ext1.sql + └── ext2.sql +---- + +==== Direct pgtle.sh Invocation + +`pgtle`, `check-pgtle`, and `run-pgtle` are thin make wrappers around `pgtle.sh`. For more targeted operations you can run it directly: + +- `pgtle.sh --extension NAME [--pgtle-version VERSION]` — generate registration SQL for one extension (what `make pgtle` wraps). +- `pgtle.sh --get-version` — print the pg_tle version installed in the target database (empty if not installed). +- `pgtle.sh --get-dir VERSION` — print which version-range directory (see <<_version_groupings>>) a given pg_tle version maps to, without generating anything. +- `pgtle.sh --run` — execute the generated registration SQL against the target database (what `make run-pgtle` wraps). + +=== How It Works +`make pgtle` does the following: + +. Parses control file(s): Extracts `comment`, `default_version`, `requires`, and `schema` fields +. Discovers SQL files: Finds all versioned files (`sql/{ext}--{version}.sql`) and upgrade scripts (`sql/{ext}--{ver1}--{ver2}.sql`) +. Wraps SQL content: Uses a fixed dollar-quote delimiter (`$_pgtle_wrap_delimiter_$`) to wrap SQL for pg_tle functions +. Generates registration SQL: Creates `pgtle.install_extension()` calls for each version, `pgtle.install_update_path()` for upgrades, and `pgtle.set_default_version()` for the default +. Version-specific output: Generates separate files for different pg_tle capability levels + +Each generated SQL file is wrapped in a transaction (`BEGIN;` ... `COMMIT;`) to ensure atomic installation. + +=== Troubleshooting + +==== "No versioned SQL files found" + +*Problem*: The script can't find `sql/{ext}--{version}.sql` files. + +*Solution*: Run `make` first to generate versioned files from your base `sql/{ext}.sql` file. + +==== "Control file not found" + +*Problem*: The script can't find `{ext}.control` in the current directory. + +*Solution*: Run `make pgtle` from your extension's root directory (where the `.control` file is). + +==== "SQL file contains reserved pg_tle delimiter" + +*Problem*: Your SQL files contain the string `$_pgtle_wrap_delimiter_$` (extremely unlikely). + +*Solution*: Don't use that dollar-quote delimiter in your code. + +==== Extension uses C code + +*Problem*: Your control file has `module_pathname`, indicating C code. + +*Solution*: pg_tle only supports trusted languages. You cannot use C extensions with pg_tle. The script will warn you but still generate files (which won't work). + +NOTE: there are several untrusted languages (such as plpython), and the only tests for C. + +== Configuration Variables + +This is the comprehensive list of make variables you can override to customize pgxntool's behavior, either in your Makefile (before `include pgxntool/base.mk`) or on the `make` command line. Several of these are also covered in more detail in the sections above; this section exists so there's one place that lists all of them. + +NOTE: Variables marked with `*` must be set to exactly `yes` or `no` (case-insensitive) when given explicitly; any other value is a hard `make` error. + +=== PGXN_REMOTE + +Default: `origin`. The git remote used by `tag`, `rmtag`, `forcetag`, and `dist` (which depends on `tag`). If your `origin` is a personal fork rather than the canonical repository -- for example, if you develop on a fork and only push release tags to the canonical repo -- set this so tagging and pushing target the right remote: + +---- +make dist PGXN_REMOTE=upstream +---- + +=== ASCIIDOC + +Default: auto-detected, the first of `asciidoctor` or `asciidoc` found on `PATH`. Path to the Asciidoc processor used to build `.html` files from `$(ASCIIDOC_EXTS)` source files. Override if the processor you want isn't first on `PATH`, or isn't on `PATH` at all. See <<_document_handling>>. + +=== PGTLE_VERSION + +Default: unset (generates every known pg_tle version range). Set on the command line to limit `make pgtle` to the single version range this value falls into. See <<_pgtle>>. + +=== PG_CONFIG + +Default: `pg_config`. Path to the `pg_config` binary used to detect the PostgreSQL version and locate PGXS. Override when the right `pg_config` isn't the one on `PATH`, such as when testing against a specific PostgreSQL install. + +=== TESTDIR + +Default: `test`. Root directory for test input files: `$(TESTDIR)/sql/`, `$(TESTDIR)/expected/`, `$(TESTDIR)/build/`, `$(TESTDIR)/install/`. Overriding this on its own is unusual; it mainly exists so `TESTOUT` has a sensible default. + +=== TESTOUT + +Default: `$(TESTDIR)`. Directory `pg_regress` writes actual test output to -- `$(TESTOUT)/results/`, `$(TESTOUT)/regression.diffs`, etc. -- via `--outputdir` in `REGRESS_OPTS`. Kept separate from `TESTDIR` so generated output can be pointed somewhere other than the directory holding your checked-in `sql`/`expected` files, if you want that separation. + +=== PGXNTOOL_VERIFY_RESULTS_MODE + +Default: `pgtap`. Controls how the <<_verify_results_safeguard,verify-results safeguard>> decides whether tests are failing. + +==== pgtap + +Scans `test/results/*.out` for pgTap `not ok` lines and plan mismatches, falling back to checking for `regression.diffs` too. Use this mode when your test suite uses pgTap. + +==== diffs + +Checks only for `regression.diffs`, matching classic pg_regress behavior. Use this mode when your tests use plain SQL expected-output comparison only. + +=== PGXNTOOL_ENABLE_TEST_BUILD * + +Default: auto-detected -- `yes` if `test/build/*.sql` files exist, `no` otherwise. Enables or disables the <<_test_build,test-build>> pre-flight SQL check. Set explicitly to `yes` if you want an error when `test/build/` unexpectedly has no SQL files (catches accidental deletion of its contents), or to `no` to disable the check even when files are present. + +=== PGXNTOOL_ENABLE_TEST_INSTALL * + +Default: auto-detected -- `yes` if `test/install/*.sql` files exist, `no` otherwise. Enables or disables the <<_testinstall,test/install>> schedule-based setup feature. Same explicit-override semantics as `PGXNTOOL_ENABLE_TEST_BUILD`. + +=== PGXNTOOL_ENABLE_VERIFY_RESULTS * + +Default: `yes`. Enables or disables the <<_verify_results_safeguard,verify-results safeguard>> that blocks `make results` when tests are failing. Setting it to empty on the command line (`make PGXNTOOL_ENABLE_VERIFY_RESULTS= results`) also disables it. + +=== PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED * + +Default: `yes`. Enables or disables the check-stale-expected safeguard, which fails `make test` if `test/expected/` (or `test/build/expected/`) contains a `.out` file with no corresponding `.sql` file -- catching a stale file left behind after a test was renamed or removed. Set to `no` to make the check a complete no-op (it's dropped from `TEST_DEPS` entirely). + +This is also the supported pattern for disabling one specific optional pgxntool feature: a documented `PGXNTOOL_ENABLE_*` variable, gating both the target's registration into `TEST_DEPS`/etc. and (where applicable) the target's own definition. Cleanly replacing or overriding an arbitrary pgxntool-generated recipe with your own entirely is a separate, larger, and not-yet-decided design question -- see https://github.com/Postgres-Extensions/pgxntool/issues/30[issue #30] (two proposed approaches) -- and is *not* what this mechanism solves. + +=== PGXNTOOL_CHECK_EXPECTED_FILE_TYPES * + +Default: `yes`. Sub-check of check-stale-expected, independent of `PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED`: fails (with a distinct error message and exit code from the orphaned-`.out` check) if `test/expected/` (or `test/build/expected/`) contains any file that isn't `*.out`. Set to `no` to disable just this sub-check while leaving the orphaned-`.out` check active. + +=== PGXNTOOL_NO_PGXS_INCLUDE + +Default: unset (PGXS is included normally). Skips including PGXS (`$(PGXS)`) entirely. This is only for advanced scenarios where you need to manage the PGXS include yourself; most projects should never set this. + == Copyright -Copyright (c) 2015 Jim Nasby +Copyright (c) 2026 Jim Nasby -PGXNtool is released under a https://github.com/decibel/pgxntool/blob/master/LICENCE[BSD license]. Note that it includes https://github.com/dominictarr/JSON.sh[JSON.sh], which is released under a https://github.com/decibel/pgxntool/blob/master/JSON.sh.LICENCE[MIT license]. +PGXNtool is released under a https://github.com/Postgres-Extensions/pgxntool/blob/master/LICENSE[BSD license]. Note that it includes https://github.com/dominictarr/JSON.sh[JSON.sh], which is released under a https://github.com/Postgres-Extensions/pgxntool/blob/master/JSON.sh.LICENSE[MIT license]. diff --git a/README.html b/README.html index ae4a597..36d3cf5 100644 --- a/README.html +++ b/README.html @@ -2,31 +2,26 @@ - + - + PGXNtool @@ -428,26 +444,75 @@

PGXNtool

Table of Contents
@@ -458,15 +523,15 @@

PGXNtool

PGXNtool is meant to make developing new Postgres extensions for PGXN easier.

-

Currently, it consists a base Makefile that you can include instead of writing your own, a template META.json, and some test framework. More features will be added over time.

+

Currently, it consists a base Makefile that you can include instead of writing your own, a template META.in.json (from which META.json is generated — you edit the former, never the latter directly), and some test framework. More features will be added over time.

-

If you find any bugs or have ideas for improvements, please open an issue.

+

If you find any bugs or have ideas for improvements, please open an issue.

-

1. Install

+

1. Install

This assumes that you’ve already initialized your extension in git.

@@ -485,7 +550,7 @@

1. Install

-
git subtree add -P pgxntool --squash git@github.com:decibel/pgxntool.git release
+
git subtree add -P pgxntool --squash git@github.com:Postgres-Extensions/pgxntool.git release
 pgxntool/setup.sh
@@ -495,7 +560,18 @@

1. Install

-

2. Usage

+

2. Development

+
+
+

If you want to contribute to pgxntool development, work from the pgxntool-test repository, not from this repository. That repository contains the test infrastructure and development tools needed to validate changes to pgxntool. This repository contains only the framework files that get embedded into extension projects via git subtree.

+
+
+

Changes are normally paired across both repos: a pgxntool change should come with a matching branch (same name, on the same account) and PR in pgxntool-test, and CI enforces this pairing. See the CI and Contributing section in pgxntool-test for the full workflow.

+
+
+
+
+

3. Usage

Typically, you can just create a simple Makefile that does nothing but include base.mk:

@@ -508,7 +584,7 @@

2. Usage

-

3. make targets

+

4. make targets

These are the make targets that are provided by base.mk

@@ -520,93 +596,280 @@

3. make targe
Note
-all the targets normally provided by Postgres PGXS still work. +all the targets normally provided by Postgres PGXS still work. If you need to skip PGXS being included entirely (for advanced/non-standard setups), set PGXNTOOL_NO_PGXS_INCLUDE in your Makefile before include pgxntool/base.mk.

-

3.1. html

+

4.1. html

This will build any .html files that can be created. See [_Document_Handling].

-

3.2. test

+

4.2. test

+
+

Runs your extension’s test suite: installs the extension and runs it through PGXS’s installcheck, first pulling in anything you’ve hooked into testdeps and, if enabled, sanity-checking your test SQL via test-build.

+
-

Runs unit tests via the PGXS installcheck target. Unlike a simple make installcheck though, the test rule has the following prerequisites: clean testdeps install installcheck. All of those are PGXS rules, except for testdeps.

+

Whether test-build runs is controlled by the PGXNTOOL_ENABLE_TEST_BUILD variable — see test-build for what it does and how to turn it on/off.

+
+
+ + + + + +
+
Note
+		 +test intentionally does not depend on clean — that caused problems with incremental/watch-based builds. If your tests need a clean build to pass, that’s a sign of a missing dependency elsewhere rather than something to fix by adding clean back. +
+
+
+ + + + + +
+
Note
+		 +While you can still run make installcheck or any other valid PGXS make target directly, it’s recommended to use make test when using pgxntool. The test target ensures proper test isolation and correct dependency installation. +
-

3.3. testdeps

+

4.3. test-build

-

This rule allows you to ensure certain actions have taken place before running tests. By default it has a single prerequisite, pgtap, which will attempt to install pgtap from PGXN. This depneds on having the pgxn client installed.

+

Validates that extension SQL files are syntactically correct before running the full test suite. This feature runs SQL files from test/build/ through pg_regress, providing better error messages than CREATE EXTENSION failures when there are syntax errors in your extension code.

-

You can add any other dependencies you want by simply adding another testdeps rule. For example:

+

How it works:

+
+
+
    +
  1. +

    Place SQL files in test/build/*.sql

    +
    2. +
  2. +

    Place expected output in test/build/expected/*.out

    +
    3. +
  3. +

    These files run through pg_regress before make test runs the main test suite

    +
    4. +
  4. +

    If any build test fails, the test run stops immediately with clear error messages

    +
    5. +
-

testdeps example from test_factory

+

Directory structure:

-
testdeps: check_control
+
test/build/
+├── *.sql              # SQL test files (checked in)
+├── expected/          # Expected output files (checked in)
+│   └── *.out
+└── sql/               # GENERATED - do not edit or check in
+    └── *.sql          # Synced from *.sql above

+
+
+
+

The sql/ subdirectory is generated automatically by make test-build. It is listed in .gitignore and removed by make clean. Do not place files directly in test/build/sql/.

+
+
+

Configuration:

+
+
+

The feature auto-detects based on whether test/build/*.sql files exist:

+
+
+
    +
  • +

    Files present → feature enabled automatically

    +
    • +
  • +

    No files → feature disabled (no impact on existing projects)

    +
    • +
+
+
+

You can override auto-detection by setting PGXNTOOL_ENABLE_TEST_BUILD:

+
+
+
+
# In your Makefile
+PGXNTOOL_ENABLE_TEST_BUILD = yes  # or no
+
+
+
+

Example: Validate extension SQL compiles

+
+
+

Create test/build/build.sql to run your extension’s SQL directly:

+
+
+
+
\set ECHO none
+-- Sets ON_ERROR_STOP, VERBOSITY verbose, and ON_ERROR_ROLLBACK
+\i test/pgxntool/psql.sql
+-- Suppress column headers and row counts for cleaner expected output
+\t
 
-.PHONY: check_control
-check_control:
-	grep -q "requires = 'pgtap, test_factory'" test_factory_pgtap.control
+BEGIN; +SET client_min_messages = WARNING; + +-- Install dependencies your extension requires +CREATE EXTENSION IF NOT EXISTS pgtap CASCADE; + +-- Clean slate +DROP EXTENSION IF EXISTS myext; +DROP SCHEMA IF EXISTS myext; +CREATE SCHEMA myext; + +-- Run the actual extension SQL (not CREATE EXTENSION) +-- psql.sql above ensures errors abort immediately with clear messages +\i sql/myext.sql + +-- If we get here, the build succeeded; ON_ERROR_STOP would have aborted on any error above +\echo # BUILD TEST SUCCEEDED +ROLLBACK;
-

If you want to over-ride the default dependency on pgtap you should be able to do that with a makefile override. If you need help with that, please open an issue.

+

This approach catches SQL syntax errors before running CREATE EXTENSION, giving clearer error messages with line numbers. The ROLLBACK ensures nothing persists—this is purely validation.

-
- - - - - -
-
Warning
-		 -It will probably cause problems if you try to create a testdeps rule that has a recipe. Instead of doing that, put the recipe in a separate rule and make that rule a prerequisite of testdeps as show in the example. -
+
+

Why use \i instead of CREATE EXTENSION?

+
+
+

When CREATE EXTENSION fails, PostgreSQL shows only "syntax error" with limited context. Running the SQL directly via \i shows the exact line and position of errors, making debugging much faster.

-

3.4. results

+

4.4. test/install

-

Because make test ultimately runs installcheck, it’s using the Postgres test suite. Unfortunately, that suite is based on running diff between a raw output file and expected results. I STRONGLY recommend you use pgTap instead! The extra effort of learning pgTap will quickly pay for itself. This example might help get you started.

+

Runs setup files before the main test suite within the same pg_regress invocation. This allows expensive one-time operations (like extension installation) to set up state that persists into the regular test files.

-

No matter what method you use, once you know that all your tests are passing correctly, you need to create or update the test output expected files. make results does that for you.

+

How it works:

+
+
    +
  1. +

    Place SQL files in test/install/*.sql

    +
    2. +
  2. +

    Place expected output alongside as test/install/*.out

    +
    3. +
  3. +

    A schedule file is auto-generated that lists install files with ../install/ relative paths

    +
    4. +
  4. +

    pg_regress processes the install schedule first, then runs regular test files — all in one invocation, so database state persists

    +
    5. +
-
-

3.5. tag

-

make tag will create a git branch for the current version of your extension, as determined by the META.json file. The reason to do this is so you can always refer to the exact code that went into a released version.

+

Directory structure:

+
+
+
+
test/install/
+├── *.sql              # SQL setup files (checked in)
+├── *.out              # Expected output (checked in, alongside .sql)
+├── .gitignore         # Ignores pg_regress artifacts (*.out.diff)
+└── schedule           # GENERATED - auto-created by make
+
-

If there’s already a tag for the current version that probably means you forgot to update META.json, so you’ll get an error. If you’re certain you want to over-write the tag, you can do make forcetag, which removes the existing tag (via make rmtag) and creates a new one.

+

The schedule file is generated automatically and listed in .gitignore. Do not edit it.

-
- - - - - -
-
Warning
-		 -You will be very unhappy if you forget to update the .control file for your extension! There is an open issue to improve this. -
+
+

Configuration:

+
+

The feature auto-detects based on whether test/install/*.sql files exist:

+
+
+
    +
  • +

    Files present → feature enabled automatically

    +
    • +
  • +

    No files → feature disabled (no impact on existing projects)

    +
    • +
+
+
+

You can override auto-detection by setting PGXNTOOL_ENABLE_TEST_INSTALL:

+
+
+
+
# In your Makefile
+PGXNTOOL_ENABLE_TEST_INSTALL = yes  # or no
+
+
+
+

Why this is useful:

+
+
+

Without test/install, each test file typically needs to run CREATE EXTENSION in its setup, which adds overhead and doesn’t allow validating the installation step separately. With test/install, setup runs once before all tests, and any state it creates (tables, extensions, etc.) is available to every subsequent test file.

+
+
+

Key detail: Install files and regular tests run in a single pg_regress invocation. This means the database is NOT dropped between install and test phases — state created by install files persists into the main test suite.

+
+
+

4.4.1. Update & Upgrade (U&U) Testing

+
+

Beyond validating a plain install, it’s worth testing that your extension behaves correctly across the two transitions every extension with more than one release eventually goes through:

+
+
+
    +
  • +

    Update: ALTER EXTENSION ext UPDATE, moving to a newer extension version on the same PostgreSQL version.

    +
    • +
  • +

    Upgrade: pg_upgrade, moving to a newer PostgreSQL major version while carrying the extension’s on-disk catalog state along with it.

    +
    • +
+
+
+

Both catch bugs that a simple "did the script run without error" check misses — wrong mappings, missing `REVOKE`s, objects that drifted from the base SQL, or catalog state that doesn’t survive a binary upgrade cleanly. This isn’t a niche concern that only applies if your extension does something unusual (adds enum types, custom operators, etc.) — it applies to any extension that will ever be updated or upgraded in place, which in practice is every extension.

+
+
+

test/install is the recommended place to build this, because its "load once, committed, before the suite" mechanic (described above) is exactly what update/upgrade testing needs.

+
+
+

The pattern: load the extension exactly once, committed, from a test/install/*.sql file, in one of several modes selected by a custom backend setting that your Makefile sets via PGOPTIONS (e.g. -c myext.test_load_mode=$(TEST_LOAD_SOURCE)) and that test/install reads with current_setting('myext.test_load_mode'):

+
+
+
    +
  • +

    fresh: CREATE EXTENSION ext;

    +
    • +
  • +

    update: CREATE EXTENSION ext VERSION 'oldest_supported'; then ALTER EXTENSION ext UPDATE; (to the current default_version, or an explicit intermediate version)

    +
    • +
  • +

    existing: the extension is already installed — by a real pg_upgrade, or an update performed outside the suite — and test/install only asserts it’s present at the expected version, without dropping or touching it

    +
    • +
+
+
+

The same test/sql/ suite and the *same test/expected/ output then run unchanged against every mode. Identical expected output passing in *update mode is the update-equivalence assertion — if the updated database behaves any differently than a fresh install, some test will diff. Running that same suite in existing mode against a database that just went through a real pg_upgrade gives you upgrade testing using the same test files, no separate suite required.

+
+
+

If you adopt this pattern, test/deps.sql should no longer run CREATE EXTENSION itself for the per-test setup — the committed install performed by test/install persists into every (rolled-back) test file, exactly as described above.

-
-

3.6. dist

-

make dist will create a .zip file for your current version that you can upload to PGXN. The file is named after the PGXN name and version (the top-level "name" and "version" attributes in META.json). The .zip file is placed in the parent directory so as not to clutter up your git repo.

+

Why the install step must be committed, not run per-test in test/deps.sql: every test/sql/ file runs inside BEGIN; …​ ROLLBACK; — a pgxntool convention (test/pgxntool/setup.sql opens the transaction; since it’s never committed, it rolls back when the session ends). Running the update inside that transaction means the suite never actually exercises a committed update — which doesn’t match production, where ALTER EXTENSION UPDATE commits before anything else touches the database, and it hides bugs that only show up once the update is durably committed. (Modifying an enum is one example of how this can turn into an outright failure rather than just a hidden bug — but it’s just an example; the reason to commit first holds regardless of what the update script does.)

+
+
+

As a bonus, sharing one committed install across every test file also removes the per-test re-install cost.

@@ -615,73 +878,640 @@

3.6. dist

Note
-Part of the clean recipe is cleaning up these .zip files. If you accidentally clean before uploading, just run make dist-only. +This pattern isn’t (yet) native to pgxntool — you wire up the mode selection and PGOPTIONS plumbing yourself in test/install and your Makefile. First-class support (a standard mode-switching toggle, and scaffolding for real pg_upgrade testing) is planned; see issue #42.
+
+

Working examples (specific files, not full PRs — both extensions' histories include plenty of unrelated changes):

+
+
+
    +
  • +

    cat_tools test/install/load.sql and the TEST_LOAD_SOURCE block in its Makefile — the fresh/update/existing pattern above, including the PGOPTIONS wiring. This is a reasonably complete example, and somewhat more elaborate than the minimum needed to get started.

    +
    • +
  • +

    pg_count_nulls test/sql/extension_tests.sql additionally demonstrates testing an extension installed into a non-default schema. It’s a useful reference for that specific problem, but it also defines its assertions as plpgsql functions rather than plain pgTAP calls — a pattern that adds complexity of its own and isn’t recommended as a model for U&U testing itself.

    +
    • +
+
+
-

3.7. pgxntool-sync

+

4.5. testdeps

-

This rule will pull down the latest released version of PGXNtool via git subtree pull.

+

This rule allows you to ensure certain actions have taken place before running tests. By default it has a single prerequisite, pgtap, which will attempt to install pgtap from PGXN. This depneds on having the pgxn client installed.

-
+
+

You can add any other dependencies you want by simply adding another testdeps rule. For example:

+
+
+

testdeps example from test_factory

+
+
+
+
testdeps: check_control
+
+.PHONY: check_control
+check_control:
+	grep -q "requires = 'pgtap, test_factory'" test_factory_pgtap.control
+
+
+
+

If you want to over-ride the default dependency on pgtap you should be able to do that with a makefile override. If you need help with that, please open an issue.

+
+
-
Note
+
Warning
 -Your repository must be clean (no modified files) in order to run this. Running this command will produce a git commit of the merge. +It will probably cause problems if you try to create a testdeps rule that has a recipe. Instead of doing that, put the recipe in a separate rule and make that rule a prerequisite of testdeps as show in the example.
-
+
+
+

4.6. results

+
+

Because make test ultimately runs installcheck, it’s using the Postgres test suite. Unfortunately, that suite is based on running diff between a raw output file and expected results. I STRONGLY recommend you use pgTap instead! With pgTap, it’s MUCH easier to determine whether a test is passing or not - tests explicitly pass or fail rather than requiring you to examine diff output. The extra effort of learning pgTap will quickly pay for itself. This example might help get you started.

+
+
+

No matter what method you use, once you know that all your tests are passing correctly, you need to create or update the test output expected files. make results does that for you.

+
+
-
Tip
+
Important
 -There is also a pgxntool-sync-% rule if you need to do more advanced things. +make results requires manual verification first. The correct workflow is:
+
+
    +
  1. +

    Run make test and examine the diff output

    +
    2. +
  2. +

    Manually verify that the differences are correct and expected

    +
    3. +
  3. +

    Only then run make results to update the expected output files in test/expected/

    +
    4. +
-
-
-
-

4. Document Handling

-
-

PGXNtool supports generation and installation of document files. There are several variables and rules that control this behavior.

+

Never run make results without first verifying the test changes are correct. The results target copies files from test/results/ to test/expected/, so running it blindly will make incorrect output become the new expected behavior.

+
+

4.6.1. verify-results safeguard

-

It is recommended that you commit any generated documentation files (such as HTML generated from Asciidoc) into git. -That way users will have these files installed when they install your extension. -If any generated files are missing (or out-of-date) during installation, PGXNtool will build them if Asciidoc is present on the system.

+

By default, make results will refuse to run if your tests are failing, so you can’t accidentally promote incorrect output into the new expected results. Which failures it looks for is controlled by PGXNTOOL_VERIFY_RESULTS_MODE:

-
-

4.1. Document Variables

-
DOC_DIRS
-
-

Directories to look for documents in. -Defined as += doc.

-
-
DOCS
+
pgtap (the default)
-

PGXS variable. -See The DOCS variable below.

+

Scans test/results/*.out for pgTAP not ok lines (ignoring # TODO items) and plan-count mismatches, then also checks test/results/regression.diffs for any output mismatch pgTAP itself wouldn’t catch.

-
DOCS_HTML
+
diffs
-

Document HTML files. -PGXNtool appends `$(ASCIIDOC_HTML) to this variable.

+

Only checks whether test/results/regression.diffs exists.

+
+
+
+
+

If tests are failing, make results prints an explanation of what failed and exits without touching test/expected/; fix the failures, then re-run make results.

+
+
+

To select a mode explicitly:

+
+
+
+
# In your Makefile
+PGXNTOOL_VERIFY_RESULTS_MODE = diffs  # or pgtap (the default)
+
+
+
+

To disable this safeguard entirely (not recommended):

+
+
+
+
# In your Makefile
+PGXNTOOL_ENABLE_VERIFY_RESULTS = no
+
+# Or on the command line
+make PGXNTOOL_ENABLE_VERIFY_RESULTS=no results
+
+
+
+
+
+

4.7. tag

+
+

make tag will create a git tag for the current version of your extension, as determined by the META.json file (generated from META.in.json — see PGXN Distributions vs. Extensions), and push it to origin. The reason to do this is so you can always refer to the exact code that went into a released version.

+
+
+

If a tag for the current version already exists and points at your current commit, make tag does nothing — this makes it safe for other targets (like dist) to depend on tag without worrying about re-running it. If the existing tag points at a different commit — meaning you likely forgot to bump the version — you’ll get an error. If you’re certain you want to over-write the tag, you can do make forcetag, which removes the existing tag (via make rmtag) and creates a new one.

+
+
+ + + + + +
+
Warning
+		 +You will be very unhappy if you forget to update the .control file for your extension! There is an open issue to improve this. +
+
+
+
+

4.8. dist

+
+

make dist will create a .zip file for your current version that you can upload to PGXN. It first runs tag (creating/pushing the version’s git tag as described above if needed), then uses git archive at that tag to build the .zip file, so the archive always matches the exact tagged commit. The file is named after the PGXN name and version (the top-level "name" and "version" attributes in META.json, generated from META.in.json). The .zip file is placed in the parent directory so as not to clutter up your git repo.

+
+
+ + + + + +
+
Note
+		 +Part of the clean recipe is cleaning up these .zip files. If you accidentally clean before uploading, just run make dist-only. +
+
+
+ + + + + +
+
Warning
+		 +If your project has a .gitattributes file, make dist/make dist-only will refuse to run unless it’s committed to git, and will tell you so. This is because git archive (which builds the .zip) only honors export-ignore attributes on committed files — an uncommitted .gitattributes would silently have no effect on the archive contents, which could leak files into your PGXN distribution that you meant to exclude. +
+
+
+

make forcedist is a shortcut for forcetag dist: it force-recreates the tag (see forcetag above) before rebuilding the distribution .zip.

+
+
+
+

4.9. pgxntool-sync

+
+

This rule will pull down the latest released version of PGXNtool via git subtree pull and then reconcile the files setup.sh copied into your project (.gitignore, test/deps.sql) with a 3-way merge.

+
+
+ + + + + +
+
Note
+		 +Your repository must be clean (no modified files) in order to run this. Running this command will produce a git commit of the merge. +
+
+
+ + + + + +
+
Note
+		 +git subtree pull copies pgxntool’s entire tree into your project, including dev-only directories (pgxntool/.github, pgxntool/.claude) that don’t belong in a project that merely embeds pgxntool. Every sync automatically removes these two directories from pgxntool/ afterward — this is expected, not a bug, and doesn’t affect anything else in your project. +
+
+
+ + + + + +
+
Tip
+		 +The actual work is done by pgxntool/pgxntool-sync.sh, so you can run it directly (pgxntool/pgxntool-sync.sh) if you’d rather not go through make. It optionally takes <repo> and <ref> arguments to pull from somewhere other than the default. +
+
+
+ + + + + +
+
Tip
+		 +pgxntool-sync.sh itself delegates the 3-way merge of setup.sh-copied files to pgxntool/update-setup-files.sh <old-pgxntool-commit>, which you can also run directly — for example, to redo or debug just the merge step without pulling again. <old-pgxntool-commit> is the pgxntool subtree commit that was current before the sync you want to re-merge. +
+
+
+ + + + + +
+
Tip
+		 +There is also a pgxntool-sync-% rule if you need to do more advanced things. make pgxntool-sync-<name> pulls from the <repo> <ref> defined by the pgxntool-sync-<name> make variable. +
+
+
+
+

4.10. pgxntool-version

+
+

make pgxntool-version prints the version of the embedded pgxntool copy, read from the first line of pgxntool/HISTORY.asc. That line is "STABLE" instead of a version number if this copy was synced from an unreleased commit rather than a tagged release.

+
+
+ + + + + +
+
Tip
+		 +The actual work is done by pgxntool/bin/version, so you can run it directly (pgxntool/bin/version) if you’d rather not go through make. +
+
+
+
+

4.11. list

+
+

make list prints every make target defined in your project — including ones provided by PGXS and pgxntool, not just ones you wrote — one per line. Useful for discovering what’s available without reading through the Makefile.

+
+
+
+

4.12. print-%

+
+

make print-VARNAME prints the current value (and origin) of any make variable, e.g. make print-PGXNVERSION. Useful for debugging why a variable isn’t set to what you expect.

+
+
+
+

4.13. distclean

+
+

make distclean removes generated configuration files (META.json — generated from META.in.json — plus meta.mk and control.mk) that survive a normal make clean.

+
+
+ + + + + +
+
Note
+		 +PGXS doesn’t provide any special support for distclean — its built-in distclean target simply depends on clean. PGXNtool uses its own PGXNTOOL_distclean variable to track files that should only be removed by distclean, not clean. +
+
+
+

If your extension generates additional files that should be removed by distclean but not clean, you can add them:

+
+
+
+
PGXNTOOL_distclean += my_generated_config.mk
+
+
+
+
+

4.14. pgtle

+
+

Generates pg_tle (Trusted Language Extensions) registration SQL files for deploying extensions in managed environments like AWS RDS/Aurora. See [_pg_tle_Support] for complete documentation.

+
+
+

make pgtle generates SQL files in pg_tle/ subdirectories organized by pg_tle version ranges. For version range details, see pgtle_versions.md.

+
+
+

Set PGTLE_VERSION on the command line to limit generation to the single version range that value falls into, instead of every known range:

+
+
+
+
make pgtle PGTLE_VERSION=1.5.0
+
+
+
+
+

4.15. check-pgtle

+
+

Checks if pg_tle is installed and reports the version. This target: +- Reports the version from pg_extension if CREATE EXTENSION pg_tle has been run in the database +- Errors if pg_tle is not available in the cluster

+
+
+

This target assumes PG* environment variables are configured for psql connectivity.

+
+
+
+
make check-pgtle
+
+
+
+
+

4.16. run-pgtle

+
+

Registers all extensions with pg_tle by executing the generated pg_tle registration SQL files in a PostgreSQL database. This target: +- Requires pg_tle to be installed in the target database. This isn’t a make-level dependency on check-pgtle — pgtle.sh checks it itself when it runs, and will tell you to run make check-pgtle if pg_tle isn’t there +- Uses pgtle.sh to determine which version range directory to use based on the installed pg_tle version +- Runs all generated SQL files via psql to register your extensions with pg_tle

+
+
+

This target assumes that running psql without any arguments will connect to the desired database. You can control this by setting the various PG* environment variables (and possibly using the .pgpassword file). See the PostgreSQL documentation for more details.

+
+
+ + + + + +
+
Note
+		 +The pgtle target is a dependency, so make run-pgtle will automatically generate the SQL files if needed. +
+
+
+
+
make run-pgtle
+
+
+
+

After running make run-pgtle, you can create your extension in the database:

+
+
+
+
CREATE EXTENSION "your-extension-name";
+
+
+
+
+
+
+

5. Version-Specific SQL Files

+
+
+

PGXNtool automatically generates version-specific SQL files from your base SQL file. These files follow the pattern sql/{extension}--{version}.sql and are used by PostgreSQL’s extension system to install specific versions of your extension.

+
+
+

5.1. PGXN Distributions vs. Extensions

+
+

PGXN distinguishes two things it’s easy to conflate: a distribution and an extension. Per the PGXN meta spec, a distribution is "a collection of extensions, source code, utilities, tests, and/or documents that are distributed together" — the thing you release and upload to PGXN, identified by the top-level name/version in META.json (generated from META.in.json — you edit the latter, never META.json directly). An extension is the individual thing installed via CREATE EXTENSION, described by its own .control file. A single distribution routinely provides more than one extension — META.in.json’s `provides map lists each one; the pgTAP distribution, for example, provides both the pgtap and schematap extensions from one release.

+
+
+

This split matters for versioning, and it’s where pgxntool has a real gap: META.json’s top-level `version is the distribution’s release version (used by make tag and make dist — it’s the git tag name and the .zip filename). Each extension’s own version, though, ends up tracked in two separate places that pgxntool does not keep in sync for you:

+
+
+
    +
  • +

    the extension’s .control file (default_version) — what PostgreSQL and pgxntool’s own build actually use; see What Controls the Version Number below

    +
    • +
  • +

    that extension’s entry under META.in.json’s `provides map (provides.{extension}.version) — which the PGXN meta spec defines as "a Version for the extension" in its own right, not the distribution’s version, and which is what PGXN’s public index reads

    +
    • +
+
+
+

Nothing generates one from the other or checks that they still agree, so it’s easy to bump one and forget the other; issue #47 tracks fixing that.

+
+
+
+

5.2. How Version Files Are Generated

+
+

When you run make (or make all), PGXNtool:

+
+
+
    +
  1. +

    Reads each extension’s own .control file to determine its version from default_version (via control.mk.sh)

    +
    2. +
  2. +

    Generates a Makefile rule that copies your base SQL file (sql/{extension}.sql) to the version-specific file (sql/{extension}--{version}.sql)

    +
    3. +
  3. +

    Executes this rule, creating the version-specific file with a header comment indicating it’s auto-generated

    +
    4. +
+
+
+

For example, if your myext.control contains:

+
+
+
+
default_version = '1.2.3'
+
+
+
+

Running make will create sql/myext—​1.2.3.sql by copying sql/myext.sql.

+
+
+
+

5.3. What Controls the Version Number

+
+

The version number comes from each extension’s own .control file → default_version, not from META.json. PostgreSQL itself uses default_version to pick which versioned SQL file to load, so control.mk.sh parses the .control file(s) directly to keep the generated filename in sync with what PostgreSQL will actually use. META.in.json has its own, separate provides.{extension}.version for that same extension (see PGXN Distributions vs. Extensions above) — pgxntool never reads it when generating SQL files, and nothing keeps it in sync with the .control file’s default_version.

+
+
+

To change the version of one of your extensions: +1. Update default_version in that extension’s .control file +2. Run make to regenerate the version-specific file +3. Update that extension’s provides.{extension}.version in META.in.json to match by hand, and regenerate META.json (make META.json) — pgxntool won’t do this for you +4. Update META.in.json’s top-level `version too if you’re also cutting a new distribution release

+
+
+
+

5.4. Committing Version Files

+
+

Version-specific SQL files are treated as permanent files that should be committed to your repository by default. This makes it much easier to test updates to extensions, as you can see exactly what SQL was included in each version.

+
+
+ + + + + +
+
Important
+		 +These files are auto-generated and include a header comment warning not to edit them. Any manual changes will be overwritten the next time you run make. To modify the extension, edit the base SQL file (sql/{extension}.sql) instead. +
+
+
+

5.4.1. Why Commit Them: Update Testing

+
+

The primary value of committing a version’s install script is update testing: with the file in place, you can install that exact version (CREATE EXTENSION ext VERSION 'x.y.z'), run ALTER EXTENSION ext UPDATE, and verify the update path actually works. In principle you would only ever need the very first committed version’s install script — every later version is reachable by chaining upgrade scripts from it. See test/install for how to wire this into your test suite.

+
+
+

In practice, you should keep most old versions tracked rather than relying purely on that chain, because you cannot predict when a new major PostgreSQL version will break the ability to install an older extension version — a system catalog column changes type, a SELECT * over a catalog starts failing, and so on. Committing the old version’s install script lets CI catch that regression directly, by attempting to install that exact historical version against the new PostgreSQL release. For any non-trivial extension, most old versions should stay tracked for this reason alone.

+
+
+
+

5.4.2. When It’s OK to Skip a Version

+
+

For a minor version change that doesn’t meaningfully alter the extension (e.g. a small bug fix), it’s unlikely to straddle a PostgreSQL supported-version boundary, so there’s much less test-coverage value in committing that specific version’s generated install script. For large extensions, deliberately not committing some of these individual version files is worth doing to keep repository size down — the base sql/{extension}.sql still fully captures the change in git history, and the install script itself is trivially regenerated from that base file at build time.

+
+
+
+

5.4.3. Don’t .gitignore a Skipped Version — rm It Once

+
+

When you decide not to track a particular version’s generated install script, do not add a .gitignore entry for it.

+
+
+

make only ever generates the current version’s file, stamped from the base source according to that extension’s .control file default_version. The moment you bump the version, make starts generating the new current version’s file and stops touching the old one — the old file left behind in your working tree is a stale, one-off artifact, not something make keeps recreating on every build.

+
+
+

Because it’s only ever transiently present, the correct cleanup is a single rm sql/{extension}--{old-version}.sql right after the version bump — not a permanent .gitignore rule. A per-version .gitignore line would be perpetual clutter added on every release, for a file that’s only ever present for one build cycle. A broad glob (e.g. sql/--.sql) is wrong for the same reason it’s wrong below in Alternative: Ignoring All Version Files: it would also hide the prior-version install scripts and upgrade scripts (sql/{extension}--{a}--{b}.sql) that you do want tracked and visible in git status.

+
+
+
+

5.4.4. Never Hand-Edit a Version File That’s No Longer Current

+
+

control.mk.sh generates a Make rule along these lines for the current version’s file:

+
+
+
+
$(EXTENSION_ext_VERSION_FILE): sql/ext.sql extension.control
+	@(echo '/* DO NOT EDIT - AUTO-GENERATED FILE */'; cat sql/ext.sql) > $(EXTENSION_ext_VERSION_FILE)
+
+
+
+

That rule only ever targets the file matching the extension’s current default_version, so editing the base sql/{extension}.sql and running make is the correct, safe way to change that one file.

+
+
+

Every other versioned file — any sql/{extension}--{version}.sql where {version} is no longer current — is a frozen historical record, not something make will ever regenerate or overwrite again. Its entire purpose is to preserve exactly what shipped in that version (see Why Commit Them: Update Testing above), so it can keep being used to test update paths and PostgreSQL-version compatibility against exactly what your users actually installed.

+
+
+

Hand-editing an old versioned file directly is never correct — it silently corrupts that historical record. If you need to change behavior after a version has already shipped, bump the version and add a proper sql/{extension}--{old}--{new}.sql upgrade script instead. Never edit sql/{extension}--{old}.sql in place.

+
+
+ + + + + +
+
Caution
+		 +This is especially relevant for AI coding agents, which won’t know this convention exists unless it’s spelled out. The generic DO NOT EDIT - AUTO-GENERATED FILE header on every version file doesn’t distinguish "regenerated on every build" (the current version) from "was generated once, now frozen" (every other version). An agent without this context may reasonably — but incorrectly — treat any auto-generated file as fair game to patch directly. Old versioned SQL files must instead be treated as append-only history: the fix is always a new version plus an upgrade script, never an in-place edit. +
+
+
+
+
+

5.5. Alternative: Ignoring All Version Files

+
+ + + + + +
+
Note
+		 +This is a different, all-or-nothing choice from When It’s OK to Skip a Version above, where the recommendation is to keep tracking most versions and selectively skip only a few low-value minor ones. This section instead covers not tracking any version-specific files at all. +
+
+
+

If you prefer not to commit version-specific SQL files, you must add them to your .gitignore to prevent make dist from failing due to untracked files. Add the following to your .gitignore:

+
+
+
+
# Auto-generated version-specific SQL files (if not committing them)
+sql/*--*.sql
+!sql/*--*--*.sql
+
+
+
+

The second line (!sql/----*.sql) ensures that upgrade scripts (which contain two version numbers and should be manually written) are still tracked.

+
+
+ + + + + +
+
Warning
+		 +If you ignore version files instead of committing them, they will NOT be included in your PGXN distribution (make dist uses git archive, which only includes tracked files). This means users installing your extension from PGXN will need make and PGXS available to build the extension - they cannot simply copy the SQL files into their PostgreSQL installation. For maximum compatibility, we recommend committing version files. +
+
+
+
+

5.6. Distribution Inclusion

+
+

Version-specific files are included in distributions created by make dist only if they are committed to git. Since make dist uses git archive, only tracked files are included in the distribution archive.

+
+
+
+

5.7. Multiple Versions

+
+

If you need to support multiple versions of your extension:

+
+
+
    +
  1. +

    Create additional version-specific files manually (e.g., sql/myext—​1.0.0.sql, sql/myext—​1.1.0.sql)

    +
    2. +
  2. +

    Create upgrade scripts for version transitions (e.g., sql/myext—​1.0.0—​1.1.0.sql)

    +
    3. +
  3. +

    Update default_version in the extension’s .control file to reflect the current version you’re working on

    +
    4. +
  4. +

    Commit all version files and upgrade scripts to your repository

    +
    5. +
+
+
+

The version file for the current version (specified in the .control file’s default_version) will be automatically regenerated when you run make, but other version files you create manually will be preserved.

+
+
+
+
+
+

6. Document Handling

+
+
+

PGXNtool supports generation and installation of document files. There are several variables and rules that control this behavior.

+
+
+

It is recommended that you commit any generated documentation files (such as HTML generated from Asciidoc) into git. +That way users will have these files installed when they install your extension. +If any generated files are missing (or out-of-date) during installation, PGXNtool will build them if Asciidoc is present on the system.

+
+
+

6.1. Document Variables

+
+
+
DOC_DIRS
+
+

Directories to look for documents in. +Defined as += doc.

+
+
DOCS
+
+

PGXS variable. +See The DOCS variable below.

+
+
DOCS_HTML
+
+

Document HTML files. +PGXNtool appends `$(ASCIIDOC_HTML) to this variable.

ASCIIDOC
@@ -691,7 +1521,7 @@

4
ASCIIDOC_EXTS

File extensions to consider as Asciidoc. -Defined as += adoc asciidoc.

+Defined as += adoc asciidoc asc.

ASCIIDOC_FILES
@@ -712,11 +1542,10 @@

4

-

4.2. Document Rules

+

6.2. Document Rules

-

If Asciidoc is found (or $(ASCIIDOC) is set), the html rule will be added as a prerequisite to the install and installchec rules. -That will ensure that docs are generated for install and test, but only if Asciidoc is available. -The dist rule will always depend on html though, to ensure html files are up-to-date before creating a distribution.

+

If Asciidoc is found (or $(ASCIIDOC) is set), html is added as a dependency of the all target, which both install and installcheck depend on — so docs get built for install and test, but only if Asciidoc is available. +The dist rule always depends on html directly, regardless of whether Asciidoc was found, to ensure html files are up-to-date before creating a distribution.

The html rule simply depends on `$(ASCIIDOC_HTML). @@ -730,12 +1559,12 @@

4.2. Docu
ASCIIDOC_template
define ASCIIDOC_template
-%.html: %.$(1) (1)
-ifndef ASCIIDOC
+%.html: %.$(1) # (1)
+ifeq (,$(strip $(ASCIIDOC)))
 	$$(warning Could not find "asciidoc" or "asciidoctor". Add one of them to your PATH,)
 	$$(warning or set ASCIIDOC to the correct location.)
 	$$(error Could not build %$$@)
-endif # ifndef ASCIIDOC
+endif # ifeq ASCIIDOC
 	$$(ASCIIDOC) $$(ASCIIDOC_FLAGS) $$<
 endef # define ASCIIDOC_template
@@ -752,9 +1581,12 @@

4.2. Docu These rules will throw an error if they are run if $(ASCIIDOC) isn’t defined. On a normal user system that should never happen, because the html rule won’t be included in install or installcheck.

+
+

make docclean removes all generated HTML files ($(DOCS_HTML)). It’s separate from make clean/make distclean, so you can clear out generated docs without touching your build artifacts.

+
-

4.3. The DOCS variable

+

6.3. The DOCS variable

This variable has special meaning to PGXS. See the Postgres documentation for full details.

@@ -782,20 +1614,347 @@

4.3

-

5. Copyright

+

7. pg_tle Support

+
+
+

pgxntool can generate pg_tle (Trusted Language Extensions) registration SQL for deploying PostgreSQL extensions in managed environments like AWS RDS and Aurora where filesystem access is not available.

+
+
+

For make targets, see: pgtle, check-pgtle, run-pgtle.

+
+
+

7.1. What is pg_tle?

+
+

pg_tle is an AWS open-source framework that enables developers to create and deploy PostgreSQL extensions without filesystem access. Traditional PostgreSQL extensions require .control and .sql files on the filesystem, which isn’t possible in managed services like RDS and Aurora.

+
+
+

pg_tle solves this by: +- Storing extension metadata and SQL in database tables +- Using the pgtle_admin role for administrative operations +- Enabling CREATE EXTENSION to work in managed environments

+
+
+
+

7.2. Quick Start

+
+

Generate pg_tle registration SQL for your extension:

+
+
+
+
make pgtle
+
+
+
+

This creates files in pg_tle/ subdirectories organized by pg_tle version ranges. See pgtle_versions.md for complete version range details and API compatibility boundaries.

+
+
+
+

7.3. Version Groupings

+
+

pgxntool creates different sets of files for different pg_tle versions to handle backward-incompatible API changes. Each version boundary represents a change to pg_tle’s API functions that we use.

+
+
+

For details on version boundaries and API changes, see pgtle_versions.md.

+
+
+
+

7.4. Installation Example

+
+ + + + + +
+
Important
+		 +This is only a basic example. Always refer to the main pg_tle documentation for complete installation instructions and best practices. +
+
+
+

Basic installation steps:

+
+
+
    +
  1. +

    Ensure pg_tle is installed and grant the pgtle_admin role to your user

    +
    2. +
  2. +

    Generate and run the pg_tle registration SQL files:

    +
    +
    +
    make run-pgtle
    +
    +
    +
    +

    This automatically detects your pg_tle version and runs the appropriate SQL files. See pgtle_versions.md for version range details.

    +
    +
    3. +
  3. +

    Create your extension: CREATE EXTENSION myextension;

    +
    4. +
+
+
+
+

7.5. Advanced Usage

+
+

7.5.1. Multi-Extension Projects

+
+

If your project has multiple extensions (multiple .control files), make pgtle generates files for all of them:

+
+
+
+
myproject/
+├── ext1.control
+├── ext2.control
+└── pg_tle/
+    ├── 1.0.0-1.5.0/
+    │   ├── ext1.sql
+    │   └── ext2.sql
+    └── 1.5.0+/
+        ├── ext1.sql
+        └── ext2.sql
+
+
+
+
+

7.5.2. Direct pgtle.sh Invocation

+
+

pgtle, check-pgtle, and run-pgtle are thin make wrappers around pgtle.sh. For more targeted operations you can run it directly:

+
+
+
    +
  • +

    pgtle.sh --extension NAME [--pgtle-version VERSION] — generate registration SQL for one extension (what make pgtle wraps).

    +
    • +
  • +

    pgtle.sh --get-version — print the pg_tle version installed in the target database (empty if not installed).

    +
    • +
  • +

    pgtle.sh --get-dir VERSION — print which version-range directory (see Version Groupings) a given pg_tle version maps to, without generating anything.

    +
    • +
  • +

    pgtle.sh --run — execute the generated registration SQL against the target database (what make run-pgtle wraps).

    +
    • +
+
+
+
+
+

7.6. How It Works

+
+

make pgtle does the following:

+
+
+
    +
  1. +

    Parses control file(s): Extracts comment, default_version, requires, and schema fields

    +
    2. +
  2. +

    Discovers SQL files: Finds all versioned files (sql/{ext}--{version}.sql) and upgrade scripts (sql/{ext}--{ver1}--{ver2}.sql)

    +
    3. +
  3. +

    Wraps SQL content: Uses a fixed dollar-quote delimiter ($pgtle_wrap_delimiter$) to wrap SQL for pg_tle functions

    +
    4. +
  4. +

    Generates registration SQL: Creates pgtle.install_extension() calls for each version, pgtle.install_update_path() for upgrades, and pgtle.set_default_version() for the default

    +
    5. +
  5. +

    Version-specific output: Generates separate files for different pg_tle capability levels

    +
    6. +
+
+
+

Each generated SQL file is wrapped in a transaction (BEGIN; …​ COMMIT;) to ensure atomic installation.

+
+
+
+

7.7. Troubleshooting

+
+

7.7.1. "No versioned SQL files found"

+
+

Problem: The script can’t find sql/{ext}--{version}.sql files.

+
+
+

Solution: Run make first to generate versioned files from your base sql/{ext}.sql file.

+
+
+
+

7.7.2. "Control file not found"

+
+

Problem: The script can’t find {ext}.control in the current directory.

+
+
+

Solution: Run make pgtle from your extension’s root directory (where the .control file is).

+
+
+
+

7.7.3. "SQL file contains reserved pg_tle delimiter"

+
+

Problem: Your SQL files contain the string $pgtle_wrap_delimiter$ (extremely unlikely).

+
+
+

Solution: Don’t use that dollar-quote delimiter in your code.

+
+
+
+

7.7.4. Extension uses C code

+
+

Problem: Your control file has module_pathname, indicating C code.

+
+
+

Solution: pg_tle only supports trusted languages. You cannot use C extensions with pg_tle. The script will warn you but still generate files (which won’t work).

+
+
+ + + + + +
+
Note
+		 +there are several untrusted languages (such as plpython), and the only tests for C. +
+
+
+
+
+
+
+

8. Configuration Variables

+
+
+

This is the comprehensive list of make variables you can override to customize pgxntool’s behavior, either in your Makefile (before include pgxntool/base.mk) or on the make command line. Several of these are also covered in more detail in the sections above; this section exists so there’s one place that lists all of them.

+
+
+ + + + + +
+
Note
+		 +Variables marked with * must be set to exactly yes or no (case-insensitive) when given explicitly; any other value is a hard make error. +
+
+
+

8.1. PGXN_REMOTE

+
+

Default: origin. The git remote used by tag, rmtag, forcetag, and dist (which depends on tag). If your origin is a personal fork rather than the canonical repository — for example, if you develop on a fork and only push release tags to the canonical repo — set this so tagging and pushing target the right remote:

+
+
+
+
make dist PGXN_REMOTE=upstream
+
+
+
+
+

8.2. ASCIIDOC

+
+

Default: auto-detected, the first of asciidoctor or asciidoc found on PATH. Path to the Asciidoc processor used to build .html files from $(ASCIIDOC_EXTS) source files. Override if the processor you want isn’t first on PATH, or isn’t on PATH at all. See Document Handling.

+
+
+
+

8.3. PGTLE_VERSION

+
+

Default: unset (generates every known pg_tle version range). Set on the command line to limit make pgtle to the single version range this value falls into. See pgtle.

+
+
+
+

8.4. PG_CONFIG

+
+

Default: pg_config. Path to the pg_config binary used to detect the PostgreSQL version and locate PGXS. Override when the right pg_config isn’t the one on PATH, such as when testing against a specific PostgreSQL install.

+
+
+
+

8.5. TESTDIR

+
+

Default: test. Root directory for test input files: $(TESTDIR)/sql/, $(TESTDIR)/expected/, $(TESTDIR)/build/, $(TESTDIR)/install/. Overriding this on its own is unusual; it mainly exists so TESTOUT has a sensible default.

+
+
+
+

8.6. TESTOUT

+
+

Default: $(TESTDIR). Directory pg_regress writes actual test output to — $(TESTOUT)/results/, $(TESTOUT)/regression.diffs, etc. — via --outputdir in REGRESS_OPTS. Kept separate from TESTDIR so generated output can be pointed somewhere other than the directory holding your checked-in sql/expected files, if you want that separation.

+
+
+
+

8.7. PGXNTOOL_VERIFY_RESULTS_MODE

+
+

Default: pgtap. Controls how the verify-results safeguard decides whether tests are failing.

+
+
+

8.7.1. pgtap

+
+

Scans test/results/*.out for pgTap not ok lines and plan mismatches, falling back to checking for regression.diffs too. Use this mode when your test suite uses pgTap.

+
+
+
+

8.7.2. diffs

+
+

Checks only for regression.diffs, matching classic pg_regress behavior. Use this mode when your tests use plain SQL expected-output comparison only.

+
+
+
+
+

8.8. PGXNTOOL_ENABLE_TEST_BUILD *

+
+

Default: auto-detected — yes if test/build/*.sql files exist, no otherwise. Enables or disables the test-build pre-flight SQL check. Set explicitly to yes if you want an error when test/build/ unexpectedly has no SQL files (catches accidental deletion of its contents), or to no to disable the check even when files are present.

+
+
+
+

8.9. PGXNTOOL_ENABLE_TEST_INSTALL *

+
+

Default: auto-detected — yes if test/install/*.sql files exist, no otherwise. Enables or disables the test/install schedule-based setup feature. Same explicit-override semantics as PGXNTOOL_ENABLE_TEST_BUILD.

+
+
+
+

8.10. PGXNTOOL_ENABLE_VERIFY_RESULTS *

+
+

Default: yes. Enables or disables the verify-results safeguard that blocks make results when tests are failing. Setting it to empty on the command line (make PGXNTOOL_ENABLE_VERIFY_RESULTS= results) also disables it.

+
+
+
+

8.11. PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED *

+
+

Default: yes. Enables or disables the check-stale-expected safeguard, which fails make test if test/expected/ (or test/build/expected/) contains a .out file with no corresponding .sql file — catching a stale file left behind after a test was renamed or removed. Set to no to make the check a complete no-op (it’s dropped from TEST_DEPS entirely).

+
+
+

This is also the supported pattern for disabling one specific optional pgxntool feature: a documented PGXNTOOL_ENABLE_* variable, gating both the target’s registration into TEST_DEPS/etc. and (where applicable) the target’s own definition. Cleanly replacing or overriding an arbitrary pgxntool-generated recipe with your own entirely is a separate, larger, and not-yet-decided design question — see issue #30 (two proposed approaches) — and is not what this mechanism solves.

+
+
+
+

8.12. PGXNTOOL_CHECK_EXPECTED_FILE_TYPES *

+
+

Default: yes. Sub-check of check-stale-expected, independent of PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED: fails (with a distinct error message and exit code from the orphaned-.out check) if test/expected/ (or test/build/expected/) contains any file that isn’t *.out. Set to no to disable just this sub-check while leaving the orphaned-.out check active.

+
+
+
+

8.13. PGXNTOOL_NO_PGXS_INCLUDE

+
+

Default: unset (PGXS is included normally). Skips including PGXS ($(PGXS)) entirely. This is only for advanced scenarios where you need to manage the PGXS include yourself; most projects should never set this.

+
+
+
+
+
+

9. Copyright

-

Copyright (c) 2015 Jim Nasby <Jim.Nasby@BlueTreble.com>

+

Copyright (c) 2026 Jim Nasby <Jim.Nasby@gmail.com>

-

PGXNtool is released under a BSD license. Note that it includes JSON.sh, which is released under a MIT license.

+

PGXNtool is released under a BSD license. Note that it includes JSON.sh, which is released under a MIT license.

diff --git a/_.gitignore b/_.gitignore index 3eb345a..1873c2c 100644 --- a/_.gitignore +++ b/_.gitignore @@ -1,11 +1,15 @@ # Editor files .*.swp +# Claude Code local settings +.claude/*.local.json + # Explicitly exclude META.json! !/META.json # Generated make files meta.mk +control.mk # Compiler output *.o @@ -13,14 +17,24 @@ meta.mk .deps/ # built targets -/sql/*--* -!/sql/*--*--*.sql +# Note: Version-specific files (sql/*--*.sql) are now tracked in git and should be committed # Test artifacts results/ regression.diffs regression.out +# Generated sql/ directory for test/build +# Created by make test-build. See README.asc for details. +test/build/sql/ + +# Auto-generated schedule file for test/install +# Created by make when test/install/*.sql files exist. +test/install/schedule + # Misc tmp/ .DS_Store + +# pg_tle generated files +/pg_tle/ diff --git a/base.mk b/base.mk index a976ebb..e77cff2 100644 --- a/base.mk +++ b/base.mk @@ -1,5 +1,17 @@ +# Include guard: base.mk can end up included twice in a single `make` run +# (e.g. an extension's own .mk module includes it, and the extension's +# Makefile also includes it directly via the line setup.sh writes). Without +# this, every target in the file gets redefined, producing +# overriding-recipe/ignoring-old-recipe warnings. A second inclusion is a +# harmless no-op. +ifndef PGXNTOOL_BASE_MK_INCLUDED +PGXNTOOL_BASE_MK_INCLUDED := 1 + PGXNTOOL_DIR := pgxntool +# Ensure 'all' is the default target (not META.json which happens to be first) +.DEFAULT_GOAL := all + # # META.json # @@ -10,39 +22,233 @@ META.json: META.in.json $(PGXNTOOL_DIR)/build_meta.sh # # meta.mk # -# Buind meta.mk, which contains info from META.json, and include it +# Build meta.mk, which contains PGXN distribution info from META.json PGXNTOOL_distclean += meta.mk meta.mk: META.json Makefile $(PGXNTOOL_DIR)/base.mk $(PGXNTOOL_DIR)/meta.mk.sh @$(PGXNTOOL_DIR)/meta.mk.sh $< >$@ -include meta.mk -DATA = $(EXTENSION_VERSION_FILES) $(wildcard sql/*--*--*.sql) +# +# control.mk +# +# Build control.mk, which contains extension info from .control files +# This is separate from meta.mk because: +# - META.json specifies PGXN distribution metadata +# - .control files specify what PostgreSQL actually uses (e.g., default_version) +# These can differ, and PostgreSQL cares about the control file version. +# +# Find all control files first (needed for dependencies) +PGXNTOOL_CONTROL_FILES := $(wildcard *.control) +PGXNTOOL_distclean += control.mk +control.mk: $(PGXNTOOL_CONTROL_FILES) Makefile $(PGXNTOOL_DIR)/base.mk $(PGXNTOOL_DIR)/control.mk.sh + @$(PGXNTOOL_DIR)/control.mk.sh $(PGXNTOOL_CONTROL_FILES) >$@ + +-include control.mk + +DATA = $(EXTENSION__CURRENT_VERSION__FILES) $(wildcard sql/*--*--*.sql) DOC_DIRS += doc # NOTE: if this is empty it gets forcibly defined to NUL before including PGXS DOCS += $(foreach dir,$(DOC_DIRS),$(wildcard $(dir)/*)) # Find all asciidoc targets ASCIIDOC ?= $(shell which asciidoctor 2>/dev/null || which asciidoc 2>/dev/null) -ASCIIDOC_EXTS += adoc asciidoc +ASCIIDOC_EXTS += adoc asciidoc asc ASCIIDOC_FILES += $(foreach dir,$(DOC_DIRS),$(foreach ext,$(ASCIIDOC_EXTS),$(wildcard $(dir)/*.$(ext)))) PG_CONFIG ?= pg_config TESTDIR ?= test TESTOUT ?= $(TESTDIR) -TEST_SOURCE_FILES += $(wildcard $(TESTDIR)/input/*.source) -TEST_OUT_FILES = $(subst input,output,$(TEST_SOURCE_FILES)) TEST_SQL_FILES += $(wildcard $(TESTDIR)/sql/*.sql) TEST_RESULT_FILES = $(patsubst $(TESTDIR)/sql/%.sql,$(TESTDIR)/expected/%.out,$(TEST_SQL_FILES)) -TEST_FILES = $(TEST_SOURCE_FILES) $(TEST_SQL_FILES) -REGRESS = $(sort $(notdir $(subst .source,,$(TEST_FILES:.sql=)))) # Sort is to get unique list +TEST_FILES = $(TEST_SQL_FILES) +REGRESS = $(sort $(notdir $(TEST_FILES:.sql=))) REGRESS_OPTS = --inputdir=$(TESTDIR) --outputdir=$(TESTOUT) # See additional setup below + +# +# OPTIONAL TEST FEATURES +# +# These sections configure optional test features. Each feature can be enabled/disabled +# via a makefile variable. If not explicitly set, features auto-detect based on +# directory existence or default behavior. The actual feature implementation is +# located later in this file (see test-build target, schedule file generation, etc.). +# + +# Helper function: normalize a yes/no variable to lowercase and validate. +# Usage: $(call pgxntool_validate_yesno,VALUE,VARIABLE_NAME) +# Returns the lowercase value ("yes" or "no"), or errors if invalid. +pgxntool_validate_yesno = $(strip \ + $(if $(filter yes no,$(shell echo "$(1)" | tr '[:upper:]' '[:lower:]')),\ + $(shell echo "$(1)" | tr '[:upper:]' '[:lower:]'),\ + $(error $(2) must be "yes" or "no", got "$(1)"))) + +# ------------------------------------------------------------------------------ +# test-build: Sanity check extension files before running full test suite +# ------------------------------------------------------------------------------ +# Purpose: Validates that extension SQL files are syntactically correct by running +# files from test/build/ through pg_regress. This provides better error +# messages than CREATE EXTENSION failures. +# +# Variable: PGXNTOOL_ENABLE_TEST_BUILD +# - Can be set manually in Makefile or command line +# - Allowed values: "yes" or "no" (case-insensitive) +# - If not set: Auto-detects based on existence of test/build/*.sql files +# - Set to "yes" explicitly to get an error if test/build/ has no SQL files +# (useful to catch accidental deletion of test/build/ contents) +# - Set to "no" explicitly to disable even when test/build/ has SQL files +# +# Implementation: See test-build target definition (search for "test-build:" in this file) +# +TEST_BUILD_SQL_FILES = $(wildcard $(TESTDIR)/build/*.sql) +TEST_BUILD_FILES = $(TEST_BUILD_SQL_FILES) +ifdef PGXNTOOL_ENABLE_TEST_BUILD + # override needed so command-line values (make VAR=YES) are normalized, not silently ignored. + # := needed for immediate evaluation of the function call (avoids infinite recursion with =). + override PGXNTOOL_ENABLE_TEST_BUILD := $(call pgxntool_validate_yesno,$(PGXNTOOL_ENABLE_TEST_BUILD),PGXNTOOL_ENABLE_TEST_BUILD) +else + # Auto-detect: enable if test/build/ directory has SQL files + ifneq ($(strip $(TEST_BUILD_FILES)),) + PGXNTOOL_ENABLE_TEST_BUILD = yes + else + PGXNTOOL_ENABLE_TEST_BUILD = no + endif +endif + +# ------------------------------------------------------------------------------ +# test/install: Run setup files before all tests in the same pg_regress session +# ------------------------------------------------------------------------------ +# Purpose: Runs files from test/install/ before all test/sql/ files within a +# SINGLE pg_regress invocation via schedule files. This ensures that +# state created by install files (tables, extensions, etc.) persists +# into the main test suite. +# +# Variable: PGXNTOOL_ENABLE_TEST_INSTALL +# - Can be set manually in Makefile or command line +# - Allowed values: "yes" or "no" (case-insensitive) +# - If not set: Auto-detects based on existence of test/install/*.sql files +# - Set to "yes" explicitly to get an error if test/install/ has no SQL files +# (useful to catch accidental deletion of test/install/ contents) +# - Set to "no" explicitly to disable even when test/install/ has SQL files +# +# Directory layout (follows ~/code/extensions/archive/ pattern): +# test/install/*.sql - Install SQL files +# test/install/*.out - Expected output (lives alongside .sql files) +# test/install/schedule - Auto-generated schedule file +# test/sql/schedule - Auto-generated schedule file for regular tests +# +# The schedule files use relative paths (../install/testname) so pg_regress +# resolves install files from their original location without copying. +# +# NOTE: The variable normalization pattern below (ifdef/NORM/error/override) is +# identical to test-build and verify-results. Refactoring options: +# 1. A $(call normalize_bool_var,VAR,DEFAULT) Make function +# 2. A small include fragment (e.g. pgxntool/mk/bool-var.mk) +# Either approach would eliminate the ~10-line block repeated for each feature. +TEST_INSTALL_SQL_FILES = $(wildcard $(TESTDIR)/install/*.sql) +ifdef PGXNTOOL_ENABLE_TEST_INSTALL + # override needed so command-line values (make VAR=YES) are normalized, not silently ignored. + # := needed for immediate evaluation of the function call (avoids infinite recursion with =). + override PGXNTOOL_ENABLE_TEST_INSTALL := $(call pgxntool_validate_yesno,$(PGXNTOOL_ENABLE_TEST_INSTALL),PGXNTOOL_ENABLE_TEST_INSTALL) +else + # Auto-detect: enable if test/install/ directory has SQL files + ifneq ($(strip $(TEST_INSTALL_SQL_FILES)),) + PGXNTOOL_ENABLE_TEST_INSTALL = yes + else + PGXNTOOL_ENABLE_TEST_INSTALL = no + endif +endif + +# ------------------------------------------------------------------------------ +# verify-results: Safeguard for make results +# ------------------------------------------------------------------------------ +# Purpose: Prevents accidentally running 'make results' when tests are failing. +# +# Variable: PGXNTOOL_ENABLE_VERIFY_RESULTS +# - Can be set manually in Makefile or command line +# - Allowed values: "yes" or "no" (case-insensitive) +# - Setting to empty on the command line (e.g. PGXNTOOL_ENABLE_VERIFY_RESULTS=) also disables the feature +# - If not set: Defaults to "yes" (enabled by default for all pgxntool projects) +# - Usage: Controls whether verify-results target exists and blocks make results +# +# Variable: PGXNTOOL_VERIFY_RESULTS_MODE +# - Controls how verify-results detects test failures +# - "pgtap" (default): scans test/results/*.out for "not ok" lines and plan +# mismatches (TAP failures). Also checks regression.diffs as a fallback. +# Use this mode when your test suite uses pgTap. +# - "diffs": checks only for regression.diffs existence (classic pg_regress behavior) +# Use this mode when your tests use plain SQL expected-output comparison only. +# +# Implementation: See verify-results target definition and results target modification +# (search for "verify-results" and "results:" in this file) +# +ifdef PGXNTOOL_ENABLE_VERIFY_RESULTS + override PGXNTOOL_ENABLE_VERIFY_RESULTS := $(call pgxntool_validate_yesno,$(PGXNTOOL_ENABLE_VERIFY_RESULTS),PGXNTOOL_ENABLE_VERIFY_RESULTS) +else + # Default to yes (enabled by default for all pgxntool projects) + PGXNTOOL_ENABLE_VERIFY_RESULTS = yes +endif + +# Default mode: pgtap (scans results/*.out for TAP failures) +PGXNTOOL_VERIFY_RESULTS_MODE ?= pgtap + +# ------------------------------------------------------------------------------ +# check-stale-expected: catch orphaned/unexpected test/expected/ files +# ------------------------------------------------------------------------------ +# Variable: PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED +# - Can be set manually in Makefile or command line +# - Allowed values: "yes" or "no" (case-insensitive) +# - Default: "yes" (enabled by default for all pgxntool projects) +# - Set to "no" to make this check a complete no-op: the target is +# dropped from TEST_DEPS entirely (see its own definition below) +# +# Variable: PGXNTOOL_CHECK_EXPECTED_FILE_TYPES +# - Sub-check, independent of the variable above: fails if test/expected/ +# (or test/build/expected/) contains any file that isn't *.out +# - Allowed values: "yes" or "no" (case-insensitive) +# - Default: "yes" +# - Set to "no" to disable just this sub-check while leaving the rest of +# check-stale-expected (the orphaned-.out check) active +# - Passed through to check-stale-expected.sh; see that script for the +# distinct error message/exit code this sub-check uses +# +# Variable: _CHECK_STALE_EXPECTED_SCRIPT (internal shim, not user-facing) +# - Path to the script the check-stale-expected target invokes +# - Default: $(PGXNTOOL_DIR)/test/bin/check-stale-expected.sh +# +# Implementation: See check-stale-expected target definition (search for +# "check-stale-expected:" in this file) +# +ifdef PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED + override PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED := $(call pgxntool_validate_yesno,$(PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED),PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED) +else + PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED = yes +endif + +ifdef PGXNTOOL_CHECK_EXPECTED_FILE_TYPES + override PGXNTOOL_CHECK_EXPECTED_FILE_TYPES := $(call pgxntool_validate_yesno,$(PGXNTOOL_CHECK_EXPECTED_FILE_TYPES),PGXNTOOL_CHECK_EXPECTED_FILE_TYPES) +else + PGXNTOOL_CHECK_EXPECTED_FILE_TYPES = yes +endif + +_CHECK_STALE_EXPECTED_SCRIPT ?= $(PGXNTOOL_DIR)/test/bin/check-stale-expected.sh + +# Generate unique database name for tests to prevent conflicts across projects +# Uses project name + first 5 chars of md5 hash of current directory +# This prevents multiple test runs in different directories from clobbering each other +REGRESS_DBHASH := $(shell echo $(CURDIR) | (md5 2>/dev/null || md5sum) | cut -c1-5) +REGRESS_DBNAME := $(or $(PGXN),regression)_$(REGRESS_DBHASH) MODULES = $(patsubst %.c,%,$(wildcard src/*.c)) ifeq ($(strip $(MODULES)),) MODULES =# Set to NUL so PGXS doesn't puke endif -EXTRA_CLEAN = $(wildcard ../$(PGXN)-*.zip) $(EXTENSION_VERSION_FILES) +EXTRA_CLEAN = $(wildcard ../$(PGXN)-*.zip) pg_tle/ +# PGXS's own pg_regress_clean_files unconditionally rm -rf's a top-level +# results/, but our tests write to $(TESTOUT)/results/ (see REGRESS_OPTS +# --outputdir above), so that's the directory that actually needs cleaning. +# filter-out (not -=, which GNU Make 4.4+ rejects as a parse error) guards +# against a stray top-level results/ entry while adding the real one. +EXTRA_CLEAN := $(filter-out results/,$(EXTRA_CLEAN)) $(TESTOUT)/results/ # Get Postgres version, as well as major (9.4, etc) version. # NOTE! In at least some versions, PGXS defines VERSION, so we intentionally don't use that variable @@ -56,11 +262,40 @@ test = $(shell test $(1) $(2) $(3) && echo yes || echo no) GE91 = $(call test, $(MAJORVER), -ge, 91) ifeq ($(GE91),yes) -all: $(EXTENSION_VERSION_FILES) +all: $(EXTENSION__CURRENT_VERSION__FILES) endif -ifeq ($($call test, $(MAJORVER), -lt 13), yes) - REGRESS_OPTS += --load-language=plpgsql +ifeq ($(call test, $(MAJORVER), -lt, 130), yes) +REGRESS_OPTS += --load-language=plpgsql +endif + +# +# test/install: Schedule-based approach +# +# When enabled, generates a schedule file listing install files, and adds it +# to REGRESS_OPTS. pg_regress processes --schedule tests before command-line +# test names, so install files run first in the SAME pg_regress invocation. +# This ensures state created by install files persists into the main test suite. +# +# The schedule uses relative paths (../install/testname) so pg_regress finds +# install files in their original location without copying. +# +ifeq ($(PGXNTOOL_ENABLE_TEST_INSTALL),yes) +PGXNTOOL_INSTALL_SCHEDULE = $(TESTDIR)/install/schedule +EXTRA_CLEAN += $(PGXNTOOL_INSTALL_SCHEDULE) + +# Add install schedule; REGRESS stays as-is (regular tests run after schedule) +REGRESS_OPTS += --schedule=$(PGXNTOOL_INSTALL_SCHEDULE) + +# Always regenerate schedule file to catch added/removed files +.PHONY: $(PGXNTOOL_INSTALL_SCHEDULE) +$(PGXNTOOL_INSTALL_SCHEDULE): + @echo "# Auto-generated - DO NOT EDIT" > $@ + @for f in $(notdir $(basename $(TEST_INSTALL_SQL_FILES))); do \ + echo "test: ../install/$$f" >> $@; \ + done + +installcheck: $(PGXNTOOL_INSTALL_SCHEDULE) endif PGXS := $(shell $(PG_CONFIG) --pgxs) @@ -70,13 +305,44 @@ DATA += $(wildcard *.control) # Don't have installcheck bomb on error .IGNORE: installcheck -installcheck: $(TEST_RESULT_FILES) $(TEST_OUT_FILES) $(TEST_SQL_FILES) $(TEST_SOURCE_FILES) +installcheck: $(TEST_RESULT_FILES) $(TEST_SQL_FILES) | $(TESTDIR)/sql/ $(TESTDIR)/expected/ $(TESTOUT)/results/ # # TEST SUPPORT # # These targets are meant to make running tests easier. +# Build test dependencies list based on enabled features. This base +# assignment (a plain `=`, not `+=`) must come before any `TEST_DEPS +=` +# line below -- Make processes the file top-to-bottom, and a later plain +# `=` would silently wipe out any `+=` that came before it. +TEST_DEPS = testdeps + +# ------------------------------------------------------------------------------ +# check-stale-expected: catch orphaned/unexpected test/expected/ files +# ------------------------------------------------------------------------------ +# Purpose: test/expected/*.out must mirror test/sql/*.sql 1:1 (likewise +# test/build/expected/*.out vs test/build/*.sql, when test-build is in use). +# It's easy to leave a stale .out behind after renaming or removing a .sql +# file; this makes `make test` fail loudly instead of letting it linger +# unnoticed. Logic lives in check-stale-expected.sh (enough of it to warrant +# a real script rather than an inline recipe). +# +# This depends on `installcheck` directly (not just position in TEST_DEPS) +# because it MUST run after pg_regress, not before: TEST_DEPS lists multiple +# independent prerequisites of `test`, and Make does not guarantee the order +# unrelated prerequisites of the same target are built in. An explicit +# dependency edge is the only ordering guarantee Make actually gives. +# +# See PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED / PGXNTOOL_CHECK_EXPECTED_FILE_TYPES +# above for how to disable this entirely or just its non-.out file sub-check. +ifeq ($(PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED),yes) +.PHONY: check-stale-expected +check-stale-expected: installcheck + @$(_CHECK_STALE_EXPECTED_SCRIPT) $(TESTDIR) $(PGXNTOOL_CHECK_EXPECTED_FILE_TYPES) +TEST_DEPS += check-stale-expected +endif + # make test: run any test dependencies, then do a `make install installcheck`. # If regressions are found, it will output them. # @@ -84,30 +350,134 @@ installcheck: $(TEST_RESULT_FILES) $(TEST_OUT_FILES) $(TEST_SQL_FILES) $(TEST_SO # watch-make if you're generating intermediate files. If tests end up needing # clean it's an indication of a missing dependency anyway. .PHONY: test -test: testdeps install installcheck +ifeq ($(PGXNTOOL_ENABLE_TEST_BUILD),yes) +TEST_DEPS += test-build +endif +TEST_DEPS += install installcheck +test: $(TEST_DEPS) @if [ -r $(TESTOUT)/regression.diffs ]; then cat $(TESTOUT)/regression.diffs; fi -# make results: runs `make test` and copy all result files to expected +# +# verify-results: Safeguard for make results +# +# Checks if tests are passing before allowing make results to proceed +ifeq ($(PGXNTOOL_ENABLE_VERIFY_RESULTS),yes) +.PHONY: verify-results +ifeq ($(PGXNTOOL_VERIFY_RESULTS_MODE),pgtap) +verify-results: + @$(PGXNTOOL_DIR)/verify-results-pgtap.sh $(TESTOUT) +else +verify-results: + @if [ -r $(TESTOUT)/regression.diffs ]; then \ + echo "ERROR: Tests are failing. Cannot run 'make results'."; \ + echo "Fix test failures first, then run 'make results'."; \ + echo ""; \ + echo "See $(TESTOUT)/regression.diffs for details:"; \ + cat $(TESTOUT)/regression.diffs; \ + exit 1; \ + fi +endif +endif + +# make results: runs `make test` and copies all result files to expected. # DO NOT RUN THIS UNLESS YOU'RE CERTAIN ALL YOUR TESTS ARE PASSING! +# +# Dependency chain (verify-results: test) guarantees test completes before verify-results +# checks regression.diffs, even under make -j. Listing both as independent prerequisites +# of results would allow them to run concurrently, letting verify-results see stale state. .PHONY: results +ifeq ($(PGXNTOOL_ENABLE_VERIFY_RESULTS),yes) +verify-results: test +results: verify-results +else results: test - rsync -rlpgovP $(TESTOUT)/results/ $(TESTDIR)/expected +endif + @mkdir -p $(TESTDIR)/expected + @for f in $(TESTOUT)/results/*.out; do \ + [ -f "$$f" ] || continue; \ + cp "$$f" $(TESTDIR)/expected/$$(basename "$$f"); \ + done # testdeps is a generic dependency target that you can add targets to .PHONY: testdeps testdeps: pgtap +# +# pg_tle support - Generate pg_tle registration SQL +# + +# PGXNTOOL_CONTROL_FILES is defined above (for control.mk dependencies) +PGXNTOOL_EXTENSIONS = $(basename $(PGXNTOOL_CONTROL_FILES)) + +# Main target +# Depend on 'all' to ensure versioned SQL files are generated first +# Depend on control.mk (which defines EXTENSION__CURRENT_VERSION__FILES) +# Depend on control files explicitly so changes trigger rebuilds +# Generates all supported pg_tle versions for each extension, unless +# PGTLE_VERSION is set on the command line to limit output to one range +.PHONY: pgtle +pgtle: all control.mk $(PGXNTOOL_CONTROL_FILES) + @$(foreach ext,$(PGXNTOOL_EXTENSIONS),\ + $(PGXNTOOL_DIR)/pgtle.sh --extension $(ext) $(if $(PGTLE_VERSION),--pgtle-version $(PGTLE_VERSION));) + +# +# pg_tle installation support +# + +# Check if pg_tle is installed and report version +# Only reports version if CREATE EXTENSION pg_tle has been run +# Errors if pg_tle extension is not installed +# Uses pgtle.sh to get version (avoids code duplication) +.PHONY: check-pgtle +check-pgtle: + @echo "Checking pg_tle installation..." + @PGTLE_VERSION=$$($(PGXNTOOL_DIR)/pgtle.sh --get-version 2>/dev/null); \ + if [ -n "$$PGTLE_VERSION" ]; then \ + echo "pg_tle extension version: $$PGTLE_VERSION"; \ + exit 0; \ + fi; \ + echo "ERROR: pg_tle extension is not installed" >&2; \ + echo " Run 'CREATE EXTENSION pg_tle;' first" >&2; \ + exit 1 + +# Run pg_tle registration SQL files +# Requires pg_tle extension to be installed (checked via check-pgtle) +# Uses pgtle.sh to determine which version range directory to use +# Assumes PG* environment variables are configured +.PHONY: run-pgtle +run-pgtle: pgtle + @$(PGXNTOOL_DIR)/pgtle.sh --run + # These targets ensure all the relevant directories exist -$(TESTDIR)/sql: - @mkdir -p $@ -$(TESTDIR)/expected/: +$(TESTDIR)/sql $(TESTDIR)/expected/ $(TESTOUT)/results/: @mkdir -p $@ +# pg_regress aborts with "could not open file" if an expected output file is +# missing, so create empty placeholders for any test that lacks one. $(TEST_RESULT_FILES): | $(TESTDIR)/expected/ + @# Create empty expected file so pg_regress doesn't abort with "file not found". + @# pg_regress requires an expected/*.out file to exist for each test; without it + @# it stops immediately rather than running the test and showing the diff. @touch $@ -$(TESTDIR)/output/: - @mkdir -p $@ -$(TEST_OUT_FILES): | $(TESTDIR)/output/ $(TESTDIR)/expected/ $(TESTDIR)/sql/ - @touch $@ + +# +# test-build: Sanity check extension files in test/build/ +# +# The sql/ subdirectory is generated - files are synced from test/build/*.sql. +# This directory should be in .gitignore and is cleaned by make clean. +# +ifeq ($(PGXNTOOL_ENABLE_TEST_BUILD),yes) +TEST_BUILD_SQL_DIR = $(TESTDIR)/build/sql +TEST_BUILD_REGRESS = $(sort $(notdir $(basename $(TEST_BUILD_SQL_FILES)))) +.PHONY: test-build +test-build: install + @$(PGXNTOOL_DIR)/run-test-build.sh $(TESTDIR) + $(MAKE) -C . REGRESS="$(TEST_BUILD_REGRESS)" REGRESS_OPTS="--inputdir=$(TESTDIR)/build --outputdir=$(TESTDIR)/build" installcheck + @if [ -r $(TESTDIR)/build/regression.diffs ]; then \ + echo "test-build failed - see $(TESTDIR)/build/regression.diffs"; \ + cat $(TESTDIR)/build/regression.diffs; \ + exit 1; \ + fi +endif # @@ -137,8 +507,9 @@ dist: html # But don't add it as an install or test dependency unless we do have asciidoc ifneq (,$(strip $(ASCIIDOC))) -# Need to do this so install & co will pick up ALL targets. Unfortunately this can result in some duplication. -DOCS += $(ASCIIDOC_HTML) +# Add HTML to DOCS for install, deduplicating against any HTML already picked +# up by the wildcard (e.g. pre-built HTML committed to the repo). +DOCS := $(sort $(filter-out $(ASCIIDOC_HTML),$(DOCS)) $(ASCIIDOC_HTML)) # Also need to add html as a dep to all (which will get picked up by install & installcheck all: html @@ -153,16 +524,29 @@ docclean: # # TAGGING SUPPORT # +# Remote used for tag/rmtag/forcetag/dist. Override on the command line or in +# your Makefile if you push tags somewhere other than origin. +PGXN_REMOTE ?= origin + rmtag: - git fetch origin # Update our remotes - @test -z "$$(git branch --list $(PGXNVERSION))" || git branch -d $(PGXNVERSION) - @test -z "$$(git branch --list -r origin/$(PGXNVERSION))" || git push --delete origin $(PGXNVERSION) + git fetch $(PGXN_REMOTE) # Update our remotes + @test -z "$$(git tag --list $(PGXNVERSION))" || git tag -d $(PGXNVERSION) + @test -z "$$(git ls-remote --tags $(PGXN_REMOTE) $(PGXNVERSION) | grep -v '{}')" || git push --delete $(PGXN_REMOTE) $(PGXNVERSION) -# TODO: Don't puke if tag already exists *and is the same* tag: @test -z "$$(git status --porcelain)" || (echo 'Untracked changes!'; echo; git status; exit 1) - git branch $(PGXNVERSION) - git push --set-upstream origin $(PGXNVERSION) + @# Skip if tag already exists and points to HEAD + @if git rev-parse $(PGXNVERSION) >/dev/null 2>&1; then \ + if [ "$$(git rev-parse $(PGXNVERSION))" = "$$(git rev-parse HEAD)" ]; then \ + echo "Tag $(PGXNVERSION) already exists at HEAD, skipping"; \ + else \ + echo "ERROR: Tag $(PGXNVERSION) exists but points to different commit" >&2; \ + exit 1; \ + fi; \ + else \ + git tag $(PGXNVERSION); \ + fi + git push $(PGXN_REMOTE) $(PGXNVERSION) .PHONY: forcetag forcetag: rmtag tag @@ -171,6 +555,13 @@ forcetag: rmtag tag dist: tag dist-only dist-only: + @# Check if .gitattributes exists but isn't committed + @if [ -f .gitattributes ] && ! git ls-files --error-unmatch .gitattributes >/dev/null 2>&1; then \ + echo "ERROR: .gitattributes exists but is not committed to git." >&2; \ + echo " git archive only respects export-ignore for committed files." >&2; \ + echo " Please commit .gitattributes for export-ignore to take effect." >&2; \ + exit 1; \ + fi git archive --prefix=$(PGXN)-$(PGXNVERSION)/ -o ../$(PGXN)-$(PGXNVERSION).zip $(PGXNVERSION) .PHONY: forcedist @@ -190,20 +581,39 @@ print-% : ; $(info $* is $(flavor $*) variable set to "$($*)") @true # # subtree sync support # -# This is setup to allow any number of pull targets by defining special -# variables. pgxntool-sync-release is an example of this. -.PHONY: pgxn-sync-% +# All the real work (git subtree pull + update-setup-files.sh) lives in +# pgxntool/pgxntool-sync.sh so it can be run directly, without make. These +# targets are thin wrappers around that script. +# +# `make pgxntool-sync` pulls the latest released version from the canonical +# repository (the script's built-in default). +# +# `make pgxntool-sync-` pulls from the " " defined by the +# pgxntool-sync- variable, allowing any number of custom pull sources. +.PHONY: pgxntool-sync pgxntool-sync-% +pgxntool-sync: + @pgxntool/pgxntool-sync.sh pgxntool-sync-%: - git subtree pull -P pgxntool --squash -m "Pull pgxntool from $($@)" $($@) -pgxntool-sync: pgxntool-sync-release + @pgxntool/pgxntool-sync.sh $($@) + +# `make pgxntool-version` prints the version of the embedded pgxntool copy. +# Delegates to bin/version so it can be run without make too. +.PHONY: pgxntool-version +pgxntool-version: + @$(PGXNTOOL_DIR)/bin/version # DANGER! Use these with caution. They may add extra crap to your history and # could make resolving merges difficult! -pgxntool-sync-release := git@github.com:decibel/pgxntool.git release -pgxntool-sync-stable := git@github.com:decibel/pgxntool.git stable -pgxntool-sync-local := ../pgxntool release # Not the same as PGXNTOOL_DIR! -pgxntool-sync-local-stable := ../pgxntool stable # Not the same as PGXNTOOL_DIR! - +# `pgxntool-sync` (no suffix) already pulls the canonical release; these are the +# alternatives. `-master` pulls the bleeding edge; `-local*` pull from a sibling +# ../pgxntool checkout (not the same as PGXNTOOL_DIR!). +pgxntool-sync-master := https://github.com/Postgres-Extensions/pgxntool.git master +pgxntool-sync-local := ../pgxntool release +pgxntool-sync-local-master := ../pgxntool master + +# PGXS doesn't provide any special support for distclean (it just depends on +# clean), so we roll our own. Files that should only be removed by distclean +# (not clean) are added to PGXNTOOL_distclean near their build rules above. distclean: rm -f $(PGXNTOOL_distclean) @@ -214,6 +624,22 @@ DOCS =# Set to NUL so PGXS doesn't puke endif include $(PGXS) + +# Override CONTRIB_TESTDB (set unconditionally by PGXS) with our unique database +# name. This must be after include $(PGXS) because PGXS uses = (not ?=). +# PGXS appends --dbname=$(CONTRIB_TESTDB) to REGRESS_OPTS, so overriding +# CONTRIB_TESTDB is the correct way to control the database name — adding our +# own --dbname would result in two --dbname flags passed to pg_regress. +CONTRIB_TESTDB = $(REGRESS_DBNAME) + +# Clean generated sql/ directory for test-build +ifeq ($(PGXNTOOL_ENABLE_TEST_BUILD),yes) +.PHONY: clean-test-build +clean-test-build: + rm -rf $(TEST_BUILD_SQL_DIR) +clean: clean-test-build +endif + # # pgtap # @@ -229,3 +655,5 @@ $(DESTDIR)$(datadir)/extension/pgtap.control: pgxn install pgtap --sudo endif # fndef PGXNTOOL_NO_PGXS_INCLUDE + +endif # ifndef PGXNTOOL_BASE_MK_INCLUDED diff --git a/bin/version b/bin/version new file mode 100755 index 0000000..c18af60 --- /dev/null +++ b/bin/version @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# bin/version - Print the pgxntool version embedded in this project +# +# The release process (see .claude/skills/release) stamps the first line of +# HISTORY.asc with the released version number, so that line is the source +# of truth for "what version of pgxntool is this?". If this copy was synced +# from an unreleased commit rather than a tagged release, the first line +# will be "STABLE" instead of a version number -- that's printed as-is, since +# it's an accurate answer too. Anything else (a corrupt or hand-edited file) +# is rejected rather than printed as if it were a real answer. +# +# Usage: bin/version [] +# +# history-file Path to HISTORY.asc to read. Defaults to the copy +# one level up from this script (i.e. pgxntool/HISTORY.asc +# in a normal checkout). + +set -o errexit -o errtrace -o pipefail +trap 'echo "Error on line ${LINENO}"' ERR + +PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")/.." +source "$PGXNTOOL_DIR/lib.sh" + +history_file="${1:-$PGXNTOOL_DIR/HISTORY.asc}" + +[[ -f "$history_file" ]] || die 1 "$history_file not found" + +version=$(head -n1 "$history_file") + +[[ -n "$version" ]] || die 1 "$history_file is empty" + +# Matches the X.Y.Z format the release process validates and stamps. +[[ "$version" == "STABLE" || "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || \ + die 1 "$history_file: first line '$version' is neither STABLE nor a X.Y.Z version" + +echo "$version" diff --git a/build_meta.sh b/build_meta.sh index 70d2273..d4ea9fa 100755 --- a/build_meta.sh +++ b/build_meta.sh @@ -1,16 +1,28 @@ -#!/bin/bash +#!/usr/bin/env bash + +# Build META.json from META.in.json template +# +# WHY META.in.json EXISTS: +# META.in.json serves as a template that: +# 1. Shows all possible PGXN metadata fields (both required and optional) with comments +# 2. Can have empty placeholder fields like "key": "" or "key": [ "", "" ] +# 3. Users edit this to fill in their extension's metadata +# +# WHY WE GENERATE META.json: +# The reason we generate META.json from a template is to eliminate empty fields that +# are optional; PGXN.org gets upset about them. In the future it's possible we'll do +# more here (for example, if we added more info to the template we could use it to +# generate control files). +# +# WHY WE COMMIT META.json: +# PGXN.org requires META.json to be present in submitted distributions. We choose +# to commit it to git instead of manually adding it to distributions for simplicity +# (and since it generally only changes once for each new version). set -e -error () { - echo $@ >&2 -} -die () { - return=$1 - shift - error $@ - exit $return -} +BASEDIR=$(dirname "$0") +source "$BASEDIR/lib.sh" [ $# -eq 2 ] || die 2 Invalid number of arguments $# diff --git a/control.mk.sh b/control.mk.sh new file mode 100755 index 0000000..aa491cf --- /dev/null +++ b/control.mk.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# +# control.mk.sh - Generate Makefile rules from PostgreSQL extension control files +# +# This script parses .control files to extract extension metadata (particularly +# default_version) and generates Make variables and rules for building versioned +# SQL files. +# +# Usage: control.mk.sh [ ...] +# +# Output (to stdout, meant to be redirected to control.mk): +# EXTENSIONS += +# EXTENSION_SQL_FILES += sql/.sql +# EXTENSION__VERSION := +# EXTENSION__VERSION_FILE = sql/--.sql +# EXTENSION__CURRENT_VERSION__FILES += $(EXTENSION__VERSION_FILE) +# +# +# Why control files instead of META.json? +# META.json's "provides" section specifies versions for PGXN distribution metadata. +# But PostgreSQL uses the control file's default_version to determine which +# versioned SQL file to load. These can differ (e.g., PGXN distribution version +# might be updated independently of extension version). Using the control file +# ensures the generated SQL files match what PostgreSQL expects. + +set -o errexit -o errtrace -o pipefail + +BASEDIR=$(dirname "$0") +source "$BASEDIR/lib.sh" + +# Extract default_version from a PostgreSQL extension control file +# Usage: get_control_default_version +# Errors if: +# - Control file doesn't exist +# - default_version is not specified (pgxntool requires it) +# - Multiple default_version lines exist +get_control_default_version() { + local control_file="$1" + + if [ ! -f "$control_file" ]; then + die 2 "Control file '$control_file' not found" + fi + + # Count default_version lines + local count + count=$(grep -cE "^[[:space:]]*default_version[[:space:]]*=" "$control_file" 2>/dev/null) || count=0 + + if [ "$count" -eq 0 ]; then + die 2 "default_version not specified in '$control_file'. PostgreSQL allows extensions without a default_version, but pgxntool requires it to generate versioned SQL files." + fi + + if [ "$count" -gt 1 ]; then + die 2 "Multiple default_version lines found in '$control_file'. Control files must have exactly one default_version." + fi + + # Extract the version value + # Handles: default_version = '1.0', default_version = "1.0", trailing comments + local version=$(grep -E "^[[:space:]]*default_version[[:space:]]*=" "$control_file" | \ + sed -e "s/^[^=]*=[[:space:]]*//" \ + -e "s/[[:space:]]*#.*//" \ + -e "s/^['\"]//;s/['\"]$//" ) + + if [ -z "$version" ]; then + die 2 "Could not parse default_version value from '$control_file'" + fi + + echo "$version" +} + +# Main: process each control file passed as argument +if [ $# -eq 0 ]; then + die 1 "Usage: control.mk.sh [ ...]" +fi + +for control_file in "$@"; do + ext=$(basename "$control_file" .control) + version=$(get_control_default_version "$control_file") + + echo "EXTENSIONS += $ext" + echo "EXTENSION_SQL_FILES += sql/${ext}.sql" + echo "EXTENSION_${ext}_VERSION := ${version}" + echo "EXTENSION_${ext}_VERSION_FILE = sql/${ext}--\$(EXTENSION_${ext}_VERSION).sql" + echo "EXTENSION__CURRENT_VERSION__FILES += \$(EXTENSION_${ext}_VERSION_FILE)" + echo "\$(EXTENSION_${ext}_VERSION_FILE): sql/${ext}.sql ${control_file}" + # Single redirect via a subshell, not `>` truncate followed by `>>` append: + # if this rule ever runs twice in the same `make` invocation (e.g. reached + # through two different dependency chains under parallel make), the `>>` + # form doubles the file's content instead of overwriting it. + echo " @(echo '/* DO NOT EDIT - AUTO-GENERATED FILE */'; cat sql/${ext}.sql) > \$(EXTENSION_${ext}_VERSION_FILE)" + echo +done + +# vi: expandtab ts=2 sw=2 diff --git a/lib.sh b/lib.sh new file mode 100644 index 0000000..3cbeb0e --- /dev/null +++ b/lib.sh @@ -0,0 +1,110 @@ +# lib.sh - Common utility functions for pgxntool scripts +# +# This file is meant to be sourced by other scripts, not executed directly. +# Usage: source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# ============================================================================= +# SETUP FILES CONFIGURATION +# ============================================================================= +# Files copied by setup.sh and tracked by update-setup-files.sh for sync updates. +# Format: "source_in_pgxntool:destination_in_project" +# ============================================================================= +SETUP_FILES=( + "_.gitignore:.gitignore" + "test/deps.sql:test/deps.sql" +) + +# Symlinks created by setup.sh and verified by update-setup-files.sh +# Format: "destination:target" +SETUP_SYMLINKS=( + "test/pgxntool:../pgxntool/test/pgxntool" +) + +# Error function - outputs to stderr but doesn't exit +# Usage: error "message" +error() { + echo "ERROR: $*" >&2 +} + +# Die function - outputs error message and exits with specified code +# Usage: die EXIT_CODE "message" +die() { + local exit_code=$1 + shift + error "$@" + exit $exit_code +} + +# Returns true if an array isn't empty. +# +# array_not_empty "${#errors[@]}" +# +# BUT WHY ON EARTH DO THIS?? +# +# This wraps a one-liner intentionally. The function forces any reader +# (human or AI agent) to navigate here and read this comment before +# "simplifying" the call site. Without it, the natural next step is to +# inline the expression — and the natural inline form breaks bash 3.2. +# +# On bash 3.2 (Mac OS default), when using `set -u`, expanding "${arr[@]}" +# on an empty array triggers "unbound variable" even when the array was +# explicitly initialized with arr=(). +# +# The comment inside the function body exists to catch any agent or human who +# navigates to the function without reading this comment first. +array_not_empty() { + # DO NOT EDIT THIS FUNCTION! DO NOT REMOVE THIS COMMENT! (see main function comment) + [ "${1:-0}" -gt 0 ] +} + +# Debug function +# Usage: debug LEVEL "message" +# Outputs message to stderr if DEBUG >= LEVEL +# +# LEVEL encodes how noisy/esoteric a message is -- roughly, how far you'd crank +# DEBUG before you'd actually want to see it. Higher = noisier, more rarely +# useful. This is signal-to-noise, NOT code nesting depth: a top-level line can +# warrant a high level if it's esoteric, and loop-body detail is usually high +# precisely because it's noisy. +# +# The tiers below are anchors, not strict multiples -- pick any value in range +# to fine-tune between existing calls without renumbering: +# - 10: Critical errors, important warnings +# - 20: Warnings, significant state changes +# - 30: General debugging, function entry/exit, array operations +# - 40: Verbose details, loop iterations +# - 50+: Maximum verbosity (per-iteration innards) +# +# Enable with: DEBUG=30 scriptname.sh +debug() { + local level=$1 + shift + local message="$*" + + if [ "${DEBUG:-0}" -ge "$level" ]; then + echo "DEBUG[$level]: $message" >&2 + fi +} + +# Remove pgxntool's own dev-only directories from a consuming project. +# +# `git subtree` copies the ENTIRE pgxntool tree into the consumer, including +# dev-only dirs like .github/ (pgxntool's CI) and .claude/. Those are +# export-ignored from `make dist` and don't belong in a project that merely +# embeds pgxntool. (GitHub only runs workflows at the repo root, so a consumer's +# pgxntool/.github never executes anyway — but it's still clutter.) git subtree +# doesn't honor export-ignore, so we prune them here after a sync. +# +# Must be run from the project root (the dir containing pgxntool/). Safe to call +# repeatedly; a no-op once the dirs are gone. +prune_pgxntool_dev_dirs() { + local d + for d in .github .claude; do + [ -e "pgxntool/$d" ] || continue + echo " pgxntool/$d: pruning (pgxntool dev-only, not for embedding projects)" + # Stage the removal if tracked; rm -rf guarantees it's gone even if not. + # || : keeps this best-effort under `set -e` (rm -rf is the real cleanup). + git rm -rq --ignore-unmatch "pgxntool/$d" >/dev/null 2>&1 || : + rm -rf "pgxntool/$d" + done +} diff --git a/meta.mk.sh b/meta.mk.sh index a5da2ec..e6cecc5 100755 --- a/meta.mk.sh +++ b/meta.mk.sh @@ -1,25 +1,38 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash +# +# meta.mk.sh - Generate Makefile variables from META.json +# +# This script parses META.json (PGXN distribution metadata) and generates +# Make variables for the distribution name and version. +# +# Usage: meta.mk.sh +# +# Output (to stdout, meant to be redirected to meta.mk): +# PGXN := +# PGXNVERSION := +# +# Note: Extension-specific variables (like EXTENSION_*_VERSION) are generated +# by control.mk.sh from .control files, not from META.json. This is because +# META.json specifies PGXN distribution metadata, while .control files specify +# what PostgreSQL actually uses. set -o errexit -o errtrace -o pipefail -trap 'echo "Error on line ${LINENO}" >&2' ERR -META=$1 -BASEDIR=`dirname $0` +BASEDIR=$(dirname "$0") +source "$BASEDIR/lib.sh" + JSON_SH=$BASEDIR/JSON.sh -error () { - echo $@ >&2 -} trap 'error "Error on line ${LINENO}"' ERR -die () { - local retval=$1 - shift - error $@ - exit $retval -} +META=$1 +if [ -z "$META" ]; then + die 1 "Usage: meta.mk.sh " +fi -REQUIRED='abstract maintainer license provides name version' +if [ ! -f "$META" ]; then + die 2 "META.json file '$META' not found" +fi #function to get value of specified key #returns empty string if not found @@ -27,7 +40,7 @@ REQUIRED='abstract maintainer license provides name version' #usage: VAR=$(getkey foo.bar) #get value of "bar" contained within "foo" # VAR=$(getkey foo[4].bar) #get value of "bar" contained in the array "foo" on position 4 # VAR=$(getkey [4].foo) #get value of "foo" contained in the root unnamed array on position 4 -function _getkey { +_getkey() { #reformat key string (parameter) to what JSON.sh uses KEYSTRING=$(sed -e 's/\[/\"\,/g' -e 's/^\"\,/\[/g' -e 's/\]\./\,\"/g' -e 's/\./\"\,\"/g' -e '/^\[/! s/^/\[\"/g' -e '/\]$/! s/$/\"\]/g' <<< "$@") #extract the key value @@ -37,60 +50,21 @@ function _getkey { FOUT="${FOUT%\"*}" echo "$FOUT" } -function getkeys { - KEYSTRING=$(sed -e 's/\[/\"\,/g' -e 's/^\"\,/\[/g' -e 's/\]\./\,\"/g' -e 's/\./\"\,\"/g' -e '/^\[/! s/^/\[\"/g' -e '/\",\"$/! s/$/\",\"/g' <<< "$@") - #extract the key value - FOUT=$(grep -F "$KEYSTRING" <<< "$JSON_PARSED") - FOUT="${FOUT%$'\t'*}" - echo "$FOUT" -} - -#function returning length of array -#returns zero if key in parameter does not exist or is not an array -#usage: VAR=$(getarrlen foo.bar) #get length of array "bar" contained within "foo" -# VAR=$(getarrlen) #get length of the root unnamed array -# VAR=$(getarrlen [2].foo.bar) #get length of array "bar" contained within "foo", which is stored in the root unnamed array on position 2 -function getarrlen { - #reformat key string (parameter) to what JSON.sh uses - KEYSTRING=$(gsed -e '/^\[/! s/\[/\"\,/g' -e 's/\]\./\,\"/g' -e 's/\./\"\,\"/g' -e '/^$/! {/^\[/! s/^/\[\"/g}' -e '/^$/! s/$/\"\,/g' -e 's/\[/\\\[/g' -e 's/\]/\\\]/g' -e 's/\,/\\\,/g' -e '/^$/ s/^/\\\[/g' <<< "$@") - #extract the key array length - get last index - LEN=$(grep -o "${KEYSTRING}[0-9]*" <<< "$JSON_PARSED" | tail -n -1 | grep -o "[0-9]*$") - #increment to get length, if empty => zero - if [ -n "$LEN" ]; then - LEN=$(($LEN+1)) - else - LEN="0" - fi - echo "$LEN" -} -JSON_PARSED=$(cat $META | $JSON_SH -l) - -function getkey { +getkey() { out=$(_getkey "$@") [ -n "$out" ] || die 2 "key $@ not found in $META" echo $out } -# Handle meta-spec specially -spec_version=`getkey meta-spec.version` -[ "$spec_version" == "1.0.0" ] || die 2 "Unknown meta-spec/version: $PGXN_meta-spec_version" +JSON_PARSED=$(cat "$META" | $JSON_SH -l) + +# Validate meta-spec version +spec_version=$(getkey meta-spec.version) +[ "$spec_version" == "1.0.0" ] || die 2 "Unknown meta-spec/version: $spec_version" +# Output distribution name and version echo "PGXN := $(getkey name)" echo "PGXNVERSION := $(getkey version)" -echo - -provides=$(getkeys provides | sed -e 's/\["provides","//' -e 's/",".*//' | uniq) -for ext in $provides; do - version=$(getkey provides.${ext}.version) - [ -n "$version" ] || die 2 "provides/${ext} does not specify a version number" - echo "EXTENSIONS += $ext" - echo "EXTENSION_SQL_FILES += sql/${ext}.sql" - echo "EXTENSION_${ext}_VERSION := ${version}" - echo "EXTENSION_${ext}_VERSION_FILE = sql/${ext}--\$(EXTENSION_${ext}_VERSION).sql" - echo "EXTENSION_VERSION_FILES += \$(EXTENSION_${ext}_VERSION_FILE)" - echo "\$(EXTENSION_${ext}_VERSION_FILE): sql/${ext}.sql META.json meta.mk" - echo ' cp $< $@' -done # vi: expandtab ts=2 sw=2 diff --git a/pgtle.sh b/pgtle.sh new file mode 100755 index 0000000..e6229fd --- /dev/null +++ b/pgtle.sh @@ -0,0 +1,866 @@ +#!/usr/bin/env bash +# +# pgtle.sh - Generate pg_tle registration SQL for PostgreSQL extensions +# +# Part of pgxntool: https://github.com/decibel/pgxntool +# +# SYNOPSIS +# pgtle.sh --extension EXTNAME [--pgtle-version VERSION] +# pgtle.sh --get-dir VERSION +# pgtle.sh --get-version +# pgtle.sh --run +# +# DESCRIPTION +# Generates pg_tle (Trusted Language Extensions) registration SQL from +# a pgxntool-based PostgreSQL extension. Reads the extension's .control +# file and SQL files, wrapping them for pg_tle deployment in managed +# environments like AWS RDS and Aurora. +# +# pg_tle enables extension installation without filesystem access by +# storing extension code in database tables. This script converts +# traditional PostgreSQL extensions into pg_tle-compatible SQL. +# +# OPTIONS +# --extension NAME +# Extension name (required). Must match a .control file basename +# in the current directory. +# +# --pgtle-version VERSION +# Generate for specific pg_tle version only (optional). +# Format: 1.0.0-1.4.0, 1.4.0-1.5.0, or 1.5.0+ +# Default: Generate all supported versions +# +# --get-dir VERSION +# Returns the directory path for the given pg_tle version. +# Format: VERSION is a version string like "1.5.2" +# Output: Directory path like "pg_tle/1.5.0+", "pg_tle/1.4.0-1.5.0", or "pg_tle/1.0.0-1.4.0" +# This option is used by make to determine which directory to use +# +# --get-version +# Returns the installed pg_tle version from the database. +# Output: Version string like "1.5.2" or empty if not installed +# Exit status: 0 if pg_tle is installed, 1 if not installed +# +# --run +# Runs the generated pg_tle registration SQL files. This option: +# - Detects the installed pg_tle version from the database +# - Determines the appropriate directory using --get-dir logic +# - Executes all SQL files in that directory via psql +# - Assumes PG* environment variables are configured for psql +# +# VERSION NOTATION +# X.Y.Z+ Works on pg_tle >= X.Y.Z +# X.Y.Z-A.B.C Works on pg_tle >= X.Y.Z and < A.B.C +# +# Note the boundary conditions: +# 1.5.0+ means >= 1.5.0 (includes 1.5.0) +# 1.4.0-1.5.0 means >= 1.4.0 and < 1.5.0 (excludes 1.5.0) +# 1.0.0-1.4.0 means >= 1.0.0 and < 1.4.0 (excludes 1.4.0) +# +# SUPPORTED VERSIONS +# 1.0.0-1.4.0 pg_tle 1.0.0 through 1.3.x (no uninstall function, no schema parameter) +# 1.4.0-1.5.0 pg_tle 1.4.0 through 1.4.x (has uninstall function, no schema parameter) +# 1.5.0+ pg_tle 1.5.0 and later (has uninstall function, schema parameter support) +# +# EXAMPLES +# # Generate all versions (default) +# pgtle.sh --extension myext +# +# # Generate only for pg_tle 1.5+ +# pgtle.sh --extension myext --pgtle-version 1.5.0+ +# +# # Get directory for a specific pg_tle version +# pgtle.sh --get-dir 1.5.2 +# # Output: pg_tle/1.5.0+ +# +# pgtle.sh --get-dir 1.4.2 +# # Output: pg_tle/1.4.0-1.5.0 +# +# # Get installed pg_tle version from database +# pgtle.sh --get-version +# # Output: 1.5.2 (or empty if not installed) +# +# # Run generated pg_tle registration SQL files +# pgtle.sh --run +# +# OUTPUT +# Creates files in version-specific subdirectories: +# pg_tle/1.0.0-1.4.0/{extension}.sql +# pg_tle/1.4.0-1.5.0/{extension}.sql +# pg_tle/1.5.0+/{extension}.sql +# +# Each file contains: +# - All versions of the extension +# - All upgrade paths between versions +# - Default version configuration +# - Complete installation instructions +# +# For --get-dir: Outputs the directory path to stdout. +# +# For --get-version: Outputs the installed pg_tle version to stdout, or empty if not installed. +# +# For --run: Executes SQL files and outputs progress messages to stderr. +# +# REQUIREMENTS +# - Must run from extension directory (where .control files are) +# - Extension must use only trusted languages (PL/pgSQL, SQL, PL/Perl, etc.) +# - No C code (module_pathname not supported by pg_tle) +# - Versioned SQL files must exist: sql/{ext}--{version}.sql +# +# EXIT STATUS +# 0 Success +# 1 Error (missing files, validation failure, C code detected, etc.) +# +# SEE ALSO +# pgxntool/README-pgtle.md - Complete user guide +# https://github.com/aws/pg_tle - pg_tle documentation +# + +set -eo pipefail + +# Source common library functions (error, die, debug) +PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")" +source "$PGXNTOOL_DIR/lib.sh" + +# Constants +PGTLE_DELIMITER='$_pgtle_wrap_delimiter_$' +PGTLE_VERSIONS=("1.0.0-1.4.0" "1.4.0-1.5.0" "1.5.0+") + +# Supported pg_tle version ranges and their capabilities +# Use a function instead of associative array for compatibility with bash < 4.0 +get_pgtle_capability() { + local version="$1" + case "$version" in + "1.0.0-1.4.0") + echo "no_uninstall_no_schema" + ;; + "1.4.0-1.5.0") + echo "has_uninstall_no_schema" + ;; + "1.5.0+") + echo "has_uninstall_has_schema" + ;; + *) + echo "unknown" + ;; + esac +} + +# Global variables (populated from control file) +EXTENSION="" +DEFAULT_VERSION="" +COMMENT="" +REQUIRES="" +SCHEMA="" +MODULE_PATHNAME="" +VERSION_FILES=() +UPGRADE_FILES=() + +debug 30 "Global arrays initialized: VERSION_FILES=${#VERSION_FILES[@]}, UPGRADE_FILES=${#UPGRADE_FILES[@]}" +PGTLE_VERSION="" # Empty = generate all +GET_DIR_VERSION="" # For --get-dir option + +# Arrays (populated from SQL discovery) +VERSION_FILES=() +UPGRADE_FILES=() + +# Parse and validate a version string +# Extracts numeric version (major.minor.patch) from version strings +# Handles versions with suffixes like "1.5.0alpha1", "2.0beta", "1.2.3dev" +# Returns: numeric version string (e.g., "1.5.0") or exits with error +parse_version() { + local version="$1" + + if [ -z "$version" ]; then + die 1 "Version string is empty" + fi + + # Extract numeric version part (major.minor.patch) + # Matches: 1.5.0, 1.5, 10.2.1alpha, 2.0beta1, etc. + # Pattern: start of string, then digits, dot, digits, optionally (dot digits), then anything + local numeric_version + if [[ "$version" =~ ^([0-9]+\.[0-9]+(\.[0-9]+)?) ]]; then + numeric_version="${BASH_REMATCH[1]}" + else + die 1 "Cannot parse version string: '$version' + Expected format: major.minor[.patch][suffix] + Examples: 1.5.0, 1.5, 2.0alpha1, 10.2.3dev" + fi + + # Ensure we have at least major.minor (add .0 if needed) + if [[ ! "$numeric_version" =~ \. ]]; then + die 1 "Invalid version format: '$version' (need at least major.minor)" + fi + + # If we only have major.minor, add .0 for patch + if [[ ! "$numeric_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + numeric_version="${numeric_version}.0" + fi + + echo "$numeric_version" +} + +# Convert version string to comparable integer +# Takes a numeric version string (major.minor.patch) and converts to integer +# Example: "1.5.0" -> 1005000 +# Encoding scheme: major * 1000000 + minor * 1000 + patch +# This limits each component to 0-999 to prevent overflow +version_to_number() { + local version="$1" + + # Parse major.minor.patch + local major minor patch + if [[ "$version" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + major="${BASH_REMATCH[1]}" + minor="${BASH_REMATCH[2]}" + patch="${BASH_REMATCH[3]}" + else + die 1 "version_to_number: Invalid numeric version format: '$version'" + fi + + # Check for overflow in encoding scheme + # Each component must be < 1000 to fit in the allocated space + if [ "$major" -ge 1000 ]; then + die 1 "version_to_number: Major version too large: $major (max 999) + Version: $version" + fi + if [ "$minor" -ge 1000 ]; then + die 1 "version_to_number: Minor version too large: $minor (max 999) + Version: $version" + fi + if [ "$patch" -ge 1000 ]; then + die 1 "version_to_number: Patch version too large: $patch (max 999) + Version: $version" + fi + + # Convert to comparable number: major * 1000000 + minor * 1000 + patch + echo $(( major * 1000000 + minor * 1000 + patch )) +} + +# Get directory for a given pg_tle version +# Takes a version string like "1.5.2" and returns the directory path +# Handles versions with suffixes (e.g., "1.5.0alpha1") +# Returns: "pg_tle/1.0.0-1.4.0", "pg_tle/1.4.0-1.5.0", or "pg_tle/1.5.0+" +get_version_dir() { + local version="$1" + + if [ -z "$version" ]; then + die 1 "Version required for --get-dir (got empty string)" + fi + + # Parse and validate version + local numeric_version + numeric_version=$(parse_version "$version") + + # Check if the original version has a pre-release suffix + # Pre-release versions (alpha, beta, rc, dev) are considered BEFORE the release + # Example: 1.4.0alpha1 comes BEFORE 1.4.0, so it should use the 1.0.0-1.4.0 range + local has_prerelease=0 + if [[ "$version" =~ (alpha|beta|rc|dev) ]]; then + has_prerelease=1 + fi + + # Convert versions to comparable numbers + local version_num + local threshold_1_4_num + local threshold_1_5_num + version_num=$(version_to_number "$numeric_version") + threshold_1_4_num=$(version_to_number "1.4.0") + threshold_1_5_num=$(version_to_number "1.5.0") + + # Compare and return appropriate directory: + # < 1.4.0 -> 1.0.0-1.4.0 + # >= 1.4.0 and < 1.5.0 -> 1.4.0-1.5.0 + # >= 1.5.0 -> 1.5.0+ + # + # Special handling for pre-release versions: + # If version equals a threshold but has a pre-release suffix, treat it as less than that threshold + # Example: 1.4.0alpha1 is treated as < 1.4.0, so it uses 1.0.0-1.4.0 + if [ "$version_num" -lt "$threshold_1_4_num" ]; then + echo "pg_tle/1.0.0-1.4.0" + elif [ "$version_num" -eq "$threshold_1_4_num" ] && [ "$has_prerelease" -eq 1 ]; then + # Pre-release of 1.4.0 is considered < 1.4.0 + echo "pg_tle/1.0.0-1.4.0" + elif [ "$version_num" -lt "$threshold_1_5_num" ]; then + echo "pg_tle/1.4.0-1.5.0" + elif [ "$version_num" -eq "$threshold_1_5_num" ] && [ "$has_prerelease" -eq 1 ]; then + # Pre-release of 1.5.0 is considered < 1.5.0 + echo "pg_tle/1.4.0-1.5.0" + else + echo "pg_tle/1.5.0+" + fi +} + +# Get pg_tle version from installed extension +# Returns version string or empty if not installed +get_pgtle_version() { + psql --no-psqlrc --tuples-only --no-align --command "SELECT extversion FROM pg_extension WHERE extname = 'pg_tle';" 2>/dev/null | tr -d '[:space:]' || echo "" +} + +# Run pg_tle registration SQL files +# Detects installed pg_tle version and runs appropriate SQL files +run_pgtle_sql() { + echo "Running pg_tle registration SQL files..." >&2 + + # Get version from installed extension + local pgtle_version=$(get_pgtle_version) + if [ -z "$pgtle_version" ]; then + die 1 "pg_tle extension is not installed + Run 'CREATE EXTENSION pg_tle;' first, or use 'make check-pgtle' to verify" + fi + + # Get directory for this version + local pgtle_dir=$(get_version_dir "$pgtle_version") + if [ -z "$pgtle_dir" ]; then + die 1 "Failed to determine pg_tle directory for version $pgtle_version" + fi + + echo "Using pg_tle files for version $pgtle_version (directory: $pgtle_dir)" >&2 + + # Check if directory exists + if [ ! -d "$pgtle_dir" ]; then + die 1 "pg_tle directory $pgtle_dir does not exist + Run 'make pgtle' first to generate files" + fi + + # Run all SQL files in the directory + local sql_file + local found=0 + for sql_file in "$pgtle_dir"/*.sql; do + if [ -f "$sql_file" ]; then + found=1 + echo "Running $sql_file..." >&2 + psql --no-psqlrc -v ON_ERROR_STOP=1 --file="$sql_file" || exit 1 + fi + done + + if [ "$found" -eq 0 ]; then + die 1 "No SQL files found in $pgtle_dir + Run 'make pgtle' first to generate files" + fi + + echo "pg_tle registration complete" >&2 +} + +# Main logic +main() { + # Handle --get-dir, --get-version, --test-function, and --run options first (early exit, before other validation) + local args=("$@") + local i=0 + while [ $i -lt ${#args[@]} ]; do + if [ "${args[$i]}" = "--get-dir" ] && [ $((i+1)) -lt ${#args[@]} ]; then + get_version_dir "${args[$((i+1))]}" + exit 0 + elif [ "${args[$i]}" = "--get-version" ]; then + local version=$(get_pgtle_version) + if [ -n "$version" ]; then + echo "$version" + exit 0 + else + exit 1 + fi + elif [ "${args[$i]}" = "--test-function" ] && [ $((i+1)) -lt ${#args[@]} ]; then + # Hidden option for testing internal functions + # NOT a supported public interface - used only by the test suite + # Usage: pgtle.sh --test-function FUNC_NAME [ARGS...] + local func_name="${args[$((i+1))]}" + shift $((i+2)) # Remove script name and --test-function and func_name + + # Check if function exists + if ! declare -f "$func_name" >/dev/null 2>&1; then + die 1 "Function '$func_name' does not exist" + fi + + # Call the function with remaining arguments + "$func_name" "${args[@]:$((i+2))}" + exit $? + elif [ "${args[$i]}" = "--run" ]; then + run_pgtle_sql + exit 0 + fi + i=$((i+1)) + done + + # Parse other arguments + parse_args "$@" + + validate_environment + parse_control_file + discover_sql_files + + if [ -z "$PGTLE_VERSION" ]; then + # Generate all versions + for version in "${PGTLE_VERSIONS[@]}"; do + generate_pgtle_sql "$version" + done + else + # Generate specific version + generate_pgtle_sql "$PGTLE_VERSION" + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + --extension) + EXTENSION="$2" + shift 2 + ;; + --pgtle-version) + PGTLE_VERSION="$2" + shift 2 + ;; + --get-dir) # This case should ideally not be hit due to early exit + GET_DIR_VERSION="$2" + shift 2 + ;; + --get-version) # This case should ideally not be hit due to early exit + shift + ;; + --test-function) # Hidden option for testing - not documented, not supported + shift 2 # Skip function name and --test-function + ;; + --run) # This case should ideally not be hit due to early exit + shift + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac + done + + if [ -z "$EXTENSION" ] && [ -z "$GET_DIR_VERSION" ]; then + die 1 "--extension is required (unless using --get-dir, --get-version, --test-function, or --run)" + fi +} + +validate_environment() { + # Check if control file exists + if [ ! -f "${EXTENSION}.control" ]; then + die 1 "Control file not found: ${EXTENSION}.control + Must run from extension directory" + fi +} + +parse_control_file() { + local control_file="${EXTENSION}.control" + + echo "Parsing control file: $control_file" >&2 + + # Parse key = value or key = 'value' format + while IFS= read -r line; do + # Skip comments and empty lines + [[ "$line" =~ ^[[:space:]]*# ]] && continue + [[ "$line" =~ ^[[:space:]]*$ ]] && continue + + # Extract key = value + if [[ "$line" =~ ^[[:space:]]*([a-z_]+)[[:space:]]*=[[:space:]]*(.*)[[:space:]]*$ ]]; then + local key="${BASH_REMATCH[1]}" + local value="${BASH_REMATCH[2]}" + + # Trim trailing comments and whitespace FIRST, then strip quotes. + # Order matters: stripping quotes before removing comments leaves a rogue + # trailing quote for values like 'version' # note. See issue #25. + value="${value%%#*}" # Remove trailing comments + # Trim all trailing whitespace (spaces and tabs). The prior %% pattern with + # a literal space only removed one character; multiple spaces or a tab before + # the comment (e.g., 'value' # note or 'value'$'\t'# note) would leave + # stray whitespace that breaks the quote-strip below. + if [[ "$value" =~ ^(.*[^[:space:]])[[:space:]]*$ ]]; then + value="${BASH_REMATCH[1]}" + else + value="" + fi + + # Strip quotes if present (both single and double) + value="${value#\'}" + value="${value%\'}" + value="${value#\"}" + value="${value%\"}" + + # Store in global variables + case "$key" in + default_version) DEFAULT_VERSION="$value" ;; + comment) COMMENT="$value" ;; + requires) REQUIRES="$value" ;; + schema) SCHEMA="$value" ;; + module_pathname) MODULE_PATHNAME="$value" ;; + esac + fi + done < "$control_file" + + # Validate required fields + if [ -z "$DEFAULT_VERSION" ]; then + die 1 "Control file missing default_version" + fi + + if [ -z "$COMMENT" ]; then + echo "WARNING: Control file missing comment, using extension name" >&2 + COMMENT="$EXTENSION extension" + fi + + # Warn about C code + if [ -n "$MODULE_PATHNAME" ]; then + cat >&2 <<-EOF + WARNING: Extension uses module_pathname (C code) + pg_tle only supports trusted languages (PL/pgSQL, SQL, etc.) + Generated SQL will likely not work + EOF + fi + + echo " default_version: $DEFAULT_VERSION" >&2 + echo " comment: $COMMENT" >&2 + if [ -n "$REQUIRES" ]; then + echo " requires: $REQUIRES" >&2 + fi + if [ -n "$SCHEMA" ]; then + echo " schema: $SCHEMA" >&2 + fi +} + +discover_sql_files() { + echo "Discovering SQL files for extension: $EXTENSION" >&2 + debug 30 "discover_sql_files: Starting discovery for extension: $EXTENSION" + + # Ensure default_version file exists and has content if base file exists + # This handles the case where make all hasn't generated it yet, or it exists but is empty + local default_version_file="sql/${EXTENSION}--${DEFAULT_VERSION}.sql" + local base_file="sql/${EXTENSION}.sql" + if [ -f "$base_file" ] && ([ ! -f "$default_version_file" ] || [ ! -s "$default_version_file" ]); then + debug 40 "discover_sql_files: Creating default_version file from base file" + cp "$base_file" "$default_version_file" + fi + + # Find versioned files: sql/{ext}--{version}.sql + # Use find to get proper null-delimited output, then filter out upgrade scripts + VERSION_FILES=() # Reset array + debug 40 "discover_sql_files: Reset VERSION_FILES array" + while IFS= read -r -d '' file; do + local basename=$(basename "$file" .sql) + local dash_count=$(echo "$basename" | grep -o -- "--" | wc -l | tr -d '[:space:]') + # Skip upgrade scripts (they have 2 dashes) + if [ "$dash_count" -ne 1 ]; then + continue + fi + # Error on empty version files + if [ ! -s "$file" ]; then + die 1 "Empty version file found: $file" + fi + VERSION_FILES+=("$file") + done < <(find sql/ -maxdepth 1 -name "${EXTENSION}--*.sql" -print0 2>/dev/null | sort -zV) + + # Find upgrade scripts: sql/{ext}--{ver1}--{ver2}.sql + # These have TWO occurrences of "--" in the filename + UPGRADE_FILES=() # Reset array + debug 40 "discover_sql_files: Reset UPGRADE_FILES array" + while IFS= read -r -d '' file; do + # Empty upgrade files are allowed (no-op upgrades) + local basename=$(basename "$file" .sql) + local dash_count=$(echo "$basename" | grep -o -- "--" | wc -l | tr -d '[:space:]') + if [ "$dash_count" -eq 2 ]; then + UPGRADE_FILES+=("$file") + fi + done < <(find sql/ -maxdepth 1 -name "${EXTENSION}--*--*.sql" -print0 2>/dev/null | sort -zV) + + if [ ${#VERSION_FILES[@]} -eq 0 ]; then + die 1 "No versioned SQL files found for $EXTENSION + Expected pattern: sql/${EXTENSION}--{version}.sql + Run 'make' first to generate versioned files from sql/${EXTENSION}.sql" + fi + + echo " Found ${#VERSION_FILES[@]} version file(s):" >&2 + for f in "${VERSION_FILES[@]}"; do + echo " - $f" >&2 + done + + debug 40 "discover_sql_files: Checking UPGRADE_FILES array, count=${#UPGRADE_FILES[@]}" + if array_not_empty "${#UPGRADE_FILES[@]}"; then + echo " Found ${#UPGRADE_FILES[@]} upgrade script(s):" >&2 + debug 30 "discover_sql_files: Iterating over ${#UPGRADE_FILES[@]} upgrade files" + for f in "${UPGRADE_FILES[@]}"; do + echo " - $f" >&2 + done + else + debug 30 "discover_sql_files: No upgrade files found" + fi +} + +extract_version_from_filename() { + local filename="$1" + local basename=$(basename "$filename" .sql) + + # Match patterns: + # - ext--1.0.0 → FROM_VERSION=1.0.0, TO_VERSION="" + # - ext--1.0.0--2.0.0 → FROM_VERSION=1.0.0, TO_VERSION=2.0.0 + # - ext--stable → FROM_VERSION=stable, TO_VERSION="" (pg_tle treats + # versions as opaque strings; see check_valid_version_name() in + # pg_tle's tleextension.c, which only forbids empty strings, "--", + # and leading/trailing "-" — no numeric requirement) + # + # Each version segment is [^-]+(-[^-]+)* : one or more non-dash runs + # joined by single dashes, so a segment can't be empty, start/end with + # "-", or contain "--" (which is reserved as the FROM/TO delimiter). + + if [[ "$basename" =~ ^${EXTENSION}--([^-]+(-[^-]+)*)(--([^-]+(-[^-]+)*))?$ ]]; then + FROM_VERSION="${BASH_REMATCH[1]}" + TO_VERSION="${BASH_REMATCH[4]}" # Empty for non-upgrade files + return 0 + else + die 1 "Cannot parse version from filename: $filename + Expected format: ${EXTENSION}--{version}.sql or ${EXTENSION}--{ver1}--{ver2}.sql" + fi +} + +validate_delimiter() { + local sql_file="$1" + + if grep -qF "$PGTLE_DELIMITER" "$sql_file"; then + die 1 "SQL file contains reserved pg_tle delimiter: $sql_file + Found: $PGTLE_DELIMITER + This delimiter is used internally by pgtle.sh to wrap SQL content. + You must modify your SQL to not contain this string. If this poses a + serious problem, please open an issue at https://github.com/decibel/pgxntool/issues" + fi +} + +wrap_sql_content() { + local sql_file="$1" + + validate_delimiter "$sql_file" + + # Output wrapped SQL with proper indentation + # Empty files are valid (no-op upgrades) + echo " ${PGTLE_DELIMITER}" + cat "$sql_file" + echo " ${PGTLE_DELIMITER}" +} + +build_requires_array() { + # Input: "plpgsql, other_ext, another" + # Output: 'plpgsql', 'other_ext', 'another' + + # Split on comma, trim whitespace, quote each element + REQUIRES_ARRAY=$(echo "$REQUIRES" | \ + sed 's/[[:space:]]*,[[:space:]]*/\n/g' | \ + sed "s/^[[:space:]]*//;s/[[:space:]]*$//" | \ + sed "s/^/'/;s/$/'/" | \ + paste -sd, -) +} + +generate_header() { + local pgtle_version="$1" + local output_file="$2" + local version_count=${#VERSION_FILES[@]} + local upgrade_count=${#UPGRADE_FILES[@]} + + # Determine version compatibility message + local compat_msg + if [[ "$pgtle_version" == *"+"* ]]; then + local base_version="${pgtle_version%+}" + compat_msg="-- Works on pg_tle >= ${base_version}" + else + local min_version="${pgtle_version%-*}" + local max_version="${pgtle_version#*-}" + compat_msg="-- Works on pg_tle >= ${min_version} and < ${max_version}" + fi + + cat < $to_ver" + echo "SELECT pgtle.install_update_path(" + echo " '${EXTENSION}'," + echo " '${from_ver}'," + echo " '${to_ver}'," + wrap_sql_content "$upgrade_file" + echo ");" + echo +} + +generate_pgtle_sql() { + local pgtle_version="$1" + debug 30 "generate_pgtle_sql: Starting for version $pgtle_version, extension $EXTENSION" + + # Get capability using function (compatible with bash < 4.0) + local capability=$(get_pgtle_capability "$pgtle_version") + local version_dir="pg_tle/${pgtle_version}" + local output_file="${version_dir}/${EXTENSION}.sql" + + # Ensure arrays are initialized (defensive programming) + # Arrays should already be initialized at top level, but ensure they exist + debug 40 "generate_pgtle_sql: Checking array initialization" + debug 30 "generate_pgtle_sql: VERSION_FILES is ${VERSION_FILES+set}, count=${#VERSION_FILES[@]}" + debug 30 "generate_pgtle_sql: UPGRADE_FILES is ${UPGRADE_FILES+set}, count=${#UPGRADE_FILES[@]}" + + if [ -z "${VERSION_FILES+set}" ]; then + echo "WARNING: VERSION_FILES not set, initializing" >&2 + VERSION_FILES=() + fi + if [ -z "${UPGRADE_FILES+set}" ]; then + echo "WARNING: UPGRADE_FILES not set, initializing" >&2 + UPGRADE_FILES=() + fi + + # Create version-specific output directory if needed + mkdir -p "$version_dir" + + echo "Generating: $output_file (pg_tle $pgtle_version)" >&2 + + # Generate SQL to file + { + generate_header "$pgtle_version" "$output_file" + + cat < "$output_file" + + echo " ✓ Generated: $output_file" >&2 +} + +main "$@" + diff --git a/pgtle_versions.md b/pgtle_versions.md new file mode 100644 index 0000000..74f8184 --- /dev/null +++ b/pgtle_versions.md @@ -0,0 +1,46 @@ +# pg_tle Version Support Matrix + +This file documents pg_tle version boundaries that affect pgxntool's pg_tle support code. Each boundary represents a backward-incompatible API change. + +## Version Ranges (pgxntool notation) + +### 1.0.0-1.4.0 +- **pg_tle versions:** 1.0.0 through 1.3.x +- **PostgreSQL support:** 11-17 +- **Features:** Basic extension management, custom data types, authentication hooks + +### 1.4.0-1.5.0 +- **pg_tle versions:** 1.4.0 through 1.4.x +- **PostgreSQL support:** 11-17 +- **API:** Added alignment/storage parameters to `pgtle.create_base_type()` +- **Features:** Custom alignment/storage, enhanced warnings + +### 1.5.0+ +- **pg_tle versions:** 1.5.0 and later (tested through 1.5.2) +- **PostgreSQL support:** 12-18 (dropped PG 11) +- **API:** BREAKING CHANGE - `pgtle.install_extension()` now requires schema parameter +- **Features:** Schema parameter support in installation + +## Key API Changes by Version + +**1.4.0:** Added alignment/storage parameters to `pgtle.create_base_type()` + +**1.5.0** +- Changed `pgtle.install_extension()` signature +- Added required `schema` parameter +- Dropped PostgreSQL 11 support + +## Version Notation + +- `X.Y.Z+` - Works on pg_tle >= X.Y.Z +- `X.Y.Z-A.B.C` - Works on pg_tle >= X.Y.Z and < A.B.C + +**Boundary conditions:** +- `1.5.0+` means >= 1.5.0 (includes 1.5.0) +- `1.4.0-1.5.0` means >= 1.4.0 and < 1.5.0 (excludes 1.5.0) +- `1.0.0-1.4.0` means >= 1.0.0 and < 1.4.0 (excludes 1.4.0) + +## For Complete Details + +- `pgtle.sh` (comments at top) +- https://github.com/aws/pg_tle diff --git a/pgxntool-sync.sh b/pgxntool-sync.sh new file mode 100755 index 0000000..56896ac --- /dev/null +++ b/pgxntool-sync.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# pgxntool-sync.sh - Pull the latest pgxntool via git subtree and reconcile setup files +# +# This performs the two steps needed to update pgxntool inside a project: +# +# 1. git subtree pull -P pgxntool ... (update the embedded pgxntool copy) +# 2. update-setup-files.sh (3-way merge files setup.sh copied out, +# like .gitignore and test/deps.sql) +# +# It is invoked by the `make pgxntool-sync` targets, but can also be run directly +# so you never need make to update pgxntool. +# +# Usage: pgxntool-sync.sh [ []] +# +# repo Git URL (or path) to pull pgxntool from. Defaults to the canonical +# pgxntool repository. +# ref Branch, tag, or commit to pull. Defaults to the `release` tag, which +# always points at the latest released version. +# +# Run from the root of your project (the directory containing pgxntool/). + +set -o errexit -o errtrace -o pipefail +trap 'echo "Error on line ${LINENO}"' ERR + +PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")" +source "$PGXNTOOL_DIR/lib.sh" + +# Canonical source for pgxntool. `release` is a floating tag that the release +# process force-moves to each new version, so it always names the latest +# released version -- exactly what a plain sync should track. +DEFAULT_REPO="https://github.com/Postgres-Extensions/pgxntool.git" +DEFAULT_REF="release" + +repo=${1:-$DEFAULT_REPO} +ref=${2:-$DEFAULT_REF} + +# We must run from the project root: git subtree pull operates on the pgxntool/ +# prefix and update-setup-files.sh resolves paths relative to the current dir. +[[ -d "pgxntool" ]] || die 1 "pgxntool directory not found. Run from your project root." +# Use rev-parse, not [ -d .git ]: in a worktree .git is a file, not a directory. +git rev-parse --git-dir >/dev/null 2>&1 || die 1 "Not in a git repository. Run from your project root." + +# The old commit is the pgxntool subtree HEAD before the pull; update-setup-files.sh +# needs it as the merge base for files that were copied out of pgxntool. +old_commit=$(git log -1 --format=%H -- pgxntool/) + +git subtree pull -P pgxntool --squash -m "Pull pgxntool from $repo $ref" "$repo" "$ref" + +"$PGXNTOOL_DIR/update-setup-files.sh" "$old_commit" diff --git a/run-test-build.sh b/run-test-build.sh new file mode 100755 index 0000000..86a61eb --- /dev/null +++ b/run-test-build.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# pgxntool/run-test-build.sh - Prepare test/build/ for the test-build target +# +# Sets up the generated sql/ directory and ensures expected/*.out files exist +# so pg_regress can run without aborting on missing files. +# +# Usage: run-test-build.sh TESTDIR +# +# Called by the test-build target in base.mk before running installcheck. + +set -e + +TESTDIR="${1:?Usage: run-test-build.sh TESTDIR}" +BUILD_DIR="$TESTDIR/build" +SQL_DIR="$BUILD_DIR/sql" +EXPECTED_DIR="$BUILD_DIR/expected" + +mkdir -p "$SQL_DIR" +mkdir -p "$EXPECTED_DIR" + +# Verify .sql files exist. This script is only called when test-build is +# enabled, so missing files indicate a misconfiguration. +files=("$BUILD_DIR"/*.sql) +if [ ! -f "${files[0]}" ]; then + echo "ERROR: no .sql files found in $BUILD_DIR/" >&2 + exit 1 +fi + +# Sync .sql files to sql/ directory for pg_regress. +# --checksum: compare by content, not size+mtime. rsync's default "quick check" +# assumes equal size+mtime means files are identical, which isn't safe here — +# builds can produce identical-sized files with different content. Checksum +# comparison also avoids unnecessary writes that could trigger antivirus. +# --times: preserve source mtimes on destination files so make's dependency +# tracking works correctly. +# --delete: remove files from sql/ that no longer exist in build/. +# --include/--exclude: select only *.sql from the directory source +# (--delete requires a directory transfer, not individual file arguments). +rsync -r --checksum --times --delete --include='*.sql' --exclude='*' "$BUILD_DIR/" "$SQL_DIR/" + +# Create empty expected/*.out files for .sql tests (if not already present). +# pg_regress requires an expected file to exist for each test; without it +# pg_regress stops immediately rather than running the test and showing the diff. +for file in "$BUILD_DIR"/*.sql; do + out="$EXPECTED_DIR/$(basename "$file" .sql).out" + [ -f "$out" ] || touch "$out" +done diff --git a/setup.sh b/setup.sh index 881ccaa..8a31d0e 100755 --- a/setup.sh +++ b/setup.sh @@ -3,7 +3,13 @@ set -o errexit -o errtrace -o pipefail trap 'echo "Error on line ${LINENO}"' ERR -[ -d .git ] || git init +# Source common library functions (error, die, debug) +PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")" +source "$PGXNTOOL_DIR/lib.sh" + +# Use rev-parse, not [ -d .git ]: in a worktree .git is a file, not a directory, +# so the old check would wrongly re-run `git init` inside a valid working tree. +git rev-parse --git-dir >/dev/null 2>&1 || git init if ! git diff --cached --exit-code; then echo "Git repository is not clean; please commit and try again." >&2 @@ -35,21 +41,44 @@ safecp () { fi } -safecp pgxntool/_.gitignore .gitignore -safecp pgxntool/META.in.json META.in.json +# ============================================================================= +# SETUP FILES +# ============================================================================= +# SETUP_FILES and SETUP_SYMLINKS are defined in lib.sh +# These are also used by update-setup-files.sh for sync updates. +# ============================================================================= + +# Copy tracked setup files (defined in lib.sh) +for entry in "${SETUP_FILES[@]}"; do + src="pgxntool/${entry%%:*}" + dest="${entry##*:}" + # Create parent directory if needed + mkdir -p "$(dirname "$dest")" + safecp "$src" "$dest" +done + +# Create tracked symlinks (defined in lib.sh) +for entry in "${SETUP_SYMLINKS[@]}"; do + dest="${entry%%:*}" + target="${entry##*:}" + mkdir -p "$(dirname "$dest")" + if [ ! -e "$dest" ]; then + echo "Creating symlink $dest -> $target" + ln -s "$target" "$dest" + git add "$dest" + else + echo "$dest already exists" + fi +done +# META.in.json and Makefile are NOT in SETUP_FILES because users heavily customize them +safecp pgxntool/META.in.json META.in.json safecreate Makefile include pgxntool/base.mk make META.json git add META.json -mkdir -p sql test src - -cd test -mkdir -p sql -safecp ../pgxntool/test/deps.sql deps.sql -[ -d pgxntool ] || ln -s ../pgxntool/test/pgxntool . -git add pgxntool +mkdir -p sql test/sql src git status echo "If you won't be creating C code then you can: @@ -58,4 +87,4 @@ rmdir src If everything looks good then -git commit -am 'Add pgxntool (https://github.com/decibel/pgxntool/tree/release)'" +git commit -am 'Add pgxntool (https://github.com/Postgres-Extensions/pgxntool/tree/release)'" diff --git a/test/bin/check-stale-expected.sh b/test/bin/check-stale-expected.sh new file mode 100755 index 0000000..6a4d221 --- /dev/null +++ b/test/bin/check-stale-expected.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# +# check-stale-expected.sh - Catch orphaned/unexpected test/expected/ files +# +# test/expected/*.out must mirror test/sql/*.sql 1:1 (likewise +# test/build/expected/*.out vs test/build/*.sql, when test-build is in use). +# It's easy to leave a stale .out behind after renaming or removing a .sql +# file; this makes `make test` fail loudly instead of letting it linger +# unnoticed. +# +# test/install/ is NOT checked here: its expected output lives alongside the +# .sql files (test/install/foo.out), not in a separate expected/ +# subdirectory, so there's no 1:1 directory mirror to compare. +# +# pg_regress supports up to 10 alternate expected-output files per test +# (test.out, test_0.out .. test_9.out - see get_alternative_expectfile() in +# pg_regress.c), tried in turn when the primary doesn't match. See the case +# block below for how that's recognized and the tradeoff involved. +# +# expected/ is also checked for files that aren't *.out at all -- there's no +# legitimate reason for anything else to live there (stray editor swap +# files, .orig files from a botched merge, etc.). This is a distinct +# failure class from an orphaned .out file: different message, different +# exit code (see below), and independently disable-able via +# PGXNTOOL_CHECK_EXPECTED_FILE_TYPES=no. +# +# Exit code is a bitmask so the two failure classes can be told apart: +# 1 - one or more orphaned .out files (no corresponding .sql) +# 2 - one or more unexpected non-.out files in expected/ +# 3 - both +# +# Usage: check-stale-expected.sh [check-file-types] +# check-file-types: yes|no (default yes) -- controls the non-*.out +# file check described above. Taken as a positional argument (rather +# than an environment variable) so it's easy to vary directly in a +# test loop instead of having to set/unset an env var around each +# invocation. + +set -o errexit -o errtrace -o pipefail + +BASEDIR=$(dirname "$0") +source "$BASEDIR/../../lib.sh" + +if [ $# -lt 1 ] || [ $# -gt 2 ]; then + die 1 "Usage: check-stale-expected.sh [check-file-types]" +fi + +testdir="$1" +failed=0 + +check_file_types=$(printf '%s' "${2:-yes}" | tr '[:upper:]' '[:lower:]') + +# Usage: check_pair +check_pair() { + local sqldir="$1" expdir="$2" + local f base + + [ -d "$expdir" ] || return 0 + + for f in "$expdir"/*; do + # Without nullglob, this glob is left as the literal unexpanded pattern + # string when $expdir has no entries at all; skip that non-existent + # "file" rather than treating it as real input. + [ -f "$f" ] || continue + + case "$f" in + *.out) + base=$(basename "$f" .out) + case "$base" in + # pg_regress supports up to 10 alternate expected-output files + # per test (test.out, test_0.out .. test_9.out -- see + # get_alternative_expectfile() in pg_regress.c), so a trailing + # _N here doesn't necessarily mean an orphaned file. Recognizing + # this is pure string matching -- it doesn't require checking + # whether a file exists -- so the suffix is stripped unconditionally, + # leaving exactly one existence check per file (below) rather than + # one to recognize the pattern and a second to validate it. + # Tradeoff: a real test literally named e.g. foo_1.sql (with no + # foo.sql at all) would be misidentified as an alternate file for + # a nonexistent "foo" and incorrectly flagged as stale. Accepted + # as a vanishingly rare edge case. + *_[0-9]) + base=${base%_*} + ;; + esac + + if [ ! -f "$sqldir/$base.sql" ]; then + error "$f has no corresponding $sqldir/$base.sql" + (( failed |= 1 )) + fi + ;; + *) + if [ "$check_file_types" = yes ]; then + error "unexpected non-.out file in $expdir: $f" + (( failed |= 2 )) + fi + ;; + esac + done +} + +check_pair "$testdir/sql" "$testdir/expected" +check_pair "$testdir/build" "$testdir/build/expected" + +exit "$failed" + +# vi: expandtab ts=2 sw=2 diff --git a/update-setup-files.sh b/update-setup-files.sh new file mode 100755 index 0000000..f814605 --- /dev/null +++ b/update-setup-files.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# +# update-setup-files.sh - Update files that were initially copied by setup.sh +# +# This script handles the 3-way merge of setup files after a pgxntool subtree +# update. It compares the old pgxntool version, new pgxntool version, and +# user's current file to determine the appropriate action: +# +# 1. If pgxntool didn't change the file: skip (nothing to do) +# 2. If user hasn't modified the file: auto-update +# 3. If both changed: 3-way merge with conflict markers +# +# Usage: update-setup-files.sh +# +# The old commit is the pgxntool subtree commit BEFORE the sync. + +set -o errexit -o errtrace -o pipefail +trap 'echo "Error on line ${LINENO}"' ERR + +PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")" +source "$PGXNTOOL_DIR/lib.sh" + +# SETUP_FILES and SETUP_SYMLINKS are defined in lib.sh + +# ============================================================================= +# Functions +# ============================================================================= + +usage() { + echo "Usage: $0 " + echo + echo "Updates setup files after a pgxntool subtree sync." + echo + echo "Arguments:" + echo " old-pgxntool-commit The pgxntool commit hash BEFORE the sync" + exit 1 +} + +# Get file content from a specific commit +# Usage: get_old_content +get_old_content() { + local commit=$1 + local path=$2 + git show "${commit}:pgxntool/${path}" 2>/dev/null +} + +# Get current file content from pgxntool directory +# Usage: get_new_content +get_new_content() { + local path=$1 + cat "pgxntool/${path}" 2>/dev/null +} + +# Process a single setup file +# Usage: process_file +process_file() { + local source=$1 + local dest=$2 + local old_commit=$3 + + # Get the three versions + local old_content new_content user_content + + old_content=$(get_old_content "$old_commit" "$source") || { + debug 20 "Could not get old version of $source (new file in pgxntool?)" + old_content="" + } + + new_content=$(get_new_content "$source") || { + error "Could not read pgxntool/$source" + return 1 + } + + # Check if destination exists + if [[ ! -e "$dest" ]]; then + echo " $dest: creating (file was missing)" + cp "pgxntool/$source" "$dest" + return 0 + fi + + user_content=$(cat "$dest") + + # Step 1: Did pgxntool change this file? + if [[ "$old_content" == "$new_content" ]]; then + debug 30 "$dest: pgxntool unchanged, skipping" + return 0 + fi + + # Step 2: Did user modify their copy? + if [[ "$user_content" == "$old_content" ]]; then + echo " $dest: updated (you hadn't modified it)" + cp "pgxntool/$source" "$dest" + return 0 + fi + + # Step 3: Both changed - need 3-way merge + echo " $dest: attempting 3-way merge..." + + # Create temp files for git merge-file + local tmp_old tmp_new + tmp_old=$(mktemp) + tmp_new=$(mktemp) + trap "rm -f '$tmp_old' '$tmp_new'" RETURN + + echo "$old_content" > "$tmp_old" + echo "$new_content" > "$tmp_new" + + # git merge-file modifies the first file in place + # Returns 0 on clean merge, >0 if conflicts (but still writes result) + if git merge-file -L "yours" -L "old pgxntool" -L "new pgxntool" \ + "$dest" "$tmp_old" "$tmp_new"; then + echo " $dest: merged cleanly (please review)" + else + echo " $dest: CONFLICTS - resolve manually" + fi +} + +# Process a symlink +# Usage: process_symlink +process_symlink() { + local dest=$1 + local target=$2 + + if [[ -L "$dest" ]]; then + local current_target + current_target=$(readlink "$dest") + if [[ "$current_target" == "$target" ]]; then + debug 30 "$dest: symlink unchanged" + else + echo " $dest: symlink points to '$current_target', expected '$target'" + echo " (not auto-fixing - please check manually)" + fi + elif [[ -e "$dest" ]]; then + echo " $dest: exists but is not a symlink (expected symlink to $target)" + else + echo " $dest: creating symlink to $target" + ln -s "$target" "$dest" + fi +} + +# ============================================================================= +# Main +# ============================================================================= + +[[ $# -eq 1 ]] || usage + +old_commit=$1 + +# Verify we're in a git repo with pgxntool subtree. +# Use rev-parse, not [ -d .git ]: in a worktree .git is a file, not a directory. +[[ -d "pgxntool" ]] || die 1 "pgxntool directory not found. Run from project root." +git rev-parse --git-dir >/dev/null 2>&1 || die 1 "Not in a git repository." + +# Verify the old commit is valid +if ! git cat-file -e "${old_commit}^{commit}" 2>/dev/null; then + die 1 "Invalid commit: $old_commit" +fi + +echo "Checking setup files for updates..." +echo + +# Process regular files +for entry in "${SETUP_FILES[@]}"; do + source="${entry%%:*}" + dest="${entry##*:}" + process_file "$source" "$dest" "$old_commit" +done + +# Process symlinks +for entry in "${SETUP_SYMLINKS[@]}"; do + dest="${entry%%:*}" + target="${entry##*:}" + process_symlink "$dest" "$target" +done + +# Prune pgxntool's own dev-only dirs (.github/, .claude/) that the subtree pull +# re-introduces but that don't belong in a consuming project (see lib.sh). +prune_pgxntool_dev_dirs + +echo +echo "Done. Review changes with 'git diff' and commit when ready." diff --git a/verify-results-pgtap.sh b/verify-results-pgtap.sh new file mode 100755 index 0000000..dd8d7e9 --- /dev/null +++ b/verify-results-pgtap.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# pgxntool/verify-results-pgtap.sh - Check pgtap results before 'make results' +# +# Scans pgtap output files for failures and plan mismatches, then checks +# regression.diffs as a fallback. Exits non-zero if any problems are found. +# +# Usage: verify-results-pgtap.sh TESTOUT +# +# Called by the verify-results target in base.mk (pgtap mode). + +set -e + +TESTOUT="${1:?Usage: verify-results-pgtap.sh TESTOUT}" + +# Check for pgtap failures in result files (excluding TODO items) +failed=0 +for f in "$TESTOUT"/results/*.out; do + [ -f "$f" ] || continue + if grep -q '^not ok' "$f"; then + notok=$(grep '^not ok' "$f" | grep -v '# TODO' || true) + if [ -n "$notok" ]; then + echo "ERROR: pgtap failure detected in $f" + echo "$notok" + failed=1 + fi + fi + if grep -q 'Looks like you planned' "$f"; then + echo "ERROR: pgtap plan mismatch in $f" + grep 'Looks like you planned' "$f" + failed=1 + fi +done +if [ $failed -ne 0 ]; then + echo + echo "pgtap failures detected. Cannot run 'make results'." + exit 1 +fi + +# Also check regression.diffs (output mismatch even if pgtap all passed) +if [ -r "$TESTOUT/regression.diffs" ]; then + echo "ERROR: Tests are failing. Cannot run 'make results'." + echo "Fix test failures first, then run 'make results'." + echo + echo "See $TESTOUT/regression.diffs for details:" + cat "$TESTOUT/regression.diffs" + exit 1 +fi From 6f27479335e32a4a4b42d9d1903a6c4d83986f21 Mon Sep 17 00:00:00 2001 From: jnasbyupgrade Date: Wed, 29 Jul 2026 16:43:31 -0500 Subject: [PATCH 2/2] Prune pgxntool dev-only dirs after sync update-setup-files.sh pruned pgxntool/.claude and pgxntool/.github, which are pgxntool's own dev tooling and not meant to be embedded in consuming projects. --- pgxntool/.claude/CLAUDE.md | 28 -- pgxntool/.claude/development.md | 47 --- pgxntool/.github/workflows/CLAUDE.md | 71 ---- pgxntool/.github/workflows/ci.yml | 336 ------------------ .../.github/workflows/claude-code-review.yml | 115 ------ pgxntool/.github/workflows/claude.yml | 51 --- pgxntool/.github/workflows/protect-label.yml | 146 -------- 7 files changed, 794 deletions(-) delete mode 100644 pgxntool/.claude/CLAUDE.md delete mode 100644 pgxntool/.claude/development.md delete mode 100644 pgxntool/.github/workflows/CLAUDE.md delete mode 100644 pgxntool/.github/workflows/ci.yml delete mode 100644 pgxntool/.github/workflows/claude-code-review.yml delete mode 100644 pgxntool/.github/workflows/claude.yml delete mode 100644 pgxntool/.github/workflows/protect-label.yml diff --git a/pgxntool/.claude/CLAUDE.md b/pgxntool/.claude/CLAUDE.md deleted file mode 100644 index e2e74fe..0000000 --- a/pgxntool/.claude/CLAUDE.md +++ /dev/null @@ -1,28 +0,0 @@ -# Claude Development Notes - -This file contains guidance for Claude Code when working in this repository. -It is excluded from distributions via `.gitattributes export-ignore`. - -## CI Monitoring After Every Push - -**REQUIRED**: After every `git push`, immediately start a background task to -monitor the CI run for that push. If you pushed to both pgxntool and -pgxntool-test, start a background task for each repo — do not monitor them -sequentially. - -Use `gh run watch` or poll with `gh run list` / `gh pr checks` in the -background task. Report failures to the user as soon as they are detected; -do not wait for all jobs to finish before reporting. - -## Multiple Concurrent Sessions - -It is common to have multiple Claude Code sessions open simultaneously across -pgxntool and pgxntool-test. To avoid cross-session interference: - -**If you are asked to do something on an existing PR that you did not open or -are not already working on in this session, immediately ask for confirmation -before proceeding.** For example: "I see PR #32 exists. Were you asking me to -work on that, or did you mean to send this to a different session?" - -This applies to: editing PR branches, pushing to them, closing/reopening them, -adding commits, modifying PR descriptions, or any other PR-level action. diff --git a/pgxntool/.claude/development.md b/pgxntool/.claude/development.md deleted file mode 100644 index 886484a..0000000 --- a/pgxntool/.claude/development.md +++ /dev/null @@ -1,47 +0,0 @@ -# pgxntool Development Guidelines - -**THIS FILE IS FOR PGXNTOOL DEVELOPERS ONLY.** - -If you are an extension developer using pgxntool in your project, this file does not -apply to you. See the top-level `CLAUDE.md` instead. - -## Critical: Work from pgxntool-test, Not Here - -**NEVER make changes to pgxntool directly from this repository.** - -pgxntool development must be done from a checkout of **pgxntool-test**, which contains -the full test infrastructure. Working here directly means you cannot run tests, and -any changes you commit cannot be validated before merging. - -**Correct workflow:** -1. Clone or use an existing checkout of `pgxntool-test` -2. Work in a worktree: both `pgxntool/` and `pgxntool-test/` will be siblings -3. Make changes to `pgxntool/` from within that pgxntool-test context -4. Run the test suite via `make test` in pgxntool-test before committing - -**See:** https://github.com/Postgres-Extensions/pgxntool-test for the full development -workflow. - ---- - -## Makefile Variable Assignment Rules - -**RULE: Do not use `:=` (simply expanded) unless you have a specific need for immediate evaluation.** - -Use `=` (recursively expanded) for standard variable assignments. Reserve `:=` for cases where the right-hand side must be evaluated exactly once at assignment time — for example, when assigning the result of a `$(call ...)` function that references the variable being set (which would cause infinite recursion with `=`). - -When a variable must also override command-line values, combine `override` with `:=` — but only where `override` is genuinely needed. - -## Debug Level Rules (lib.sh `debug` function) - -`debug LEVEL "msg"` prints when `DEBUG >= LEVEL`. LEVEL encodes how noisy/esoteric a message is — how far you'd crank `DEBUG` before you'd want to see it — **not** code nesting depth. A top-level line can warrant a high level if it's esoteric, and loop-body detail is usually high precisely because it's noisy. Judge by signal-to-noise. - -The tiers are anchors, not strict multiples of 10 — any value in range is fine, leaving room to fine-tune between existing calls without renumbering: - -- **10**: Critical errors, important warnings -- **20**: Warnings, significant state changes -- **30**: General debugging, function entry/exit, array operations -- **40**: Verbose details, loop iterations -- **50+**: Maximum verbosity (per-iteration innards) - -Note: The BATS test helper `debug` function (in `tests/lib/helpers.bash` in pgxntool-test) uses a separate 1–5 scale controlled by `$TESTDEBUG`. The two systems are independent. diff --git a/pgxntool/.github/workflows/CLAUDE.md b/pgxntool/.github/workflows/CLAUDE.md deleted file mode 100644 index 1d5a6f0..0000000 --- a/pgxntool/.github/workflows/CLAUDE.md +++ /dev/null @@ -1,71 +0,0 @@ -# .github/workflows — CI Architecture - -## Workflow files - -- **`ci.yml`** — main CI for pgxntool pull requests. Runs `check-test-pr` (verifies - the paired pgxntool-test PR's CI passed), then optionally runs `test` (only for the - commit-with-no-tests path — see below). -- **`protect-label.yml`** — enforces that only maintainers with write access can apply - or remove the `commit-with-no-tests` label. - -## Normal CI flow (paired test PR exists) - -When a pgxntool PR has a corresponding open PR in pgxntool-test with the same branch -name, the `check-test-pr` job polls (up to 20 minutes) for that test PR's CI to -complete and pass. If it passes, pgxntool CI passes — **no tests run here**. Tests run -exactly once, in pgxntool-test's own CI. - -## commit-with-no-tests path - -When a maintainer applies the `commit-with-no-tests` label (and no paired test PR -exists), the `test` job runs tests directly in pgxntool CI against pgxntool-test/master. -This is the rare exception, not the norm. - -## Doc-only bypass - -`check-test-pr` checks this first, before the paired-test-PR lookup or the -`commit-with-no-tests` label: if every changed file in the PR is pure -documentation (`*.md`, `*.asc`, `*.adoc`, `*.asciidoc`, anywhere including -under `.claude/`, but never under `.github/` — workflow definitions carry -real behavioral weight regardless of extension), it skips both the paired -branch requirement and the `test` job entirely. `claude-code-review.yml` is -a separate workflow gated by its own `if:` and always still runs. - -## Cross-repo reusable workflow — tradeoffs and constraints - -The `test` job calls a reusable workflow from pgxntool-test: -```yaml -uses: Postgres-Extensions/pgxntool-test/.github/workflows/run-tests.yml@ -``` - -GitHub Actions requires the `uses:` ref to be a **static string** — expressions like -`${{ }}` are not supported in the repo/path portion or the `@ref` suffix in practice. - -### The @branch → @master ref - -While developing on a feature branch where pgxntool-test also has changes, this ref -is set to `@` so CI can find `run-tests.yml` before it lands on master. - -**IMPORTANT**: This ref must be updated to `@master` before pgxntool merges. The -correct merge order is: **pgxntool-test merges first**, then update this ref to -`@master`, then pgxntool merges. - -**For Claude**: Do NOT leave a `@` ref without explicit user approval. The -user merges directly from the PR page — there are no manual steps between merges. -See `.github/workflows/CLAUDE.md` in pgxntool-test for the full picture. - -### Changes to run-tests.yml - -`run-tests.yml` lives in pgxntool-test and is the single source of truth for all test -steps. If it changes, pgxntool's CI uses `@master` — so it won't see the new version -until pgxntool-test merges. This is acceptable because: -- Changes to `run-tests.yml` require a paired test PR (not commit-with-no-tests) -- When a paired test PR exists, pgxntool's `test` job is skipped anyway -- The two scenarios are mutually exclusive in practice - -## Label name - -The label `commit-with-no-tests` is defined as a const (`NO_TEST_LABEL`) in `ci.yml` -and as `LABEL` in `protect-label.yml`. The job-level `if:` condition in -`protect-label.yml` must also use the literal string (YAML can't reference JS consts) -— keep these in sync if the label name ever changes. diff --git a/pgxntool/.github/workflows/ci.yml b/pgxntool/.github/workflows/ci.yml deleted file mode 100644 index 2bfcbaa..0000000 --- a/pgxntool/.github/workflows/ci.yml +++ /dev/null @@ -1,336 +0,0 @@ -name: CI - -on: - pull_request: - # We use 'pull_request' (not 'pull_request_target') deliberately. - # 'pull_request_target' runs with write access to the base repo, which is - # a security risk for untrusted fork code. Since this workflow only reads - # from other public repos (no secrets needed), 'pull_request' is correct - # and safe even for fork PRs. - -permissions: - contents: read # required by actions/checkout in the reusable test workflow - pull-requests: read - checks: read - -concurrency: - group: ci-pr-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - check-test-pr: - name: Check for paired pgxntool-test PR - runs-on: ubuntu-latest - # This check polls until the paired pgxntool-test CI run completes - # (up to 20 minutes). The job timeout gives a few minutes of headroom. - timeout-minutes: 25 - outputs: - run-tests: ${{ steps.check.outputs.run_tests }} - test-ref: ${{ steps.check.outputs.test_ref }} - - steps: - - name: Find paired pgxntool-test PR or check commit-with-no-tests label - id: check - # Pinned to an immutable SHA (supply-chain hardening); comment tracks the tag. - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - # GITHUB_TOKEN is sufficient for reading public repos. If these repos - # are ever made private, replace with a PAT stored as a secret with - # 'repo' scope on both repos. Note: PAT expiration causes silent - # failures here — the API returns 401 and the job errors out instead - # of failing gracefully with a useful message. - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const branch = context.payload.pull_request.head.ref; - const prNumber = context.payload.pull_request.number; - // Single source of truth for the label name. Must also match the - // literal string in the protect-label.yml job-level `if:` condition - // (YAML expressions can't reference JS constants). - const NO_TEST_LABEL = 'commit-with-no-tests'; - - // DOC-ONLY BYPASS: skip both the paired-test-PR requirement and - // the actual Postgres test run when every changed file is pure - // documentation. This is independent of, and takes priority - // over, everything below — a doc-only PR needs neither a paired - // branch nor the NO_TEST_LABEL override. - // - // Files under .github/ are never doc-only even if their - // extension matches (they're workflow definitions with real - // behavioral weight, some running with pull_request_target - // privileges). Everything else — including .claude/*.md prompt - // and command docs, and README.html (a generated rendering of - // README.asc, no execution weight of its own) — counts. - // - // This does NOT skip claude-review: that's a separate workflow - // gated by its own `if:`, unaffected by this check's outputs. - const DOC_EXTENSIONS = /\.(md|asc|adoc|asciidoc|html)$/i; - const changedFiles = await github.paginate(github.rest.pulls.listFiles, { - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - per_page: 100 - }); - // Check both filename and previous_filename: a rename like - // src/foo.sql -> docs/foo.md must not read as doc-only just - // because the new name matches — the old path is a real change. - const changedPaths = changedFiles.flatMap(f => - f.previous_filename ? [f.filename, f.previous_filename] : [f.filename] - ); - const isDocOnly = changedPaths.length > 0 && changedPaths.every(p => - DOC_EXTENSIONS.test(p) && !p.startsWith('.github/') - ); - if (isDocOnly) { - core.info( - `All ${changedFiles.length} changed file(s) are documentation-only ` + - `(matched ${DOC_EXTENSIONS}, none under .github/); skipping the ` + - `paired-test-PR requirement and the Postgres test matrix.` - ); - core.setOutput('run_tests', 'false'); - core.setOutput('test_ref', ''); - return; - } - - // master-to-master PRs have no paired test PR by convention. - // Run tests against pgxntool-test/master directly. - // - // If a fork PR's branch is named 'master', that's almost certainly - // a mistake (contributors should use a feature branch), but we - // don't block it — just warn visibly as an annotation on the run. - // Note: pull_request gives a read-only token for fork PRs, so we - // can't post a PR comment back to the upstream repo from here. - // Gate on the BASE branch too: this shortcut is only for - // master-to-master PRs. A PR from master into some other base must - // still go through the normal paired-test lookup below. - if (branch === 'master' && context.payload.pull_request.base.ref === 'master') { - const headRepo = context.payload.pull_request.head.repo; - const isBaseRepo = - headRepo?.owner?.login === context.repo.owner && - headRepo?.name === context.repo.repo; - if (!isBaseRepo) { - core.warning( - `PR head branch is named 'master' but comes from a fork ` + - `(${headRepo?.full_name ?? 'unknown'}). Contributors should ` + - `use a feature branch, not master. Proceeding with tests ` + - `against pgxntool-test/master.` - ); - } - core.setOutput('run_tests', 'true'); - core.setOutput('test_ref', 'master'); - return; - } - - // The owner of this PR's head repo — the contributor's fork owner - // for fork PRs, or the base repo owner for maintainer PRs. - // The paired pgxntool-test PR must come from the SAME owner. - // We never cross-match PRs across different contributors' forks. - const prOwner = context.payload.pull_request.head.repo?.owner?.login; - - // Look for open pgxntool-test PRs with the SAME branch name AND - // the same fork owner. Branch names must match exactly. - // - // The GitHub API's 'head' filter requires "owner:branch" format. - // We list all open PRs and filter locally — safe for repos with - // few open PRs, and avoids needing to know the fork repo name. - // paginate() fetches all pages automatically, so this is correct - // even if pgxntool-test ever exceeds 100 open PRs (the per_page cap). - const prs = await github.paginate(github.rest.pulls.list, { - owner: context.repo.owner, - repo: 'pgxntool-test', - state: 'open', - per_page: 100 - }); - - const matching = prs.filter(pr => - pr.head.ref === branch && - pr.head.repo?.owner?.login === prOwner - ); - if (matching.length > 1) { - core.setFailed( - `Multiple open pgxntool-test PRs from ${prOwner} match branch ` + - `'${branch}'. Cannot determine which one to use.\n\n` + - `Close all but one, then re-run this check.` - ); - return; - } - - const testPR = matching.length === 1 ? matching[0] : null; - - if (testPR) { - // Error if the no-test label is also set — that's contradictory. - // Re-fetch the PR live (not from payload) in case the label was - // added after this workflow was triggered. - const { data: currentPR } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber - }); - if (currentPR.labels.some(l => l.name === NO_TEST_LABEL)) { - core.setFailed( - `PR has the '${NO_TEST_LABEL}' label, but a paired ` + - `pgxntool-test PR #${testPR.number} exists on branch '${branch}'.\n\n` + - `Remove the '${NO_TEST_LABEL}' label — it should only be used ` + - `when there is genuinely no paired test PR.` - ); - return; - } - - // A paired test PR exists. Verify its CI passed for the exact - // current HEAD SHA and that the run is recent enough to be valid. - const sha = testPR.head.sha; - const testPRUrl = - `https://github.com/${context.repo.owner}/pgxntool-test/pull/${testPR.number}`; - const recheckUrl = - `https://github.com/${context.repo.owner}/${context.repo.repo}/pull/${prNumber}/checks`; - - core.info(`Found pgxntool-test PR #${testPR.number} (${sha.slice(0, 7)})`); - - // Poll until all check runs for the exact HEAD SHA complete. - // Using 'ref: sha' (not branch name) ensures we only see runs for - // this commit — never stale runs from an older push on the same branch. - // - // We poll rather than fail immediately because both repos are often - // pushed close together. When that happens, pgxntool CI starts while - // pgxntool-test CI may not have queued yet. We wait up to 20 minutes. - const POLL_INTERVAL_MS = 30 * 1000; - const MAX_WAIT_MS = 20 * 60 * 1000; - const waitStart = Date.now(); - let runs; - - while (true) { - // per_page: 100 is intentional here — a single commit will - // not realistically have 100+ CI check runs, so pagination - // is unnecessary. (pulls.list uses paginate() above because - // an active repo could have many open PRs.) - const { data: checks } = await github.rest.checks.listForRef({ - owner: context.repo.owner, - repo: 'pgxntool-test', - ref: sha, - per_page: 100 - }); - runs = checks.check_runs; - - const incomplete = runs.filter(r => r.status !== 'completed'); - if (runs.length > 0 && incomplete.length === 0) break; - - const elapsed = Date.now() - waitStart; - if (elapsed >= MAX_WAIT_MS) { - const mins = Math.round(elapsed / 60000); - if (runs.length === 0) { - core.setFailed( - `pgxntool-test PR #${testPR.number} has no CI runs for ` + - `SHA ${sha.slice(0, 7)} after waiting ${mins} min.\n\n` + - `Push a commit (or manually re-run CI) on the test PR:\n` + - ` Test PR: ${testPRUrl}\n` + - ` Re-run this check: ${recheckUrl}` - ); - } else { - const names = incomplete.map(r => r.name).join(', '); - core.setFailed( - `pgxntool-test PR #${testPR.number} CI did not finish within ` + - `${mins} min for SHA ${sha.slice(0, 7)}: ${names}\n\n` + - ` Test PR: ${testPRUrl}\n` + - ` Re-run this check: ${recheckUrl}` - ); - } - return; - } - - if (runs.length === 0) { - core.info(`No CI runs yet for pgxntool-test PR #${testPR.number} (${sha.slice(0, 7)}); waiting 30s...`); - } else { - const names = incomplete.map(r => r.name).join(', '); - core.info(`pgxntool-test CI still running (${names}); waiting 30s...`); - } - await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL_MS)); - } - - // All checks complete — look for failures. - // 'success', 'skipped', 'neutral' are non-blocking. - const failed = runs.filter( - r => !['success', 'skipped', 'neutral'].includes(r.conclusion) - ); - if (failed.length > 0) { - const names = failed.map(r => `${r.name} (${r.conclusion})`).join(', '); - core.setFailed( - `pgxntool-test PR #${testPR.number} CI failed for ` + - `SHA ${sha.slice(0, 7)}: ${names}\n\n` + - `Fix the test PR CI, then re-run this check:\n` + - ` Test PR: ${testPRUrl}\n` + - ` Re-run this check: ${recheckUrl}` - ); - return; - } - - core.info( - `pgxntool-test PR #${testPR.number} CI passed for ` + - `SHA ${sha.slice(0, 7)} — tests run there, not here.` - ); - core.setOutput('run_tests', 'false'); - core.setOutput('test_ref', sha); - return; - } - - // No paired test PR found. Check for the NO_TEST_LABEL label, - // which a maintainer can apply when a pgxntool change genuinely - // needs no test changes (unusual). - // - // We make a live API call rather than reading from the event - // payload. The payload is a snapshot from when this workflow was - // triggered — a maintainer may have added the label after that. - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber - }); - - if (pr.labels.some(l => l.name === NO_TEST_LABEL)) { - core.info( - `'${NO_TEST_LABEL}' label is present; running tests ` + - "against pgxntool-test/master. The protect-label workflow " + - "ensures only maintainers can apply this label." - ); - core.setOutput('run_tests', 'true'); - core.setOutput('test_ref', 'master'); - return; - } - - // Neither a paired test PR nor the override label was found. - // Fail with a clear, actionable message. - core.setFailed( - `No paired pgxntool-test PR found for branch '${branch}', ` + - `and no '${NO_TEST_LABEL}' label on this PR.\n\n` + - `pgxntool changes should always be paired with matching test\n` + - `changes in pgxntool-test. This check enforces that pairing.\n\n` + - `To resolve:\n` + - ` 1. Open a PR in pgxntool-test from the SAME account (${prOwner}),\n` + - ` on a branch ALSO named '${branch}'. Both the branch name and\n` + - ` the head owner must match exactly for the pairing to work.\n\n` + - ` 2. If this pgxntool change truly needs no test updates (unusual),\n` + - ` ask a maintainer to apply the '${NO_TEST_LABEL}' label.\n` + - ` Only maintainers can apply this label. It is not a normal\n` + - ` shortcut — most pgxntool changes require test updates.\n\n` + - `See: https://github.com/Postgres-Extensions/pgxntool-test#ci-and-contributing` - ); - - test: - needs: check-test-pr - if: needs.check-test-pr.outputs.run-tests == 'true' - # ----------------------------------------------------------------------- - # CROSS-REPO REUSABLE WORKFLOW — READ BEFORE CHANGING THIS REF - # See: .github/workflows/CLAUDE.md for full architecture notes. - # - # The ref must be a static string — GitHub Actions does not support - # expressions in uses:. It points at pgxntool-test's run-tests.yml on - # master. (During feature-branch development this is temporarily set to - # @ so CI can find run-tests.yml before it lands on master, and - # flipped back to @master once pgxntool-test/ has merged.) - # ----------------------------------------------------------------------- - uses: Postgres-Extensions/pgxntool-test/.github/workflows/run-tests.yml@master - with: - # pgxntool: this PR's own branch, on its own account (a fork for fork PRs). - pgxntool-owner: ${{ github.event.pull_request.head.repo.owner.login }} - pgxntool-branch: ${{ github.event.pull_request.head.ref }} - # pgxntool-test: no paired test PR in this path, so use canonical master - # from Postgres-Extensions only (never a fork's master). - pgxntool-test-owner: Postgres-Extensions - pgxntool-test-ref: master diff --git a/pgxntool/.github/workflows/claude-code-review.yml b/pgxntool/.github/workflows/claude-code-review.yml deleted file mode 100644 index afa5099..0000000 --- a/pgxntool/.github/workflows/claude-code-review.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: Claude Code Review - -# Runs on PRs INTO this repo. We use pull_request_target (not pull_request) so -# that PRs from a fork can access CLAUDE_CODE_OAUTH_TOKEN — GitHub withholds -# secrets from `pull_request` runs triggered by forks, which is why the plain -# `pull_request` version never worked for fork PRs. -# -# SECURITY: pull_request_target runs in the BASE repo with secrets and a -# write-capable token. The job is gated to PRs from the trusted `jnasbyupgrade` -# fork only — an arbitrary external fork can never trigger this secret-bearing -# job. The workflow file always comes from the base branch (master), so a PR -# cannot modify the reviewer that runs on it. We never check out the fork's PR -# head: GitHub Actions refuses that combination by default (the "pwn request" -# guard — see actions/checkout's allow-unsafe-pr-checkout), and -# anthropics/claude-code-action's own docs (docs/security.md) recommend -# checking out the base ref and letting the action read PR content via the -# GitHub API instead. The code-review prompt passes the PR number; the action -# has a GitHub token and pull-requests read/write, so it fetches the diff -# itself (e.g. `gh pr diff`) without ever writing fork code to disk. -on: - pull_request_target: - types: [opened, synchronize, reopened, ready_for_review] - -concurrency: - group: claude-review-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - claude-review: - # Trusted fork only, and skip drafts (don't spend API/CI on unfinished PRs). - # To add more trusted owners, extend the head-owner check. - if: >- - github.event.pull_request.draft == false && - github.event.pull_request.head.repo.owner.login == 'jnasbyupgrade' - runs-on: ubuntu-latest - timeout-minutes: 60 - permissions: - contents: read - pull-requests: write # post the review comments - checks: read # read sibling check-runs for the cost gate - # write (not just read) needed so claude-code-action's internal - # bun-setup step can save its cache; read-only causes a harmless but - # noisy "Cache reservation failed: cache write denied: token has no - # writable scopes" warning. - actions: write - steps: - # COST GATE: the paid Claude review is the last thing to run. Wait for the - # PR head's OTHER check-runs to finish and only proceed if they are clean. - # If any sibling check failed we skip the review to avoid spending money - # reviewing a PR that is already known-broken. Uniform across all repos: - # it discovers sibling checks dynamically (no per-repo workflow names). - # - decision=run : all sibling checks completed with a good conclusion, - # OR no sibling checks exist after a short grace window - # (nothing to gate on), OR the poll timed out is treated - # as skip (see below). - # - decision=skip : at least one sibling check failed/cancelled/etc, or - # we timed out waiting for still-pending checks. - # We exclude this workflow's own check-run (job name `claude-review`) so the - # gate never waits on or fails because of itself. - - name: Wait for CI; skip the paid review if any check failed - id: gate - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - SHA: ${{ github.event.pull_request.head.sha }} - run: | - decision=skip - for i in $(seq 1 72); do # ~24 min max - json=$(gh api "repos/$REPO/commits/$SHA/check-runs" --paginate \ - --jq '[.check_runs[] | select(.name != "claude-review")]' 2>/dev/null) || json='' - [ -z "$json" ] && { sleep 20; continue; } - total=$(jq 'length' <<<"$json") - if [ "$total" -eq 0 ]; then - [ "$i" -ge 9 ] && { decision=run; break; } # ~3 min grace: nothing to gate on - sleep 20; continue - fi - pending=$(jq '[.[]|select(.status!="completed")]|length' <<<"$json") - if [ "$pending" -eq 0 ]; then - bad=$(jq '[.[]|select((.conclusion//"")|test("^(failure|cancelled|timed_out|action_required|stale)$"))]|length' <<<"$json") - [ "$bad" -eq 0 ] && decision=run || decision=skip - break - fi - sleep 20 - done - echo "decision=$decision" >> "$GITHUB_OUTPUT" - echo "gate decision: $decision" - - - name: Check out base branch - if: steps.gate.outputs.decision == 'run' - # Intentionally tracks the major-version tag (not a pinned SHA) so - # upstream fixes are picked up automatically. - # - # No `repository:`/`ref:` here on purpose — this checks out the base - # branch (master), never the fork's PR head. See the SECURITY note - # above. - uses: actions/checkout@v6 - with: - fetch-depth: 1 - persist-credentials: false - - - name: Run Claude Code Review - if: steps.gate.outputs.decision == 'run' - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - # Provide github_token so the action uses it directly for GitHub API - # calls instead of the OIDC->GitHub-App-token exchange, which 401s under - # pull_request_target. GITHUB_TOKEN is repo/workflow-scoped (independent - # of the actor's role) and has pull-requests: write here. - github_token: ${{ secrets.GITHUB_TOKEN }} - # NOTE: plugin_marketplaces can't be pinned — it tracks the - # marketplace repo's default branch (upstream anthropics/claude-code). - plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' - plugins: 'code-review@claude-code-plugins' - prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' diff --git a/pgxntool/.github/workflows/claude.yml b/pgxntool/.github/workflows/claude.yml deleted file mode 100644 index f5b9ce9..0000000 --- a/pgxntool/.github/workflows/claude.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Claude Code - -on: - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - issues: - types: [opened, assigned] - pull_request_review: - types: [submitted] - -# No concurrency limit: @claude mentions are independent, read-only requests; -# serializing would only delay responses and cancelling would drop them. -jobs: - claude: - if: | - (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || - (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - # Required for Claude to read CI results on PRs. write (not just read) - # is also needed so claude-code-action's internal bun-setup step can - # save its cache; read-only causes a harmless but noisy - # "Cache reservation failed: cache write denied: token has no writable - # scopes" warning. - actions: write - steps: - - name: Checkout repository - # Intentionally tracks the major-version tag (not a pinned SHA) so - # upstream fixes are picked up automatically. - uses: actions/checkout@v4 - with: - fetch-depth: 1 - persist-credentials: false - - - name: Run Claude Code - id: claude - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - # Allows Claude to read CI results on PRs - additional_permissions: | - actions: read diff --git a/pgxntool/.github/workflows/protect-label.yml b/pgxntool/.github/workflows/protect-label.yml deleted file mode 100644 index de71ea3..0000000 --- a/pgxntool/.github/workflows/protect-label.yml +++ /dev/null @@ -1,146 +0,0 @@ -name: Protect 'commit-with-no-tests' label - -on: - # IMPORTANT: Must use pull_request_target, NOT pull_request. - # - # 'pull_request' from a fork runs with a read-only GITHUB_TOKEN scoped to - # the fork. It cannot add or remove labels on the upstream repo (write - # operation), and cannot call getCollaboratorPermissionLevel (requires write - # permission to the target repo). - # - # 'pull_request_target' runs in the base repo's context with a token that - # has write access — exactly what we need here. - # - # Security: because pull_request_target has write access, never check out - # or execute code from the PR head in this workflow. This workflow only calls - # the GitHub API via actions/github-script and is safe. - pull_request_target: - types: [labeled, unlabeled] - -jobs: - protect: - # Only fire for the label we care about. All other label changes are - # unaffected by this workflow. - # Note: this literal must match the LABEL const defined in the script below. - if: github.event.label.name == 'commit-with-no-tests' - runs-on: ubuntu-latest - permissions: - pull-requests: write # To add/remove labels - issues: write # GitHub label API goes through the issues endpoint - - steps: - - name: Enforce write-access-only on 'commit-with-no-tests' label - # Pinned to an immutable SHA: this workflow runs as pull_request_target - # with write access, so a moved upstream tag must not change what runs. - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - script: | - const actor = context.actor; - const prNumber = context.payload.pull_request.number; - const action = context.payload.action; // 'labeled' or 'unlabeled' - // Single source of truth for the label name within this script. - // Must also match the literal in the job-level `if:` condition above - // (YAML expressions can't reference JS constants). - const LABEL = 'commit-with-no-tests'; - - // When this workflow re-adds or removes the label itself, that fires - // this event again with actor = 'github-actions[bot]'. Without this - // guard the job loops forever. We match any '[bot]' suffix to also - // cover other automation (Dependabot, Renovate, etc.). - if (actor.endsWith('[bot]')) { - core.info(`Actor is a bot (${actor}); skipping permission check`); - return; - } - - // Check the actor's effective permission level in this repo. - // - // EDGE CASE — 404 for non-collaborators: This API returns 404 when - // the user is not an explicit collaborator. This is the normal case - // for contributors who forked and opened a PR. If we don't catch - // this error, the job crashes with an unhandled exception and the - // label stays in whatever state the contributor put it in — - // defeating the entire protection. - // - // EDGE CASE — org team members: Users with write access via org - // team membership (not a direct collaborator invite) correctly show - // as 'write' here because the API returns effective permission. - // Exception: if the org has "private member visibility" set and the - // token can't enumerate team membership, they may get a 404 instead. - // If that becomes an issue, add a fallback to - // github.rest.orgs.getMembershipForUser(). - // - // EDGE CASE — other errors: Network blips, API outages, and rate - // limiting all throw here. We fail safe by treating any unexpected - // error as "no write access" and logging for debugging. - let hasWrite = false; - try { - const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({ - owner: context.repo.owner, - repo: context.repo.repo, - username: actor - }); - hasWrite = ['admin', 'write'].includes(perm.permission); - } catch (e) { - if (e.status === 404) { - // Not a collaborator — no write access. Expected and normal. - hasWrite = false; - } else { - core.warning( - `Unexpected error checking permissions for ${actor} ` + - `(HTTP ${e.status}): ${e.message}. Treating as no write access.` - ); - hasWrite = false; - } - } - - if (action === 'labeled' && !hasWrite) { - core.info(`${actor} lacks write access; removing '${LABEL}' label`); - await github.rest.issues.removeLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - name: LABEL - }); - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: - `@${actor} The \`commit-with-no-tests\` label can only be applied by ` + - `maintainers with write access to this repository.\n\n` + - `If you believe no test changes are needed for this PR, please ask a ` + - `maintainer to apply the label after reviewing. Note that most pgxntool ` + - `changes do require paired test updates — this label should be used sparingly.` - }); - - } else if (action === 'unlabeled' && !hasWrite) { - // Non-writer removed the label. Put it back. - // - // EDGE CASE — brief label-absent window: There is a short window - // between removal and this workflow re-adding the label. During - // that window the label genuinely does not exist. This is harmless - // in practice: the ci.yml workflow reads labels via a live API - // call (not from its cached payload), so a re-run after the label - // is restored will pick it up correctly. - core.info(`${actor} lacks write access; re-adding '${LABEL}' label`); - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - labels: [LABEL] - }); - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: - `@${actor} The \`commit-with-no-tests\` label can only be removed by ` + - `maintainers with write access to this repository.\n\n` + - `Contact a maintainer if you believe this label was applied in error.` - }); - - } else if (hasWrite) { - core.info( - `${actor} has write access; '${action}' on '${LABEL}' label is approved` - ); - }