diff --git a/pgxntool/CLAUDE.md b/pgxntool/CLAUDE.md index 75c9d16..5db8c95 100644 --- a/pgxntool/CLAUDE.md +++ b/pgxntool/CLAUDE.md @@ -101,7 +101,7 @@ include pgxntool/base.mk - `PGXNVERSION` - version number - `EXTENSIONS` - list of extensions provided - `EXTENSION_*_VERSION` - per-extension versions - - `EXTENSION_VERSION_FILES` - auto-generated versioned SQL files + - `EXTENSION__CURRENT_VERSION__FILES` - the current/most-recent auto-generated versioned SQL file for each extension (not all version files) - `base.mk` includes `meta.mk` via `-include` ### The Magic of base.mk @@ -144,11 +144,11 @@ make # Build extension (generates versioned SQL, docs) make test # Full test: testdeps → install → installcheck → show diffs make results # Run tests and update expected output files make html # Generate HTML from Asciidoc sources -make tag # Create git branch for current META.json version +make tag # Create git tag for current META.json version make dist # Create PGXN .zip (auto-tags, places in ../) make pgtle # Generate pg_tle registration SQL (see pg_tle Support below) make check-pgtle # Check pg_tle installation and report version -make install-pgtle # Install pg_tle registration SQL files into database +make run-pgtle # Register extensions with pg_tle in the database make pgxntool-sync # Update to latest pgxntool via git subtree pull ``` @@ -204,7 +204,7 @@ When tests fail, examine the diff output carefully. The actual test output in `t ### Distribution Packaging - `make dist` creates `../PGXN-VERSION.zip` -- Always creates git branch tag matching version +- Always creates git tag matching version - Uses `git archive` to package - Validates repo is clean before tagging @@ -227,16 +227,16 @@ pgxntool can generate pg_tle (Trusted Language Extensions) registration SQL for **Installation targets:** -- `make check-pgtle` - Checks if pg_tle is installed and reports the version. Reports version from `pg_extension` if extension has been created, or newest available version from `pg_available_extension_versions` if available but not created. Errors if pg_tle not available in cluster. Assumes `PG*` environment variables are configured. +- `make check-pgtle` - Checks if pg_tle is installed and reports the version. Reports the version from `pg_extension` if `CREATE EXTENSION pg_tle` has been run in the database. Errors if pg_tle is not available in the cluster. Assumes `PG*` environment variables are configured. -- `make install-pgtle` - Auto-detects pg_tle version and installs appropriate registration SQL files. Updates or creates pg_tle extension as needed. Determines which version range files to install based on detected version. Runs all generated SQL files via `psql` to register extensions with pg_tle. Assumes `PG*` environment variables are configured. +- `make run-pgtle` - Registers all extensions with pg_tle by executing the generated pg_tle registration SQL files. Requires pg_tle to already be installed in the target database (`CREATE EXTENSION pg_tle;`) -- it does not create the extension itself, and errors out telling you to run `make check-pgtle` if it's missing. Depends on `pgtle`, so it generates the SQL files first if needed. Assumes `PG*` environment variables are configured. **Version notation:** - `X.Y.Z+` means >= X.Y.Z - `X.Y.Z-A.B.C` means >= X.Y.Z and < A.B.C (note boundary) **Key implementation details:** -- Script: `pgxntool/pgtle-wrap.sh` (bash) +- Script: `pgxntool/pgtle.sh` (bash) - Parses `.control` files for metadata (NOT META.json) - Fixed delimiter: `$_pgtle_wrap_delimiter_$` (validated not in source) - Each output file contains ALL versions and ALL upgrade paths @@ -269,6 +269,7 @@ Generated files depend on: 4. **Control File Versions**: No automatic validation that `.control` matches `META.json` version 5. **PGXNTOOL_NO_PGXS_INCLUDE**: Setting this skips PGXS inclusion (for special scenarios) 6. **Distribution Placement**: `.zip` files go in parent directory (`../`) to avoid repo clutter +7. **Never hand-edit an old versioned SQL file**: `sql/{ext}--{version}.sql` is only auto-regenerated by `make` for the extension's *current* `default_version` (from its `.control` file — not `META.json`; see "PGXN Distributions vs. Extensions" in README.asc). Every other `sql/{ext}--{version}.sql` (any version that is no longer current) is a frozen historical record used to test update paths and PostgreSQL-version compatibility — editing one directly silently corrupts that record. If you need to change something after a version has shipped, bump the version and add a `sql/{ext}--{old}--{new}.sql` upgrade script instead; never edit `sql/{ext}--{old}.sql` in place. This applies even though the file carries a generic `DO NOT EDIT - AUTO-GENERATED FILE` header — that header doesn't distinguish "regenerated every build" (current version) from "generated once, now frozen" (every other version). See "Version-Specific SQL Files" in README.asc for the full reasoning, including when it's acceptable to not commit a given version's file at all, and why that should be a plain `rm` rather than a `.gitignore` entry. ## Scripts diff --git a/pgxntool/HISTORY.asc b/pgxntool/HISTORY.asc index b0894da..231809f 100644 --- a/pgxntool/HISTORY.asc +++ b/pgxntool/HISTORY.asc @@ -1,3 +1,55 @@ +2.2.0 +----- +== Add `check-stale-expected` to catch orphaned test/expected files +`make test` now fails if `test/expected/*.out` (or `test/build/expected/*.out`) +has no corresponding `test/sql/*.sql` (or `test/build/*.sql`). If your project +has genuinely stale expected-output files sitting around, `make test` will +newly fail until they're removed. pg_regress's alternate expected-output +files (`test_0.out` .. `test_9.out`) are recognized and not flagged as stale. + +== Rename `EXTENSION_VERSION_FILES` to `EXTENSION__CURRENT_VERSION__FILES` +The old name implied it listed every version file for an extension; it +actually only ever holds the current/most-recent one per extension. If your +Makefile references `EXTENSION_VERSION_FILES` directly (e.g. to add extra +dependencies to `DATA`), update it to `EXTENSION__CURRENT_VERSION__FILES`. + +== Add `make pgxntool-version` to print the embedded pgxntool version +There was no way to tell which pgxntool version a project had embedded via +`git subtree` short of digging through git history. `make pgxntool-version` +(and `pgxntool/bin/version` directly) now prints it, read from the +first line of `pgxntool/HISTORY.asc` -- the same line the release process +stamps with the version number. The line is rejected unless it's `STABLE` +or a plain `X.Y.Z` version (digits only, no pre-release suffixes). + +== Add `PGXN_REMOTE` to override the remote used by tag/rmtag/forcetag/dist +These targets hardcoded the `origin` remote, silently re-tagging the wrong +repository for maintainers whose `origin` is a personal fork. Defaults to +`origin`, so existing behavior is unchanged unless you set it. + +== Fix `make pgtle PGTLE_VERSION=X` generating all versions instead of one +`base.mk`'s `pgtle` target never passed the `PGTLE_VERSION` make variable +through to `pgtle.sh --pgtle-version`, so it silently generated all pg_tle +version ranges regardless of the value given on the command line. + +== Fix `pgtle.sh` rejecting non-numeric version aliases like `stable` +`extract_version_from_filename()` required every version segment to start +with a digit, so extensions using a persistent `stable` pseudo-version +(`default_version = 'stable'`) failed with "Cannot parse version from +filename". Widened to accept any non-empty, `--`-free version string, +matching how pg_tle itself treats extension versions. + +== Fix versioned SQL file corruption under overlapping make invocations +The rule generating `sql/--.sql` used a `>` truncate followed +by a separate `>>` append, so a rule that fired more than once (e.g. +overlapping `make` processes) doubled the file's content. Now uses a single +atomic redirect. + +== Add include guard to base.mk +`base.mk` could end up included twice in one `make` run (for example, an +extension's own `.mk` module includes it in addition to the Makefile's own +include), which redefined every target and printed overriding-recipe +warnings. A second inclusion is now a harmless no-op. + 2.1.0 ----- == Fix setup.sh / pgxntool-sync.sh / update-setup-files.sh inside a git worktree diff --git a/pgxntool/README.asc b/pgxntool/README.asc index db6162f..a799c7c 100644 --- a/pgxntool/README.asc +++ b/pgxntool/README.asc @@ -7,9 +7,9 @@ Easier PGXN development PGXNtool is meant to make developing new Postgres extensions for http://pgxn.org[PGXN] easier. -Currently, it consists a base Makefile that you can include instead of writing your own, a template META.json, and some test framework. More features will be added over time. +Currently, it consists a base Makefile that you can include instead of writing your own, a template `META.in.json` (from which `META.json` is generated — you edit the former, never the latter directly), and some test framework. More features will be added over time. -If you find any bugs or have ideas for improvements, please https://github.com/decibel/pgxntool/issues[*open an issue*]. +If you find any bugs or have ideas for improvements, please https://github.com/Postgres-Extensions/pgxntool/issues[*open an issue*]. == Install This assumes that you've already initialized your extension in git. @@ -17,7 +17,7 @@ This assumes that you've already initialized your extension in git. NOTE: The `--squash` is important! Otherwise you'll clutter your repo with a bunch of commits you probably don't want. ---- -git subtree add -P pgxntool --squash git@github.com:decibel/pgxntool.git release +git subtree add -P pgxntool --squash git@github.com:Postgres-Extensions/pgxntool.git release pgxntool/setup.sh ---- @@ -39,15 +39,19 @@ include pgxntool/base.mk == make targets These are the make targets that are provided by base.mk -NOTE: all the targets normally provided by Postgres http://www.postgresql.org/docs/current/static/extend-pgxs.html[PGXS] still work. +NOTE: all the targets normally provided by Postgres http://www.postgresql.org/docs/current/static/extend-pgxs.html[PGXS] still work. If you need to skip PGXS being included entirely (for advanced/non-standard setups), set `PGXNTOOL_NO_PGXS_INCLUDE` in your Makefile before `include pgxntool/base.mk`. === html This will build any .html files that can be created. See <<_Document_Handling>>. === test -Runs unit tests via the PGXS `installcheck` target. Unlike a simple `make installcheck` though, the `test` rule has the following prerequisites: clean testdeps install installcheck. All of those are PGXS rules, except for `testdeps`. +Runs your extension's test suite: installs the extension and runs it through PGXS's `installcheck`, first pulling in anything you've hooked into <<_testdeps>> and, if enabled, sanity-checking your test SQL via <<_test_build>>. -NOTE: While you can still run `make installcheck` or any other valid PGXS make target directly, it's recommended to use `make test` when using pgxntool. The `test` target ensures clean builds, proper test isolation, and correct dependency installation. +Whether `test-build` runs is controlled by the `PGXNTOOL_ENABLE_TEST_BUILD` variable — see <<_test_build>> for what it does and how to turn it on/off. + +NOTE: `test` intentionally does *not* depend on `clean` — that caused problems with incremental/watch-based builds. If your tests need a clean build to pass, that's a sign of a missing dependency elsewhere rather than something to fix by adding `clean` back. + +NOTE: While you can still run `make installcheck` or any other valid PGXS make target directly, it's recommended to use `make test` when using pgxntool. The `test` target ensures proper test isolation and correct dependency installation. === test-build Validates that extension SQL files are syntactically correct before running the full test suite. This feature runs SQL files from `test/build/` through `pg_regress`, providing better error messages than `CREATE EXTENSION` failures when there are syntax errors in your extension code. @@ -163,6 +167,38 @@ Without `test/install`, each test file typically needs to run `CREATE EXTENSION` **Key detail:** Install files and regular tests run in a single `pg_regress` invocation. This means the database is NOT dropped between install and test phases — state created by install files persists into the main test suite. +==== Update & Upgrade (U&U) Testing + +Beyond validating a plain install, it's worth testing that your extension behaves correctly across the two transitions every extension with more than one release eventually goes through: + +- **Update**: `ALTER EXTENSION ext UPDATE`, moving to a newer extension version on the same PostgreSQL version. +- **Upgrade**: `pg_upgrade`, moving to a newer PostgreSQL major version while carrying the extension's on-disk catalog state along with it. + +Both catch bugs that a simple "did the script run without error" check misses — wrong mappings, missing `REVOKE`s, objects that drifted from the base SQL, or catalog state that doesn't survive a binary upgrade cleanly. This isn't a niche concern that only applies if your extension does something unusual (adds enum types, custom operators, etc.) — it applies to any extension that will ever be updated or upgraded in place, which in practice is every extension. + +`test/install` is the recommended place to build this, because its "load once, committed, before the suite" mechanic (described above) is exactly what update/upgrade testing needs. + +**The pattern**: load the extension exactly once, committed, from a `test/install/*.sql` file, in one of several modes selected by a custom backend setting that your `Makefile` sets via `PGOPTIONS` (e.g. `-c myext.test_load_mode=$(TEST_LOAD_SOURCE)`) and that `test/install` reads with `current_setting('myext.test_load_mode')`: + +- *fresh*: `CREATE EXTENSION ext;` +- *update*: `CREATE EXTENSION ext VERSION 'oldest_supported';` then `ALTER EXTENSION ext UPDATE;` (to the current `default_version`, or an explicit intermediate version) +- *existing*: the extension is already installed — by a real `pg_upgrade`, or an update performed outside the suite — and `test/install` only asserts it's present at the expected version, without dropping or touching it + +The *same* `test/sql/*` suite and the *same* `test/expected/*` output then run unchanged against every mode. Identical expected output passing in *update* mode *is* the update-equivalence assertion — if the updated database behaves any differently than a fresh install, some test will diff. Running that same suite in *existing* mode against a database that just went through a real `pg_upgrade` gives you upgrade testing using the same test files, no separate suite required. + +If you adopt this pattern, `test/deps.sql` should no longer run `CREATE EXTENSION` itself for the per-test setup — the committed install performed by `test/install` persists into every (rolled-back) test file, exactly as described above. + +**Why the install step must be committed, not run per-test in `test/deps.sql`:** every `test/sql/` file runs inside `BEGIN; ... ROLLBACK;` — a pgxntool convention (`test/pgxntool/setup.sql` opens the transaction; since it's never committed, it rolls back when the session ends). Running the update inside that transaction means the suite never actually exercises a *committed* update — which doesn't match production, where `ALTER EXTENSION UPDATE` commits before anything else touches the database, and it hides bugs that only show up once the update is durably committed. (Modifying an `enum` is one example of how this can turn into an outright failure rather than just a hidden bug — but it's just an example; the reason to commit first holds regardless of what the update script does.) + +As a bonus, sharing one committed install across every test file also removes the per-test re-install cost. + +NOTE: This pattern isn't (yet) native to pgxntool — you wire up the mode selection and `PGOPTIONS` plumbing yourself in `test/install` and your `Makefile`. First-class support (a standard mode-switching toggle, and scaffolding for real `pg_upgrade` testing) is planned; see https://github.com/Postgres-Extensions/pgxntool/issues/42[issue #42]. + +**Working examples** (specific files, not full PRs — both extensions' histories include plenty of unrelated changes): + +- https://github.com/Postgres-Extensions/cat_tools/blob/d01c933a3fcbf28d3bdd2bf6f2bd7b116e95e604/test/install/load.sql[cat_tools `test/install/load.sql`] and the `TEST_LOAD_SOURCE` block in its https://github.com/Postgres-Extensions/cat_tools/blob/d01c933a3fcbf28d3bdd2bf6f2bd7b116e95e604/Makefile[`Makefile`] — the fresh/update/existing pattern above, including the `PGOPTIONS` wiring. This is a reasonably complete example, and somewhat more elaborate than the minimum needed to get started. +- https://github.com/Postgres-Extensions/pg_count_nulls/blob/9f088ac36f6345023bb5b3d20ad0e4aa6ade0d69/test/sql/extension_tests.sql[pg_count_nulls `test/sql/extension_tests.sql`] additionally demonstrates testing an extension installed into a non-default schema. It's a useful reference for that specific problem, but it also defines its assertions as plpgsql functions rather than plain pgTAP calls — a pattern that adds complexity of its own and isn't recommended as a model for U&U testing itself. + === testdeps This rule allows you to ensure certain actions have taken place before running tests. By default it has a single prerequisite, `pgtap`, which will attempt to install http://pgtap.org[pgtap] from PGXN. This depneds on having the pgxn client installed. @@ -177,7 +213,7 @@ check_control: grep -q "requires = 'pgtap, test_factory'" test_factory_pgtap.control ---- -If you want to over-ride the default dependency on `pgtap` you should be able to do that with a makefile override. If you need help with that, please https://github.com/decibel/pgxntool/issues[open an issue]. +If you want to over-ride the default dependency on `pgtap` you should be able to do that with a makefile override. If you need help with that, please https://github.com/Postgres-Extensions/pgxntool/issues[open an issue]. WARNING: It will probably cause problems if you try to create a `testdeps` rule that has a recipe. Instead of doing that, put the recipe in a separate rule and make that rule a prerequisite of `testdeps` as show in the example. @@ -195,15 +231,20 @@ IMPORTANT: *`make results` requires manual verification first*. The correct work Never run `make results` without first verifying the test changes are correct. The `results` target copies files from `test/results/` to `test/expected/`, so running it blindly will make incorrect output become the new expected behavior. ==== verify-results safeguard -By default, `make results` will refuse to run if `test/results/regression.diffs` exists (indicating failing tests). This prevents accidentally updating expected files with incorrect output. +By default, `make results` will refuse to run if your tests are failing, so you can't accidentally promote incorrect output into the new expected results. Which failures it looks for is controlled by `PGXNTOOL_VERIFY_RESULTS_MODE`: -If tests are failing, you'll see: +`pgtap` (the default):: Scans `test/results/*.out` for pgTAP `not ok` lines (ignoring `# TODO` items) and plan-count mismatches, then also checks `test/results/regression.diffs` for any output mismatch pgTAP itself wouldn't catch. +`diffs`:: Only checks whether `test/results/regression.diffs` exists. + +If tests are failing, `make results` prints an explanation of what failed and exits without touching `test/expected/`; fix the failures, then re-run `make results`. + +To select a mode explicitly: ---- -ERROR: Tests are failing. Cannot run 'make results'. -Fix test failures first, then run 'make results'. +# In your Makefile +PGXNTOOL_VERIFY_RESULTS_MODE = diffs # or pgtap (the default) ---- -To disable this safeguard (not recommended): +To disable this safeguard entirely (not recommended): ---- # In your Makefile PGXNTOOL_ENABLE_VERIFY_RESULTS = no @@ -213,28 +254,47 @@ make PGXNTOOL_ENABLE_VERIFY_RESULTS=no results ---- === tag -`make tag` will create a git branch for the current version of your extension, as determined by the META.json file. The reason to do this is so you can always refer to the exact code that went into a released version. +`make tag` will create a git tag for the current version of your extension, as determined by the META.json file (generated from `META.in.json` — see <<_pgxn_distributions_vs_extensions>>), and push it to `origin`. The reason to do this is so you can always refer to the exact code that went into a released version. -If there's already a tag for the current version that probably means you forgot to update META.json, so you'll get an error. If you're certain you want to over-write the tag, you can do `make forcetag`, which removes the existing tag (via `make rmtag`) and creates a new one. +If a tag for the current version already exists and points at your current commit, `make tag` does nothing — this makes it safe for other targets (like `dist`) to depend on `tag` without worrying about re-running it. If the existing tag points at a *different* commit — meaning you likely forgot to bump the version — you'll get an error. If you're certain you want to over-write the tag, you can do `make forcetag`, which removes the existing tag (via `make rmtag`) and creates a new one. -WARNING: You will be very unhappy if you forget to update the .control file for your extension! There is an https://github.com/decibel/pgxntool/issues/1[open issue] to improve this. +WARNING: You will be very unhappy if you forget to update the .control file for your extension! There is an https://github.com/Postgres-Extensions/pgxntool/issues/1[open issue] to improve this. === dist -`make dist` will create a .zip file for your current version that you can upload to PGXN. The file is named after the PGXN name and version (the top-level "name" and "version" attributes in META.json). The .zip file is placed in the *parent* directory so as not to clutter up your git repo. +`make dist` will create a .zip file for your current version that you can upload to PGXN. It first runs `tag` (creating/pushing the version's git tag as described above if needed), then uses `git archive` at that tag to build the .zip file, so the archive always matches the exact tagged commit. The file is named after the PGXN name and version (the top-level "name" and "version" attributes in META.json, generated from `META.in.json`). The .zip file is placed in the *parent* directory so as not to clutter up your git repo. NOTE: Part of the `clean` recipe is cleaning up these .zip files. If you accidentally clean before uploading, just run `make dist-only`. +WARNING: If your project has a `.gitattributes` file, `make dist`/`make dist-only` will refuse to run unless it's committed to git, and will tell you so. This is because `git archive` (which builds the .zip) only honors `export-ignore` attributes on *committed* files — an uncommitted `.gitattributes` would silently have no effect on the archive contents, which could leak files into your PGXN distribution that you meant to exclude. + +`make forcedist` is a shortcut for `forcetag dist`: it force-recreates the tag (see `forcetag` above) before rebuilding the distribution .zip. + === pgxntool-sync This rule will pull down the latest released version of PGXNtool via `git subtree pull` and then reconcile the files `setup.sh` copied into your project (`.gitignore`, `test/deps.sql`) with a 3-way merge. NOTE: Your repository must be clean (no modified files) in order to run this. Running this command will produce a git commit of the merge. +NOTE: `git subtree pull` copies pgxntool's entire tree into your project, including dev-only directories (`pgxntool/.github`, `pgxntool/.claude`) that don't belong in a project that merely embeds pgxntool. Every sync automatically removes these two directories from `pgxntool/` afterward — this is expected, not a bug, and doesn't affect anything else in your project. + TIP: The actual work is done by `pgxntool/pgxntool-sync.sh`, so you can run it directly (`pgxntool/pgxntool-sync.sh`) if you'd rather not go through `make`. It optionally takes `` and `` arguments to pull from somewhere other than the default. +TIP: `pgxntool-sync.sh` itself delegates the 3-way merge of `setup.sh`-copied files to `pgxntool/update-setup-files.sh `, which you can also run directly -- for example, to redo or debug just the merge step without pulling again. `` is the pgxntool subtree commit that was current *before* the sync you want to re-merge. + TIP: There is also a `pgxntool-sync-%` rule if you need to do more advanced things. `make pgxntool-sync-` pulls from the ` ` defined by the `pgxntool-sync-` make variable. +=== pgxntool-version +`make pgxntool-version` prints the version of the embedded pgxntool copy, read from the first line of `pgxntool/HISTORY.asc`. That line is "STABLE" instead of a version number if this copy was synced from an unreleased commit rather than a tagged release. + +TIP: The actual work is done by `pgxntool/bin/version`, so you can run it directly (`pgxntool/bin/version`) if you'd rather not go through `make`. + +=== list +`make list` prints every make target defined in your project — including ones provided by PGXS and pgxntool, not just ones you wrote — one per line. Useful for discovering what's available without reading through the Makefile. + +=== print-% +`make print-VARNAME` prints the current value (and origin) of any make variable, e.g. `make print-PGXNVERSION`. Useful for debugging why a variable isn't set to what you expect. + === distclean -`make distclean` removes generated configuration files (`META.json`, `meta.mk`, `control.mk`) that survive a normal `make clean`. +`make distclean` removes generated configuration files (`META.json` — generated from `META.in.json` — plus `meta.mk` and `control.mk`) that survive a normal `make clean`. NOTE: PGXS doesn't provide any special support for `distclean` — its built-in `distclean` target simply depends on `clean`. PGXNtool uses its own `PGXNTOOL_distclean` variable to track files that should only be removed by `distclean`, not `clean`. @@ -248,6 +308,12 @@ Generates pg_tle (Trusted Language Extensions) registration SQL files for deploy `make pgtle` generates SQL files in `pg_tle/` subdirectories organized by pg_tle version ranges. For version range details, see `pgtle_versions.md`. +Set `PGTLE_VERSION` on the command line to limit generation to the single version range that value falls into, instead of every known range: + +---- +make pgtle PGTLE_VERSION=1.5.0 +---- + === check-pgtle Checks if pg_tle is installed and reports the version. This target: - Reports the version from `pg_extension` if `CREATE EXTENSION pg_tle` has been run in the database @@ -261,7 +327,7 @@ make check-pgtle === run-pgtle Registers all extensions with pg_tle by executing the generated pg_tle registration SQL files in a PostgreSQL database. This target: -- Requires pg_tle extension to be installed (checked via `check-pgtle`) +- Requires pg_tle to be installed in the target database. This isn't a make-level dependency on `check-pgtle` — `pgtle.sh` checks it itself when it runs, and will tell you to run `make check-pgtle` if pg_tle isn't there - Uses `pgtle.sh` to determine which version range directory to use based on the installed pg_tle version - Runs all generated SQL files via `psql` to register your extensions with pg_tle @@ -282,42 +348,87 @@ CREATE EXTENSION "your-extension-name"; PGXNtool automatically generates version-specific SQL files from your base SQL file. These files follow the pattern `sql/{extension}--{version}.sql` and are used by PostgreSQL's extension system to install specific versions of your extension. +=== PGXN Distributions vs. Extensions + +PGXN distinguishes two things it's easy to conflate: a *distribution* and an *extension*. Per https://pgxn.org/spec/[the PGXN meta spec], a distribution is "a collection of extensions, source code, utilities, tests, and/or documents that are distributed together" — the thing you release and upload to PGXN, identified by the top-level `name`/`version` in `META.json` (generated from `META.in.json` — you edit the latter, never `META.json` directly). An extension is the individual thing installed via `CREATE EXTENSION`, described by its own `.control` file. A single distribution routinely provides *more than one* extension — `META.in.json`'s `provides` map lists each one; the pgTAP distribution, for example, provides both the `pgtap` and `schematap` extensions from one release. + +This split matters for versioning, and it's where pgxntool has a real gap: `META.json`'s top-level `version` is the *distribution's* release version (used by `make tag` and `make dist` — it's the git tag name and the `.zip` filename). Each extension's *own* version, though, ends up tracked in *two* separate places that pgxntool does not keep in sync for you: + +- the extension's `.control` file (`default_version`) — what PostgreSQL and pgxntool's own build actually use; see <<_what_controls_the_version_number>> below +- that extension's entry under `META.in.json`'s `provides` map (`provides.{extension}.version`) — which the PGXN meta spec defines as "a Version for the extension" in its own right, *not* the distribution's version, and which is what PGXN's public index reads + +Nothing generates one from the other or checks that they still agree, so it's easy to bump one and forget the other; https://github.com/Postgres-Extensions/pgxntool/issues/47[issue #47] tracks fixing that. + === How Version Files Are Generated When you run `make` (or `make all`), PGXNtool: -1. Reads your `META.json` file to determine the extension version from `provides.{extension}.version` +1. Reads each extension's own `.control` file to determine its version from `default_version` (via `control.mk.sh`) 2. Generates a Makefile rule that copies your base SQL file (`sql/{extension}.sql`) to the version-specific file (`sql/{extension}--{version}.sql`) 3. Executes this rule, creating the version-specific file with a header comment indicating it's auto-generated -For example, if your `META.json` contains: +For example, if your `myext.control` contains: ---- -"provides": { - "myext": { - "version": "1.2.3", - ... - } -} +default_version = '1.2.3' ---- Running `make` will create `sql/myext--1.2.3.sql` by copying `sql/myext.sql`. === What Controls the Version Number -The version number comes from `META.json` → `provides.{extension}.version`, *not* from your `.control` file's `default_version` field. The `.control` file's `default_version` is used by PostgreSQL to determine which version to install by default, but the actual version-specific file that gets generated is determined by what's in `META.json`. +The version number comes from each extension's *own* `.control` file → `default_version`, *not* from `META.json`. PostgreSQL itself uses `default_version` to pick which versioned SQL file to load, so `control.mk.sh` parses the `.control` file(s) directly to keep the generated filename in sync with what PostgreSQL will actually use. `META.in.json` has its own, separate `provides.{extension}.version` for that same extension (see <<_pgxn_distributions_vs_extensions>> above) — pgxntool never reads it when generating SQL files, and nothing keeps it in sync with the `.control` file's `default_version`. -To change the version of your extension: -1. Update `provides.{extension}.version` in `META.json` +To change the version of one of your extensions: +1. Update `default_version` in that extension's `.control` file 2. Run `make` to regenerate the version-specific file -3. Update `default_version` in your `.control` file to match (if needed) +3. Update that extension's `provides.{extension}.version` in `META.in.json` to match by hand, and regenerate `META.json` (`make META.json`) — pgxntool won't do this for you +4. Update `META.in.json`'s top-level `version` too if you're also cutting a new distribution release === Committing Version Files -Version-specific SQL files are now treated as permanent files that should be committed to your repository. This makes it much easier to test updates to extensions, as you can see exactly what SQL was included in each version. +Version-specific SQL files are treated as permanent files that should be committed to your repository by default. This makes it much easier to test updates to extensions, as you can see exactly what SQL was included in each version. IMPORTANT: These files are auto-generated and include a header comment warning not to edit them. Any manual changes will be overwritten the next time you run `make`. To modify the extension, edit the base SQL file (`sql/{extension}.sql`) instead. -=== Alternative: Ignoring Version Files +==== Why Commit Them: Update Testing + +The primary value of committing a version's install script is *update testing*: with the file in place, you can install that exact version (`CREATE EXTENSION ext VERSION 'x.y.z'`), run `ALTER EXTENSION ext UPDATE`, and verify the update path actually works. In principle you would only ever need the very first committed version's install script — every later version is reachable by chaining upgrade scripts from it. See <<_testinstall>> for how to wire this into your test suite. + +In practice, you should keep *most* old versions tracked rather than relying purely on that chain, because you cannot predict when a new *major PostgreSQL version* will break the ability to install an *older* extension version — a system catalog column changes type, a `SELECT *` over a catalog starts failing, and so on. Committing the old version's install script lets CI catch that regression directly, by attempting to install that exact historical version against the new PostgreSQL release. For any non-trivial extension, most old versions should stay tracked for this reason alone. + +==== When It's OK to Skip a Version + +For a *minor version change that doesn't meaningfully alter the extension* (e.g. a small bug fix), it's unlikely to straddle a PostgreSQL supported-version boundary, so there's much less test-coverage value in committing that specific version's generated install script. For large extensions, deliberately not committing some of these individual version files is worth doing to keep repository size down — the base `sql/{extension}.sql` still fully captures the change in git history, and the install script itself is trivially regenerated from that base file at build time. + +==== Don't `.gitignore` a Skipped Version — `rm` It Once + +When you decide not to track a particular version's generated install script, do *not* add a `.gitignore` entry for it. + +`make` only ever generates the *current* version's file, stamped from the base source according to that extension's `.control` file `default_version`. The moment you bump the version, `make` starts generating the *new* current version's file and stops touching the old one — the old file left behind in your working tree is a stale, one-off artifact, not something `make` keeps recreating on every build. + +Because it's only ever transiently present, the correct cleanup is a single `rm sql/{extension}--{old-version}.sql` right after the version bump — not a permanent `.gitignore` rule. A per-version `.gitignore` line would be perpetual clutter added on every release, for a file that's only ever present for one build cycle. A broad glob (e.g. `sql/*--*.sql`) is wrong for the same reason it's wrong below in <<_alternative_ignoring_all_version_files>>: it would also hide the prior-version install scripts and upgrade scripts (`sql/{extension}--{a}--{b}.sql`) that you *do* want tracked and visible in `git status`. + +==== Never Hand-Edit a Version File That's No Longer Current + +`control.mk.sh` generates a Make rule along these lines for the current version's file: + +---- +$(EXTENSION_ext_VERSION_FILE): sql/ext.sql extension.control + @(echo '/* DO NOT EDIT - AUTO-GENERATED FILE */'; cat sql/ext.sql) > $(EXTENSION_ext_VERSION_FILE) +---- + +That rule only ever targets the file matching the extension's *current* `default_version`, so editing the base `sql/{extension}.sql` and running `make` is the correct, safe way to change that one file. + +Every *other* versioned file — any `sql/{extension}--{version}.sql` where `{version}` is no longer current — is a frozen historical record, not something `make` will ever regenerate or overwrite again. Its entire purpose is to preserve exactly what shipped in that version (see <<_why_commit_them_update_testing>> above), so it can keep being used to test update paths and PostgreSQL-version compatibility against exactly what your users actually installed. + +**Hand-editing an old versioned file directly is never correct** — it silently corrupts that historical record. If you need to change behavior after a version has already shipped, bump the version and add a proper `sql/{extension}--{old}--{new}.sql` upgrade script instead. Never edit `sql/{extension}--{old}.sql` in place. + +CAUTION: This is especially relevant for AI coding agents, which won't know this convention exists unless it's spelled out. The generic `DO NOT EDIT - AUTO-GENERATED FILE` header on every version file doesn't distinguish "regenerated on every build" (the current version) from "was generated once, now frozen" (every other version). An agent without this context may reasonably — but incorrectly — treat any auto-generated file as fair game to patch directly. Old versioned SQL files must instead be treated as append-only history: the fix is always a new version plus an upgrade script, never an in-place edit. + +[[_alternative_ignoring_all_version_files]] +=== Alternative: Ignoring All Version Files + +NOTE: This is a different, all-or-nothing choice from <<_when_its_ok_to_skip_a_version>> above, where the recommendation is to keep tracking most versions and selectively skip only a few low-value minor ones. This section instead covers not tracking *any* version-specific files at all. If you prefer not to commit version-specific SQL files, you must add them to your `.gitignore` to prevent `make dist` from failing due to untracked files. Add the following to your `.gitignore`: @@ -341,10 +452,10 @@ If you need to support multiple versions of your extension: 1. Create additional version-specific files manually (e.g., `sql/myext--1.0.0.sql`, `sql/myext--1.1.0.sql`) 2. Create upgrade scripts for version transitions (e.g., `sql/myext--1.0.0--1.1.0.sql`) -3. Update `META.json` to reflect the current version you're working on +3. Update `default_version` in the extension's `.control` file to reflect the current version you're working on 4. Commit all version files and upgrade scripts to your repository -The version file for the current version (specified in `META.json`) will be automatically regenerated when you run `make`, but other version files you create manually will be preserved. +The version file for the current version (specified in the `.control` file's `default_version`) will be automatically regenerated when you run `make`, but other version files you create manually will be preserved. == Document Handling PGXNtool supports generation and installation of document files. There are several variables and rules that control this behavior. @@ -381,9 +492,8 @@ PGXNtool replaces each `$(ASCIIDOC_EXTS)` in `$(ASCIIDOC_FILES)` with `html`. The result is appended to `ASCIIDOC_HTML` using `+=`. === Document Rules -If Asciidoc is found (or `$(ASCIIDOC)` is set), the `html` rule will be added as a prerequisite to the `install` and `installchec` rules. -That will ensure that docs are generated for install and test, but only if Asciidoc is available. -The `dist` rule will always depend on `html` though, to ensure html files are up-to-date before creating a distribution. +If Asciidoc is found (or `$(ASCIIDOC)` is set), `html` is added as a dependency of the `all` target, which both `install` and `installcheck` depend on — so docs get built for install and test, but only if Asciidoc is available. +The `dist` rule always depends on `html` directly, regardless of whether Asciidoc was found, to ensure html files are up-to-date before creating a distribution. The `html` rule simply depends on `$(ASCIIDOC_HTML). This rule is always present. @@ -396,11 +506,11 @@ These rules are generated from `ASCIIDOC_template`: ---- define ASCIIDOC_template %.html: %.$(1) # <1> -ifndef ASCIIDOC +ifeq (,$(strip $(ASCIIDOC))) $$(warning Could not find "asciidoc" or "asciidoctor". Add one of them to your PATH,) $$(warning or set ASCIIDOC to the correct location.) $$(error Could not build %$$@) -endif # ifndef ASCIIDOC +endif # ifeq ASCIIDOC $$(ASCIIDOC) $$(ASCIIDOC_FLAGS) $$< endef # define ASCIIDOC_template ---- @@ -410,6 +520,8 @@ These rules will *always* exist, even if `$(ASCIIDOC)` isn't set (ie: if Asciido These rules will throw an error if they are run if `$(ASCIIDOC)` isn't defined. On a normal user system that should never happen, because the `html` rule won't be included in `install` or `installcheck`. +`make docclean` removes all generated HTML files (`$(DOCS_HTML)`). It's separate from `make clean`/`make distclean`, so you can clear out generated docs without touching your build artifacts. + === The DOCS variable This variable has special meaning to PGXS. See the Postgres documentation for full details. @@ -487,6 +599,15 @@ myproject/ └── ext2.sql ---- +==== Direct pgtle.sh Invocation + +`pgtle`, `check-pgtle`, and `run-pgtle` are thin make wrappers around `pgtle.sh`. For more targeted operations you can run it directly: + +- `pgtle.sh --extension NAME [--pgtle-version VERSION]` — generate registration SQL for one extension (what `make pgtle` wraps). +- `pgtle.sh --get-version` — print the pg_tle version installed in the target database (empty if not installed). +- `pgtle.sh --get-dir VERSION` — print which version-range directory (see <<_version_groupings>>) a given pg_tle version maps to, without generating anything. +- `pgtle.sh --run` — execute the generated registration SQL against the target database (what `make run-pgtle` wraps). + === How It Works `make pgtle` does the following: @@ -525,7 +646,80 @@ Each generated SQL file is wrapped in a transaction (`BEGIN;` ... `COMMIT;`) to *Solution*: pg_tle only supports trusted languages. You cannot use C extensions with pg_tle. The script will warn you but still generate files (which won't work). NOTE: there are several untrusted languages (such as plpython), and the only tests for C. + +== Configuration Variables + +This is the comprehensive list of make variables you can override to customize pgxntool's behavior, either in your Makefile (before `include pgxntool/base.mk`) or on the `make` command line. Several of these are also covered in more detail in the sections above; this section exists so there's one place that lists all of them. + +NOTE: Variables marked with `*` must be set to exactly `yes` or `no` (case-insensitive) when given explicitly; any other value is a hard `make` error. + +=== PGXN_REMOTE + +Default: `origin`. The git remote used by `tag`, `rmtag`, `forcetag`, and `dist` (which depends on `tag`). If your `origin` is a personal fork rather than the canonical repository -- for example, if you develop on a fork and only push release tags to the canonical repo -- set this so tagging and pushing target the right remote: + +---- +make dist PGXN_REMOTE=upstream +---- + +=== ASCIIDOC + +Default: auto-detected, the first of `asciidoctor` or `asciidoc` found on `PATH`. Path to the Asciidoc processor used to build `.html` files from `$(ASCIIDOC_EXTS)` source files. Override if the processor you want isn't first on `PATH`, or isn't on `PATH` at all. See <<_document_handling>>. + +=== PGTLE_VERSION + +Default: unset (generates every known pg_tle version range). Set on the command line to limit `make pgtle` to the single version range this value falls into. See <<_pgtle>>. + +=== PG_CONFIG + +Default: `pg_config`. Path to the `pg_config` binary used to detect the PostgreSQL version and locate PGXS. Override when the right `pg_config` isn't the one on `PATH`, such as when testing against a specific PostgreSQL install. + +=== TESTDIR + +Default: `test`. Root directory for test input files: `$(TESTDIR)/sql/`, `$(TESTDIR)/expected/`, `$(TESTDIR)/build/`, `$(TESTDIR)/install/`. Overriding this on its own is unusual; it mainly exists so `TESTOUT` has a sensible default. + +=== TESTOUT + +Default: `$(TESTDIR)`. Directory `pg_regress` writes actual test output to -- `$(TESTOUT)/results/`, `$(TESTOUT)/regression.diffs`, etc. -- via `--outputdir` in `REGRESS_OPTS`. Kept separate from `TESTDIR` so generated output can be pointed somewhere other than the directory holding your checked-in `sql`/`expected` files, if you want that separation. + +=== PGXNTOOL_VERIFY_RESULTS_MODE + +Default: `pgtap`. Controls how the <<_verify_results_safeguard,verify-results safeguard>> decides whether tests are failing. + +==== pgtap + +Scans `test/results/*.out` for pgTap `not ok` lines and plan mismatches, falling back to checking for `regression.diffs` too. Use this mode when your test suite uses pgTap. + +==== diffs + +Checks only for `regression.diffs`, matching classic pg_regress behavior. Use this mode when your tests use plain SQL expected-output comparison only. + +=== PGXNTOOL_ENABLE_TEST_BUILD * + +Default: auto-detected -- `yes` if `test/build/*.sql` files exist, `no` otherwise. Enables or disables the <<_test_build,test-build>> pre-flight SQL check. Set explicitly to `yes` if you want an error when `test/build/` unexpectedly has no SQL files (catches accidental deletion of its contents), or to `no` to disable the check even when files are present. + +=== PGXNTOOL_ENABLE_TEST_INSTALL * + +Default: auto-detected -- `yes` if `test/install/*.sql` files exist, `no` otherwise. Enables or disables the <<_testinstall,test/install>> schedule-based setup feature. Same explicit-override semantics as `PGXNTOOL_ENABLE_TEST_BUILD`. + +=== PGXNTOOL_ENABLE_VERIFY_RESULTS * + +Default: `yes`. Enables or disables the <<_verify_results_safeguard,verify-results safeguard>> that blocks `make results` when tests are failing. Setting it to empty on the command line (`make PGXNTOOL_ENABLE_VERIFY_RESULTS= results`) also disables it. + +=== PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED * + +Default: `yes`. Enables or disables the check-stale-expected safeguard, which fails `make test` if `test/expected/` (or `test/build/expected/`) contains a `.out` file with no corresponding `.sql` file -- catching a stale file left behind after a test was renamed or removed. Set to `no` to make the check a complete no-op (it's dropped from `TEST_DEPS` entirely). + +This is also the supported pattern for disabling one specific optional pgxntool feature: a documented `PGXNTOOL_ENABLE_*` variable, gating both the target's registration into `TEST_DEPS`/etc. and (where applicable) the target's own definition. Cleanly replacing or overriding an arbitrary pgxntool-generated recipe with your own entirely is a separate, larger, and not-yet-decided design question -- see https://github.com/Postgres-Extensions/pgxntool/issues/30[issue #30] (two proposed approaches) -- and is *not* what this mechanism solves. + +=== PGXNTOOL_CHECK_EXPECTED_FILE_TYPES * + +Default: `yes`. Sub-check of check-stale-expected, independent of `PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED`: fails (with a distinct error message and exit code from the orphaned-`.out` check) if `test/expected/` (or `test/build/expected/`) contains any file that isn't `*.out`. Set to `no` to disable just this sub-check while leaving the orphaned-`.out` check active. + +=== PGXNTOOL_NO_PGXS_INCLUDE + +Default: unset (PGXS is included normally). Skips including PGXS (`$(PGXS)`) entirely. This is only for advanced scenarios where you need to manage the PGXS include yourself; most projects should never set this. + == Copyright Copyright (c) 2026 Jim Nasby -PGXNtool is released under a https://github.com/decibel/pgxntool/blob/master/LICENCE[BSD license]. Note that it includes https://github.com/dominictarr/JSON.sh[JSON.sh], which is released under a https://github.com/decibel/pgxntool/blob/master/JSON.sh.LICENCE[MIT license]. +PGXNtool is released under a https://github.com/Postgres-Extensions/pgxntool/blob/master/LICENSE[BSD license]. Note that it includes https://github.com/dominictarr/JSON.sh[JSON.sh], which is released under a https://github.com/Postgres-Extensions/pgxntool/blob/master/JSON.sh.LICENSE[MIT license]. diff --git a/pgxntool/README.html b/pgxntool/README.html index 431358f..36d3cf5 100644 --- a/pgxntool/README.html +++ b/pgxntool/README.html @@ -4,7 +4,7 @@ - + PGXNtool @@ -141,7 +141,7 @@ #content::before{content:none} #header>h1:first-child{color:rgba(0,0,0,.85);margin-top:2.25rem;margin-bottom:0} #header>h1:first-child+#toc{margin-top:8px;border-top:1px solid #dddddf} -#header>h1:only-child{border-bottom:1px solid #dddddf;padding-bottom:8px} +#header>h1:only-child,body.toc2 #header>h1:nth-last-child(2){border-bottom:1px solid #dddddf;padding-bottom:8px} #header .details{border-bottom:1px solid #dddddf;line-height:1.45;padding-top:.25em;padding-bottom:.25em;padding-left:.25em;color:rgba(0,0,0,.6);display:flex;flex-flow:row wrap} #header .details span:first-child{margin-left:-.125em} #header .details span.email a{color:rgba(0,0,0,.85)} @@ -163,7 +163,6 @@ #toctitle{color:#7a2518;font-size:1.2em} @media screen and (min-width:768px){#toctitle{font-size:1.375em} body.toc2{padding-left:15em;padding-right:0} -body.toc2 #header>h1:nth-last-child(2){border-bottom:1px solid #dddddf;padding-bottom:8px} #toc.toc2{margin-top:0!important;background:#f8f8f7;position:fixed;width:15em;left:0;top:0;border-right:1px solid #e7e7e9;border-top-width:0!important;border-bottom-width:0!important;z-index:1000;padding:1.25em 1em;height:100%;overflow:auto} #toc.toc2 #toctitle{margin-top:0;margin-bottom:.8rem;font-size:1.2em} #toc.toc2>ul{font-size:.9em;margin-bottom:0} @@ -329,7 +328,7 @@ a.image object{pointer-events:none} sup.footnote,sup.footnoteref{font-size:.875em;position:static;vertical-align:super} sup.footnote a,sup.footnoteref a{text-decoration:none} -sup.footnote a:active,sup.footnoteref a:active,#footnotes .footnote a:first-of-type:active{text-decoration:underline} +sup.footnote a:active,sup.footnoteref a:active{text-decoration:underline} #footnotes{padding-top:.75em;padding-bottom:.75em;margin-bottom:.625em} #footnotes hr{width:20%;min-width:6.25em;margin:-.25em 0 .75em;border-width:1px 0 0} #footnotes .footnote{padding:0 .375em 0 .225em;line-height:1.3334;font-size:.875em;margin-left:1.2em;margin-bottom:.2em} @@ -458,20 +457,24 @@

PGXNtool

  • 4.7. tag
  • 4.8. dist
  • 4.9. pgxntool-sync
    • -
  • 4.10. distclean
    • -
  • 4.11. pgtle
    • -
  • 4.12. check-pgtle
    • -
  • 4.13. run-pgtle
    • +
  • 4.10. pgxntool-version
    • +
  • 4.11. list
    • +
  • 4.12. print-%
    • +
  • 4.13. distclean
    • +
  • 4.14. pgtle
    • +
  • 4.15. check-pgtle
    • +
  • 4.16. run-pgtle
  • 5. Version-Specific SQL Files
  • 6. Document Handling @@ -492,6 +495,24 @@

    PGXNtool

  • 7.7. Troubleshooting
    • +
  • 8. Configuration Variables + +
    • +
  • 9. Copyright
    • @@ -502,10 +523,10 @@

    PGXNtool

    PGXNtool is meant to make developing new Postgres extensions for PGXN easier.

    -

    Currently, it consists a base Makefile that you can include instead of writing your own, a template META.json, and some test framework. More features will be added over time.

    +

    Currently, it consists a base Makefile that you can include instead of writing your own, a template META.in.json (from which META.json is generated — you edit the former, never the latter directly), and some test framework. More features will be added over time.

    -

    If you find any bugs or have ideas for improvements, please open an issue.

    +

    If you find any bugs or have ideas for improvements, please open an issue.

    @@ -529,7 +550,7 @@

    -
    git subtree add -P pgxntool --squash git@github.com:decibel/pgxntool.git release
+
    git subtree add -P pgxntool --squash git@github.com:Postgres-Extensions/pgxntool.git release
 pgxntool/setup.sh
    @@ -575,7 +596,7 @@

    Note -all the targets normally provided by Postgres PGXS still work. +all the targets normally provided by Postgres PGXS still work. If you need to skip PGXS being included entirely (for advanced/non-standard setups), set PGXNTOOL_NO_PGXS_INCLUDE in your Makefile before include pgxntool/base.mk. @@ -589,7 +610,10 @@

    4.2. test

    -

    Runs unit tests via the PGXS installcheck target. Unlike a simple make installcheck though, the test rule has the following prerequisites: clean testdeps install installcheck. All of those are PGXS rules, except for testdeps.

    +

    Runs your extension’s test suite: installs the extension and runs it through PGXS’s installcheck, first pulling in anything you’ve hooked into testdeps and, if enabled, sanity-checking your test SQL via test-build.

    +
    +
    +

    Whether test-build runs is controlled by the PGXNTOOL_ENABLE_TEST_BUILD variable — see test-build for what it does and how to turn it on/off.

    @@ -598,7 +622,19 @@

    Note

    + +
    -While you can still run make installcheck or any other valid PGXS make target directly, it’s recommended to use make test when using pgxntool. The test target ensures clean builds, proper test isolation, and correct dependency installation. +test intentionally does not depend on clean — that caused problems with incremental/watch-based builds. If your tests need a clean build to pass, that’s a sign of a missing dependency elsewhere rather than something to fix by adding clean back. +
    +
    +
    + + + +
    +
    Note
    +    		 +While you can still run make installcheck or any other valid PGXS make target directly, it’s recommended to use make test when using pgxntool. The test target ensures proper test isolation and correct dependency installation.
    @@ -786,6 +822,81 @@

    Key detail: Install files and regular tests run in a single pg_regress invocation. This means the database is NOT dropped between install and test phases — state created by install files persists into the main test suite.

    +
    +

    4.4.1. Update & Upgrade (U&U) Testing

    +
    +

    Beyond validating a plain install, it’s worth testing that your extension behaves correctly across the two transitions every extension with more than one release eventually goes through:

    +
    +
    +
      +
    • +

      Update: ALTER EXTENSION ext UPDATE, moving to a newer extension version on the same PostgreSQL version.

      +
      • +
    • +

      Upgrade: pg_upgrade, moving to a newer PostgreSQL major version while carrying the extension’s on-disk catalog state along with it.

      +
      • +
    +
    +
    +

    Both catch bugs that a simple "did the script run without error" check misses — wrong mappings, missing `REVOKE`s, objects that drifted from the base SQL, or catalog state that doesn’t survive a binary upgrade cleanly. This isn’t a niche concern that only applies if your extension does something unusual (adds enum types, custom operators, etc.) — it applies to any extension that will ever be updated or upgraded in place, which in practice is every extension.

    +
    +
    +

    test/install is the recommended place to build this, because its "load once, committed, before the suite" mechanic (described above) is exactly what update/upgrade testing needs.

    +
    +
    +

    The pattern: load the extension exactly once, committed, from a test/install/*.sql file, in one of several modes selected by a custom backend setting that your Makefile sets via PGOPTIONS (e.g. -c myext.test_load_mode=$(TEST_LOAD_SOURCE)) and that test/install reads with current_setting('myext.test_load_mode'):

    +
    +
    +
      +
    • +

      fresh: CREATE EXTENSION ext;

      +
      • +
    • +

      update: CREATE EXTENSION ext VERSION 'oldest_supported'; then ALTER EXTENSION ext UPDATE; (to the current default_version, or an explicit intermediate version)

      +
      • +
    • +

      existing: the extension is already installed — by a real pg_upgrade, or an update performed outside the suite — and test/install only asserts it’s present at the expected version, without dropping or touching it

      +
      • +
    +
    +
    +

    The same test/sql/ suite and the *same test/expected/ output then run unchanged against every mode. Identical expected output passing in *update mode is the update-equivalence assertion — if the updated database behaves any differently than a fresh install, some test will diff. Running that same suite in existing mode against a database that just went through a real pg_upgrade gives you upgrade testing using the same test files, no separate suite required.

    +
    +
    +

    If you adopt this pattern, test/deps.sql should no longer run CREATE EXTENSION itself for the per-test setup — the committed install performed by test/install persists into every (rolled-back) test file, exactly as described above.

    +
    +
    +

    Why the install step must be committed, not run per-test in test/deps.sql: every test/sql/ file runs inside BEGIN; …​ ROLLBACK; — a pgxntool convention (test/pgxntool/setup.sql opens the transaction; since it’s never committed, it rolls back when the session ends). Running the update inside that transaction means the suite never actually exercises a committed update — which doesn’t match production, where ALTER EXTENSION UPDATE commits before anything else touches the database, and it hides bugs that only show up once the update is durably committed. (Modifying an enum is one example of how this can turn into an outright failure rather than just a hidden bug — but it’s just an example; the reason to commit first holds regardless of what the update script does.)

    +
    +
    +

    As a bonus, sharing one committed install across every test file also removes the per-test re-install cost.

    +
    +
    + + + + + +
    +
    Note
    +    		 +This pattern isn’t (yet) native to pgxntool — you wire up the mode selection and PGOPTIONS plumbing yourself in test/install and your Makefile. First-class support (a standard mode-switching toggle, and scaffolding for real pg_upgrade testing) is planned; see issue #42. +
    +
    +
    +

    Working examples (specific files, not full PRs — both extensions' histories include plenty of unrelated changes):

    +
    +
    +
      +
    • +

      cat_tools test/install/load.sql and the TEST_LOAD_SOURCE block in its Makefile — the fresh/update/existing pattern above, including the PGOPTIONS wiring. This is a reasonably complete example, and somewhat more elaborate than the minimum needed to get started.

      +
      • +
    • +

      pg_count_nulls test/sql/extension_tests.sql additionally demonstrates testing an extension installed into a non-default schema. It’s a useful reference for that specific problem, but it also defines its assertions as plpgsql functions rather than plain pgTAP calls — a pattern that adds complexity of its own and isn’t recommended as a model for U&U testing itself.

      +
      • +
    +
    +

    4.5. testdeps

    @@ -808,7 +919,7 @@

    -

    If you want to over-ride the default dependency on pgtap you should be able to do that with a makefile override. If you need help with that, please open an issue.

    +

    If you want to over-ride the default dependency on pgtap you should be able to do that with a makefile override. If you need help with that, please open an issue.

    @@ -862,19 +973,34 @@

    4.6.1. verify-results safeguard

    -

    By default, make results will refuse to run if test/results/regression.diffs exists (indicating failing tests). This prevents accidentally updating expected files with incorrect output.

    +

    By default, make results will refuse to run if your tests are failing, so you can’t accidentally promote incorrect output into the new expected results. Which failures it looks for is controlled by PGXNTOOL_VERIFY_RESULTS_MODE:

    +
    +
    +
    +
    pgtap (the default)
    +
    +

    Scans test/results/*.out for pgTAP not ok lines (ignoring # TODO items) and plan-count mismatches, then also checks test/results/regression.diffs for any output mismatch pgTAP itself wouldn’t catch.

    +
    +
    diffs
    +
    +

    Only checks whether test/results/regression.diffs exists.

    +
    +
    +
    +
    +

    If tests are failing, make results prints an explanation of what failed and exits without touching test/expected/; fix the failures, then re-run make results.

    -

    If tests are failing, you’ll see:

    +

    To select a mode explicitly:

    -
    ERROR: Tests are failing. Cannot run 'make results'.
-Fix test failures first, then run 'make results'.
    +
    # In your Makefile
+PGXNTOOL_VERIFY_RESULTS_MODE = diffs  # or pgtap (the default)
    -

    To disable this safeguard (not recommended):

    +

    To disable this safeguard entirely (not recommended):

    @@ -890,10 +1016,10 @@

    4.7. tag

    -

    make tag will create a git branch for the current version of your extension, as determined by the META.json file. The reason to do this is so you can always refer to the exact code that went into a released version.

    +

    make tag will create a git tag for the current version of your extension, as determined by the META.json file (generated from META.in.json — see PGXN Distributions vs. Extensions), and push it to origin. The reason to do this is so you can always refer to the exact code that went into a released version.

    -

    If there’s already a tag for the current version that probably means you forgot to update META.json, so you’ll get an error. If you’re certain you want to over-write the tag, you can do make forcetag, which removes the existing tag (via make rmtag) and creates a new one.

    +

    If a tag for the current version already exists and points at your current commit, make tag does nothing — this makes it safe for other targets (like dist) to depend on tag without worrying about re-running it. If the existing tag points at a different commit — meaning you likely forgot to bump the version — you’ll get an error. If you’re certain you want to over-write the tag, you can do make forcetag, which removes the existing tag (via make rmtag) and creates a new one.

    @@ -902,7 +1028,7 @@

    4.
    Warning

    -You will be very unhappy if you forget to update the .control file for your extension! There is an open issue to improve this. +You will be very unhappy if you forget to update the .control file for your extension! There is an open issue to improve this.
    @@ -911,7 +1037,7 @@

    4.

    4.8. dist

    -

    make dist will create a .zip file for your current version that you can upload to PGXN. The file is named after the PGXN name and version (the top-level "name" and "version" attributes in META.json). The .zip file is placed in the parent directory so as not to clutter up your git repo.

    +

    make dist will create a .zip file for your current version that you can upload to PGXN. It first runs tag (creating/pushing the version’s git tag as described above if needed), then uses git archive at that tag to build the .zip file, so the archive always matches the exact tagged commit. The file is named after the PGXN name and version (the top-level "name" and "version" attributes in META.json, generated from META.in.json). The .zip file is placed in the parent directory so as not to clutter up your git repo.

    @@ -925,6 +1051,21 @@

    +
    + + + + + +
    +
    Warning
    +    		 +If your project has a .gitattributes file, make dist/make dist-only will refuse to run unless it’s committed to git, and will tell you so. This is because git archive (which builds the .zip) only honors export-ignore attributes on committed files — an uncommitted .gitattributes would silently have no effect on the archive contents, which could leak files into your PGXN distribution that you meant to exclude. +
    +
    +
    +

    make forcedist is a shortcut for forcetag dist: it force-recreates the tag (see forcetag above) before rebuilding the distribution .zip.

    +

    4.9. pgxntool-sync

    @@ -943,6 +1084,18 @@

    + + + + + +
    +
    Note
    +    		 +git subtree pull copies pgxntool’s entire tree into your project, including dev-only directories (pgxntool/.github, pgxntool/.claude) that don’t belong in a project that merely embeds pgxntool. Every sync automatically removes these two directories from pgxntool/ afterward — this is expected, not a bug, and doesn’t affect anything else in your project. +
    +

    @@ -962,6 +1115,18 @@

    Tip

    + +
    +pgxntool-sync.sh itself delegates the 3-way merge of setup.sh-copied files to pgxntool/update-setup-files.sh <old-pgxntool-commit>, which you can also run directly — for example, to redo or debug just the merge step without pulling again. <old-pgxntool-commit> is the pgxntool subtree commit that was current before the sync you want to re-merge. +
    +
     +
    + + + + @@ -969,9 +1134,39 @@

    -

    4.10. distclean

    +

    4.10. pgxntool-version

    -

    make distclean removes generated configuration files (META.json, meta.mk, control.mk) that survive a normal make clean.

    +

    make pgxntool-version prints the version of the embedded pgxntool copy, read from the first line of pgxntool/HISTORY.asc. That line is "STABLE" instead of a version number if this copy was synced from an unreleased commit rather than a tagged release.

    +
    +
    +
    +
    Tip
    +    		 There is also a pgxntool-sync-% rule if you need to do more advanced things. make pgxntool-sync-<name> pulls from the <repo> <ref> defined by the pgxntool-sync-<name> make variable.
    + + + + +
    +
    Tip
    +    		 +The actual work is done by pgxntool/bin/version, so you can run it directly (pgxntool/bin/version) if you’d rather not go through make. +
    +
     +

    +
    +

    4.11. list

    +
    +

    make list prints every make target defined in your project — including ones provided by PGXS and pgxntool, not just ones you wrote — one per line. Useful for discovering what’s available without reading through the Makefile.

    +
    +
    +
    +

    4.12. print-%

    +
    +

    make print-VARNAME prints the current value (and origin) of any make variable, e.g. make print-PGXNVERSION. Useful for debugging why a variable isn’t set to what you expect.

    +
    +
    +
    +

    4.13. distclean

    +
    +

    make distclean removes generated configuration files (META.json — generated from META.in.json — plus meta.mk and control.mk) that survive a normal make clean.

    @@ -995,16 +1190,24 @@

    -

    4.11. pgtle

    +

    4.14. pgtle

    Generates pg_tle (Trusted Language Extensions) registration SQL files for deploying extensions in managed environments like AWS RDS/Aurora. See [_pg_tle_Support] for complete documentation.

    make pgtle generates SQL files in pg_tle/ subdirectories organized by pg_tle version ranges. For version range details, see pgtle_versions.md.

    +
    +

    Set PGTLE_VERSION on the command line to limit generation to the single version range that value falls into, instead of every known range:

    +
    +
    +
    +
    make pgtle PGTLE_VERSION=1.5.0
    +
    +
    -

    4.12. check-pgtle

    +

    4.15. check-pgtle

    Checks if pg_tle is installed and reports the version. This target: - Reports the version from pg_extension if CREATE EXTENSION pg_tle has been run in the database @@ -1020,10 +1223,10 @@

    -

    4.13. run-pgtle

    +

    4.16. run-pgtle

    Registers all extensions with pg_tle by executing the generated pg_tle registration SQL files in a PostgreSQL database. This target: -- Requires pg_tle extension to be installed (checked via check-pgtle) +- Requires pg_tle to be installed in the target database. This isn’t a make-level dependency on check-pgtle — pgtle.sh checks it itself when it runs, and will tell you to run make check-pgtle if pg_tle isn’t there - Uses pgtle.sh to determine which version range directory to use based on the installed pg_tle version - Runs all generated SQL files via psql to register your extensions with pg_tle

    @@ -1065,14 +1268,36 @@

    -

    5.1. How Version Files Are Generated

    +

    5.1. PGXN Distributions vs. Extensions

    +
    +

    PGXN distinguishes two things it’s easy to conflate: a distribution and an extension. Per the PGXN meta spec, a distribution is "a collection of extensions, source code, utilities, tests, and/or documents that are distributed together" — the thing you release and upload to PGXN, identified by the top-level name/version in META.json (generated from META.in.json — you edit the latter, never META.json directly). An extension is the individual thing installed via CREATE EXTENSION, described by its own .control file. A single distribution routinely provides more than one extension — META.in.json’s `provides map lists each one; the pgTAP distribution, for example, provides both the pgtap and schematap extensions from one release.

    +
    +
    +

    This split matters for versioning, and it’s where pgxntool has a real gap: META.json’s top-level `version is the distribution’s release version (used by make tag and make dist — it’s the git tag name and the .zip filename). Each extension’s own version, though, ends up tracked in two separate places that pgxntool does not keep in sync for you:

    +
    +
    +
      +
    • +

      the extension’s .control file (default_version) — what PostgreSQL and pgxntool’s own build actually use; see What Controls the Version Number below

      +
      • +
    • +

      that extension’s entry under META.in.json’s `provides map (provides.{extension}.version) — which the PGXN meta spec defines as "a Version for the extension" in its own right, not the distribution’s version, and which is what PGXN’s public index reads

      +
      • +
    +
    +
    +

    Nothing generates one from the other or checks that they still agree, so it’s easy to bump one and forget the other; issue #47 tracks fixing that.

    +
    +
    +
    +

    5.2. How Version Files Are Generated

    When you run make (or make all), PGXNtool:

    1. -

      Reads your META.json file to determine the extension version from provides.{extension}.version

      +

      Reads each extension’s own .control file to determine its version from default_version (via control.mk.sh)

    2. Generates a Makefile rule that copies your base SQL file (sql/{extension}.sql) to the version-specific file (sql/{extension}--{version}.sql)

      @@ -1083,16 +1308,11 @@

      -

      For example, if your META.json contains:

      +

      For example, if your myext.control contains:

    -
    "provides": {
-  "myext": {
-    "version": "1.2.3",
-    ...
-  }
-}
    +
    default_version = '1.2.3'
    @@ -1100,21 +1320,22 @@

    -

    5.2. What Controls the Version Number

    +

    5.3. What Controls the Version Number

    -

    The version number comes from META.json → provides.{extension}.version, not from your .control file’s default_version field. The .control file’s default_version is used by PostgreSQL to determine which version to install by default, but the actual version-specific file that gets generated is determined by what’s in META.json.

    +

    The version number comes from each extension’s own .control file → default_version, not from META.json. PostgreSQL itself uses default_version to pick which versioned SQL file to load, so control.mk.sh parses the .control file(s) directly to keep the generated filename in sync with what PostgreSQL will actually use. META.in.json has its own, separate provides.{extension}.version for that same extension (see PGXN Distributions vs. Extensions above) — pgxntool never reads it when generating SQL files, and nothing keeps it in sync with the .control file’s default_version.

    -

    To change the version of your extension: -1. Update provides.{extension}.version in META.json +

    To change the version of one of your extensions: +1. Update default_version in that extension’s .control file 2. Run make to regenerate the version-specific file -3. Update default_version in your .control file to match (if needed)

    +3. Update that extension’s provides.{extension}.version in META.in.json to match by hand, and regenerate META.json (make META.json) — pgxntool won’t do this for you +4. Update META.in.json’s top-level `version too if you’re also cutting a new distribution release

    -

    5.3. Committing Version Files

    +

    5.4. Committing Version Files

    -

    Version-specific SQL files are now treated as permanent files that should be committed to your repository. This makes it much easier to test updates to extensions, as you can see exactly what SQL was included in each version.

    +

    Version-specific SQL files are treated as permanent files that should be committed to your repository by default. This makes it much easier to test updates to extensions, as you can see exactly what SQL was included in each version.

    @@ -1128,9 +1349,81 @@

    +

    5.4.1. Why Commit Them: Update Testing

    +
    +

    The primary value of committing a version’s install script is update testing: with the file in place, you can install that exact version (CREATE EXTENSION ext VERSION 'x.y.z'), run ALTER EXTENSION ext UPDATE, and verify the update path actually works. In principle you would only ever need the very first committed version’s install script — every later version is reachable by chaining upgrade scripts from it. See test/install for how to wire this into your test suite.

    +
    +
    +

    In practice, you should keep most old versions tracked rather than relying purely on that chain, because you cannot predict when a new major PostgreSQL version will break the ability to install an older extension version — a system catalog column changes type, a SELECT * over a catalog starts failing, and so on. Committing the old version’s install script lets CI catch that regression directly, by attempting to install that exact historical version against the new PostgreSQL release. For any non-trivial extension, most old versions should stay tracked for this reason alone.

    +
    + +
    +

    5.4.2. When It’s OK to Skip a Version

    +
    +

    For a minor version change that doesn’t meaningfully alter the extension (e.g. a small bug fix), it’s unlikely to straddle a PostgreSQL supported-version boundary, so there’s much less test-coverage value in committing that specific version’s generated install script. For large extensions, deliberately not committing some of these individual version files is worth doing to keep repository size down — the base sql/{extension}.sql still fully captures the change in git history, and the install script itself is trivially regenerated from that base file at build time.

    +
    +
    +
    +

    5.4.3. Don’t .gitignore a Skipped Version — rm It Once

    +
    +

    When you decide not to track a particular version’s generated install script, do not add a .gitignore entry for it.

    +
    +
    +

    make only ever generates the current version’s file, stamped from the base source according to that extension’s .control file default_version. The moment you bump the version, make starts generating the new current version’s file and stops touching the old one — the old file left behind in your working tree is a stale, one-off artifact, not something make keeps recreating on every build.

    +
    +
    +

    Because it’s only ever transiently present, the correct cleanup is a single rm sql/{extension}--{old-version}.sql right after the version bump — not a permanent .gitignore rule. A per-version .gitignore line would be perpetual clutter added on every release, for a file that’s only ever present for one build cycle. A broad glob (e.g. sql/--.sql) is wrong for the same reason it’s wrong below in Alternative: Ignoring All Version Files: it would also hide the prior-version install scripts and upgrade scripts (sql/{extension}--{a}--{b}.sql) that you do want tracked and visible in git status.

    +
    +
    +
    +

    5.4.4. Never Hand-Edit a Version File That’s No Longer Current

    +
    +

    control.mk.sh generates a Make rule along these lines for the current version’s file:

    +
    +
    +
    +
    $(EXTENSION_ext_VERSION_FILE): sql/ext.sql extension.control
+	@(echo '/* DO NOT EDIT - AUTO-GENERATED FILE */'; cat sql/ext.sql) > $(EXTENSION_ext_VERSION_FILE)
    +
    +
    +
    +

    That rule only ever targets the file matching the extension’s current default_version, so editing the base sql/{extension}.sql and running make is the correct, safe way to change that one file.

    +
    +
    +

    Every other versioned file — any sql/{extension}--{version}.sql where {version} is no longer current — is a frozen historical record, not something make will ever regenerate or overwrite again. Its entire purpose is to preserve exactly what shipped in that version (see Why Commit Them: Update Testing above), so it can keep being used to test update paths and PostgreSQL-version compatibility against exactly what your users actually installed.

    +
    +
    +

    Hand-editing an old versioned file directly is never correct — it silently corrupts that historical record. If you need to change behavior after a version has already shipped, bump the version and add a proper sql/{extension}--{old}--{new}.sql upgrade script instead. Never edit sql/{extension}--{old}.sql in place.

    +
    +
    +

    + + + + +
    +
    Caution
    +    		 +This is especially relevant for AI coding agents, which won’t know this convention exists unless it’s spelled out. The generic DO NOT EDIT - AUTO-GENERATED FILE header on every version file doesn’t distinguish "regenerated on every build" (the current version) from "was generated once, now frozen" (every other version). An agent without this context may reasonably — but incorrectly — treat any auto-generated file as fair game to patch directly. Old versioned SQL files must instead be treated as append-only history: the fix is always a new version plus an upgrade script, never an in-place edit. +
    +
    +
    -

    5.4. Alternative: Ignoring Version Files

    +

    5.5. Alternative: Ignoring All Version Files

    +
    + + + + + +
    +
    Note
    +    		 +This is a different, all-or-nothing choice from When It’s OK to Skip a Version above, where the recommendation is to keep tracking most versions and selectively skip only a few low-value minor ones. This section instead covers not tracking any version-specific files at all. +
    +

    If you prefer not to commit version-specific SQL files, you must add them to your .gitignore to prevent make dist from failing due to untracked files. Add the following to your .gitignore:

    @@ -1158,13 +1451,13 @@

    -

    5.5. Distribution Inclusion

    +

    5.6. Distribution Inclusion

    Version-specific files are included in distributions created by make dist only if they are committed to git. Since make dist uses git archive, only tracked files are included in the distribution archive.

    -

    5.6. Multiple Versions

    +

    5.7. Multiple Versions

    If you need to support multiple versions of your extension:

    @@ -1177,7 +1470,7 @@

    Create upgrade scripts for version transitions (e.g., sql/myext—​1.0.0—​1.1.0.sql)

  • -

    Update META.json to reflect the current version you’re working on

    +

    Update default_version in the extension’s .control file to reflect the current version you’re working on

  • Commit all version files and upgrade scripts to your repository

    @@ -1185,7 +1478,7 @@

    • -

    The version file for the current version (specified in META.json) will be automatically regenerated when you run make, but other version files you create manually will be preserved.

    +

    The version file for the current version (specified in the .control file’s default_version) will be automatically regenerated when you run make, but other version files you create manually will be preserved.

    @@ -1251,9 +1544,8 @@

    <

    6.2. Document Rules

    -

    If Asciidoc is found (or $(ASCIIDOC) is set), the html rule will be added as a prerequisite to the install and installchec rules. -That will ensure that docs are generated for install and test, but only if Asciidoc is available. -The dist rule will always depend on html though, to ensure html files are up-to-date before creating a distribution.

    +

    If Asciidoc is found (or $(ASCIIDOC) is set), html is added as a dependency of the all target, which both install and installcheck depend on — so docs get built for install and test, but only if Asciidoc is available. +The dist rule always depends on html directly, regardless of whether Asciidoc was found, to ensure html files are up-to-date before creating a distribution.

    The html rule simply depends on `$(ASCIIDOC_HTML). @@ -1268,11 +1560,11 @@ 

    define ASCIIDOC_template
 %.html: %.$(1) # (1)
-ifndef ASCIIDOC
+ifeq (,$(strip $(ASCIIDOC)))
 	$$(warning Could not find "asciidoc" or "asciidoctor". Add one of them to your PATH,)
 	$$(warning or set ASCIIDOC to the correct location.)
 	$$(error Could not build %$$@)
-endif # ifndef ASCIIDOC
+endif # ifeq ASCIIDOC
 	$$(ASCIIDOC) $$(ASCIIDOC_FLAGS) $$<
 endef # define ASCIIDOC_template

    @@ -1289,6 +1581,9 @@

    $(ASCIIDOC) isn’t defined. On a normal user system that should never happen, because the html rule won’t be included in install or installcheck.

    +
    +

    make docclean removes all generated HTML files ($(DOCS_HTML)). It’s separate from make clean/make distclean, so you can clear out generated docs without touching your build artifacts.

    +

    6.3. The DOCS variable

    @@ -1423,6 +1718,28 @@

    +

    7.5.2. Direct pgtle.sh Invocation

    +
    +

    pgtle, check-pgtle, and run-pgtle are thin make wrappers around pgtle.sh. For more targeted operations you can run it directly:

    +
    +
    +
      +
    • +

      pgtle.sh --extension NAME [--pgtle-version VERSION] — generate registration SQL for one extension (what make pgtle wraps).

      +
      • +
    • +

      pgtle.sh --get-version — print the pg_tle version installed in the target database (empty if not installed).

      +
      • +
    • +

      pgtle.sh --get-dir VERSION — print which version-range directory (see Version Groupings) a given pg_tle version maps to, without generating anything.

      +
      • +
    • +

      pgtle.sh --run — execute the generated registration SQL against the target database (what make run-pgtle wraps).

      +
      • +
    +
    +

    7.6. How It Works

    @@ -1497,23 +1814,147 @@

    there are several untrusted languages (such as plpython), and the only tests for C. -== Copyright -Copyright (c) 2026 Jim Nasby <Jim.Nasby@gmail.com>

    + + + + +
    +

    8. Configuration Variables

    +
    -

    PGXNtool is released under a BSD license. Note that it includes JSON.sh, which is released under a MIT license.

    +

    This is the comprehensive list of make variables you can override to customize pgxntool’s behavior, either in your Makefile (before include pgxntool/base.mk) or on the make command line. Several of these are also covered in more detail in the sections above; this section exists so there’s one place that lists all of them.

    +
    +
    + + + + + +
    +
    Note
    +    		 +Variables marked with * must be set to exactly yes or no (case-insensitive) when given explicitly; any other value is a hard make error. +
    +
    +
    +

    8.1. PGXN_REMOTE

    +
    +

    Default: origin. The git remote used by tag, rmtag, forcetag, and dist (which depends on tag). If your origin is a personal fork rather than the canonical repository — for example, if you develop on a fork and only push release tags to the canonical repo — set this so tagging and pushing target the right remote:

    +
    +
    +
    +
    make dist PGXN_REMOTE=upstream
    +
    +
    +
    +
    +

    8.2. ASCIIDOC

    +
    +

    Default: auto-detected, the first of asciidoctor or asciidoc found on PATH. Path to the Asciidoc processor used to build .html files from $(ASCIIDOC_EXTS) source files. Override if the processor you want isn’t first on PATH, or isn’t on PATH at all. See Document Handling.

    +
    +
    +
    +

    8.3. PGTLE_VERSION

    +
    +

    Default: unset (generates every known pg_tle version range). Set on the command line to limit make pgtle to the single version range this value falls into. See pgtle.

    +
    +
    +
    +

    8.4. PG_CONFIG

    +
    +

    Default: pg_config. Path to the pg_config binary used to detect the PostgreSQL version and locate PGXS. Override when the right pg_config isn’t the one on PATH, such as when testing against a specific PostgreSQL install.

    +
    +
    +
    +

    8.5. TESTDIR

    +
    +

    Default: test. Root directory for test input files: $(TESTDIR)/sql/, $(TESTDIR)/expected/, $(TESTDIR)/build/, $(TESTDIR)/install/. Overriding this on its own is unusual; it mainly exists so TESTOUT has a sensible default.

    +
    +
    +
    +

    8.6. TESTOUT

    +
    +

    Default: $(TESTDIR). Directory pg_regress writes actual test output to — $(TESTOUT)/results/, $(TESTOUT)/regression.diffs, etc. — via --outputdir in REGRESS_OPTS. Kept separate from TESTDIR so generated output can be pointed somewhere other than the directory holding your checked-in sql/expected files, if you want that separation.

    +
    +
    +
    +

    8.7. PGXNTOOL_VERIFY_RESULTS_MODE

    +
    +

    Default: pgtap. Controls how the verify-results safeguard decides whether tests are failing.

    +
    +
    +

    8.7.1. pgtap

    +
    +

    Scans test/results/*.out for pgTap not ok lines and plan mismatches, falling back to checking for regression.diffs too. Use this mode when your test suite uses pgTap.

    +
    +
    +
    +

    8.7.2. diffs

    +
    +

    Checks only for regression.diffs, matching classic pg_regress behavior. Use this mode when your tests use plain SQL expected-output comparison only.

    +
    +
    +
    +
    +

    8.8. PGXNTOOL_ENABLE_TEST_BUILD *

    +
    +

    Default: auto-detected — yes if test/build/*.sql files exist, no otherwise. Enables or disables the test-build pre-flight SQL check. Set explicitly to yes if you want an error when test/build/ unexpectedly has no SQL files (catches accidental deletion of its contents), or to no to disable the check even when files are present.

    +
    +
    +
    +

    8.9. PGXNTOOL_ENABLE_TEST_INSTALL *

    +
    +

    Default: auto-detected — yes if test/install/*.sql files exist, no otherwise. Enables or disables the test/install schedule-based setup feature. Same explicit-override semantics as PGXNTOOL_ENABLE_TEST_BUILD.

    +
    +
    +
    +

    8.10. PGXNTOOL_ENABLE_VERIFY_RESULTS *

    +
    +

    Default: yes. Enables or disables the verify-results safeguard that blocks make results when tests are failing. Setting it to empty on the command line (make PGXNTOOL_ENABLE_VERIFY_RESULTS= results) also disables it.

    +
    +
    +
    +

    8.11. PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED *

    +
    +

    Default: yes. Enables or disables the check-stale-expected safeguard, which fails make test if test/expected/ (or test/build/expected/) contains a .out file with no corresponding .sql file — catching a stale file left behind after a test was renamed or removed. Set to no to make the check a complete no-op (it’s dropped from TEST_DEPS entirely).

    +
    +
    +

    This is also the supported pattern for disabling one specific optional pgxntool feature: a documented PGXNTOOL_ENABLE_* variable, gating both the target’s registration into TEST_DEPS/etc. and (where applicable) the target’s own definition. Cleanly replacing or overriding an arbitrary pgxntool-generated recipe with your own entirely is a separate, larger, and not-yet-decided design question — see issue #30 (two proposed approaches) — and is not what this mechanism solves.

    +
    +
    +
    +

    8.12. PGXNTOOL_CHECK_EXPECTED_FILE_TYPES *

    +
    +

    Default: yes. Sub-check of check-stale-expected, independent of PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED: fails (with a distinct error message and exit code from the orphaned-.out check) if test/expected/ (or test/build/expected/) contains any file that isn’t *.out. Set to no to disable just this sub-check while leaving the orphaned-.out check active.

    +
    +
    +
    +

    8.13. PGXNTOOL_NO_PGXS_INCLUDE

    +
    +

    Default: unset (PGXS is included normally). Skips including PGXS ($(PGXS)) entirely. This is only for advanced scenarios where you need to manage the PGXS include yourself; most projects should never set this.

    +
    +
    +
    +

    9. Copyright

    +
    +
    +

    Copyright (c) 2026 Jim Nasby <Jim.Nasby@gmail.com>

    +
    +
    +

    PGXNtool is released under a BSD license. Note that it includes JSON.sh, which is released under a MIT license.

    diff --git a/pgxntool/base.mk b/pgxntool/base.mk index a7ab93c..e77cff2 100644 --- a/pgxntool/base.mk +++ b/pgxntool/base.mk @@ -1,3 +1,12 @@ +# Include guard: base.mk can end up included twice in a single `make` run +# (e.g. an extension's own .mk module includes it, and the extension's +# Makefile also includes it directly via the line setup.sh writes). Without +# this, every target in the file gets redefined, producing +# overriding-recipe/ignoring-old-recipe warnings. A second inclusion is a +# harmless no-op. +ifndef PGXNTOOL_BASE_MK_INCLUDED +PGXNTOOL_BASE_MK_INCLUDED := 1 + PGXNTOOL_DIR := pgxntool # Ensure 'all' is the default target (not META.json which happens to be first) @@ -37,7 +46,7 @@ control.mk: $(PGXNTOOL_CONTROL_FILES) Makefile $(PGXNTOOL_DIR)/base.mk $(PGXNTOO -include control.mk -DATA = $(EXTENSION_VERSION_FILES) $(wildcard sql/*--*--*.sql) +DATA = $(EXTENSION__CURRENT_VERSION__FILES) $(wildcard sql/*--*--*.sql) DOC_DIRS += doc # NOTE: if this is empty it gets forcibly defined to NUL before including PGXS DOCS += $(foreach dir,$(DOC_DIRS),$(wildcard $(dir)/*)) @@ -182,6 +191,47 @@ endif # Default mode: pgtap (scans results/*.out for TAP failures) PGXNTOOL_VERIFY_RESULTS_MODE ?= pgtap +# ------------------------------------------------------------------------------ +# check-stale-expected: catch orphaned/unexpected test/expected/ files +# ------------------------------------------------------------------------------ +# Variable: PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED +# - Can be set manually in Makefile or command line +# - Allowed values: "yes" or "no" (case-insensitive) +# - Default: "yes" (enabled by default for all pgxntool projects) +# - Set to "no" to make this check a complete no-op: the target is +# dropped from TEST_DEPS entirely (see its own definition below) +# +# Variable: PGXNTOOL_CHECK_EXPECTED_FILE_TYPES +# - Sub-check, independent of the variable above: fails if test/expected/ +# (or test/build/expected/) contains any file that isn't *.out +# - Allowed values: "yes" or "no" (case-insensitive) +# - Default: "yes" +# - Set to "no" to disable just this sub-check while leaving the rest of +# check-stale-expected (the orphaned-.out check) active +# - Passed through to check-stale-expected.sh; see that script for the +# distinct error message/exit code this sub-check uses +# +# Variable: _CHECK_STALE_EXPECTED_SCRIPT (internal shim, not user-facing) +# - Path to the script the check-stale-expected target invokes +# - Default: $(PGXNTOOL_DIR)/test/bin/check-stale-expected.sh +# +# Implementation: See check-stale-expected target definition (search for +# "check-stale-expected:" in this file) +# +ifdef PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED + override PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED := $(call pgxntool_validate_yesno,$(PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED),PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED) +else + PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED = yes +endif + +ifdef PGXNTOOL_CHECK_EXPECTED_FILE_TYPES + override PGXNTOOL_CHECK_EXPECTED_FILE_TYPES := $(call pgxntool_validate_yesno,$(PGXNTOOL_CHECK_EXPECTED_FILE_TYPES),PGXNTOOL_CHECK_EXPECTED_FILE_TYPES) +else + PGXNTOOL_CHECK_EXPECTED_FILE_TYPES = yes +endif + +_CHECK_STALE_EXPECTED_SCRIPT ?= $(PGXNTOOL_DIR)/test/bin/check-stale-expected.sh + # Generate unique database name for tests to prevent conflicts across projects # Uses project name + first 5 chars of md5 hash of current directory # This prevents multiple test runs in different directories from clobbering each other @@ -193,6 +243,12 @@ MODULES =# Set to NUL so PGXS doesn't puke endif EXTRA_CLEAN = $(wildcard ../$(PGXN)-*.zip) pg_tle/ +# PGXS's own pg_regress_clean_files unconditionally rm -rf's a top-level +# results/, but our tests write to $(TESTOUT)/results/ (see REGRESS_OPTS +# --outputdir above), so that's the directory that actually needs cleaning. +# filter-out (not -=, which GNU Make 4.4+ rejects as a parse error) guards +# against a stray top-level results/ entry while adding the real one. +EXTRA_CLEAN := $(filter-out results/,$(EXTRA_CLEAN)) $(TESTOUT)/results/ # Get Postgres version, as well as major (9.4, etc) version. # NOTE! In at least some versions, PGXS defines VERSION, so we intentionally don't use that variable @@ -206,7 +262,7 @@ test = $(shell test $(1) $(2) $(3) && echo yes || echo no) GE91 = $(call test, $(MAJORVER), -ge, 91) ifeq ($(GE91),yes) -all: $(EXTENSION_VERSION_FILES) +all: $(EXTENSION__CURRENT_VERSION__FILES) endif ifeq ($(call test, $(MAJORVER), -lt, 130), yes) @@ -256,6 +312,37 @@ installcheck: $(TEST_RESULT_FILES) $(TEST_SQL_FILES) | $(TESTDIR)/sql/ $(TESTDIR # # These targets are meant to make running tests easier. +# Build test dependencies list based on enabled features. This base +# assignment (a plain `=`, not `+=`) must come before any `TEST_DEPS +=` +# line below -- Make processes the file top-to-bottom, and a later plain +# `=` would silently wipe out any `+=` that came before it. +TEST_DEPS = testdeps + +# ------------------------------------------------------------------------------ +# check-stale-expected: catch orphaned/unexpected test/expected/ files +# ------------------------------------------------------------------------------ +# Purpose: test/expected/*.out must mirror test/sql/*.sql 1:1 (likewise +# test/build/expected/*.out vs test/build/*.sql, when test-build is in use). +# It's easy to leave a stale .out behind after renaming or removing a .sql +# file; this makes `make test` fail loudly instead of letting it linger +# unnoticed. Logic lives in check-stale-expected.sh (enough of it to warrant +# a real script rather than an inline recipe). +# +# This depends on `installcheck` directly (not just position in TEST_DEPS) +# because it MUST run after pg_regress, not before: TEST_DEPS lists multiple +# independent prerequisites of `test`, and Make does not guarantee the order +# unrelated prerequisites of the same target are built in. An explicit +# dependency edge is the only ordering guarantee Make actually gives. +# +# See PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED / PGXNTOOL_CHECK_EXPECTED_FILE_TYPES +# above for how to disable this entirely or just its non-.out file sub-check. +ifeq ($(PGXNTOOL_ENABLE_CHECK_STALE_EXPECTED),yes) +.PHONY: check-stale-expected +check-stale-expected: installcheck + @$(_CHECK_STALE_EXPECTED_SCRIPT) $(TESTDIR) $(PGXNTOOL_CHECK_EXPECTED_FILE_TYPES) +TEST_DEPS += check-stale-expected +endif + # make test: run any test dependencies, then do a `make install installcheck`. # If regressions are found, it will output them. # @@ -263,8 +350,6 @@ installcheck: $(TEST_RESULT_FILES) $(TEST_SQL_FILES) | $(TESTDIR)/sql/ $(TESTDIR # watch-make if you're generating intermediate files. If tests end up needing # clean it's an indication of a missing dependency anyway. .PHONY: test -# Build test dependencies list based on enabled features -TEST_DEPS = testdeps ifeq ($(PGXNTOOL_ENABLE_TEST_BUILD),yes) TEST_DEPS += test-build endif @@ -326,13 +411,14 @@ PGXNTOOL_EXTENSIONS = $(basename $(PGXNTOOL_CONTROL_FILES)) # Main target # Depend on 'all' to ensure versioned SQL files are generated first -# Depend on control.mk (which defines EXTENSION_VERSION_FILES) +# Depend on control.mk (which defines EXTENSION__CURRENT_VERSION__FILES) # Depend on control files explicitly so changes trigger rebuilds -# Generates all supported pg_tle versions for each extension +# Generates all supported pg_tle versions for each extension, unless +# PGTLE_VERSION is set on the command line to limit output to one range .PHONY: pgtle pgtle: all control.mk $(PGXNTOOL_CONTROL_FILES) @$(foreach ext,$(PGXNTOOL_EXTENSIONS),\ - $(PGXNTOOL_DIR)/pgtle.sh --extension $(ext);) + $(PGXNTOOL_DIR)/pgtle.sh --extension $(ext) $(if $(PGTLE_VERSION),--pgtle-version $(PGTLE_VERSION));) # # pg_tle installation support @@ -438,10 +524,14 @@ docclean: # # TAGGING SUPPORT # +# Remote used for tag/rmtag/forcetag/dist. Override on the command line or in +# your Makefile if you push tags somewhere other than origin. +PGXN_REMOTE ?= origin + rmtag: - git fetch origin # Update our remotes + git fetch $(PGXN_REMOTE) # Update our remotes @test -z "$$(git tag --list $(PGXNVERSION))" || git tag -d $(PGXNVERSION) - @test -z "$$(git ls-remote --tags origin $(PGXNVERSION) | grep -v '{}')" || git push --delete origin $(PGXNVERSION) + @test -z "$$(git ls-remote --tags $(PGXN_REMOTE) $(PGXNVERSION) | grep -v '{}')" || git push --delete $(PGXN_REMOTE) $(PGXNVERSION) tag: @test -z "$$(git status --porcelain)" || (echo 'Untracked changes!'; echo; git status; exit 1) @@ -456,7 +546,7 @@ tag: else \ git tag $(PGXNVERSION); \ fi - git push origin $(PGXNVERSION) + git push $(PGXN_REMOTE) $(PGXNVERSION) .PHONY: forcetag forcetag: rmtag tag @@ -506,6 +596,12 @@ pgxntool-sync: pgxntool-sync-%: @pgxntool/pgxntool-sync.sh $($@) +# `make pgxntool-version` prints the version of the embedded pgxntool copy. +# Delegates to bin/version so it can be run without make too. +.PHONY: pgxntool-version +pgxntool-version: + @$(PGXNTOOL_DIR)/bin/version + # DANGER! Use these with caution. They may add extra crap to your history and # could make resolving merges difficult! # `pgxntool-sync` (no suffix) already pulls the canonical release; these are the @@ -559,3 +655,5 @@ $(DESTDIR)$(datadir)/extension/pgtap.control: pgxn install pgtap --sudo endif # fndef PGXNTOOL_NO_PGXS_INCLUDE + +endif # ifndef PGXNTOOL_BASE_MK_INCLUDED diff --git a/pgxntool/bin/version b/pgxntool/bin/version new file mode 100755 index 0000000..c18af60 --- /dev/null +++ b/pgxntool/bin/version @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# bin/version - Print the pgxntool version embedded in this project +# +# The release process (see .claude/skills/release) stamps the first line of +# HISTORY.asc with the released version number, so that line is the source +# of truth for "what version of pgxntool is this?". If this copy was synced +# from an unreleased commit rather than a tagged release, the first line +# will be "STABLE" instead of a version number -- that's printed as-is, since +# it's an accurate answer too. Anything else (a corrupt or hand-edited file) +# is rejected rather than printed as if it were a real answer. +# +# Usage: bin/version [] +# +# history-file Path to HISTORY.asc to read. Defaults to the copy +# one level up from this script (i.e. pgxntool/HISTORY.asc +# in a normal checkout). + +set -o errexit -o errtrace -o pipefail +trap 'echo "Error on line ${LINENO}"' ERR + +PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")/.." +source "$PGXNTOOL_DIR/lib.sh" + +history_file="${1:-$PGXNTOOL_DIR/HISTORY.asc}" + +[[ -f "$history_file" ]] || die 1 "$history_file not found" + +version=$(head -n1 "$history_file") + +[[ -n "$version" ]] || die 1 "$history_file is empty" + +# Matches the X.Y.Z format the release process validates and stamps. +[[ "$version" == "STABLE" || "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || \ + die 1 "$history_file: first line '$version' is neither STABLE nor a X.Y.Z version" + +echo "$version" diff --git a/pgxntool/control.mk.sh b/pgxntool/control.mk.sh index cc63cea..aa491cf 100755 --- a/pgxntool/control.mk.sh +++ b/pgxntool/control.mk.sh @@ -13,7 +13,7 @@ # EXTENSION_SQL_FILES += sql/.sql # EXTENSION__VERSION := # EXTENSION__VERSION_FILE = sql/--.sql -# EXTENSION_VERSION_FILES += $(EXTENSION__VERSION_FILE) +# EXTENSION__CURRENT_VERSION__FILES += $(EXTENSION__VERSION_FILE) # # # Why control files instead of META.json? @@ -80,10 +80,13 @@ for control_file in "$@"; do echo "EXTENSION_SQL_FILES += sql/${ext}.sql" echo "EXTENSION_${ext}_VERSION := ${version}" echo "EXTENSION_${ext}_VERSION_FILE = sql/${ext}--\$(EXTENSION_${ext}_VERSION).sql" - echo "EXTENSION_VERSION_FILES += \$(EXTENSION_${ext}_VERSION_FILE)" + echo "EXTENSION__CURRENT_VERSION__FILES += \$(EXTENSION_${ext}_VERSION_FILE)" echo "\$(EXTENSION_${ext}_VERSION_FILE): sql/${ext}.sql ${control_file}" - echo " @echo '/* DO NOT EDIT - AUTO-GENERATED FILE */' > \$(EXTENSION_${ext}_VERSION_FILE)" - echo " @cat sql/${ext}.sql >> \$(EXTENSION_${ext}_VERSION_FILE)" + # Single redirect via a subshell, not `>` truncate followed by `>>` append: + # if this rule ever runs twice in the same `make` invocation (e.g. reached + # through two different dependency chains under parallel make), the `>>` + # form doubles the file's content instead of overwriting it. + echo " @(echo '/* DO NOT EDIT - AUTO-GENERATED FILE */'; cat sql/${ext}.sql) > \$(EXTENSION_${ext}_VERSION_FILE)" echo done diff --git a/pgxntool/pgtle.sh b/pgxntool/pgtle.sh index 124ca84..e6229fd 100755 --- a/pgxntool/pgtle.sh +++ b/pgxntool/pgtle.sh @@ -593,10 +593,18 @@ extract_version_from_filename() { # Match patterns: # - ext--1.0.0 → FROM_VERSION=1.0.0, TO_VERSION="" # - ext--1.0.0--2.0.0 → FROM_VERSION=1.0.0, TO_VERSION=2.0.0 + # - ext--stable → FROM_VERSION=stable, TO_VERSION="" (pg_tle treats + # versions as opaque strings; see check_valid_version_name() in + # pg_tle's tleextension.c, which only forbids empty strings, "--", + # and leading/trailing "-" — no numeric requirement) + # + # Each version segment is [^-]+(-[^-]+)* : one or more non-dash runs + # joined by single dashes, so a segment can't be empty, start/end with + # "-", or contain "--" (which is reserved as the FROM/TO delimiter). - if [[ "$basename" =~ ^${EXTENSION}--([0-9][0-9.]*)(--([0-9][0-9.]*))?$ ]]; then + if [[ "$basename" =~ ^${EXTENSION}--([^-]+(-[^-]+)*)(--([^-]+(-[^-]+)*))?$ ]]; then FROM_VERSION="${BASH_REMATCH[1]}" - TO_VERSION="${BASH_REMATCH[3]}" # Empty for non-upgrade files + TO_VERSION="${BASH_REMATCH[4]}" # Empty for non-upgrade files return 0 else die 1 "Cannot parse version from filename: $filename diff --git a/pgxntool/pgtle_versions.md b/pgxntool/pgtle_versions.md index d2c5c03..74f8184 100644 --- a/pgxntool/pgtle_versions.md +++ b/pgxntool/pgtle_versions.md @@ -7,13 +7,12 @@ This file documents pg_tle version boundaries that affect pgxntool's pg_tle supp ### 1.0.0-1.4.0 - **pg_tle versions:** 1.0.0 through 1.3.x - **PostgreSQL support:** 11-17 -- **API:** No `pgtle.uninstall_extension()` function, no schema parameter - **Features:** Basic extension management, custom data types, authentication hooks ### 1.4.0-1.5.0 - **pg_tle versions:** 1.4.0 through 1.4.x - **PostgreSQL support:** 11-17 -- **API:** Added `pgtle.uninstall_extension()` function, no schema parameter +- **API:** Added alignment/storage parameters to `pgtle.create_base_type()` - **Features:** Custom alignment/storage, enhanced warnings ### 1.5.0+ @@ -24,10 +23,10 @@ This file documents pg_tle version boundaries that affect pgxntool's pg_tle supp ## Key API Changes by Version -**1.4.0:** Added `pgtle.uninstall_extension()` -- Versions before 1.4.0 cannot uninstall extensions +**1.4.0:** Added alignment/storage parameters to `pgtle.create_base_type()` -**1.5.0:** Changed `pgtle.install_extension()` signature +**1.5.0** +- Changed `pgtle.install_extension()` signature - Added required `schema` parameter - Dropped PostgreSQL 11 support diff --git a/pgxntool/setup.sh b/pgxntool/setup.sh index 54b3633..8a31d0e 100755 --- a/pgxntool/setup.sh +++ b/pgxntool/setup.sh @@ -87,4 +87,4 @@ rmdir src If everything looks good then -git commit -am 'Add pgxntool (https://github.com/decibel/pgxntool/tree/release)'" +git commit -am 'Add pgxntool (https://github.com/Postgres-Extensions/pgxntool/tree/release)'" diff --git a/pgxntool/test/bin/check-stale-expected.sh b/pgxntool/test/bin/check-stale-expected.sh new file mode 100755 index 0000000..6a4d221 --- /dev/null +++ b/pgxntool/test/bin/check-stale-expected.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# +# check-stale-expected.sh - Catch orphaned/unexpected test/expected/ files +# +# test/expected/*.out must mirror test/sql/*.sql 1:1 (likewise +# test/build/expected/*.out vs test/build/*.sql, when test-build is in use). +# It's easy to leave a stale .out behind after renaming or removing a .sql +# file; this makes `make test` fail loudly instead of letting it linger +# unnoticed. +# +# test/install/ is NOT checked here: its expected output lives alongside the +# .sql files (test/install/foo.out), not in a separate expected/ +# subdirectory, so there's no 1:1 directory mirror to compare. +# +# pg_regress supports up to 10 alternate expected-output files per test +# (test.out, test_0.out .. test_9.out - see get_alternative_expectfile() in +# pg_regress.c), tried in turn when the primary doesn't match. See the case +# block below for how that's recognized and the tradeoff involved. +# +# expected/ is also checked for files that aren't *.out at all -- there's no +# legitimate reason for anything else to live there (stray editor swap +# files, .orig files from a botched merge, etc.). This is a distinct +# failure class from an orphaned .out file: different message, different +# exit code (see below), and independently disable-able via +# PGXNTOOL_CHECK_EXPECTED_FILE_TYPES=no. +# +# Exit code is a bitmask so the two failure classes can be told apart: +# 1 - one or more orphaned .out files (no corresponding .sql) +# 2 - one or more unexpected non-.out files in expected/ +# 3 - both +# +# Usage: check-stale-expected.sh [check-file-types] +# check-file-types: yes|no (default yes) -- controls the non-*.out +# file check described above. Taken as a positional argument (rather +# than an environment variable) so it's easy to vary directly in a +# test loop instead of having to set/unset an env var around each +# invocation. + +set -o errexit -o errtrace -o pipefail + +BASEDIR=$(dirname "$0") +source "$BASEDIR/../../lib.sh" + +if [ $# -lt 1 ] || [ $# -gt 2 ]; then + die 1 "Usage: check-stale-expected.sh [check-file-types]" +fi + +testdir="$1" +failed=0 + +check_file_types=$(printf '%s' "${2:-yes}" | tr '[:upper:]' '[:lower:]') + +# Usage: check_pair +check_pair() { + local sqldir="$1" expdir="$2" + local f base + + [ -d "$expdir" ] || return 0 + + for f in "$expdir"/*; do + # Without nullglob, this glob is left as the literal unexpanded pattern + # string when $expdir has no entries at all; skip that non-existent + # "file" rather than treating it as real input. + [ -f "$f" ] || continue + + case "$f" in + *.out) + base=$(basename "$f" .out) + case "$base" in + # pg_regress supports up to 10 alternate expected-output files + # per test (test.out, test_0.out .. test_9.out -- see + # get_alternative_expectfile() in pg_regress.c), so a trailing + # _N here doesn't necessarily mean an orphaned file. Recognizing + # this is pure string matching -- it doesn't require checking + # whether a file exists -- so the suffix is stripped unconditionally, + # leaving exactly one existence check per file (below) rather than + # one to recognize the pattern and a second to validate it. + # Tradeoff: a real test literally named e.g. foo_1.sql (with no + # foo.sql at all) would be misidentified as an alternate file for + # a nonexistent "foo" and incorrectly flagged as stale. Accepted + # as a vanishingly rare edge case. + *_[0-9]) + base=${base%_*} + ;; + esac + + if [ ! -f "$sqldir/$base.sql" ]; then + error "$f has no corresponding $sqldir/$base.sql" + (( failed |= 1 )) + fi + ;; + *) + if [ "$check_file_types" = yes ]; then + error "unexpected non-.out file in $expdir: $f" + (( failed |= 2 )) + fi + ;; + esac + done +} + +check_pair "$testdir/sql" "$testdir/expected" +check_pair "$testdir/build" "$testdir/build/expected" + +exit "$failed" + +# vi: expandtab ts=2 sw=2