diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..9191891 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,6 @@ +# actionlint 1.7.12 doesn't know concurrency.queue, which GitHub documents and this workflow +# relies on. Drop this once it does: https://github.com/rhysd/actionlint/issues/657 +paths: + .github/workflows/release-tags.yml: + ignore: + - 'unexpected key "queue" for "concurrency" section' diff --git a/.github/actions/wporg-svn/README.md b/.github/actions/wporg-svn/README.md new file mode 100644 index 0000000..8ede7b1 --- /dev/null +++ b/.github/actions/wporg-svn/README.md @@ -0,0 +1,240 @@ +# WordPress.org SVN release actions + +Three composite actions that release a plugin to plugins.svn.wordpress.org from a built ZIP. They split a release into a prepared tag, a separate switch of the stable pointer, and trunk housekeeping afterwards: + +| Action | Inputs | What it does | +|---|---|---| +| `tag` | `plugin-slug`, `zip-url` | Creates `tags/` in one commit, as a copy of the current stable tag that matches the ZIP exactly. Only changed files are uploaded. trunk and the stable pointer don't move. | +| `set-stable` | `plugin-slug`, `version` | Copies `tags//readme.txt` over `trunk/readme.txt` in one server-side commit. WordPress.org reads the Stable Tag from that file, so this is the release. | +| `update-trunk` | `plugin-slug`, `version` | Replaces trunk with `tags/` in one atomic server-side commit. Housekeeping after the release. | + +All three also take `wporg-username` and `wporg-password`. The version always comes from the plugin's PHP header (`Version:`), never from the ZIP's file name. + +## Read this first + +- **This is not WordPress.org's documented flow.** [Using Subversion](https://developer.wordpress.org/plugins/wordpress-org/how-to-use-subversion/) updates trunk first and copies trunk to a tag. These actions prepare the tag first, on purpose, so that it can be checked before anything points at it. +- **A new tag is public the moment it exists.** It is not private staging. If [Release Confirmation](https://developer.wordpress.org/plugins/wordpress-org/release-confirmation-emails/) is on, WordPress.org may see the new tag and email about a pending release before `set-stable` runs. Never confirm a tag that `tag` did not verify. +- **`tag` takes one commit.** It prepares the new tag in a local working copy and commits it once, so `tags/` appears complete or not at all. If the outcome of that commit is unclear, the action says so loudly and never cleans up or reuses the tag. See [Recovery](#recovery). +- **Released tags never change.** `tag` refuses a version whose tag already exists, under any spelling (`1.2` and `1.2.0` are the same version). + +## Order + +```text +tag -> QA the new tag, approve -> set-stable -> update-trunk +``` + +- QA `tags/` at the `revision` the tag job reports, not just the ZIP you started from. Pass that revision as `expected-revision`, as the example does, and both later actions refuse a tag that changed after QA. +- `update-trunk` also replaces `trunk/readme.txt`, so it refuses to run until `set-stable` has pointed trunk's Stable Tag at the version. It can't release anything by itself. +- `set-stable` has no upgrade-only rule. Pointing it at an older tag rolls the release back. + +## Example + +A plugin repository's release workflow. The `concurrency` lock and the protected `environment` are the caller's job: a composite action can't enforce either. + +```yaml +# A plugin repository's release workflow. Replace my-plugin, and replace +# with a full commit SHA of stellarwp/plugin-toolbox. +name: Release to WordPress.org + +on: + workflow_dispatch: + inputs: + plugin-slug: + description: The plugin's wordpress.org slug + required: true + zip-url: + description: HTTPS URL of the built plugin ZIP (prefer a short-lived signed URL) + required: true + +# One release at a time per plugin, held for the whole sequence. This lock only +# covers workflows in this repository: release each plugin from one place. +concurrency: + group: wporg-${{ inputs.plugin-slug }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + tag: + runs-on: ubuntu-24.04 + outputs: + version: ${{ steps.tag.outputs.version }} + revision: ${{ steps.tag.outputs.revision }} + steps: + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + - id: tag + uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/tag@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + zip-url: ${{ inputs.zip-url }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} + + release: + needs: tag + runs-on: ubuntu-24.04 + # A protected environment: its reviewers QA tags/ at the tag job's + # revision output before approving this job. + environment: wordpress-org + steps: + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + - uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/set-stable@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + version: ${{ needs.tag.outputs.version }} + expected-revision: ${{ needs.tag.outputs.revision }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} + - uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/update-trunk@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + version: ${{ needs.tag.outputs.version }} + expected-revision: ${{ needs.tag.outputs.revision }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} +``` + +[GitHub concurrency](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency) only covers one repository, isn't first-in-first-out, and is no lock on SVN. Release each plugin from one workflow in one repository. Don't add job-level locks with the same group while the workflow holds it. + +## Inputs and outputs + +| Input | Used by | Value | +|---|---|---| +| `plugin-slug` | all | The wordpress.org slug: lowercase letters and digits, single hyphens between them. | +| `zip-url` | `tag` | HTTPS URL of the ZIP. Pass a private URL from a secret, or mask it in an earlier step: see [Security](#security). | +| `version` | `set-stable`, `update-trunk` | The `version` output of `tag`. | +| `expected-revision` | `set-stable`, `update-trunk` | Optional: the `revision` output of `tag`. When set, the action refuses unless `tags/` last changed at exactly that revision, so what gets released is the tag that was QA'd. Leave it empty for a manual run or a rollback. | +| `wporg-username` | all | `${{ secrets.WPORG_USERNAME }}`: an account with commit access to the plugin. | +| `wporg-password` | all | `${{ secrets.WPORG_PASSWORD }}`: that account's [SVN password](https://make.wordpress.org/plugins/2024/09/04/upcoming-security-changes-for-plugin-and-theme-authors-on-wordpress-org/), not its login password. | + +Every action outputs `version` and `revision` once it has verified its result. A failed action outputs nothing. + +| Output | Meaning | +|---|---| +| `version` | The version released (`tag`: read from the plugin header). | +| `revision` | `tag`: the commit that created the new tag. `set-stable`, `update-trunk`: their commit. For a verified no-op, the revision that was checked; the step summary says it was a no-op. | +| `previous-stable` | `tag` only: the tag it copied from. | + +## Requirements + +An Ubuntu runner with bash, Subversion 1.10 or newer (`svn`, and `svnmucc` for `set-stable` and `update-trunk`), Python 3.8 or newer, and, for `tag`, curl. Ubuntu runner images may not include Subversion: install the `subversion` package as in the example. Each action checks its tools and stops with a clear error if one is missing or the runner isn't Linux. + +## What gets checked + +Before any write, every action pins one repository revision and reads everything at that revision. A failed read always stops the action: an authentication, TLS, connection or permission error is never taken to mean a path is absent. + +`tag`: + +- trunk's readme names a numeric stable tag S, and `tags/S` exists. A stable tag of `trunk` (a first release) is not supported. +- The ZIP passes every check in [ZIP limits](#zip-limits). The plugin header version V is numeric and newer than S, and the ZIP's readme Stable Tag is exactly V. +- `tags/V` doesn't exist under any spelling, and the copy of `tags/S` sets no property that changes file bytes (`svn:eol-style`, `svn:keywords`, `svn:special`, `svn:externals`). +- It copies `tags/S` at the pinned revision into an otherwise empty working copy as `tags/V`, syncs the ZIP byte for byte (additions, deletions, dotfiles, file/directory swaps), adds files even where ignore patterns would hide them, and re-validates the version, readme and full tree. An artifact identical to `tags/S` is refused. +- Just before the commit it checks again that the stable pointer, `tags/S` and the absence of `tags/V` haven't changed. If someone creates `tags/V` before the commit lands, the commit fails instead of overwriting it. +- After the commit it exports the committed tag and compares it with the ZIP again. + +`set-stable`: `tags/V/readme.txt` and `trunk/readme.txt` are regular files without byte-changing properties, and the tag readme's Stable Tag is exactly V. The write is pinned to the snapshot revision, so a competing change to `trunk/readme.txt` makes it fail instead of being overwritten. Identical bytes are a verified no-op. + +`update-trunk`: `trunk` and `tags/V` exist, and trunk's readme already names V as its Stable Tag. The remove and copy are one commit, copying `tags/V` as it was at the snapshot revision, and a change anywhere inside trunk since then makes the whole commit fail. The result is compared on the server, files and properties both. A trunk that already matches is a verified no-op. + +## ZIP limits + +| Limit | Value | +|---|---| +| Download | HTTPS only, redirects too (at most 5); 30 s to connect, 10 min per attempt, 3 retries | +| ZIP size | 256 MiB | +| Files and directories | 50,000, counting parent directories the ZIP doesn't list | +| Expanded size | 1 GiB | +| Expansion ratio | 100 to 1 | +| Path depth | 64 levels | +| Path length | 1,024 bytes | + +Every entry is checked before anything is extracted. Rejected: + +- absolute paths, `..`, backslashes, empty or `.` path segments, and names that only differ by case or Unicode form; +- control characters; +- names that aren't valid UTF-8. Names without the ZIP's UTF-8 flag are read as UTF-8, which is what Info-ZIP's `zip` writes on macOS, and on Linux depending on its locales; +- entries with two different names, where an Info-ZIP Unicode Path field disagrees with the header; +- symlinks and other special files; +- encrypted entries; +- `.svn` or `.git` anywhere. + +The ZIP is either the plugin itself or one wrapper directory of any name holding it. `__MACOSX/` and `.DS_Store` are skipped as packaging noise. The plugin root must hold `readme.txt` and exactly one root-level PHP file with a `Plugin Name` header. That file must have exactly one `Version` header in its first 8 KiB, the same window WordPress reads. + +New files are added without SVN auto-props, including any a repository inherits. Like any `svn add`, binary files still get `svn:mime-type: application/octet-stream`. Nothing else is set, `svn:executable` included. Files carried over from the previous tag keep their properties. Before committing, `tag` checks that nothing in the new tag sets a property that changes file bytes. + +## Limitations + +- **Not supported:** first releases, prerelease versions (`1.2-beta`), building the ZIP, confirming a release on WordPress.org, and Git tags or releases. +- **Races are narrowed, not prevented.** The pinned base revision protects only the paths each action writes. The rechecks shrink the windows. Neither replaces running one release at a time. +- **Success means the commit landed and was verified.** WordPress.org processing, ZIP building and Release Confirmation happen afterwards and aren't checked. +- **Not tested against wordpress.org writes.** The tests write only to throwaway local repositories; reads were checked against real plugins. Still unverified: + - authentication with an SVN password; + - out-of-date detection over HTTPS; + - real lost responses; + - whether wordpress.org's server-side rules accept a new tag committed as a copy with changes, the shape common deploy scripts use. + + Supervise the first real release, with this page at hand. + +## Recovery + +A failure prints an error naming the stage and the revisions known so far, and repeats it in the step summary. Don't rerun a write blindly: each action has already inspected the server after a failed write and says what it found. + +**Nothing was written.** Every check runs before the only write, so any failure before the commit, and `the commit failed ... tags/V does not exist`, mean nothing changed. Fix the cause and rerun. + +**`tag` can't vouch for `tags/V`.** One of these: + +- `... cannot confirm that it created it`: the commit reported an error, but `tags/V` now exists. Either the response got lost after the commit landed, or someone else created the tag. +- `The commit's outcome is unknown`: the job stopped during the commit, usually because it was cancelled. If `tags/V` doesn't exist, nothing was written; rerun. +- `tags/V was created at rN but not verified`: the commit landed, but comparing the committed tag with the ZIP failed or didn't finish. + +Then: + +1. Don't confirm it in Release Confirmation, and don't run `set-stable` for it. +2. Look at what happened (reads only): + + ```sh + svn log -v -l 5 https://plugins.svn.wordpress.org//tags/ + svn diff --summarize https://plugins.svn.wordpress.org//tags/ https://plugins.svn.wordpress.org//tags/ + ``` + +3. A maintainer decides what to do: release it once someone has compared it with the ZIP, or delete it and run `tag` again. Deleting is a write to the live plugin, so it needs that maintainer's explicit go-ahead: + + ```sh + svn rm -m "Remove unverified tag " https://plugins.svn.wordpress.org//tags/ + ``` + + If WordPress.org already lists the tag as a pending release, follow its Release Confirmation guidance for a release you won't ship. + +**`set-stable` or `update-trunk` failed.** + +- `... does not match ...; check who else is committing`: the write failed, and the target doesn't hold the release. Usually another commit got there first or the server refused the write; `svn log` shows which. Find out who else is releasing before you rerun. +- `... now matches ... Rerunning is safe`: the write most likely landed but its response was lost. A rerun verifies it and reports a no-op. + +## Security + +- The password reaches `svn` and `svnmucc` on stdin only. It isn't in any process's arguments or in its child processes' environment. Each action masks it, along with the ZIP URL. +- The actions never print the ZIP URL. They mask it too, but only from their own steps on: GitHub logs a step's `with:` inputs before the action runs. If the URL must stay private, pass it from a secret, or mask it with `::add-mask::` in an earlier step. Nothing sends WordPress.org credentials to the ZIP host, and curl ignores `.curlrc`. +- SVN runs non-interactively with a private, throwaway config directory and no credential cache. Certificate checks are never skipped. Temporary files are removed on success, failure and cancellation. +- Nothing from the ZIP runs: no PHP, scripts or hooks. +- The credentials are repository secrets, because the `tag` job writes to SVN without an approval gate. The protected environment in the example gates the release job's approval; it doesn't protect the secrets. Anyone who can push a workflow to the repository can read repository secrets, so limit who can push, and never let the secrets reach workflows that run pull request code from forks. Prefer short-lived signed ZIP URLs: `workflow_dispatch` inputs are visible to anyone who can see the run. + +## Tests + +Locally, with Subversion, Python 3 and openssl installed: + +```sh +python3 -m unittest discover -s tests/wporg-svn -v +shellcheck -x .github/actions/wporg-svn/*/*.sh .github/actions/wporg-svn/lib/svn.sh tests/wporg-svn/guard/bin/* +actionlint && actionlint .github/actions/wporg-svn/examples/release.yml +``` + +The tests run the real scripts against throwaway `file://` repositories and a local HTTPS server. A test-only guard (`tests/wporg-svn/guard`) sits first on `PATH` and refuses any SVN write whose target isn't `file://`. + +Read-only checks against real plugins are opt-in. They read from wordpress.org, and every write goes to a local repository: + +```sh +WPORG_LIVE=1 python3 -m unittest discover -s tests/wporg-svn -p test_live.py -v +``` diff --git a/.github/actions/wporg-svn/examples/release.yml b/.github/actions/wporg-svn/examples/release.yml new file mode 100644 index 0000000..76c7903 --- /dev/null +++ b/.github/actions/wporg-svn/examples/release.yml @@ -0,0 +1,63 @@ +# A plugin repository's release workflow. Replace my-plugin, and replace +# with a full commit SHA of stellarwp/plugin-toolbox. +name: Release to WordPress.org + +on: + workflow_dispatch: + inputs: + plugin-slug: + description: The plugin's wordpress.org slug + required: true + zip-url: + description: HTTPS URL of the built plugin ZIP (prefer a short-lived signed URL) + required: true + +# One release at a time per plugin, held for the whole sequence. This lock only +# covers workflows in this repository: release each plugin from one place. +concurrency: + group: wporg-${{ inputs.plugin-slug }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + tag: + runs-on: ubuntu-24.04 + outputs: + version: ${{ steps.tag.outputs.version }} + revision: ${{ steps.tag.outputs.revision }} + steps: + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + - id: tag + uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/tag@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + zip-url: ${{ inputs.zip-url }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} + + release: + needs: tag + runs-on: ubuntu-24.04 + # A protected environment: its reviewers QA tags/ at the tag job's + # revision output before approving this job. + environment: wordpress-org + steps: + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + - uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/set-stable@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + version: ${{ needs.tag.outputs.version }} + expected-revision: ${{ needs.tag.outputs.revision }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} + - uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/update-trunk@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + version: ${{ needs.tag.outputs.version }} + expected-revision: ${{ needs.tag.outputs.revision }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} diff --git a/.github/actions/wporg-svn/lib/archive.py b/.github/actions/wporg-svn/lib/archive.py new file mode 100644 index 0000000..82b0cd3 --- /dev/null +++ b/.github/actions/wporg-svn/lib/archive.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Inspect a plugin ZIP before extracting anything, then extract it safely. + +Usage: + archive.py extract ZIP DEST extract into the empty directory DEST and print + the plugin root (DEST or its one wrapper directory) + +Every entry is checked before the first byte is written. Errors go to stderr +without echoing entry names, and exit 1. +""" + +from __future__ import annotations + +import os +import re +import stat +import struct +import sys +import unicodedata +import zipfile +import zlib + +MAX_ARCHIVE_BYTES = 256 * 1024 * 1024 +MAX_ENTRIES = 50_000 +MAX_EXPANDED_BYTES = 1024 * 1024 * 1024 +MAX_RATIO = 100 +MAX_DEPTH = 64 +MAX_PATH_BYTES = 1024 + +# Packaging noise skipped (never extracted): macOS resource forks and Finder files. +NOISE_TOP = "__MACOSX" +NOISE_NAME = ".DS_Store" +VCS = {".svn", ".git"} + + +class ArchiveError(Exception): + pass + + +def _check_control(name: str) -> None: + if any(ord(c) < 0x20 or 0x7F <= ord(c) <= 0x9F for c in name): + raise ArchiveError("an entry name contains a control character") + + +def _unicode_paths(info: zipfile.ZipInfo, header: bytes): + """Yield the name in each of the entry's Info-ZIP Unicode Path fields (0x7075). + + Only version 1 fields whose CRC matches the header name count; any other is + stale, as unzip and zipfile treat it. + """ + extra, header_crc = info.extra, zlib.crc32(header) + while len(extra) >= 4: + kind, size = struct.unpack(" str: + """The entry's name as its author wrote it. + + zipfile reads a name without the UTF-8 flag as CP437, but Info-ZIP's zip + writes UTF-8 there without setting the flag: on macOS, and on Ubuntu without + en_US.UTF-8. Such names are taken as UTF-8; one that isn't valid UTF-8 is + refused rather than guessed. + """ + raw = info.orig_filename + _check_control(raw) + flagged = info.flag_bits & 0x800 + name = raw + if not flagged and not raw.isascii(): + try: + name = raw.encode("cp437").decode("utf-8") + except UnicodeError: + raise ArchiveError("an entry name is not UTF-8") from None + _check_control(name) + # unzip and zipfile 3.12+ take the name from the last valid Unicode Path + # field; older zipfile ignores them. Every one must agree, on every Python. + if any(other != name for other in _unicode_paths(info, raw.encode("utf-8" if flagged else "cp437"))): + raise ArchiveError("an entry has two different names") + # Otherwise filename differs from both only where zipfile changed the name. + if info.filename not in (raw, name): + raise ArchiveError("an entry has an unsafe path") + return name + + +def _check_name(info: zipfile.ZipInfo) -> list: + """Return the path components of a safe entry name, or raise.""" + name = _name(info) + if "\\" in name or name.startswith("/") or re.match(r"[A-Za-z]:", name): + raise ArchiveError("an entry has an unsafe path") + if len(name.encode()) > MAX_PATH_BYTES: + raise ArchiveError(f"an entry path is longer than {MAX_PATH_BYTES} bytes") + parts = (name[:-1] if name.endswith("/") else name).split("/") + if len(parts) > MAX_DEPTH: + raise ArchiveError(f"an entry path is deeper than {MAX_DEPTH} levels") + if any(part in ("", ".", "..") for part in parts): + raise ArchiveError("an entry has an unsafe path") + return parts + + +def _check_type(info: zipfile.ZipInfo) -> None: + if info.flag_bits & 0x1: + raise ArchiveError("the archive has encrypted entries") + kind = stat.S_IFMT(info.external_attr >> 16) + # Hosts that store Unix modes: 3 is Unix, 19 is OS X. + if info.create_system in (3, 19) and kind not in (0, stat.S_IFREG, stat.S_IFDIR): + raise ArchiveError("an entry is not a regular file or directory") + if kind == stat.S_IFDIR and not info.is_dir(): + raise ArchiveError("an entry is not a regular file or directory") + + +def _fold(parts) -> tuple: + return tuple(unicodedata.normalize("NFC", part).casefold() for part in parts) + + +def inspect(zf: zipfile.ZipFile, archive_bytes: int) -> list: + """Validate every entry; return (info, parts) for the entries to extract.""" + infos = zf.infolist() + if len(infos) > MAX_ENTRIES: + raise ArchiveError(f"the archive has more than {MAX_ENTRIES} entries") + + kept, raw_names, kinds, spellings, total = [], set(), {}, {}, 0 + for info in infos: + parts = _check_name(info) + _check_type(info) + if info.orig_filename in raw_names: + raise ArchiveError("the archive has a duplicate or ambiguous path") + raw_names.add(info.orig_filename) + if parts[0] == NOISE_TOP or parts[-1] == NOISE_NAME: + continue + if any(part.lower() in VCS for part in parts): + raise ArchiveError("the archive contains version control metadata") + + # Every prefix is a directory; the last component is the entry itself. + # Each distinct one becomes a file or directory on disk, listed or not. + folded = _fold(parts) + for depth in range(1, len(parts) + 1): + kind = "file" if depth == len(parts) and not info.is_dir() else "dir" + key, spelling = folded[:depth], "/".join(parts[:depth]) + if key not in kinds: + kinds[key], spellings[key] = kind, spelling + if len(kinds) > MAX_ENTRIES: + raise ArchiveError(f"the archive has more than {MAX_ENTRIES} files and directories") + elif spellings[key] != spelling: + raise ArchiveError("the archive has a duplicate or ambiguous path") + elif kind == "file" or kinds[key] == "file": + raise ArchiveError("the archive has a file/directory conflict") + + total += info.file_size + kept.append((info, parts)) + + if total > MAX_EXPANDED_BYTES: + raise ArchiveError(f"the archive expands past {MAX_EXPANDED_BYTES} bytes") + if total > MAX_RATIO * max(archive_bytes, 1): + raise ArchiveError(f"the archive's expansion ratio exceeds {MAX_RATIO}") + return kept + + +def plugin_root(kept: list) -> tuple: + """Path components of the plugin root: () when flat, (wrapper,) when wrapped.""" + top = {parts[0] for _, parts in kept} + top_files = {parts[0] for info, parts in kept if len(parts) == 1 and not info.is_dir()} + if "readme.txt" in top_files: + return () + if len(top) == 1 and not top_files: + wrapper = next(iter(top)) + if any(parts == [wrapper, "readme.txt"] and not info.is_dir() for info, parts in kept): + return (wrapper,) + raise ArchiveError("the plugin root has no readme.txt") + raise ArchiveError( + "cannot find one plugin root: readme.txt must be at the top level or inside one wrapper directory") + + +def extract(zip_path: str, dest: str) -> str: + """Extract zip_path into the empty directory dest; return the plugin root.""" + size = os.path.getsize(zip_path) + if size > MAX_ARCHIVE_BYTES: + raise ArchiveError(f"the archive is larger than {MAX_ARCHIVE_BYTES} bytes") + dest = os.path.realpath(dest) + try: + with zipfile.ZipFile(zip_path) as zf: + kept = inspect(zf, size) + root = plugin_root(kept) + written = 0 + for info, parts in kept: + target = os.path.join(dest, *parts) + if not os.path.realpath(target).startswith(dest + os.sep): + raise ArchiveError("an entry has an unsafe path") + if info.is_dir(): + os.makedirs(target, exist_ok=True) + continue + os.makedirs(os.path.dirname(target), exist_ok=True) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + with zf.open(info) as source, os.fdopen(os.open(target, flags, 0o644), "wb") as out: + while True: + chunk = source.read(1024 * 1024) + if not chunk: + break + written += len(chunk) + if written > MAX_EXPANDED_BYTES: + raise ArchiveError(f"the archive expands past {MAX_EXPANDED_BYTES} bytes") + out.write(chunk) + except (zipfile.BadZipFile, zipfile.LargeZipFile, NotImplementedError, RuntimeError, EOFError, + zlib.error, ValueError) as error: + raise ArchiveError("the archive is malformed") from error + return os.path.join(dest, *root) + + +def main(argv) -> int: + if len(argv) != 3 or argv[0] != "extract": + print(__doc__, file=sys.stderr) + return 2 + try: + print(extract(argv[1], argv[2])) + except (ArchiveError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/actions/wporg-svn/lib/release.py b/.github/actions/wporg-svn/lib/release.py new file mode 100644 index 0000000..35e5810 --- /dev/null +++ b/.github/actions/wporg-svn/lib/release.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Release metadata: numeric versions, the readme Stable Tag, the plugin header. + +Usage: + release.py version V exit 0 when V is a numeric version + release.py compare A B print lt, eq or gt + release.py equivalents V print the stdin lines numerically equal to V + release.py stable-tag README print the readme's Stable Tag value + release.py plugin-version DIR print the Version header of DIR's plugin file + +Errors go to stderr without echoing file contents, and exit 1. +""" + +from __future__ import annotations + +import collections +import os +import re +import sys + +VERSION = re.compile(r"[0-9]+(?:\.[0-9]+)*") + +# WordPress reads plugin headers from the first 8 KiB only (get_file_data). +HEADER_BYTES = 8192 + +# Header names the wordpress.org readme parser recognises (class-parser.php). +README_HEADERS = { + "tested", "tested up to", "requires", "requires at least", "requires php", "tags", + "contributors", "donate link", "stable tag", "license", "license uri", +} + +# PCRE's \R, which the readme parser splits lines on. +NEWLINES_UTF8 = re.compile("\r\n|[\n\x0b\x0c\r\x85

]") +NEWLINES_BYTES = re.compile("\r\n|[\n\x0b\x0c\r\x85]") + + +class ReleaseError(Exception): + pass + + +def is_version(value: str) -> bool: + return VERSION.fullmatch(value) is not None + + +def _key(value: str) -> list: + if not is_version(value): + raise ReleaseError("not a numeric version") + # Compare digit strings by (length, text) after dropping leading zeros: + # arbitrary precision without int(), which caps digit count on 3.11+. + return [(len(part), part) for part in (p.lstrip("0") or "0" for p in value.split("."))] + + +def compare(left: str, right: str) -> int: + a, b = _key(left), _key(right) + width = max(len(a), len(b)) + a += [(1, "0")] * (width - len(a)) + b += [(1, "0")] * (width - len(b)) + return (a > b) - (a < b) + + +def equivalents(version: str, names) -> list: + return [name for name in names if is_version(name) and compare(name, version) == 0] + + +def _php_empty(value) -> bool: + return value in ("", "0", b"", b"0") + + +def _ascii_lower(value: str) -> str: + return "".join(c.lower() if "A" <= c <= "Z" else c for c in value) + + +def _readme_header(line: str, only_valid: bool = False): + if ":" not in line or line.startswith(("#", "=")): + return None + key, value = line.split(":", 1) + key = _ascii_lower(key.strip(" \t*-\r\n")) + if only_valid and key not in README_HEADERS: + return None + return key, value + + +def stable_tag(data: bytes) -> str: + """Return the readme's single Stable Tag value, whitespace-trimmed. + + The header section ends where the wordpress.org readme parser ends it, so + examples further down the readme never count. + """ + if data.startswith(b"\xef\xbb\xbf"): + data = data[3:] + if data.startswith((b"\xff\xfe", b"\xfe\xff")): + raise ReleaseError("readme.txt is UTF-16; only UTF-8 is supported") + try: + encoding, lines = "utf-8", NEWLINES_UTF8.split(data.decode("utf-8")) + except UnicodeDecodeError: + encoding, lines = "latin-1", NEWLINES_BYTES.split(data.decode("latin-1")) + stack = collections.deque(lines) + + def first_nonwhitespace(): + while stack: + line = stack.popleft() + if not _php_empty(line.strip(" \t\n\r\0\x0b")): + return line + return "" + + # Title line, unless the readme starts straight with a header field. + line = first_nonwhitespace() + name = line.strip("#= \t\0\x0b") + if _readme_header(line, only_valid=True): + stack.appendleft(line) + name = "" + if stack and stack[0].strip("=-") == "": + stack.popleft() + if _ascii_lower(name) == "plugin name": + line = first_nonwhitespace() + if len(line.encode(encoding)) >= 50 or _readme_header(line, only_valid=True): + stack.appendleft(line) + + found = [] + line = first_nonwhitespace() + last_blank = False + while True: + header = _readme_header(line) + if header is None: + if not _php_empty(line): + break # The short description: the header section is over. + last_blank = True + else: + key, value = header + if key == "stable tag": + found.append(value) + elif key not in README_HEADERS and last_blank: + break + last_blank = False + if not stack: + break + line = stack.popleft() + + if not found: + raise ReleaseError("readme.txt has no Stable Tag field in its header") + if len(found) > 1: + raise ReleaseError(f"readme.txt has {len(found)} Stable Tag fields in its header") + return found[0].strip(" \t") + + +def _plugin_headers(data: bytes, name: bytes) -> list: + """Every match of WordPress's header regex, cleaned like get_file_data.""" + pattern = re.compile(rb"^(?:[ \t]*<\?php)?[ \t/*#@]*" + re.escape(name) + rb":(.*)$", re.M | re.I) + values = [] + for match in pattern.finditer(data): + raw = match.group(1) + cleaned = re.sub(rb"\s*(?:\*/|\?>).*", b"", raw).strip(b" \t\n\r\0\x0b") + values.append(b"" if _php_empty(raw) else cleaned) + return values + + +def plugin_version(root: str) -> str: + """Return the Version header of the one root PHP file declaring a plugin.""" + plugins = [] + for entry in sorted(os.listdir(root)): + path = os.path.join(root, entry) + if not entry.endswith(".php") or os.path.islink(path) or not os.path.isfile(path): + continue + with open(path, "rb") as handle: + data = handle.read(HEADER_BYTES).replace(b"\r", b"\n") + names = _plugin_headers(data, b"Plugin Name") + if not any(not _php_empty(value) for value in names): + continue + if len(names) > 1: + raise ReleaseError(f"a root PHP file has {len(names)} Plugin Name headers") + plugins.append(data) + + if not plugins: + raise ReleaseError("no root PHP file has a Plugin Name header") + if len(plugins) > 1: + raise ReleaseError(f"{len(plugins)} root PHP files have a Plugin Name header") + + versions = _plugin_headers(plugins[0], b"Version") + if len(versions) > 1: + raise ReleaseError(f"the plugin file has {len(versions)} Version headers") + if not versions or _php_empty(versions[0]): + raise ReleaseError("the plugin file has no Version header") + version = versions[0].decode("latin-1") + if not is_version(version): + raise ReleaseError("the plugin Version header is not a numeric version") + return version + + +def main(argv) -> int: + command, args = (argv[0], argv[1:]) if argv else ("", []) + try: + if command == "version" and len(args) == 1: + return 0 if is_version(args[0]) else 1 + if command == "compare" and len(args) == 2: + print({-1: "lt", 0: "eq", 1: "gt"}[compare(*args)]) + elif command == "equivalents" and len(args) == 1: + for name in equivalents(args[0], sys.stdin.read().splitlines()): + print(name) + elif command == "stable-tag" and len(args) == 1: + with open(args[0], "rb") as handle: + print(stable_tag(handle.read())) + elif command == "plugin-version" and len(args) == 1: + print(plugin_version(args[0])) + else: + print(__doc__, file=sys.stderr) + return 2 + except (ReleaseError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/actions/wporg-svn/lib/svn.sh b/.github/actions/wporg-svn/lib/svn.sh new file mode 100644 index 0000000..de6dd8c --- /dev/null +++ b/.github/actions/wporg-svn/lib/svn.sh @@ -0,0 +1,238 @@ +# shellcheck shell=bash +# Shared by the wporg-svn actions: strict mode, input checks, a private SVN +# configuration, revision-pinned reads, authenticated writes and failure +# reports. Set ACTION_NAME, then source this file; it does nothing on its own. +# +# Inputs arrive as environment variables, mapped by each action.yml: +# WPORG_SLUG WPORG_VERSION WPORG_ZIP_URL WPORG_USERNAME WPORG_PASSWORD +# Test seam, not an action input: WPORG_SVN_TEST_ROOT replaces the production +# root, and only a file:// URL is accepted, so it can never send credentials +# anywhere. +# +# Helpers return values in REPLY rather than through $(...), so a failure +# inside them stops the whole action. + +set -euo pipefail + +# Keep the password out of every child's environment before any child starts: +# svn reads it from stdin only. +SVN_PASSWORD=${WPORG_PASSWORD:-} +export -n SVN_PASSWORD +unset WPORG_PASSWORD + +# English messages and ASCII regex ranges, but UTF-8: in plain C, svn on Linux +# cannot convert non-ASCII file names in a working copy. +export LC_ALL=C.UTF-8 +umask 077 + +LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PRODUCTION_ROOT=https://plugins.svn.wordpress.org + +STAGE="checking inputs" +WORK= +BASE= +SLUG= +VERSION= +REVISIONS= +RECOVERY= + +# Only fixed text and validated values ever reach these messages. +fail() { + printf '::error title=%s::%s\n' "$ACTION_NAME" "$1" + exit 1 +} + +note_revision() { + REVISIONS="${REVISIONS:+$REVISIONS, }$1" +} + +summary() { + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '%s\n' "$@" >>"$GITHUB_STEP_SUMMARY" + fi +} + +output() { + if [ -n "${GITHUB_OUTPUT:-}" ]; then + printf '%s=%s\n' "$1" "$2" >>"$GITHUB_OUTPUT" + else + printf '%s=%s\n' "$1" "$2" + fi +} + +on_exit() { + local status=$? + trap - EXIT + if [ "$status" -ne 0 ]; then + local subject="${SLUG:-the plugin}${VERSION:+ $VERSION}" + printf '::error title=%s::%s\n' "$ACTION_NAME" \ + "Failed while $STAGE for $subject. Known revisions: ${REVISIONS:-none}.${RECOVERY:+ $RECOVERY}" + summary "### $ACTION_NAME failed" "" "- Plugin: $subject" "- Stage: $STAGE" \ + "- Known revisions: ${REVISIONS:-none}" ${RECOVERY:+"- Recovery: $RECOVERY"} + fi + if [ -n "$WORK" ]; then + rm -rf -- "$WORK" + fi + exit "$status" +} +trap on_exit EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# Usage: require_tools TOOL... (each script names everything it runs) +require_tools() { + if [ -n "${RUNNER_OS:-}" ] && [ "$RUNNER_OS" != Linux ]; then + fail "unsupported runner: use an Ubuntu runner" + fi + local tool + for tool in "$@"; do + command -v "$tool" >/dev/null 2>&1 || + fail "missing required tool: $tool (on Ubuntu: sudo apt-get install -y subversion)" + done + python3 -c 'import sys; sys.exit(sys.version_info < (3, 8))' || fail "Python 3.8 or newer is required" + case $(svn --version --quiet) in + 1.[0-9].*) fail "Subversion 1.10 or newer is required for --password-from-stdin" ;; + esac +} + +validate_slug() { + local pattern='^[a-z0-9]+(-[a-z0-9]+)*$' + [[ ${WPORG_SLUG:-} =~ $pattern ]] || + fail "plugin-slug must be lowercase letters and digits, with single hyphens between them" + SLUG=$WPORG_SLUG +} + +validate_version() { + python3 "$LIB_DIR/release.py" version "${WPORG_VERSION:-}" || + fail "version must be numeric dot-separated digits, like 1.2.3" + VERSION=$WPORG_VERSION +} + +# Optional: empty means "not given". +validate_expected_revision() { + local pattern='^[1-9][0-9]*$' + [ -z "${WPORG_EXPECTED_REVISION:-}" ] || [[ $WPORG_EXPECTED_REVISION =~ $pattern ]] || + fail "expected-revision must be a revision number, like the tag action's revision output" +} + +# With expected-revision given, stop unless tags/ last changed at +# exactly that revision, as of REV: what gets released is the tag that was QA'd. +check_expected_revision() { + [ -n "${WPORG_EXPECTED_REVISION:-}" ] || return 0 + last_changed "$BASE/tags/$VERSION" "$1" + [ "$REPLY" = "$WPORG_EXPECTED_REVISION" ] || + fail "tags/$VERSION last changed at r$REPLY, not at the expected r$WPORG_EXPECTED_REVISION; it is not the tag that was approved" +} + +validate_credentials() { + if [ -z "${WPORG_USERNAME:-}" ] || [ -z "$SVN_PASSWORD" ]; then + fail "wporg-username and wporg-password are required" + fi + case $WPORG_USERNAME$SVN_PASSWORD in + *$'\n'* | *$'\r'*) fail "wporg-username and wporg-password must be single lines" ;; + esac +} + +# Resolve the repository URL and create the private workspace. +prepare() { + local root=$PRODUCTION_ROOT tmp=${RUNNER_TEMP:-${TMPDIR:-/tmp}} + if [ -n "${WPORG_SVN_TEST_ROOT:-}" ]; then + case $WPORG_SVN_TEST_ROOT in + file:///*) root=${WPORG_SVN_TEST_ROOT%/} ;; + *) fail "WPORG_SVN_TEST_ROOT only accepts file:// URLs" ;; + esac + fi + BASE="$root/$SLUG" + WORK=$(mktemp -d "${tmp%/}/wporg-svn.XXXXXXXX") + mkdir "$WORK/svn-config" + printf '[auth]\npassword-stores =\n[miscellany]\nenable-auto-props = no\n' >"$WORK/svn-config/config" + # Bulk updates: one streamed response per checkout or export instead of one + # request per file, 2-5x faster from plugins.svn.wordpress.org. + printf '[global]\nstore-passwords = no\nstore-auth-creds = no\nhttp-bulk-updates = yes\n' >"$WORK/svn-config/servers" +} + +# A short, sanitized description of the last SVN error: a class and the codes. +svn_error() { + local codes class=unclassified + codes=$(grep -oE 'E[0-9]{6}' "$WORK/err" | sort -u | tr '\n' ' ') || true + case " $codes" in + *" E170001 "* | *" E215004 "*) class="authentication failed" ;; + *" E230001 "* | *" E120171 "*) class="TLS certificate verification failed" ;; + *" E160024 "* | *" E160028 "* | *" E170004 "* | *" E160020 "* | *" E155011 "*) + class="out of date: another commit changed the destination" ;; + *" E175013 "* | *" E000013 "* | *" E220001 "*) class="permission denied" ;; + *" E170013 "* | *" E175002 "* | *" E175012 "* | *" E670002 "* | *" E670008 "* | *" E120108 "*) + class="could not connect" ;; + *" E160013 "* | *" E170000 "* | *" E200009 "*) class="path not found" ;; + esac + REPLY="$class${codes:+ (${codes% })}" +} + +# Anonymous read into $WORK/out. A failed read is never taken to mean +# "absent": it stops the action. +svn_read() { + if ! svn --non-interactive --no-auth-cache --config-dir "$WORK/svn-config" "$@" >"$WORK/out" 2>"$WORK/err"; then + svn_error + fail "SVN read failed while $STAGE: $REPLY" + fi +} + +# REPLY = the repository's current (operative) revision. Pin every dependent +# read to it. +snapshot() { + local pattern='^[0-9]+$' + svn_read info --show-item revision -- "$BASE" + REPLY=$(tr -d '[:space:]' <"$WORK/out") + [[ $REPLY =~ $pattern ]] || fail "could not read the repository revision" +} + +# REPLY = dir, file or absent: NAME inside DIR_URL at REV. +svn_kind() { + svn_read list --xml -r "$3" -- "$1@$3" + REPLY=$(python3 "$LIB_DIR/svnxml.py" kind "$2" <"$WORK/out") +} + +# REPLY = every entry name inside DIR_URL at REV, one per line. +svn_names() { + svn_read list --xml -r "$2" -- "$1@$2" + REPLY=$(python3 "$LIB_DIR/svnxml.py" names <"$WORK/out") +} + +# REPLY = the last revision that changed URL, as of REV. +last_changed() { + svn_read info --show-item last-changed-revision -r "$2" -- "$1@$2" + REPLY=$(tr -d '[:space:]' <"$WORK/out") +} + +# Copy the file URL at REV to DEST, byte for byte. +svn_cat() { + svn_read cat -r "$2" -- "$1@$2" + mv -- "$WORK/out" "$3" +} + +# Stop when anything at URL (recursively) sets a content-transforming property. +check_props() { + svn_read proplist -R --xml -r "$2" -- "$1@$2" + python3 "$LIB_DIR/svnxml.py" check-props <"$WORK/out" || + fail "$3 has SVN properties that change file contents; an exact release cannot keep them" +} + +# Authenticated writes. The password travels on stdin, never in argv. On +# success REPLY is the new revision. On failure they return 1 and the outcome +# is unknown: the caller inspects, reports and stops. Writes are never retried. +mucc_write() { + REPLY= + printf '%s\n' "$SVN_PASSWORD" | svnmucc --non-interactive --no-auth-cache --config-dir "$WORK/svn-config" \ + --username "$WPORG_USERNAME" --password-from-stdin "$@" >"$WORK/out" 2>"$WORK/err" || return 1 + REPLY=$(sed -n 's/^r\([0-9][0-9]*\) committed.*/\1/p' "$WORK/out") + [ -n "$REPLY" ] +} + +commit_write() { + REPLY= + printf '%s\n' "$SVN_PASSWORD" | svn --non-interactive --no-auth-cache --config-dir "$WORK/svn-config" \ + --username "$WPORG_USERNAME" --password-from-stdin commit -m "$2" -- "$1" >"$WORK/out" 2>"$WORK/err" || + return 1 + REPLY=$(sed -n 's/^Committed revision \([0-9][0-9]*\)\.$/\1/p' "$WORK/out") + [ -n "$REPLY" ] +} diff --git a/.github/actions/wporg-svn/lib/svnxml.py b/.github/actions/wporg-svn/lib/svnxml.py new file mode 100644 index 0000000..d44e743 --- /dev/null +++ b/.github/actions/wporg-svn/lib/svnxml.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Answer questions about the XML that svn prints. The XML arrives on stdin. + +Usage: + svnxml.py kind NAME print dir, file or absent for NAME in `svn list --xml` + svnxml.py names print every entry name in `svn list --xml` + svnxml.py check-props exit 1 when `svn proplist -R --xml` shows a property + that rewrites file bytes or pulls content from elsewhere + svnxml.py diff-empty exit 0 when `svn diff --summarize --xml` lists nothing + +Unreadable XML exits 3, so it is never mistaken for an answer. +""" + +from __future__ import annotations + +import sys +import xml.etree.ElementTree as ET + +# Checked-out bytes differ from repository bytes under these, or content comes +# from another location. An exact artifact release cannot keep them. +TRANSFORMING = ("svn:eol-style", "svn:externals", "svn:keywords", "svn:special") + + +def main(argv) -> int: + command, args = (argv[0], argv[1:]) if argv else ("", []) + if (command, len(args)) not in (("kind", 1), ("names", 0), ("check-props", 0), ("diff-empty", 0)): + print(__doc__, file=sys.stderr) + return 2 + try: + doc = ET.parse(sys.stdin.buffer).getroot() + except ET.ParseError: + print("error: unreadable svn XML", file=sys.stderr) + return 3 + + if command == "kind": + kinds = [entry.get("kind") for entry in doc.iter("entry") if entry.findtext("name") == args[0]] + print(kinds[0] if kinds else "absent") + elif command == "names": + for entry in doc.iter("entry"): + print(entry.findtext("name")) + elif command == "check-props": + counts = {} + for prop in doc.iter("property"): + if prop.get("name") in TRANSFORMING: + counts[prop.get("name")] = counts.get(prop.get("name"), 0) + 1 + for name, count in sorted(counts.items()): + print(f"error: {count} path(s) set {name}", file=sys.stderr) + return 1 if counts else 0 + elif command == "diff-empty": + changed = len(list(doc.iter("path"))) + if changed: + print(f"{changed} path(s) differ", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/actions/wporg-svn/lib/tree.py b/.github/actions/wporg-svn/lib/tree.py new file mode 100644 index 0000000..c3653e4 --- /dev/null +++ b/.github/actions/wporg-svn/lib/tree.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Make a working copy match an artifact tree exactly, and compare trees. + +Usage: + tree.py sync SRC WC make WC's files match SRC; WC/.svn is untouched + tree.py reconcile WC CONFIG_DIR schedule the SVN adds, deletes and replacements + that make WC commit as it now is; print the + number of changed paths + tree.py compare A B exit 0 when A and B hold the same paths, kinds + and bytes (a top-level .svn is ignored) +""" + +from __future__ import annotations + +import os +import shutil +import stat +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET + +FIXABLE = {"unversioned", "ignored", "missing", "obstructed", "modified"} +COMMITTABLE = {"added", "deleted", "replaced", "modified"} +# Not "copied": tag reconciles a fresh copy, where every changed node is copied. +FLAGS = ("tree-conflicted", "switched", "wc-locked") + + +class TreeError(Exception): + pass + + +def _kind(path): + try: + mode = os.lstat(path).st_mode + except FileNotFoundError: + return None + if stat.S_ISDIR(mode): + return "dir" + return "file" if stat.S_ISREG(mode) else "other" + + +def entries(root) -> dict: + """{relative path: kind} under root, skipping a top-level .svn.""" + found = {} + for dirpath, dirnames, filenames in os.walk(root): + rel = os.path.relpath(dirpath, root) + if rel == ".": + dirnames[:] = [d for d in dirnames if d != ".svn"] + for name in dirnames + filenames: + path = os.path.normpath(os.path.join(rel, name)) + found[path] = _kind(os.path.join(root, path)) + return found + + +def _depth(path): + return path.count(os.sep) + + +def _same_bytes(a, b) -> bool: + if os.path.getsize(a) != os.path.getsize(b): + return False + with open(a, "rb") as left, open(b, "rb") as right: + while True: + chunk = left.read(1024 * 1024) + if chunk != right.read(1024 * 1024): + return False + if not chunk: + return True + + +def _remove(path): + if _kind(path) == "dir": + shutil.rmtree(path) + else: + os.unlink(path) + + +def sync(src, wc): + """Make wc's tree (outside .svn) equal src's: kinds, paths and bytes.""" + want, have = entries(src), entries(wc) + if "other" in want.values(): + raise TreeError("the artifact contains something other than files and directories") + for rel in sorted(have, key=_depth): + if want.get(rel) != have[rel] and os.path.lexists(os.path.join(wc, rel)): + _remove(os.path.join(wc, rel)) + for rel in sorted(want, key=_depth): + source, target = os.path.join(src, rel), os.path.join(wc, rel) + if want[rel] == "dir": + os.makedirs(target, exist_ok=True) + elif not os.path.lexists(target) or not _same_bytes(source, target): + if os.path.lexists(target): + os.unlink(target) + shutil.copyfile(source, target) + + +class _Svn: + def __init__(self, config): + self.base = ["svn", "--non-interactive", "--config-dir", config] + + def run(self, *args) -> bytes: + result = subprocess.run(self.base + list(args), capture_output=True) + if result.returncode: + raise TreeError(f"svn {args[0]} failed: " + result.stderr.decode("utf-8", "replace").strip()[-300:]) + return result.stdout + + def status(self, wc) -> list: + """(path, item, props, flags) for every entry svn reports. + + A copy scheduled for addition (svn copy URL WC) reports its root as + added; that is the copy itself, not a change to reconcile or count. + """ + doc = ET.fromstring(self.run("status", "--xml", "--no-ignore", "--", wc + "@")) + found = [] + for entry in doc.iter("entry"): + status = entry.find("wc-status") + if entry.get("path") == wc and status.get("item") == "added" and status.get("copied") == "true": + continue + flags = {flag for flag in FLAGS if status.get(flag) == "true"} + found.append((entry.get("path"), status.get("item"), status.get("props"), flags)) + return found + + def _each(self, args, paths): + """Run svn args once for all paths, listed in a --targets file.""" + if not paths: + return + with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".targets") as targets: + targets.write("".join(path + "@\n" for path in paths)) + targets.flush() + self.run(*args, "--targets", targets.name) + + def add(self, *paths): + self._each(["add", "--depth", "infinity", "--no-ignore", "--no-auto-props"], paths) + + def rm(self, *paths): + self._each(["rm", "--force"], paths) + + +def _check(found, allowed, when): + # svn add itself sets svn:mime-type on binary files, so added items may carry + # properties; the caller checks the working copy's properties separately. + bad = sorted({f"{item} (props {props}{', ' + ', '.join(sorted(flags)) if flags else ''})" + for _, item, props, flags in found if item not in allowed or props == "conflicted" or flags}) + if bad: + raise TreeError(f"unexpected working copy state {when}: " + "; ".join(bad)) + + +def _tops(paths) -> list: + """paths without those inside another of them: svn add and rm recurse.""" + tops = set() + for path in sorted(paths, key=_depth): + parent = os.path.dirname(path) + while parent not in tops and parent != os.path.dirname(parent): + parent = os.path.dirname(parent) + if parent not in tops: + tops.add(path) + return sorted(tops) + + +def reconcile(wc, config) -> int: + """Schedule SVN operations so wc commits as it is on disk; return the change count.""" + svn = _Svn(config) + _check(svn.status(wc), FIXABLE, "before reconciling") + + # A file became a directory or the reverse: delete the old node, add the new. + for path, item, _, _ in svn.status(wc): + if item == "obstructed": + aside = tempfile.mkdtemp(dir=os.path.dirname(wc)) + os.rename(path, os.path.join(aside, "item")) + svn.rm(path) + os.rename(os.path.join(aside, "item"), path) + os.rmdir(aside) + svn.add(path) + + svn.rm(*_tops(p for p, item, _, _ in svn.status(wc) if item == "missing")) + svn.add(*_tops(p for p, item, _, _ in svn.status(wc) if item in ("unversioned", "ignored"))) + + final = svn.status(wc) + _check(final, COMMITTABLE, "after reconciling") + return len(final) + + +def compare(a, b) -> list: + """Relative paths whose presence, kind or bytes differ between a and b.""" + left, right = entries(a), entries(b) + differ = [rel for rel in sorted(set(left) | set(right)) if left.get(rel) != right.get(rel)] + for rel in sorted(set(left) & set(right)): + if left[rel] == "file" == right[rel] and not _same_bytes(os.path.join(a, rel), os.path.join(b, rel)): + differ.append(rel) + return differ + + +def main(argv) -> int: + command, args = (argv[0], argv[1:]) if argv else ("", []) + try: + if command == "sync" and len(args) == 2: + sync(*args) + elif command == "reconcile" and len(args) == 2: + print(reconcile(*args)) + elif command == "compare" and len(args) == 2: + differ = compare(*args) + if differ: + print(f"error: {len(differ)} path(s) differ, for example:", file=sys.stderr) + for rel in differ[:10]: + print(f" differs: {rel}", file=sys.stderr) + return 1 + else: + print(__doc__, file=sys.stderr) + return 2 + except (TreeError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/actions/wporg-svn/set-stable/action.yml b/.github/actions/wporg-svn/set-stable/action.yml new file mode 100644 index 0000000..1391efb --- /dev/null +++ b/.github/actions/wporg-svn/set-stable/action.yml @@ -0,0 +1,46 @@ +name: "WordPress.org SVN: set stable" +description: >- + Release a prepared tag: copy tags//readme.txt over trunk/readme.txt in one server-side commit, + which points the Stable Tag at that version. Choosing an older tag rolls the release back. +inputs: + plugin-slug: + description: The plugin's wordpress.org slug. + required: true + version: + description: The tag to release, usually the tag action's version output. + required: true + expected-revision: + description: >- + Optional: the tag action's revision output. When set, refuse unless tags/ last changed at + exactly this revision, so the release is the tag that was QA'd. Leave it empty for a manual run or rollback. + required: false + wporg-username: + description: WordPress.org account with commit access to the plugin. + required: true + wporg-password: + description: That account's SVN password, not its login password. + required: true +outputs: + version: + description: The version now set as stable. + value: ${{ steps.release.outputs.version }} + revision: + description: The revision of the readme commit, or the observed revision for a verified no-op. + value: ${{ steps.release.outputs.revision }} +runs: + using: composite + steps: + - name: Mask the password + shell: bash + env: + PASSWORD: ${{ inputs.wporg-password }} + run: printf '%s\n' "$PASSWORD" | while IFS= read -r line; do [ -z "$line" ] || { line=${line//%/%25}; echo "::add-mask::${line//$'\r'/%0D}"; }; done + - id: release + shell: bash + env: + WPORG_SLUG: ${{ inputs.plugin-slug }} + WPORG_VERSION: ${{ inputs.version }} + WPORG_EXPECTED_REVISION: ${{ inputs.expected-revision }} + WPORG_USERNAME: ${{ inputs.wporg-username }} + WPORG_PASSWORD: ${{ inputs.wporg-password }} + run: bash "$GITHUB_ACTION_PATH/set-stable.sh" diff --git a/.github/actions/wporg-svn/set-stable/set-stable.sh b/.github/actions/wporg-svn/set-stable/set-stable.sh new file mode 100644 index 0000000..0eb8cbf --- /dev/null +++ b/.github/actions/wporg-svn/set-stable/set-stable.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Move the release pointer: put tags//readme.txt over trunk/readme.txt +# in one server-side commit, with no checkout. There is no upgrade-only rule: +# choosing an older tag rolls the release back. +ACTION_NAME="wporg-svn set-stable" +# shellcheck source=SCRIPTDIR/../lib/svn.sh +. "${BASH_SOURCE[0]%/*}/../lib/svn.sh" + +require_tools svn svnmucc python3 cmp +validate_slug +validate_version +validate_expected_revision +validate_credentials +prepare + +STAGE="reading tags/$VERSION and trunk" +snapshot +R=$REPLY +note_revision "snapshot r$R" +svn_kind "$BASE/tags" "$VERSION" "$R" +[ "$REPLY" = dir ] || fail "tags/$VERSION does not exist at r$R" +check_expected_revision "$R" +svn_kind "$BASE/tags/$VERSION" readme.txt "$R" +[ "$REPLY" = file ] || fail "tags/$VERSION/readme.txt is not a file at r$R" +svn_kind "$BASE/trunk" readme.txt "$R" +[ "$REPLY" = file ] || fail "trunk/readme.txt is not a file at r$R" +check_props "$BASE/tags/$VERSION/readme.txt" "$R" "tags/$VERSION/readme.txt" +check_props "$BASE/trunk/readme.txt" "$R" "trunk/readme.txt" +svn_cat "$BASE/tags/$VERSION/readme.txt" "$R" "$WORK/release-readme.txt" +svn_cat "$BASE/trunk/readme.txt" "$R" "$WORK/trunk-readme.txt" +STABLE=$(python3 "$LIB_DIR/release.py" stable-tag "$WORK/release-readme.txt") +[ "$STABLE" = "$VERSION" ] || fail "the Stable Tag in tags/$VERSION/readme.txt is not $VERSION" + +if cmp -s "$WORK/release-readme.txt" "$WORK/trunk-readme.txt"; then + output version "$VERSION" + output revision "$R" + summary "### $ACTION_NAME: verified no-op" "" \ + "trunk/readme.txt already matches tags/$VERSION/readme.txt at r$R. Nothing was committed." + exit 0 +fi + +STAGE="committing trunk/readme.txt" +if ! mucc_write -r "$R" -m "Release $VERSION: set the Stable Tag" \ + put "$WORK/release-readme.txt" "$BASE/trunk/readme.txt"; then + svn_error + ERROR=$REPLY + STAGE="inspecting trunk/readme.txt after a failed write" + snapshot + HEAD=$REPLY + svn_cat "$BASE/trunk/readme.txt" "$HEAD" "$WORK/after-readme.txt" + STAGE="committing trunk/readme.txt" + if cmp -s "$WORK/release-readme.txt" "$WORK/after-readme.txt"; then + fail "the write reported an error ($ERROR), but trunk/readme.txt now matches tags/$VERSION/readme.txt. Rerunning is safe: it verifies and reports a no-op." + fi + fail "the write failed ($ERROR) and trunk/readme.txt does not match tags/$VERSION/readme.txt at r$HEAD; check who else is committing (svn log) before rerunning." +fi +N=$REPLY +note_revision "commit r$N" + +STAGE="verifying trunk/readme.txt at r$N" +svn_cat "$BASE/trunk/readme.txt" "$N" "$WORK/committed-readme.txt" +cmp -s "$WORK/release-readme.txt" "$WORK/committed-readme.txt" || + fail "trunk/readme.txt at r$N does not match tags/$VERSION/readme.txt" + +output version "$VERSION" +output revision "$N" +summary "### $ACTION_NAME: $SLUG $VERSION" "" \ + "trunk/readme.txt now matches tags/$VERSION/readme.txt (r$N). WordPress.org processing happens separately." diff --git a/.github/actions/wporg-svn/tag/action.yml b/.github/actions/wporg-svn/tag/action.yml new file mode 100644 index 0000000..69459d3 --- /dev/null +++ b/.github/actions/wporg-svn/tag/action.yml @@ -0,0 +1,46 @@ +name: "WordPress.org SVN: tag" +description: >- + Prepare a release: create tags/ in one commit, as a copy of the current stable tag that matches + a plugin ZIP exactly. Does not touch trunk or move the stable pointer. +inputs: + plugin-slug: + description: The plugin's wordpress.org slug. + required: true + zip-url: + description: >- + HTTPS URL of the built plugin ZIP. The action never prints it and masks it, but GitHub logs step inputs + before the action runs: pass it from a secret, or mask it in an earlier step. + required: true + wporg-username: + description: WordPress.org account with commit access to the plugin. + required: true + wporg-password: + description: That account's SVN password, not its login password. + required: true +outputs: + version: + description: The version released, read from the plugin header. + value: ${{ steps.release.outputs.version }} + revision: + description: The revision that created tags/. + value: ${{ steps.release.outputs.revision }} + previous-stable: + description: The stable tag that tags/ was copied from. + value: ${{ steps.release.outputs.previous-stable }} +runs: + using: composite + steps: + - name: Mask the artifact URL and the password + shell: bash + env: + ZIP_URL: ${{ inputs.zip-url }} + PASSWORD: ${{ inputs.wporg-password }} + run: printf '%s\n' "$ZIP_URL" "$PASSWORD" | while IFS= read -r line; do [ -z "$line" ] || { line=${line//%/%25}; echo "::add-mask::${line//$'\r'/%0D}"; }; done + - id: release + shell: bash + env: + WPORG_SLUG: ${{ inputs.plugin-slug }} + WPORG_ZIP_URL: ${{ inputs.zip-url }} + WPORG_USERNAME: ${{ inputs.wporg-username }} + WPORG_PASSWORD: ${{ inputs.wporg-password }} + run: bash "$GITHUB_ACTION_PATH/tag.sh" diff --git a/.github/actions/wporg-svn/tag/tag.sh b/.github/actions/wporg-svn/tag/tag.sh new file mode 100644 index 0000000..855b827 --- /dev/null +++ b/.github/actions/wporg-svn/tag/tag.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Prepare a release without moving the stable pointer: copy the stable tag into +# an otherwise empty working copy as tags/, make it match the ZIP +# exactly and commit it. trunk and existing tags stay as they are. +# +# This is one commit: tags/ appears complete, or not at all. The +# commit records a copy of the stable tag, so only changed files are uploaded. +ACTION_NAME="wporg-svn tag" +# shellcheck source=SCRIPTDIR/../lib/svn.sh +. "${BASH_SOURCE[0]%/*}/../lib/svn.sh" + +# Keep in step with MAX_ARCHIVE_BYTES in lib/archive.py. +MAX_ZIP_BYTES=268435456 + +require_tools svn python3 curl +validate_slug +validate_credentials +[ -n "${WPORG_ZIP_URL:-}" ] || fail "zip-url is required" +case $WPORG_ZIP_URL in + https://*) ;; + *) fail "zip-url must be an https:// URL" ;; +esac +case $WPORG_ZIP_URL in + *[[:space:][:cntrl:]]*) fail "zip-url must not contain spaces or control characters" ;; +esac +prepare + +release_py() { + python3 "$LIB_DIR/release.py" "$@" +} + +# REPLY = the numeric Stable Tag in trunk/readme.txt at REV. +stable_at() { + svn_kind "$BASE/trunk" readme.txt "$1" + [ "$REPLY" = file ] || fail "trunk/readme.txt is not a file at r$1" + svn_cat "$BASE/trunk/readme.txt" "$1" "$WORK/trunk-readme.txt" + REPLY=$(release_py stable-tag "$WORK/trunk-readme.txt") + release_py version "$REPLY" || + fail "the trunk/readme.txt Stable Tag is not a numeric version; first releases are not supported" +} + +# Fail when tags/, or another spelling of the same version, exists at REV. +destination_absent() { + local found + svn_names "$BASE/tags" "$1" + found=$(printf '%s\n' "$REPLY" | release_py equivalents "$VERSION") + if [ -n "$found" ]; then + fail "tags/$VERSION already exists at r$1 (or under an equivalent spelling); released tags are never reused" + fi +} + +# Fetch zip-url over HTTPS only. Nothing about the URL is ever printed: signed +# URLs are secrets. No credentials are sent to the artifact host. +download() { + local code status=0 + # curl before 8.4 ignores --max-filesize when the server sends no length, so a + # file size limit (1 KiB blocks, this subshell only) stops it too: SIGXFSZ, 153. + code=$(ulimit -f $((MAX_ZIP_BYTES / 1024 + 1)) && + curl -q --silent --show-error --fail --location --proto '=https' --proto-redir '=https' \ + --max-redirs 5 --connect-timeout 30 --max-time 600 --retry 3 --retry-delay 5 --retry-max-time 900 \ + --max-filesize "$MAX_ZIP_BYTES" --write-out '%{http_code}' --output "$WORK/artifact.zip" \ + "$WPORG_ZIP_URL" 2>"$WORK/err") || status=$? + if [ "$status" -eq 63 ] || [ "$status" -eq 153 ]; then + fail "downloading zip-url failed: the ZIP is larger than $((MAX_ZIP_BYTES / 1048576)) MiB" + fi + if [ "$status" -ne 0 ]; then + if [[ $code =~ ^[1-9][0-9][0-9]$ ]]; then + fail "downloading zip-url failed (HTTP $code)" + fi + fail "downloading zip-url failed (curl exit $status)" + fi +} + +STAGE="reading the current release" +snapshot +R=$REPLY +note_revision "snapshot r$R" +stable_at "$R" +S=$REPLY +svn_kind "$BASE/tags" "$S" "$R" +[ "$REPLY" = dir ] || fail "the stable tag tags/$S does not exist at r$R" + +STAGE="downloading the artifact" +download + +STAGE="inspecting the artifact" +mkdir "$WORK/artifact" +ROOT=$(python3 "$LIB_DIR/archive.py" extract "$WORK/artifact.zip" "$WORK/artifact") +VERSION=$(release_py plugin-version "$ROOT") +[ "$(release_py compare "$VERSION" "$S")" = gt ] || fail "the plugin version $VERSION is not newer than the stable $S" +[ "$(release_py stable-tag "$ROOT/readme.txt")" = "$VERSION" ] || + fail "the Stable Tag in the artifact readme.txt is not $VERSION" + +STAGE="checking tags/$VERSION and tags/$S" +destination_absent "$R" +last_changed "$BASE/tags/$S" "$R" +SOURCE_CHANGED=$REPLY + +# Stop with MESSAGE when anything in the new tag sets a content-transforming property. +check_tag_props() { + svn_read proplist -R --xml -- "$TAG" + python3 "$LIB_DIR/svnxml.py" check-props <"$WORK/out" || fail "$1" +} + +STAGE="copying tags/$S into a working copy" +WC="$WORK/wc" +TAG="$WC/$VERSION" +svn_read checkout --quiet --depth empty -r "$R" -- "$BASE/tags@$R" "$WC" +svn_read copy --quiet --ignore-externals -r "$R" -- "$BASE/tags/$S@$R" "$TAG" +check_tag_props "tags/$S has SVN properties that change file contents; an exact release cannot keep them" + +STAGE="synchronising tags/$VERSION with the artifact" +python3 "$LIB_DIR/tree.py" sync "$ROOT" "$TAG" +CHANGES=$(python3 "$LIB_DIR/tree.py" reconcile "$TAG" "$WORK/svn-config") +check_tag_props "the working copy of tags/$VERSION has SVN properties that change file contents" + +STAGE="validating tags/$VERSION before the commit" +[ "$(release_py plugin-version "$TAG")" = "$VERSION" ] || fail "the working copy's plugin Version is not $VERSION" +[ "$(release_py stable-tag "$TAG/readme.txt")" = "$VERSION" ] || fail "the working copy's Stable Tag is not $VERSION" +python3 "$LIB_DIR/tree.py" compare "$ROOT" "$TAG" || fail "the working copy does not match the artifact" +[ "$CHANGES" -gt 0 ] || fail "the artifact is identical to tags/$S; there is nothing to release" + +STAGE="rechecking before the commit" +snapshot +note_revision "recheck r$REPLY" +RECHECK=$REPLY +stable_at "$RECHECK" +[ "$REPLY" = "$S" ] || fail "the trunk/readme.txt Stable Tag changed after r$R; another release is in progress" +last_changed "$BASE/tags/$S" "$RECHECK" +[ "$REPLY" = "$SOURCE_CHANGED" ] || fail "tags/$S changed after r$R" +destination_absent "$RECHECK" + +STAGE="committing tags/$VERSION" +RECOVERY="The commit's outcome is unknown: if tags/$VERSION exists, do not approve or release it until it is verified against the ZIP (wporg-svn README, Recovery)." +if ! commit_write "$TAG" "Release $VERSION: create tags/$VERSION from the artifact"; then + svn_error + ERROR=$REPLY + STAGE="inspecting tags/$VERSION after a failed commit" + snapshot + HEAD=$REPLY + svn_kind "$BASE/tags" "$VERSION" "$HEAD" + STAGE="committing tags/$VERSION" + if [ "$REPLY" = absent ]; then + RECOVERY= + fail "the commit failed ($ERROR) and tags/$VERSION does not exist at r$HEAD, so nothing was written. Rerun once the cause is fixed." + fi + RECOVERY="tags/$VERSION exists at r$HEAD, but this run cannot confirm that it created it. Do not approve or release it: inspect it first (wporg-svn README, Recovery)." + fail "the commit reported an error ($ERROR) and tags/$VERSION now exists" +fi +N=$REPLY +note_revision "commit r$N" +RECOVERY="tags/$VERSION was created at r$N but not verified against the artifact. Do not approve or release it until it is (wporg-svn README, Recovery)." + +STAGE="verifying tags/$VERSION at r$N" +svn_read export --quiet --ignore-externals -r "$N" -- "$BASE/tags/$VERSION@$N" "$WORK/verify" +python3 "$LIB_DIR/tree.py" compare "$ROOT" "$WORK/verify" || fail "tags/$VERSION at r$N does not match the artifact" + +RECOVERY= +output version "$VERSION" +output revision "$N" +output previous-stable "$S" +summary "### $ACTION_NAME: $SLUG $VERSION" "" \ + "- tags/$VERSION created at r$N as a copy of tags/$S@$R that matches the artifact, and verified." \ + "- The stable pointer is unchanged. QA tags/$VERSION@$N, then run set-stable and update-trunk." diff --git a/.github/actions/wporg-svn/update-trunk/action.yml b/.github/actions/wporg-svn/update-trunk/action.yml new file mode 100644 index 0000000..82bcb20 --- /dev/null +++ b/.github/actions/wporg-svn/update-trunk/action.yml @@ -0,0 +1,46 @@ +name: "WordPress.org SVN: update trunk" +description: >- + Post-release housekeeping: replace trunk with tags/ in one atomic server-side commit. This also + replaces trunk/readme.txt, so it refuses to run until set-stable has pointed the Stable Tag at the version. +inputs: + plugin-slug: + description: The plugin's wordpress.org slug. + required: true + version: + description: The released tag to copy into trunk. + required: true + expected-revision: + description: >- + Optional: the tag action's revision output. When set, refuse unless tags/ last changed at + exactly this revision, so the release is the tag that was QA'd. Leave it empty for a manual run or rollback. + required: false + wporg-username: + description: WordPress.org account with commit access to the plugin. + required: true + wporg-password: + description: That account's SVN password, not its login password. + required: true +outputs: + version: + description: The version trunk now matches. + value: ${{ steps.release.outputs.version }} + revision: + description: The revision that replaced trunk, or the observed revision for a verified no-op. + value: ${{ steps.release.outputs.revision }} +runs: + using: composite + steps: + - name: Mask the password + shell: bash + env: + PASSWORD: ${{ inputs.wporg-password }} + run: printf '%s\n' "$PASSWORD" | while IFS= read -r line; do [ -z "$line" ] || { line=${line//%/%25}; echo "::add-mask::${line//$'\r'/%0D}"; }; done + - id: release + shell: bash + env: + WPORG_SLUG: ${{ inputs.plugin-slug }} + WPORG_VERSION: ${{ inputs.version }} + WPORG_EXPECTED_REVISION: ${{ inputs.expected-revision }} + WPORG_USERNAME: ${{ inputs.wporg-username }} + WPORG_PASSWORD: ${{ inputs.wporg-password }} + run: bash "$GITHUB_ACTION_PATH/update-trunk.sh" diff --git a/.github/actions/wporg-svn/update-trunk/update-trunk.sh b/.github/actions/wporg-svn/update-trunk/update-trunk.sh new file mode 100644 index 0000000..463c0a7 --- /dev/null +++ b/.github/actions/wporg-svn/update-trunk/update-trunk.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Post-release housekeeping: replace trunk with tags/ in one atomic +# commit (rm + copy of the tag pinned at the snapshot revision). No checkout, +# export or local transfer. This also replaces trunk/readme.txt, so it refuses +# to run until set-stable has pointed trunk's Stable Tag at the version. +ACTION_NAME="wporg-svn update-trunk" +# shellcheck source=SCRIPTDIR/../lib/svn.sh +. "${BASH_SOURCE[0]%/*}/../lib/svn.sh" + +require_tools svn svnmucc python3 +validate_slug +validate_version +validate_expected_revision +validate_credentials +prepare + +STAGE="reading trunk and tags/$VERSION" +snapshot +R=$REPLY +note_revision "snapshot r$R" +svn_kind "$BASE" trunk "$R" +[ "$REPLY" = dir ] || fail "trunk does not exist at r$R" +svn_kind "$BASE/tags" "$VERSION" "$R" +[ "$REPLY" = dir ] || fail "tags/$VERSION does not exist at r$R" +check_expected_revision "$R" +svn_kind "$BASE/trunk" readme.txt "$R" +[ "$REPLY" = file ] || fail "trunk/readme.txt is not a file at r$R" +svn_cat "$BASE/trunk/readme.txt" "$R" "$WORK/trunk-readme.txt" +[ "$(python3 "$LIB_DIR/release.py" stable-tag "$WORK/trunk-readme.txt")" = "$VERSION" ] || + fail "the trunk/readme.txt Stable Tag is not $VERSION at r$R; run set-stable for $VERSION first" + +# trunk and the tag hold the same tree, files and properties, at REV? +same_tree() { + local status=0 + svn_read diff --summarize --xml --old="$BASE/tags/$VERSION@$R" --new="$BASE/trunk@$1" + python3 "$LIB_DIR/svnxml.py" diff-empty <"$WORK/out" 2>/dev/null || status=$? + [ "$status" -le 1 ] || fail "could not read svn's diff output" + return "$status" +} + +if same_tree "$R"; then + output version "$VERSION" + output revision "$R" + summary "### $ACTION_NAME: verified no-op" "" "trunk already matches tags/$VERSION at r$R. Nothing was committed." + exit 0 +fi + +STAGE="replacing trunk with tags/$VERSION@$R" +if ! mucc_write -r "$R" -m "Release $VERSION: sync trunk with tags/$VERSION" \ + rm "$BASE/trunk" cp "$R" "$BASE/tags/$VERSION" "$BASE/trunk"; then + svn_error + ERROR=$REPLY + STAGE="inspecting trunk after a failed write" + snapshot + HEAD=$REPLY + if same_tree "$HEAD"; then + STAGE="replacing trunk with tags/$VERSION@$R" + fail "the write reported an error ($ERROR), but trunk now matches tags/$VERSION. Rerunning is safe: it verifies and reports a no-op." + fi + STAGE="replacing trunk with tags/$VERSION@$R" + fail "the write failed ($ERROR) and trunk does not match tags/$VERSION at r$HEAD; check who else is committing (svn log) before rerunning." +fi +N=$REPLY +note_revision "commit r$N" + +STAGE="verifying trunk at r$N" +same_tree "$N" || fail "trunk at r$N does not match tags/$VERSION@$R" + +output version "$VERSION" +output revision "$N" +summary "### $ACTION_NAME: $SLUG $VERSION" "" "trunk is now a copy of tags/$VERSION@$R (r$N)." diff --git a/.github/workflows/wporg-svn.yml b/.github/workflows/wporg-svn.yml new file mode 100644 index 0000000..54346a2 --- /dev/null +++ b/.github/workflows/wporg-svn.yml @@ -0,0 +1,98 @@ +# Self-checks for the wporg-svn actions. Nothing here touches wordpress.org: +# every SVN write goes to a throwaway file:// repository, behind the test-only +# write guard (tests/wporg-svn/guard). +name: wporg-svn actions + +on: + pull_request: + paths: + - .github/actions/wporg-svn/** + - .github/workflows/wporg-svn.yml + - tests/wporg-svn/** + push: + branches: [main] + paths: + - .github/actions/wporg-svn/** + - .github/workflows/wporg-svn.yml + - tests/wporg-svn/** + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # Locally: shellcheck -x .github/actions/wporg-svn/*/*.sh .github/actions/wporg-svn/lib/svn.sh tests/wporg-svn/guard/bin/* + - name: shellcheck + run: shellcheck -x .github/actions/wporg-svn/*/*.sh .github/actions/wporg-svn/lib/svn.sh tests/wporg-svn/guard/bin/* + # Locally: actionlint && actionlint .github/actions/wporg-svn/examples/release.yml + - name: actionlint + run: | + curl -fsSL -o "$RUNNER_TEMP/actionlint.tar.gz" \ + https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz + echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $RUNNER_TEMP/actionlint.tar.gz" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/actionlint.tar.gz" -C "$RUNNER_TEMP" actionlint + "$RUNNER_TEMP/actionlint" + "$RUNNER_TEMP/actionlint" .github/actions/wporg-svn/examples/release.yml + + test: + strategy: + fail-fast: false + matrix: + runner: [ubuntu-22.04, ubuntu-24.04] + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + # Locally: python3 -m unittest discover -s tests/wporg-svn -v + - name: Unit and integration tests + run: python3 -m unittest discover -s tests/wporg-svn -v + + # Runs the three composite actions the way a caller would, against the local fixture. + end-to-end: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + - name: Start the fixture + run: | + echo "$PWD/tests/wporg-svn/guard/bin" >> "$GITHUB_PATH" + python3 tests/wporg-svn/fixture.py up "$RUNNER_TEMP/fixture" & + for _ in $(seq 60); do [ -f "$RUNNER_TEMP/fixture/env" ] && break; sleep 1; done + cat "$RUNNER_TEMP/fixture/env" >> "$GITHUB_ENV" + - id: tag + uses: ./.github/actions/wporg-svn/tag + with: + plugin-slug: fixture-plugin + zip-url: ${{ env.FIXTURE_ZIP_URL }} + wporg-username: fixture-user + wporg-password: fixture-password + - uses: ./.github/actions/wporg-svn/set-stable + with: + plugin-slug: fixture-plugin + version: ${{ steps.tag.outputs.version }} + expected-revision: ${{ steps.tag.outputs.revision }} + wporg-username: fixture-user + wporg-password: fixture-password + - uses: ./.github/actions/wporg-svn/update-trunk + with: + plugin-slug: fixture-plugin + version: ${{ steps.tag.outputs.version }} + expected-revision: ${{ steps.tag.outputs.revision }} + wporg-username: fixture-user + wporg-password: fixture-password + - name: Check the end state + env: + VERSION: ${{ steps.tag.outputs.version }} + run: python3 tests/wporg-svn/fixture.py check "$RUNNER_TEMP/fixture" "$VERSION" diff --git a/.gitignore b/.gitignore index da63dc1..dca8d65 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +__pycache__/ + node_modules/ # No lockfile is committed. Each install resolves the newest versions the ranges in package.json diff --git a/tests/wporg-svn/fixture.py b/tests/wporg-svn/fixture.py new file mode 100644 index 0000000..b9fed88 --- /dev/null +++ b/tests/wporg-svn/fixture.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""The end-to-end fixture for CI: a file:// plugin repository at release 1.0, +and a local HTTPS host serving the 1.1 ZIP. Nothing here talks to wordpress.org. + +Usage: + fixture.py up DIR create the fixture in DIR, write DIR/env (KEY=value + lines for $GITHUB_ENV), then serve until killed + fixture.py check DIR VERSION exit 0 when tags/VERSION and trunk both match the + artifact and trunk's Stable Tag is VERSION +""" + +import os +import sys +import time + +import support +from test_tag import NEXT, STABLE + + +def up(folder): + home = os.path.join(folder, "home") + os.makedirs(home) + repo = support.Repo(folder, support.base_env(home)) + repo.seed("1.0", STABLE) + https = support.Https() + https.routes["/release.zip"] = support.make_zip(NEXT, wrapper="fixture-plugin") + with open(os.path.join(folder, "env.tmp"), "w") as handle: + handle.write(f"WPORG_SVN_TEST_ROOT={repo.root}\n" + f"CURL_CA_BUNDLE={https.ca}\n" + f"FIXTURE_ZIP_URL={https.url}/release.zip?sig=fixture\n") + os.rename(os.path.join(folder, "env.tmp"), os.path.join(folder, "env")) + while True: + time.sleep(3600) + + +def check(folder, version): + repo = support.Repo(folder, support.base_env(os.path.join(folder, "home")), create=False) + expected = support.expected_tree(NEXT) + problems = [path for path in (f"tags/{version}", "trunk") if repo.tree(path) != expected] + if b"Stable tag: " + version.encode() not in repo.cat("trunk/readme.txt"): + problems.append("trunk/readme.txt Stable Tag") + print("end state ok" if not problems else "end state wrong: " + ", ".join(problems)) + return 1 if problems else 0 + + +if __name__ == "__main__": + if sys.argv[1:2] == ["up"] and len(sys.argv) == 3: + up(sys.argv[2]) + elif sys.argv[1:2] == ["check"] and len(sys.argv) == 4: + sys.exit(check(sys.argv[2], sys.argv[3])) + else: + sys.exit(__doc__) diff --git a/tests/wporg-svn/guard/bin/svn b/tests/wporg-svn/guard/bin/svn new file mode 100755 index 0000000..572dc90 --- /dev/null +++ b/tests/wporg-svn/guard/bin/svn @@ -0,0 +1,3 @@ +#!/bin/sh +# Test-only: route svn through the write guard. +exec python3 "$(dirname "$0")/../write_guard.py" svn "$@" diff --git a/tests/wporg-svn/guard/bin/svnmucc b/tests/wporg-svn/guard/bin/svnmucc new file mode 100755 index 0000000..e97b3ea --- /dev/null +++ b/tests/wporg-svn/guard/bin/svnmucc @@ -0,0 +1,3 @@ +#!/bin/sh +# Test-only: route svnmucc through the write guard. +exec python3 "$(dirname "$0")/../write_guard.py" svnmucc "$@" diff --git a/tests/wporg-svn/guard/bin/svnrdump b/tests/wporg-svn/guard/bin/svnrdump new file mode 100755 index 0000000..b61ca5f --- /dev/null +++ b/tests/wporg-svn/guard/bin/svnrdump @@ -0,0 +1,3 @@ +#!/bin/sh +# Test-only: route svnrdump through the write guard. +exec python3 "$(dirname "$0")/../write_guard.py" svnrdump "$@" diff --git a/tests/wporg-svn/guard/write_guard.py b/tests/wporg-svn/guard/write_guard.py new file mode 100644 index 0000000..3a6c230 --- /dev/null +++ b/tests/wporg-svn/guard/write_guard.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Refuse SVN writes to anything but file:// repositories. + +Test-only. The bin/ shims put this in front of the real svn, svnmucc and +svnrdump, so no test, fixture or QA run can write to a production repository, +even by mistake. +Reads pass straight through. The rule is deliberately conservative: anything +that looks like a write is checked, and a false refusal only fails loudly. + +Usage: write_guard.py svn|svnmucc|svnrdump ARGS... +With WRITE_GUARD_LOG set, every argv is appended to that file as a JSON line. +""" + +import json +import os +import re +import subprocess +import sys + +BIN = os.path.join(os.path.dirname(os.path.abspath(__file__)), "bin") +URL = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*)://") + +# svn subcommands that commit, or that commit when given a URL. +SVN_WRITES = { + "commit", "ci", "import", "mkdir", "copy", "cp", "move", "mv", "rename", "ren", "delete", "del", + "remove", "rm", "propset", "pset", "ps", "propdel", "pdel", "pd", "propedit", "pedit", "pe", + "lock", "unlock", +} + + +def is_remote(value): + match = URL.match(value) + return bool(match) and match.group(1).lower() != "file" + + +def real(tool): + """The tool from the PATH entries after the guard's own (wrappers come before it).""" + guard = os.path.realpath(BIN) + path = [d for d in os.environ.get("PATH", "").split(os.pathsep) if d] + resolved = [os.path.realpath(d) for d in path] + if guard in resolved: + path = path[resolved.index(guard) + 1:] + for directory in path: + if os.path.realpath(directory) == guard: + continue + candidate = os.path.join(directory, tool) + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + sys.exit(f"write-guard: no real {tool} on PATH") + + +def repo_root(path): + result = subprocess.run([real("svn"), "info", "--non-interactive", "--show-item", "repos-root-url", "--", path + "@"], + capture_output=True, text=True) + return result.stdout.strip() if result.returncode == 0 else None + + +def refusal(tool, args, root_of=repo_root): + """Why this command must not run, or None when it is a read or a local write.""" + if tool == "svnmucc": + urls = [arg for arg in args if URL.match(arg)] + if not urls: + return "svnmucc without an explicit file:// URL" + if any(is_remote(url) for url in urls): + return "svnmucc to a non-file:// repository" + return None + + if tool == "svnrdump": + if "load" not in args: + return None + urls = [arg for arg in args if URL.match(arg)] + if not urls or any(is_remote(url) for url in urls): + return "svnrdump load into anything but a file:// repository" + return None + + if not SVN_WRITES.intersection(args) and "--revprop" not in args: + return None + if any(is_remote(arg) for arg in args): + return "svn write to a non-file:// repository" + paths = [arg for arg in args if not arg.startswith("-") and os.path.exists(arg.rsplit("@", 1)[0])] + for path in paths + ["."]: + root = root_of(path) + if root and is_remote(root): + return "svn write from a working copy of a non-file:// repository" + return None + + +def main(): + tool, args = sys.argv[1], sys.argv[2:] + if os.environ.get("WRITE_GUARD_LOG"): + with open(os.environ["WRITE_GUARD_LOG"], "a") as log: + log.write(json.dumps([tool] + args) + "\n") + reason = refusal(tool, args) + if reason: + print(f"write-guard: refused: {reason}", file=sys.stderr) + sys.exit(99) + os.execv(real(tool), [tool] + args) + + +if __name__ == "__main__": + main() diff --git a/tests/wporg-svn/live_reads.sh b/tests/wporg-svn/live_reads.sh new file mode 100755 index 0000000..8e785fc --- /dev/null +++ b/tests/wporg-svn/live_reads.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Test-only and read-only: run the actions' read helpers against a real +# wordpress.org plugin and print what they saw. It never calls a write helper, +# and the tests run it with the write guard first on PATH. +# Usage: live_reads.sh SLUG +ACTION_NAME="wporg-svn live reads" +# shellcheck source=SCRIPTDIR/../../.github/actions/wporg-svn/lib/svn.sh +. "${BASH_SOURCE[0]%/*}/../../.github/actions/wporg-svn/lib/svn.sh" + +[ -z "${WPORG_SVN_TEST_ROOT:-}" ] || fail "live reads go to wordpress.org: unset WPORG_SVN_TEST_ROOT" +WPORG_SLUG=${1:-} +validate_slug +prepare + +STAGE="reading $SLUG" +snapshot +R=$REPLY +echo "revision=$R" +svn_kind "$BASE/trunk" readme.txt "$R" +echo "trunk-readme=$REPLY" +svn_cat "$BASE/trunk/readme.txt" "$R" "$WORK/readme.txt" +S=$(python3 "$LIB_DIR/release.py" stable-tag "$WORK/readme.txt") +python3 "$LIB_DIR/release.py" version "$S" || fail "the Stable Tag is not numeric" +echo "stable=$S" +svn_kind "$BASE/tags" "$S" "$R" +echo "stable-kind=$REPLY" +svn_kind "$BASE/tags" 0.0.0.0.0.1 "$R" +echo "probe-kind=$REPLY" +svn_names "$BASE/tags" "$R" +echo "tags=$(printf '%s\n' "$REPLY" | python3 -c 'import sys; print(",".join(l.strip() for l in sys.stdin))')" +last_changed "$BASE/tags/$S" "$R" +echo "stable-last-changed=$REPLY" diff --git a/tests/wporg-svn/support.py b/tests/wporg-svn/support.py new file mode 100644 index 0000000..5a7fa9b --- /dev/null +++ b/tests/wporg-svn/support.py @@ -0,0 +1,412 @@ +"""Shared paths, fixtures and the action runner for the wporg-svn tests. + +Every SVN write here goes to a throwaway file:// repository, and the write +guard (guard/bin) sits first on PATH for every subprocess, hooks included. +""" + +import atexit +import http.server +import io +import json +import os +import pathlib +import re +import shutil +import ssl +import struct +import subprocess +import sys +import tempfile +import threading +import unittest +import xml.etree.ElementTree as ET +import zipfile +import zlib + +TESTS = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.dirname(os.path.dirname(TESTS)) +ACTIONS = os.path.join(REPO, ".github", "actions", "wporg-svn") +LIB = os.path.join(ACTIONS, "lib") +GUARD_BIN = os.path.join(TESTS, "guard", "bin") + +sys.path.insert(0, LIB) + +SLUG = "fixture-plugin" +USERNAME = "fixture-user" +# Sentinels: neither may ever appear in anything an action prints or writes. +PASSWORD = "sentinel-password-7c1e" +SIGNATURE = "sentinel-signature-93ab" + +README = """=== Fixture Plugin === +Contributors: fixture +Requires at least: 6.0 +Tested up to: 6.6 +Stable tag: {stable} +License: GPLv2 + +A fixture. + +== Changelog == + += {version} = +* Release {version}. +""" + +PHP = """ hold the same release; assets holds a banner.""" + self.put_tree("trunk-new", files or plugin(stable)) + self.mucc("rm", self.url("trunk"), "mv", self.url("trunk-new"), self.url("trunk")) + self.mucc("put", write(self.staging, "banner.png", b"\x89PNG banner"), self.url("assets/banner.png")) + return self.mucc("cp", "HEAD", self.url("trunk"), self.url(f"tags/{stable}")) + + def tree(self, path, rev="HEAD"): + dest = tempfile.mktemp(dir=self.staging) + self.svn("export", "-q", "--ignore-externals", "-r", str(rev), f"{self.url(path)}@{rev}", dest) + try: + return read_tree(dest) + finally: + shutil.rmtree(dest) + + def cat(self, path, rev="HEAD"): + return self.svn("cat", "-r", str(rev), f"{self.url(path)}@{rev}") + + def last_changed(self, path, rev="HEAD"): + return int(self.svn("info", "--show-item", "last-changed-revision", f"{self.url(path)}@{rev}").decode()) + + def exists(self, path, rev="HEAD"): + return self.svn("info", f"{self.url(path)}@{rev}", check=False) != b"" + + def props(self, path, rev="HEAD"): + """{relative path: {name: value}} for every node under path.""" + target = self.url(path) + doc = ET.fromstring(self.svn("proplist", "-R", "-v", "--xml", "-r", str(rev), f"{target}@{rev}")) + found = {} + for node in doc.iter("target"): + rel = node.get("path")[len(target):].lstrip("/") + found[rel] = {p.get("name"): p.text or "" for p in node.iter("property")} + return found + + def changed(self, rev): + """Sorted (action, path) pairs committed in rev.""" + doc = ET.fromstring(self.svn("log", "-v", "--xml", "-r", str(rev), self.root)) + return sorted((p.get("action"), p.text) for p in doc.iter("path")) + + def hook(self, name, body): + """Install a repository hook; hooks run with an empty environment.""" + exports = "".join(f"export {k}='{self.env[k]}'\n" for k in ("PATH", "HOME", "LC_ALL")) + path = os.path.join(self.dir, "hooks", name) + with open(path, "w") as handle: + handle.write("#!/bin/sh\n" + exports + body) + os.chmod(path, 0o755) + + +_CERT = {} + + +def certificate(): + """A self-signed certificate for 127.0.0.1, created once per test run.""" + if not _CERT: + folder = tempfile.mkdtemp() + atexit.register(shutil.rmtree, folder, True) + cert, key = os.path.join(folder, "cert.pem"), os.path.join(folder, "key.pem") + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", key, "-out", cert, + "-days", "2", "-subj", "/CN=127.0.0.1", "-addext", "subjectAltName=IP:127.0.0.1"], + check=True, capture_output=True) + _CERT.update(cert=cert, key=key) + return _CERT["cert"], _CERT["key"] + + +class Https: + """A local HTTPS server. routes: path -> bytes, or callable(handler).""" + + def __init__(self): + self.routes, self.requests = {}, [] + owner = self + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + owner.requests.append({"path": self.path, "headers": dict(self.headers)}) + route = owner.routes.get(self.path.split("?")[0]) + if route is None: + self.send_error(404) + elif callable(route): + route(self) + else: + self.send_response(200) + self.send_header("Content-Length", str(len(route))) + self.end_headers() + self.wfile.write(route) + + def log_message(self, *args): + pass + + cert, key = certificate() + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert, key) + self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.server.socket = context.wrap_socket(self.server.socket, server_side=True) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.url = f"https://127.0.0.1:{self.server.server_address[1]}" + self.ca = cert + + def close(self): + self.server.shutdown() + self.server.server_close() + + +class Result: + def __init__(self, process, outputs, summary): + self.code = process.returncode + self.stdout, self.stderr = process.stdout, process.stderr + self.log = process.stdout + process.stderr + self.outputs, self.summary = outputs, summary + + +class ActionTest(unittest.TestCase): + """Runs the real action scripts against a throwaway repository.""" + + def setUp(self): + self.tmp = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.home = os.path.join(self.tmp, "home") + self.scratch = os.path.join(self.tmp, "scratch") # the actions' TMPDIR + os.makedirs(self.home) + os.makedirs(self.scratch) + self.env = base_env(self.home) + self.repo = Repo(self.tmp, self.env) + self._https = None + + @property + def https(self): + if self._https is None: + self._https = Https() + self.addCleanup(self._https.close) + return self._https + + def action_env(self, **env): + merged = dict(self.env, WPORG_SVN_TEST_ROOT=self.repo.root, WPORG_SLUG=SLUG, WPORG_USERNAME=USERNAME, + WPORG_PASSWORD=PASSWORD, TMPDIR=self.scratch, + GITHUB_OUTPUT=os.path.join(self.tmp, "github-output"), + GITHUB_STEP_SUMMARY=os.path.join(self.tmp, "github-summary"), + WRITE_GUARD_LOG=os.path.join(self.tmp, "svn-calls.jsonl")) + if self._https: + merged["CURL_CA_BUNDLE"] = self._https.ca + merged.update(env) + return {k: v for k, v in merged.items() if v is not None} + + def wrap(self, tool, body): + """Put a test-only wrapper for tool first on the actions' PATH. + + The body runs under sh; "$NEXT" is the guarded tool it may exec. + """ + folder = os.path.join(self.tmp, "wrappers") + os.makedirs(folder, exist_ok=True) + path = os.path.join(folder, tool) + nxt = os.path.join(GUARD_BIN, tool) if tool in ("svn", "svnmucc", "svnrdump") else shutil.which(tool) + with open(path, "w") as handle: + handle.write(f"#!/bin/sh\nNEXT='{nxt}'\n{body}\n") + os.chmod(path, 0o755) + self.env["PATH"] = folder + os.pathsep + self.env["PATH"].replace(folder + os.pathsep, "") + + def start(self, action, **env): + """Start an action without waiting; it leads its own process group.""" + for name in ("github-output", "github-summary"): + open(os.path.join(self.tmp, name), "w").close() + script = os.path.join(ACTIONS, action, f"{action}.sh") + return subprocess.Popen(["bash", script], env=self.action_env(**env), stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, start_new_session=True) + + def finish(self, process, timeout=180): + stdout, stderr = process.communicate(timeout=timeout) + process.stdout, process.stderr = stdout, stderr + with open(os.path.join(self.tmp, "github-output")) as handle: + outputs = dict(line.rstrip("\n").split("=", 1) for line in handle if "=" in line) + with open(os.path.join(self.tmp, "github-summary")) as handle: + summary = handle.read() + result = Result(process, outputs, summary) + + # Invariants for every run, successful or not. + self.assertEqual(os.listdir(self.scratch), [], "the action left temporary files behind") + everything = result.log + summary + repr(outputs) + for secret in (PASSWORD, SIGNATURE): + self.assertNotIn(secret, everything, "a secret leaked into the action's output") + self.assertNotRegex(everything, r"(?m)^::(?!error|warning|notice|add-mask)", + "unexpected workflow command") + return result + + def svn_calls(self): + """Every svn/svnmucc argv the actions ran, in order.""" + path = os.path.join(self.tmp, "svn-calls.jsonl") + if not os.path.exists(path): + return [] + with open(path) as handle: + return [json.loads(line) for line in handle] + + def run_action(self, action, **env): + return self.finish(self.start(action, **env)) + + def assert_success(self, result): + self.assertEqual(result.code, 0, result.log) + + def assert_failure(self, result, pattern): + self.assertNotEqual(result.code, 0, result.log) + self.assertRegex(result.log, pattern) + self.assertEqual(result.outputs, {}, "a failed action must not expose outputs") diff --git a/tests/wporg-svn/test_actions.py b/tests/wporg-svn/test_actions.py new file mode 100644 index 0000000..3e219db --- /dev/null +++ b/tests/wporg-svn/test_actions.py @@ -0,0 +1,103 @@ +"""The action.yml files wire inputs, scripts and outputs together correctly. + +actionlint checks the syntax; this checks the contract with the scripts, which +no linter knows about. The files are read as text: they are ours and simple. +""" + +import os +import re +import subprocess +import unittest + +from support import ACTIONS + +CREDENTIALS = {"wporg-username": "WPORG_USERNAME", "wporg-password": "WPORG_PASSWORD"} +EXPECTED = {"expected-revision": "WPORG_EXPECTED_REVISION"} +OPTIONAL = set(EXPECTED) +ACTIONS_SPEC = { + "tag": ({"plugin-slug": "WPORG_SLUG", "zip-url": "WPORG_ZIP_URL", **CREDENTIALS}, + {"version", "revision", "previous-stable"}), + "set-stable": ({"plugin-slug": "WPORG_SLUG", "version": "WPORG_VERSION", **EXPECTED, **CREDENTIALS}, + {"version", "revision"}), + "update-trunk": ({"plugin-slug": "WPORG_SLUG", "version": "WPORG_VERSION", **EXPECTED, **CREDENTIALS}, + {"version", "revision"}), +} + + +def block(text, key): + """The lines under a top-level key, up to the next top-level key.""" + match = re.search(rf"(?ms)^{key}:\n(.*?)(?=^\S|\Z)", text) + return match.group(1) if match else "" + + +def named(section): + """{name: its indented body} for the two-space-indented keys of a section.""" + parts = re.split(r"(?m)^ ([a-z-]+):\n", section) + return dict(zip(parts[1::2], parts[2::2])) + + +class ActionMetadataTest(unittest.TestCase): + def read(self, *path): + with open(os.path.join(*path)) as handle: + return handle.read() + + def test_each_action_matches_its_script(self): + # The integration tests prove the scripts read exactly these env names. + for action, (inputs, outputs) in ACTIONS_SPEC.items(): + with self.subTest(action=action): + text = self.read(ACTIONS, action, "action.yml") + script = self.read(ACTIONS, action, f"{action}.sh") + declared = named(block(text, "inputs")) + self.assertEqual(set(declared), set(inputs)) + for name, body in declared.items(): + self.assertIn("required: false" if name in OPTIONAL else "required: true", body, name) + + self.assertRegex(text, r"(?m)^ using: composite$") + steps = block(text, "runs") + self.assertEqual(steps.count("shell: bash"), steps.count("run:"), "every step needs shell: bash") + for run in re.findall(r"run: (.*)", steps): + self.assertNotIn("${{", run, "inputs reach scripts through env, never the script text") + self.assertIn(f'run: bash "$GITHUB_ACTION_PATH/{action}.sh"', steps) + + main = steps[steps.index("id: release"):] + for name, variable in inputs.items(): + self.assertIn(f"{variable}: ${{{{ inputs.{name} }}}}", main) + + declared_outputs = named(block(text, "outputs")) + self.assertEqual(set(declared_outputs), outputs) + emitted = set(re.findall(r"^output ([a-z-]+) ", script, re.M)) + self.assertEqual(emitted, outputs) + for name, body in declared_outputs.items(): + self.assertIn(f"value: ${{{{ steps.release.outputs.{name} }}}}", body) + + def test_the_zip_url_and_password_are_masked_first(self): + for action in ACTIONS_SPEC: + with self.subTest(action=action): + steps = block(self.read(ACTIONS, action, "action.yml"), "runs") + self.assertLess(steps.index("::add-mask::"), steps.index("id: release")) + self.assertIn("${{ inputs.wporg-password }}", steps[:steps.index("id: release")]) + if action == "tag": + self.assertIn("${{ inputs.zip-url }}", steps[:steps.index("id: release")]) + + def test_mask_values_are_escaped_for_the_runner(self): + # The runner unescapes %25, %0D and %0A in ::add-mask:: data before it masks + # the value, so a secret containing "%25" would otherwise mask the wrong string. + env = {"PATH": os.environ["PATH"], "ZIP_URL": "https://h/a%25b?sig=x%2F", "PASSWORD": "p%41ss\r"} + for action in ACTIONS_SPEC: + with self.subTest(action=action): + steps = block(self.read(ACTIONS, action, "action.yml"), "runs") + script = re.search(r"run: (.*::add-mask::.*)", steps).group(1) + out = subprocess.run(["bash", "-c", script], env=env, capture_output=True, text=True, + check=True).stdout + self.assertIn("::add-mask::p%2541ss%0D\n", out) + if action == "tag": + self.assertIn("::add-mask::https://h/a%2525b?sig=x%252F\n", out) + + def test_the_readme_example_is_the_linted_example_file(self): + readme = self.read(ACTIONS, "README.md") + example = self.read(ACTIONS, "examples", "release.yml") + self.assertIn("```yaml\n" + example + "```\n", readme, "README example drifted from examples/release.yml") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_archive.py b/tests/wporg-svn/test_archive.py new file mode 100644 index 0000000..7b54429 --- /dev/null +++ b/tests/wporg-svn/test_archive.py @@ -0,0 +1,346 @@ +"""ZIP inspection, safe extraction and plugin root resolution.""" + +import os +import shutil +import stat +import subprocess +import tempfile +import unittest +import zipfile +from unittest import mock + +import support # noqa: F401 (puts lib/ on sys.path) + +import archive + +PLUGIN = { + "readme.txt": "=== X ===\nStable tag: 1.1\n\nShort.\n", + "fixture.php": " bytes/str, or name -> ZipInfo-tweaking dict.""" + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as zf: + for name, value in entries.items(): + info = zipfile.ZipInfo(name) + info.compress_type = zipfile.ZIP_DEFLATED + data = value + if isinstance(value, dict): + data = value.get("data", b"") + if "mode" in value: + info.external_attr = value["mode"] << 16 + info.create_system = value.get("host", info.create_system) + if name.endswith("/"): + info.external_attr = info.external_attr or (stat.S_IFDIR | 0o755) << 16 + zf.writestr(info, data.encode() if isinstance(data, str) else data) + return path + + +def set_flag_bits(path, bits, on=True): + """Set, or clear, general purpose flag bits in every header.""" + with open(path, "r+b") as handle: + blob = bytearray(handle.read()) + for signature, offset in ((b"PK\x03\x04", 6), (b"PK\x01\x02", 8)): + start = blob.find(signature) + while start != -1: + at = start + offset + flags = int.from_bytes(blob[at:at + 2], "little") + blob[at:at + 2] = (flags | bits if on else flags & ~bits).to_bytes(2, "little") + start = blob.find(signature, start + 4) + handle.seek(0) + handle.write(blob) + + +class ArchiveTest(unittest.TestCase): + def setUp(self): + self.tmp = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp) + self.zip = os.path.join(self.tmp, "plugin.zip") + self.dest = os.path.join(self.tmp, "out") + os.mkdir(self.dest) + + def extract(self, entries): + build_zip(self.zip, entries) + return archive.extract(self.zip, self.dest) + + def assert_rejected(self, entries, message): + with self.assertRaisesRegex(archive.ArchiveError, message): + self.extract(entries) + self.assertEqual(os.listdir(self.dest), [], "nothing may be extracted from a rejected archive") + self.assertEqual(sorted(os.listdir(self.tmp)), ["out", "plugin.zip"], "nothing written outside") + + def read(self, *parts): + with open(os.path.join(*parts), "rb") as handle: + return handle.read() + + def test_flat_archive(self): + entries = dict(PLUGIN) + entries.update({".htaccess": "deny", "sub dir/a@b.txt": "x", "deep/er/empty.txt": "", "assets/": b""}) + root = self.extract(entries) + self.assertEqual(root, self.dest) + self.assertEqual(self.read(root, ".htaccess"), b"deny") + self.assertEqual(self.read(root, "sub dir", "a@b.txt"), b"x") + self.assertEqual(self.read(root, "deep", "er", "empty.txt"), b"") + self.assertTrue(os.path.isdir(os.path.join(root, "assets"))) + + def test_single_wrapper_directory_of_any_name(self): + root = self.extract({"whatever-name/" + k: v for k, v in PLUGIN.items()}) + self.assertEqual(root, os.path.join(self.dest, "whatever-name")) + self.assertEqual(self.read(root, "readme.txt"), PLUGIN["readme.txt"].encode()) + + def test_ignores_documented_packaging_noise(self): + entries = {"w/" + k: v for k, v in PLUGIN.items()} + entries.update({"__MACOSX/w/._readme.txt": "junk", "w/.DS_Store": "junk", ".DS_Store": "junk"}) + root = self.extract(entries) + self.assertEqual(root, os.path.join(self.dest, "w")) + self.assertEqual(sorted(os.listdir(self.dest)), ["w"]) + self.assertNotIn(".DS_Store", os.listdir(root)) + + def test_rejects_multiple_roots(self): + entries = {"a/" + k: v for k, v in PLUGIN.items()} + entries.update({"b/" + k: v for k, v in PLUGIN.items()}) + self.assert_rejected(entries, "plugin root") + + def test_rejects_deep_wrappers(self): + self.assert_rejected({"a/b/" + k: v for k, v in PLUGIN.items()}, "readme.txt") + + def test_rejects_unrelated_siblings_of_a_wrapper(self): + entries = {"w/" + k: v for k, v in PLUGIN.items()} + entries["LICENSE"] = "gpl" + self.assert_rejected(entries, "plugin root") + + def test_rejects_missing_readme(self): + self.assert_rejected({"fixture.php": PLUGIN["fixture.php"]}, "plugin root") + + def test_rejects_readme_directory(self): + self.assert_rejected({"readme.txt/x": "x", "fixture.php": "x"}, "plugin root") + + def test_rejects_traversal_and_absolute_paths(self): + for name in ("../evil.php", "a/../../evil.php", "/abs/evil.php", "C:/evil.php", "a\\..\\evil.php", + "a/./b.php", "a//b.php", "./b.php"): + with self.subTest(name=name): + entries = dict(PLUGIN) + entries[name] = "x" + self.assert_rejected(entries, "unsafe path") + + def test_rejects_control_characters(self): + for name in ("a\nb.txt", "a\x7fb.txt", "a\x1bb.txt", "a\x85b.txt"): + with self.subTest(name=repr(name)): + entries = dict(PLUGIN) + entries[name] = "x" + self.assert_rejected(entries, "control character") + + def test_rejects_nul_in_names(self): + # zipfile truncates names at NUL when writing, so patch the raw bytes. + entries = dict(PLUGIN) + entries["evil.php#.txt"] = "x" + build_zip(self.zip, entries) + with open(self.zip, "r+b") as handle: + blob = handle.read().replace(b"evil.php#.txt", b"evil.php\0.txt") + handle.seek(0) + handle.write(blob) + with self.assertRaisesRegex(archive.ArchiveError, "control character"): + archive.extract(self.zip, self.dest) + self.assertEqual(os.listdir(self.dest), []) + + def test_rejects_duplicates_and_ambiguous_spellings(self): + pairs = [ + ("README.TXT", "readme.txt"), + ("Dir/a.txt", "dir/b.txt"), + ("caf\u00e9.txt", "cafe\u0301.txt"), + ("a", "a/b.txt"), + ] + for first, second in pairs: + with self.subTest(first=first, second=second): + entries = dict(PLUGIN) + entries[first] = "1" + entries[second] = "2" + self.assert_rejected(entries, "duplicate|conflict") + + def test_rejects_exact_duplicate_entries(self): + build_zip(self.zip, PLUGIN) + with zipfile.ZipFile(self.zip, "a") as zf, self.assertWarns(UserWarning): + zf.writestr("readme.txt", "again") + with self.assertRaisesRegex(archive.ArchiveError, "duplicate"): + archive.extract(self.zip, self.dest) + + def test_rejects_symlinks_and_special_files(self): + # Host 19 (OS X) stores Unix modes too; Go's archive/zip reads them that way. + for host in (3, 19): + for kind in (stat.S_IFLNK, stat.S_IFIFO, stat.S_IFCHR, stat.S_IFBLK, stat.S_IFSOCK): + with self.subTest(host=host, kind=kind): + entries = dict(PLUGIN) + entries["link"] = {"data": "/etc/passwd", "mode": kind | 0o777, "host": host} + self.assert_rejected(entries, "not a regular file") + + def test_rejects_vcs_metadata(self): + for name in (".svn/entries", "w/.git/config", ".git", "x/.SVN/wc.db"): + with self.subTest(name=name): + entries = dict(PLUGIN) + entries[name] = "x" + self.assert_rejected(entries, "version control") + + def test_keeps_vcs_lookalikes(self): + entries = dict(PLUGIN) + entries.update({".gitignore": "x", ".svnignore": "y", "git/x": "z"}) + root = self.extract(entries) + self.assertEqual(self.read(root, ".gitignore"), b"x") + + def test_rejects_encrypted_entries(self): + build_zip(self.zip, PLUGIN) + set_flag_bits(self.zip, 0x1) + with self.assertRaisesRegex(archive.ArchiveError, "encrypted"): + archive.extract(self.zip, self.dest) + self.assertEqual(os.listdir(self.dest), []) + + def test_rejects_malformed_archives(self): + with open(self.zip, "wb") as handle: + handle.write(b"this is not a zip file at all") + with self.assertRaisesRegex(archive.ArchiveError, "malformed"): + archive.extract(self.zip, self.dest) + + def test_rejects_truncated_archives(self): + build_zip(self.zip, PLUGIN) + with open(self.zip, "r+b") as handle: + handle.truncate(os.path.getsize(self.zip) // 2) + with self.assertRaisesRegex(archive.ArchiveError, "malformed"): + archive.extract(self.zip, self.dest) + + def test_rejects_corrupt_content(self): + with zipfile.ZipFile(self.zip, "w", zipfile.ZIP_STORED) as zf: + for name, data in PLUGIN.items(): + zf.writestr(name, data) + with open(self.zip, "r+b") as handle: + blob = bytearray(handle.read()) + at = blob.find(b"Stable tag") + blob[at] ^= 0xFF + handle.seek(0) + handle.write(blob) + with self.assertRaisesRegex(archive.ArchiveError, "malformed"): + archive.extract(self.zip, self.dest) + + def test_unflagged_utf8_names_keep_their_spelling(self): + # What Info-ZIP's zip writes on macOS, and on Ubuntu without en_US.UTF-8. + build_zip(self.zip, {**PLUGIN, "caf\u00e9/na\u00efve.txt": "x", "\u65e5\u672c.txt": "y"}) + set_flag_bits(self.zip, 0x800, on=False) + with zipfile.ZipFile(self.zip) as zf: + self.assertFalse(any(info.flag_bits & 0x800 for info in zf.infolist())) + root = archive.extract(self.zip, self.dest) + self.assertEqual(self.read(root, "caf\u00e9", "na\u00efve.txt"), b"x") + self.assertEqual(self.read(root, "\u65e5\u672c.txt"), b"y") + + def test_utf8_names_from_info_zip_keep_their_spelling(self): + # Flagged or not: Ubuntu's zip sets the UTF-8 flag once en_US.UTF-8 is installed. + if not shutil.which("zip"): + self.skipTest("Info-ZIP zip is not installed") + source = os.path.join(self.tmp, "src") + for name, data in {**PLUGIN, "caf\u00e9/na\u00efve.txt": "x", "\u65e5\u672c.txt": "y"}.items(): + support.write(os.path.join(source, "plugin"), name, data) + subprocess.run(["zip", "-qr", self.zip, "plugin"], cwd=source, check=True) + root = archive.extract(self.zip, self.dest) + self.assertEqual(self.read(root, "caf\u00e9", "na\u00efve.txt"), b"x") + self.assertEqual(self.read(root, "\u65e5\u672c.txt"), b"y") + + def test_flagged_utf8_names_keep_their_spelling(self): + root = self.extract({**PLUGIN, "caf\u00e9.php": "x"}) + self.assertEqual(self.read(root, "caf\u00e9.php"), b"x") + + def unicode_path_zip(self, header_raw, *unicode_names, **field): + build_zip(self.zip, PLUGIN) + support.add_unicode_path_entry(self.zip, header_raw, *unicode_names, **field) + + def test_a_unicode_path_field_that_agrees_is_accepted(self): + self.unicode_path_zip("caf\u00e9.php".encode(), "caf\u00e9.php") + root = archive.extract(self.zip, self.dest) + self.assertEqual(self.read(root, "caf\u00e9.php"), b"z") + + def test_a_unicode_path_field_that_disagrees_is_rejected(self): + # On every Python: zipfile before 3.12 ignores the field. + for header in ("caf\u00e9.php", "plain.php"): + with self.subTest(header=header): + self.unicode_path_zip(header.encode(), "other.php") + with self.assertRaisesRegex(archive.ArchiveError, "two different names"): + archive.extract(self.zip, self.dest) + self.assertEqual(os.listdir(self.dest), []) + + def test_every_unicode_path_field_must_agree(self): + # unzip and zipfile 3.12+ take the last valid field: an agreeing one must + # not hide a later one that disagrees, nor the reverse. + for names in (("caf\u00e9.php", "other.php"), ("other.php", "caf\u00e9.php")): + with self.subTest(names=names): + shutil.rmtree(self.dest) + os.mkdir(self.dest) + self.unicode_path_zip("caf\u00e9.php".encode(), *names) + with self.assertRaisesRegex(archive.ArchiveError, "two different names"): + archive.extract(self.zip, self.dest) + self.assertEqual(os.listdir(self.dest), []) + + def test_repeated_agreeing_unicode_path_fields_are_accepted(self): + self.unicode_path_zip("caf\u00e9.php".encode(), "caf\u00e9.php", "caf\u00e9.php") + root = archive.extract(self.zip, self.dest) + self.assertEqual(self.read(root, "caf\u00e9.php"), b"z") + + def test_a_stale_unicode_path_field_is_ignored(self): + # Its CRC no longer matches the header name, which a later tool changed. + self.unicode_path_zip("caf\u00e9.php".encode(), "other.php", crc=0) + root = archive.extract(self.zip, self.dest) + self.assertEqual(sorted(os.listdir(root)), sorted(["caf\u00e9.php", *PLUGIN])) + self.assertEqual(self.read(root, "caf\u00e9.php"), b"z") + + def test_names_that_are_not_utf8_are_rejected(self): + # A legacy tool's CP437 name: its e-acute is byte 0x82, which is not UTF-8. + build_zip(self.zip, {**PLUGIN, "cafX.txt": "x"}) + support.patch_name(self.zip, "cafX.txt", b"caf\x82.txt") + with self.assertRaisesRegex(archive.ArchiveError, "not UTF-8"): + archive.extract(self.zip, self.dest) + self.assertEqual(os.listdir(self.dest), []) + + def test_path_depth_and_length_limits(self): + root = self.extract({**PLUGIN, "d/" * 63 + "f.txt": "x"}) + self.assertEqual(self.read(root, *["d"] * 63, "f.txt"), b"x") + for name in ("d/" * 64 + "f.txt", "d/" * 4000 + "f.txt", "/".join(["a" * 200] * 6)): + with self.subTest(levels=name.count("/") + 1, length=len(name)): + shutil.rmtree(self.dest) + os.mkdir(self.dest) + self.assert_rejected({**PLUGIN, name: "x"}, "deeper than 64 levels|longer than 1024 bytes") + + def test_implicit_parent_directories_count_toward_the_limit(self): + # Three entries, but nine files and directories once the parents exist. + entries = {**PLUGIN, "b/a/a/a/a/a/x.txt": ""} + with mock.patch.object(archive, "MAX_ENTRIES", 8): + self.assert_rejected(entries, "more than 8 files and directories") + with mock.patch.object(archive, "MAX_ENTRIES", 9): + self.extract(entries) + + def test_shared_parent_directories_count_once(self): + with mock.patch.object(archive, "MAX_ENTRIES", 5): + root = self.extract({**PLUGIN, "d/a.txt": "1", "d/b.txt": "2"}) + self.assertEqual(self.read(root, "d", "b.txt"), b"2") + + def test_entry_count_limit(self): + entries = dict(PLUGIN) + entries.update({f"f{i}.txt": "x" for i in range(5)}) + with mock.patch.object(archive, "MAX_ENTRIES", 6): + self.assert_rejected(entries, "entries") + + def test_expanded_size_limit(self): + entries = dict(PLUGIN) + entries["big.bin"] = os.urandom(4096) + with mock.patch.object(archive, "MAX_EXPANDED_BYTES", 4096): + self.assert_rejected(entries, "expands") + + def test_expansion_ratio_limit(self): + entries = dict(PLUGIN) + entries["zeros.bin"] = b"\0" * (1024 * 1024) + self.assert_rejected(entries, "ratio") + + def test_archive_size_limit(self): + build_zip(self.zip, PLUGIN) + with mock.patch.object(archive, "MAX_ARCHIVE_BYTES", 10): + with self.assertRaisesRegex(archive.ArchiveError, "larger"): + archive.extract(self.zip, self.dest) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_live.py b/tests/wporg-svn/test_live.py new file mode 100644 index 0000000..3c78eae --- /dev/null +++ b/tests/wporg-svn/test_live.py @@ -0,0 +1,159 @@ +"""Read-only checks against real wordpress.org plugins. Opt-in: WPORG_LIVE=1. + +Nothing here writes to wordpress.org. The read helpers never write, every write +in the mirror test goes to a local file:// repository, and the write guard is +first on PATH for all of it. + + WPORG_LIVE=1 python3 -m unittest discover -s tests/wporg-svn -p test_live.py -v + WPORG_LIVE_PLUGINS plugins for the read checks (comma-separated) + WPORG_LIVE_MIRROR plugins for the mirror release (comma-separated) +""" + +import os +import shutil +import subprocess +import tempfile +import time +import unittest + +from support import TESTS, ActionTest, Repo, base_env, read_tree # first: puts lib/ on sys.path + +import archive # noqa: E402 +import release # noqa: E402 + +LIVE = os.environ.get("WPORG_LIVE") == "1" +PLUGINS = os.environ.get("WPORG_LIVE_PLUGINS", "the-events-calendar,event-tickets,kadence-blocks,give").split(",") +MIRROR = os.environ.get("WPORG_LIVE_MIRROR", "image-widget,advanced-post-manager,the-events-calendar").split(",") +SVN_ROOT = "https://plugins.svn.wordpress.org" + + +def zip_url(slug, version): + return f"https://downloads.wordpress.org/plugin/{slug}.{version}.zip" + + +@unittest.skipUnless(LIVE, "set WPORG_LIVE=1 to read from wordpress.org") +class LiveReadTest(unittest.TestCase): + def setUp(self): + self.tmp = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.env = dict(base_env(self.tmp), TMPDIR=self.tmp) + + cache = {} + + def read(self, slug): + """Run the read harness once per plugin.""" + if slug not in self.cache: + started = time.time() + result = subprocess.run(["bash", os.path.join(TESTS, "live_reads.sh"), slug], env=self.env, + capture_output=True, text=True, timeout=1200) + seen = dict(line.split("=", 1) for line in result.stdout.splitlines() if "=" in line) + self.cache[slug] = (result, seen, time.time() - started) + return self.cache[slug][:2] + + def test_read_helpers_against_real_plugins(self): + for slug in PLUGINS: + with self.subTest(slug=slug): + result, seen = self.read(slug) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertTrue(seen["revision"].isdigit()) + self.assertEqual(seen["trunk-readme"], "file") + self.assertEqual(seen["stable-kind"], "dir") + self.assertEqual(seen["probe-kind"], "absent") + self.assertIn(seen["stable"], seen["tags"].split(",")) + print(f"\n {slug}: r{seen['revision']} stable {seen['stable']} " + f"({len(seen['tags'].split(','))} tags), read helpers took {self.cache[slug][2]:.0f}s") + + def test_a_missing_plugin_is_an_error_not_an_absence(self): + result, seen = self.read("zz-no-such-plugin-0000") + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("revision", seen) + self.assertRegex(result.stdout, r"SVN read failed while reading zz-no-such-plugin-0000: \w") + print("\n " + [line for line in result.stdout.splitlines() if "SVN read failed" in line][0]) + + def test_parsers_on_real_release_zips(self): + for slug in PLUGINS: + with self.subTest(slug=slug): + _, seen = self.read(slug) + stable, folder = seen["stable"], tempfile.mkdtemp(dir=self.tmp) + path = os.path.join(folder, "release.zip") + subprocess.run(["curl", "-q", "-fsSL", "--proto", "=https", "-o", path, zip_url(slug, stable)], + check=True, timeout=600) + os.mkdir(os.path.join(folder, "out")) + root = archive.extract(path, os.path.join(folder, "out")) + self.assertEqual(release.plugin_version(root), stable) + with open(os.path.join(root, "readme.txt"), "rb") as handle: + self.assertEqual(release.stable_tag(handle.read()), stable) + files = sum(len(names) for _, _, names in os.walk(root)) + print(f"\n {slug} {stable}: {os.path.getsize(path)} bytes zipped, {files} files, " + f"wrapper {os.path.basename(root)!r}") + + def test_the_guard_refuses_a_commit_from_a_real_working_copy(self): + wc = os.path.join(self.tmp, "wc") + subprocess.run(["svn", "checkout", "--non-interactive", "--depth", "empty", + f"{SVN_ROOT}/{PLUGINS[0]}/trunk", wc], env=self.env, check=True, capture_output=True) + # Even if the guard failed, an unmodified checkout has nothing to commit. + result = subprocess.run(["svn", "commit", "--non-interactive", "-m", "never", wc], env=self.env, + capture_output=True, text=True) + self.assertEqual(result.returncode, 99, result.stderr) + self.assertIn("write-guard: refused", result.stderr) + + +@unittest.skipUnless(LIVE, "set WPORG_LIVE=1 to read from wordpress.org") +class LiveMirrorReleaseTest(ActionTest): + """QA on real releases: real artifacts in, every write to a local repository.""" + + def real(self, *args): + return subprocess.run(["svn", "--non-interactive", *args], env=self.env, check=True, capture_output=True, + timeout=1800).stdout + + def test_real_releases_through_all_three_actions(self): + for slug in MIRROR: + with self.subTest(slug=slug): + self.mirror(slug) + + def mirror(self, slug): + readme = self.real("cat", f"{SVN_ROOT}/{slug}/trunk/readme.txt") + version = release.stable_tag(readme) + names = self.real("list", f"{SVN_ROOT}/{slug}/tags/").decode().replace("/", "").split() + older = [n for n in names if release.is_version(n) and release.compare(n, version) < 0] + previous = max(older, key=lambda n: [int(p) for p in n.split(".")]) + started = time.time() + + # Seed a local repository with the real previous release (reads from wordpress.org). + work = tempfile.mkdtemp(dir=self.tmp) + exported = os.path.join(work, "previous") + self.real("export", "-q", "--ignore-externals", f"{SVN_ROOT}/{slug}/tags/{previous}", exported) + with open(os.path.join(exported, "readme.txt"), "rb") as handle: + if release.stable_tag(handle.read()) != previous: + self.skipTest(f"{slug} {previous}: the tag's readme does not name its own version") + local = Repo(work, self.env, slug=slug) + local.svn("import", "-q", "--no-ignore", "--no-auto-props", "-m", "seed", exported, local.url("seed")) + local.mucc("rm", local.url("trunk"), "mv", local.url("seed"), local.url("trunk")) + local.mucc("cp", "HEAD", local.url("trunk"), local.url(f"tags/{previous}")) + + common = {"WPORG_SLUG": slug, "WPORG_SVN_TEST_ROOT": local.root} + tagged = self.run_action("tag", WPORG_ZIP_URL=zip_url(slug, version), **common) + self.assert_success(tagged) + self.assertEqual(tagged.outputs["version"], version) + self.assertEqual(tagged.outputs["previous-stable"], previous) + + # The local tag must match the real one byte for byte (another read). + real_tag = os.path.join(work, "real-tag") + self.real("export", "-q", "--ignore-externals", f"{SVN_ROOT}/{slug}/tags/{version}", real_tag) + local_tag = local.tree(f"tags/{version}") + expected = read_tree(real_tag) + differ = sorted(set(local_tag) ^ set(expected)) + sorted( + k for k in set(local_tag) & set(expected) if local_tag[k] != expected[k]) + self.assertEqual(differ, [], f"local tags/{version} differs from the real one") + + self.assert_success(self.run_action("set-stable", WPORG_VERSION=version, **common)) + self.assert_success(self.run_action("update-trunk", WPORG_VERSION=version, **common)) + self.assertEqual(local.tree("trunk"), local_tag) + self.assertEqual(release.stable_tag(local.cat("trunk/readme.txt")), version) + print(f"\n {slug}: {previous} -> {version}, {len(local_tag)} paths, " + f"{len(local.changed(int(tagged.outputs['revision'])))} changed by the tag commit, " + f"{time.time() - started:.0f}s") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_recovery.py b/tests/wporg-svn/test_recovery.py new file mode 100644 index 0000000..2ddd7c2 --- /dev/null +++ b/tests/wporg-svn/test_recovery.py @@ -0,0 +1,294 @@ +"""Races, failed commits, cancellation, ambiguous writes and secret handling. + +Most failures come from real repository hooks. A few come from PATH wrappers: +those simulate what a file:// repository cannot produce (a lost network +response, an authentication or TLS failure), and each says so. +""" + +import os +import signal +import time +import unittest + +from support import PASSWORD, SIGNATURE, ActionTest, expected_tree, make_zip, plugin +from test_tag import NEXT, STABLE + + +class Base(ActionTest): + def setUp(self): + super().setUp() + self.repo.seed("1.0", STABLE) + self.https.routes["/release.zip"] = make_zip(NEXT) + self.zip_url = f"{self.https.url}/release.zip?sig={SIGNATURE}" + + def tag(self, **env): + env.setdefault("WPORG_ZIP_URL", self.zip_url) + return self.run_action("tag", **env) + + def point_trunk_at(self, stable): + """What set-stable does: trunk's readme names stable.""" + self.repo.mucc("put", self.staged(f"readme-{stable}.txt", plugin(stable)["readme.txt"]), + self.repo.url("trunk/readme.txt")) + + def staged(self, name, text): + path = os.path.join(self.repo.staging, name) + with open(path, "w") as handle: + handle.write(text) + return path + + def on_commit(self, number, command): + """Run command from a start-commit hook, on the number-th commit only.""" + count = os.path.join(self.repo.staging, "commit-count") + self.repo.hook("start-commit", f""" +n=$(cat '{count}' 2>/dev/null || echo 0); n=$((n + 1)); echo $n > '{count}' +if [ "$n" = {number} ]; then {command}; fi +""") + + +class TagCommitTest(Base): + """The tag's only write: tags/ lands complete, or not at all.""" + + def cancel_during(self, hook, marker): + process = self.start("tag", WPORG_ZIP_URL=self.zip_url) + deadline = time.time() + 120 + while not os.path.exists(marker) and time.time() < deadline: + time.sleep(0.2) + self.assertTrue(os.path.exists(marker), f"the {hook} hook never ran") + os.killpg(process.pid, signal.SIGTERM) # what a runner does when the job is cancelled + result = self.finish(process, timeout=60) # also checks temp cleanup and secrets + self.assertEqual(result.code, 143, result.log) + self.assertEqual(result.outputs, {}) + return result + + def test_a_rejected_commit_writes_nothing_and_a_rerun_succeeds(self): + head = self.repo.head() + self.repo.hook("pre-commit", "echo 'rejected by policy' >&2; exit 1\n") + result = self.tag() + self.assert_failure(result, r"the commit failed \(.*\) and tags/1\.1 does not exist at r\d+, " + r"so nothing was written") + self.assertRegex(result.log, r"Known revisions: snapshot r\d+, recheck r\d+\.") + self.assertEqual(self.repo.head(), head) + self.repo.hook("pre-commit", "exit 0\n") + self.assert_success(self.tag()) + self.assertEqual(self.repo.tree("tags/1.1"), expected_tree(NEXT)) + + def test_cancellation_during_the_commit_leaves_no_tag(self): + marker = os.path.join(self.repo.staging, "committing") + self.repo.hook("pre-commit", f"touch '{marker}'; sleep 30\n") + result = self.cancel_during("pre-commit", marker) + self.assertRegex(result.log, r"Failed while committing tags/1\.1") + self.assertRegex(result.log, r"The commit's outcome is unknown: if tags/1\.1 exists") + self.assertFalse(self.repo.exists("tags/1.1")) + + def test_cancellation_while_the_commit_lands_still_warns(self): + marker = os.path.join(self.repo.staging, "committed") + self.repo.hook("post-commit", f"touch '{marker}'; sleep 30\n") # the commit is durable now + result = self.cancel_during("post-commit", marker) + self.assertTrue(self.repo.exists("tags/1.1")) + self.assertRegex(result.log, r"if tags/1\.1 exists, do not approve or release it") + + def test_a_failed_verification_reports_what_the_tag_now_holds(self): + # Simulated: a lost connection while exporting the committed tag. + self.wrap("svn", 'case " $* " in *" export "*) echo "svn: E175002: Connection reset" >&2; exit 1;; esac\n' + 'exec "$NEXT" "$@"') + result = self.tag() + self.assert_failure(result, r"Failed while verifying tags/1\.1") + self.assertRegex(result.log, r"tags/1\.1 was created at r\d+ but not verified.*Do not approve or release it") + + +class RaceBeforeTheCommitTest(Base): + def race_during_download(self, *actions): + payload = self.https.routes["/release.zip"] + + def handler(request): + self.repo.mucc(*actions, message="racer") + request.send_response(200) + request.send_header("Content-Length", str(len(payload))) + request.end_headers() + request.wfile.write(payload) + + self.https.routes["/release.zip"] = handler + + def assert_nothing_written(self, result, pattern, head): + self.assert_failure(result, pattern) + self.assertEqual(self.repo.head(), head + 1, "only the competing commit may land") + self.assertFalse(self.repo.exists("tags/1.1")) + + def test_stable_pointer_moved_during_the_download(self): + readme = self.staged("readme.txt", plugin("1.0", stable="0.9")["readme.txt"]) + self.repo.mucc("cp", "HEAD", self.repo.url("tags/1.0"), self.repo.url("tags/0.9")) + head = self.repo.head() + self.race_during_download("put", readme, self.repo.url("trunk/readme.txt")) + self.assert_nothing_written(self.tag(), r"Stable Tag changed after r\d+", head) + + def test_stable_tag_edited_during_the_download(self): + head = self.repo.head() + self.race_during_download("put", self.staged("x.txt", "edited"), self.repo.url("tags/1.0/keep.txt")) + self.assert_nothing_written(self.tag(), r"tags/1\.0 changed after r\d+", head) + + def test_destination_created_during_the_download(self): + head = self.repo.head() + self.race_during_download("mkdir", self.repo.url("tags/1.1")) + result = self.tag() + self.assert_failure(result, r"tags/1\.1 already exists at r\d+") + self.assertEqual(self.repo.head(), head + 1) + + def test_destination_created_at_write_time_is_never_overwritten(self): + self.on_commit(1, f"svnmucc --non-interactive -m racer mkdir '{self.repo.url('tags/1.1')}'") + result = self.tag() + self.assert_failure(result, r"the commit reported an error .* and tags/1\.1 now exists") + self.assertRegex(result.log, "cannot confirm that it created it. Do not approve or release it") + self.assertEqual(self.repo.tree("tags/1.1"), {}, "the other writer's tag stays as it was") + + def test_an_unrelated_new_tag_does_not_block_the_commit(self): + self.on_commit(1, f"svnmucc --non-interactive -m racer mkdir '{self.repo.url('tags/0.5')}'") + result = self.tag() + self.assert_success(result) + self.assertEqual(self.repo.tree("tags/1.1"), expected_tree(NEXT)) + + +class AmbiguousWriteTest(Base): + """A lost response after a successful write. file:// always answers, so a + wrapper runs the real (guarded) write and then reports a timeout.""" + + def lose_the_response(self): + self.wrap("svnmucc", '"$NEXT" "$@" >/dev/null 2>&1\necho "svnmucc: E175012: Connection timed out" >&2\nexit 1') + + def test_tag_commit_with_a_lost_response_is_not_trusted(self): + self.wrap("svn", 'case " $* " in *" commit "*) "$NEXT" "$@" >/dev/null 2>&1\n' + 'echo "svn: E175012: Connection timed out" >&2; exit 1;; esac\nexec "$NEXT" "$@"') + result = self.tag() + self.assert_failure(result, r"the commit reported an error \(could not connect \(E175012\)\) and tags/1\.1 now exists") + self.assertRegex(result.log, "cannot confirm that it created it") + self.assertFalse(any("export" in call for call in self.svn_calls()), "never continue past an unconfirmed write") + self.assertTrue(self.repo.exists("tags/1.1")) + + def test_set_stable_with_a_lost_response_says_a_rerun_is_safe(self): + self.repo.put_tree("tags/1.1", plugin("1.1")) + self.lose_the_response() + result = self.run_action("set-stable", WPORG_VERSION="1.1") + self.assert_failure(result, r"now matches tags/1\.1/readme\.txt\. Rerunning is safe") + self.env["PATH"] = self.env["PATH"].split(os.pathsep, 1)[1] + rerun = self.run_action("set-stable", WPORG_VERSION="1.1") + self.assert_success(rerun) + self.assertRegex(rerun.summary, "no-op") + + def test_update_trunk_with_a_lost_response_says_a_rerun_is_safe(self): + self.repo.put_tree("tags/1.1", plugin("1.1")) + self.point_trunk_at("1.1") + self.lose_the_response() + result = self.run_action("update-trunk", WPORG_VERSION="1.1") + self.assert_failure(result, r"trunk now matches tags/1\.1\. Rerunning is safe") + + +class ErrorClassificationTest(Base): + """Failed reads stop the action; they are never taken to mean "absent".""" + + def test_authentication_failure_on_the_commit(self): + # Simulated: file:// never asks for credentials. + self.wrap("svn", 'case " $* " in *" commit "*) echo "svn: E170001: Authorization failed" >&2; exit 1;; esac\n' + 'exec "$NEXT" "$@"') + head = self.repo.head() + result = self.tag() + self.assert_failure(result, r"the commit failed \(authentication failed \(E170001\)\) and tags/1\.1 does not exist") + self.assertEqual(self.repo.head(), head) + + def test_tls_failure_on_a_read_is_not_absence(self): + # Simulated: file:// has no TLS. + self.wrap("svn", 'case " $* " in *" list "*) echo "svn: E170013: Unable to connect" >&2; ' + 'echo "svn: E230001: Server SSL certificate verification failed" >&2; exit 1;; esac\n' + 'exec "$NEXT" "$@"') + self.repo.put_tree("tags/1.1", plugin("1.1")) + for action, env in (("set-stable", {"WPORG_VERSION": "1.1"}), ("tag", {"WPORG_ZIP_URL": self.zip_url})): + with self.subTest(action=action): + result = self.run_action(action, **env) + self.assert_failure(result, r"SVN read failed while .*TLS certificate verification failed \(E170013 E230001\)") + self.assertNotRegex(result.log, "does not exist") + + def test_connection_failure_on_a_read_is_not_absence(self): + # Simulated: file:// cannot drop a connection. + self.wrap("svn", 'case " $* " in *" list "*) echo "svn: E170013: Unable to connect" >&2; ' + 'echo "svn: E175002: Connection reset" >&2; exit 1;; esac\nexec "$NEXT" "$@"') + result = self.run_action("update-trunk", WPORG_VERSION="1.1") + self.assert_failure(result, r"could not connect \(E170013 E175002\)") + self.assertNotRegex(result.log, "does not exist") + + def test_connection_failure_while_copying_the_stable_tag_stops_the_tag(self): + # Simulated: file:// cannot drop a connection. + self.wrap("svn", 'case " $* " in *" copy "*) echo "svn: E170013: Unable to connect" >&2; ' + 'echo "svn: E175002: Connection reset" >&2; exit 1;; esac\nexec "$NEXT" "$@"') + head = self.repo.head() + result = self.tag() + self.assert_failure(result, r"SVN read failed while copying tags/1\.0 into a working copy: " + r"could not connect \(E170013 E175002\)") + self.assertEqual(self.repo.head(), head) + + def test_permission_failure_is_not_absence(self): + db = os.path.join(self.repo.dir, "db") + os.chmod(db, 0) + self.addCleanup(os.chmod, db, 0o755) + result = self.run_action("set-stable", WPORG_VERSION="1.1") + self.assert_failure(result, r"SVN read failed while reading .*permission denied") + self.assertNotRegex(result.log, "does not exist") + + +class HelperFailureTest(Base): + """A helper that crashes must stop the action, never read as "nothing found".""" + + def test_a_crash_while_looking_for_the_destination_stops_the_tag(self): + self.wrap("python3", 'case " $* " in *" equivalents "*) exit 1;; esac\nexec "$NEXT" "$@"') + head = self.repo.head() + self.assert_failure(self.tag(), r"Failed while checking tags/1\.1") + self.assertEqual(self.repo.head(), head) + + def test_unreadable_diff_output_stops_update_trunk(self): + self.repo.put_tree("tags/1.1", plugin("1.1")) + self.point_trunk_at("1.1") + self.wrap("svn", 'case " $* " in *" diff "*) echo "> \'{dump}\'\nexec "$NEXT" "$@"') + result = self.tag() + self.assert_success(result) + argv = repr(self.svn_calls()) + self.assertIn("--password-from-stdin", argv) + self.assertNotIn(PASSWORD, argv) + with open(dump) as handle: + inherited = PASSWORD in handle.read() # never dump the environment into test output + self.assertFalse(inherited, "child processes must not inherit the password") + + def test_signed_url_is_never_echoed_even_on_failure(self): + self.https.routes["/release.zip"] = make_zip({"readme.txt": "broken"}) + result = self.tag() # finish() asserts the signature appears nowhere + self.assertNotEqual(result.code, 0) + + +class RunnerTest(Base): + def test_unsupported_runner(self): + self.assert_failure(self.tag(RUNNER_OS="Windows"), "unsupported runner") + + def test_missing_tool(self): + path = self.env["PATH"] + for missing in ("svn", "curl"): + with self.subTest(missing=missing): + # A PATH holding every tool except the missing one. + farm = os.path.join(self.tmp, "farm-" + missing) + os.makedirs(farm) + for folder in path.split(os.pathsep): + for name in (os.listdir(folder) if os.path.isdir(folder) else []): + link, target = os.path.join(farm, name), os.path.join(folder, name) + if name != missing and not os.path.lexists(link) and os.access(target, os.X_OK): + os.symlink(target, link) + self.env["PATH"] = farm + self.assert_failure(self.tag(), "missing required tool: " + missing) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_release.py b/tests/wporg-svn/test_release.py new file mode 100644 index 0000000..4fb8045 --- /dev/null +++ b/tests/wporg-svn/test_release.py @@ -0,0 +1,214 @@ +"""Version, readme Stable Tag and plugin header parsing.""" + +import tempfile +import unittest + +from support import write + +import release + +README = """=== Fixture Plugin === +Contributors: someone +Tags: events +Requires at least: 6.0 +Tested up to: 6.6 +Stable tag: {stable} +License: GPLv2 + +A short description. + +== Changelog == + += 1.0 = +* Stable tag: 9.9 is only an example in the body. +""" + +PLUGIN = """/readme.txt over trunk/readme.txt, server-side only.""" + +import unittest + +from support import ActionTest, SLUG, plugin, write + + +class SetStableTest(ActionTest): + def setUp(self): + super().setUp() + self.repo.seed("1.0") + self.repo.put_tree("tags/1.1", plugin("1.1")) + + def set_stable(self, version="1.1", **env): + return self.run_action("set-stable", WPORG_VERSION=version, **env) + + def test_copies_the_tag_readme_into_trunk_and_nothing_else(self): + before = self.repo.head() + trunk_before = self.repo.tree("trunk") + result = self.set_stable() + self.assert_success(result) + + revision = int(result.outputs["revision"]) + self.assertEqual(result.outputs["version"], "1.1") + self.assertEqual(revision, before + 1) + self.assertEqual(self.repo.changed(revision), [("M", f"/{SLUG}/trunk/readme.txt")]) + self.assertEqual(self.repo.cat("trunk/readme.txt"), self.repo.cat("tags/1.1/readme.txt")) + trunk_after = self.repo.tree("trunk") + del trunk_before["readme.txt"], trunk_after["readme.txt"] + self.assertEqual(trunk_after, trunk_before) + + def test_preserves_the_exact_readme_bytes(self): + readme = b"=== X ===\r\nStable tag: 1.2\r\n\r\nShort.\r\n\xe2\x9c\x93 bytes \t \r\n" + self.repo.put_tree("tags/1.2", {**plugin("1.2"), "readme.txt": readme}) + self.assert_success(self.set_stable("1.2")) + self.assertEqual(self.repo.cat("trunk/readme.txt"), readme) + + def test_identical_readme_is_a_verified_no_op(self): + self.assert_success(self.set_stable()) + head = self.repo.head() + result = self.set_stable() + self.assert_success(result) + self.assertEqual(self.repo.head(), head, "a no-op must not commit") + self.assertEqual(result.outputs, {"version": "1.1", "revision": str(head)}) + self.assertRegex(result.summary, "(?i)no-op") + + def test_can_select_an_older_tag(self): + # Documented: set-stable has no upgrade-only policy, so this rolls back. + self.assert_success(self.set_stable("1.1")) + self.assert_success(self.set_stable("1.0")) + self.assertEqual(self.repo.cat("trunk/readme.txt"), self.repo.cat("tags/1.0/readme.txt")) + + def test_expected_revision_refuses_a_tag_changed_after_approval(self): + approved = self.repo.last_changed("tags/1.1") + self.repo.mucc("put", write(self.repo.staging, "late.php", "/dev/null +fi +""") + result = self.set_stable() + self.assert_failure(result, r"(?i)out of date|conflict") + self.assertEqual(self.repo.cat("trunk/readme.txt").decode(), open(racer).read()) + self.assertRegex(result.log, r"trunk/readme\.txt does not match tags/1\.1/readme\.txt at r\d+; " + r"check who else is committing \(svn log\) before rerunning") + self.assertNotRegex(result.log, "not changed by this run") + + + def test_edit_between_snapshot_and_write_is_never_overwritten(self): + # The -r baseline's own window: another commit lands after our + # reads and before our write starts. The wrapper only times that real + # commit; it fakes nothing about SVN. + racer = self.repo.staging + "/racer-readme.txt" + with open(racer, "w") as handle: + handle.write("=== X ===\nStable tag: 1.0\n\nEdited after the snapshot.\n") + marker = self.repo.staging + "/raced" + self.wrap("svnmucc", f""" +if [ ! -e '{marker}' ]; then + touch '{marker}' + "$NEXT" --non-interactive -m racer put '{racer}' '{self.repo.url("trunk/readme.txt")}' >/dev/null +fi +exec "$NEXT" "$@" +""") + result = self.set_stable() + self.assert_failure(result, r"out of date") + self.assertEqual(self.repo.cat("trunk/readme.txt").decode(), open(racer).read()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_tag.py b/tests/wporg-svn/test_tag.py new file mode 100644 index 0000000..cae4996 --- /dev/null +++ b/tests/wporg-svn/test_tag.py @@ -0,0 +1,289 @@ +"""tag: create tags/ in one commit, as a copy of the stable tag that matches the ZIP exactly.""" + +import os +import shutil +import subprocess +import tempfile +import unittest + +from support import ActionTest, SIGNATURE, SLUG, add_unicode_path_entry, as_bytes, expected_tree, make_zip, plugin, write + +STABLE = plugin("1.0", extra={ + "keep.txt": "unchanged", + "same-size.txt": "AAAA", + "src/a.php": " \'{seen}\'\nexec "$NEXT" "$@"') + self.assert_success(self.tag()) + with open(seen) as handle: + self.assertEqual(int(handle.read()), 256 * 1024 + 1, "in 1 KiB blocks") + + def test_downloads_past_the_size_limit_are_named(self): + # Simulated: serving 256 MiB is too slow for a test. 63 is curl's own + # --max-filesize exit; 153 is the SIGXFSZ kill from the file size limit. + for status in (63, 153): + with self.subTest(status=status): + self.wrap("curl", f"exit {status}") + self.rejected(r"downloading zip-url failed: the ZIP is larger than 256 MiB") + + def test_credentials_never_reach_the_artifact_host(self): + self.assert_success(self.tag()) + for request in self.https.requests: + headers = {k.lower(): v for k, v in request["headers"].items()} + self.assertNotIn("authorization", headers) + self.assertNotIn("fixture-user", repr(request)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_update_trunk.py b/tests/wporg-svn/test_update_trunk.py new file mode 100644 index 0000000..22fa5ee --- /dev/null +++ b/tests/wporg-svn/test_update_trunk.py @@ -0,0 +1,180 @@ +"""update-trunk: replace trunk with tags/ in one atomic, revision-pinned commit.""" + +import unittest + +from support import ActionTest, SLUG, plugin, write + + +class UpdateTrunkTest(ActionTest): + def setUp(self): + super().setUp() + self.repo.seed("1.0", plugin("1.0", extra={"dev-only.txt": "trunk work", "src/old.php": "/dev/null +fi +""") + result = self.update_trunk() + self.assert_failure(result, r"out of date") + self.assertEqual(self.repo.cat("trunk/src/old.php").decode(), open(racer).read()) + self.assertIn("dev-only.txt", self.repo.tree("trunk"), "trunk must be left intact") + self.assertEqual(self.repo.head(), self.snapshot + 1, "only the competing commit may land") + + def test_copies_the_snapshot_even_if_the_tag_changes_afterwards(self): + # The tag changes after our reads and before our write starts (a real + # commit, timed by the wrapper). The copy must still take the snapshot. + tag_at_snapshot = self.repo.tree("tags/1.1") + racer = self.repo.staging + "/edited.php" + with open(racer, "w") as handle: + handle.write("/dev/null +fi +exec "$NEXT" "$@" +""") + result = self.update_trunk() + self.assert_success(result) + self.assertEqual(self.repo.tree("trunk"), tag_at_snapshot) + self.assertNotEqual(self.repo.tree("trunk"), self.repo.tree("tags/1.1")) + + def test_rejected_transaction_leaves_trunk_intact(self): + trunk = self.repo.tree("trunk") + self.repo.hook("pre-commit", "echo 'rejected by policy' >&2\nexit 1\n") + result = self.update_trunk() + self.assert_failure(result, r"trunk does not match tags/1\.1 at r\d+; " + r"check who else is committing \(svn log\) before rerunning") + self.assertEqual(self.repo.head(), self.snapshot) + self.assertEqual(self.repo.tree("trunk"), trunk) + + + def test_edit_between_snapshot_and_write_aborts_the_replacement(self): + # As in set-stable: a real competing commit, timed by the wrapper to land + # after our reads and before our write starts. + racer = self.repo.staging + "/racer.php" + with open(racer, "w") as handle: + handle.write("/dev/null +fi +exec "$NEXT" "$@" +""") + result = self.update_trunk() + self.assert_failure(result, r"out of date") + self.assertEqual(self.repo.cat("trunk/src/old.php").decode(), open(racer).read()) + self.assertIn("dev-only.txt", self.repo.tree("trunk")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_write_guard.py b/tests/wporg-svn/test_write_guard.py new file mode 100644 index 0000000..6c191fd --- /dev/null +++ b/tests/wporg-svn/test_write_guard.py @@ -0,0 +1,109 @@ +"""The test-only write guard: no test or QA run may write to a non-file:// repository.""" + +import os +import subprocess +import sys +import tempfile +import unittest + +import support + +sys.path.insert(0, os.path.join(support.TESTS, "guard")) + +import write_guard # noqa: E402 + +REMOTE = "https://plugins.svn.wordpress.org.invalid/fixture-plugin" # .invalid never resolves +LOCAL = "file:///tmp/repo/fixture-plugin" + + +def root_of(remote): + return lambda path: REMOTE if remote else LOCAL + + +class RefusalTest(unittest.TestCase): + def refused(self, tool, *args, remote_wc=False): + return write_guard.refusal(tool, list(args), root_of(remote_wc)) is not None + + def test_svnmucc_only_writes_to_file_urls(self): + self.assertTrue(self.refused("svnmucc", "-m", "x", "mkdir", REMOTE + "/tags/1.0")) + self.assertTrue(self.refused("svnmucc", "-U", REMOTE, "-m", "x", "rm", "trunk")) + self.assertTrue(self.refused("svnmucc", "-m", "x", "rm", "trunk")) + self.assertTrue(self.refused("svnmucc", "cp", "1", LOCAL + "/a", "svn://example.invalid/b")) + self.assertFalse(self.refused("svnmucc", "-m", "x", "cp", "3", LOCAL + "/tags/1", LOCAL + "/tags/2")) + + def test_svnrdump_only_loads_into_file_urls(self): + self.assertTrue(self.refused("svnrdump", "load", REMOTE)) + self.assertTrue(self.refused("svnrdump", "--non-interactive", "--config-dir", "/tmp/c", "load", "--", REMOTE)) + self.assertTrue(self.refused("svnrdump", "load")) + self.assertFalse(self.refused("svnrdump", "load", LOCAL)) + self.assertFalse(self.refused("svnrdump", "dump", "--quiet", "-r", "5", "--", REMOTE + "/tags/1.0")) + + def test_svn_url_writes_to_remote_repositories_are_refused(self): + for args in ( + ["mkdir", "-m", "x", REMOTE + "/tags/9"], + ["rm", "-m", "x", REMOTE + "/trunk"], + ["copy", "-m", "x", REMOTE + "/trunk", REMOTE + "/tags/9"], + ["import", "-m", "x", ".", REMOTE + "/trunk"], + ["propset", "--revprop", "-r", "5", "svn:log", "x", REMOTE], + ["--non-interactive", "--config-dir", "/tmp/c", "move", "-m", "x", REMOTE + "/a", REMOTE + "/b"], + ["mkdir", "-m", "x", "http://plugins.svn.wordpress.org.invalid/x"], + ): + with self.subTest(args=args): + self.assertTrue(self.refused("svn", *args)) + + def test_svn_working_copy_writes_check_the_repository_root(self): + for args in (["commit", "-m", "x"], ["ci", "-m", "x", "wc"], ["lock", "wc/f"], ["unlock", "wc/f"], + ["--config-dir", "/tmp/c", "commit", "-m", "x", "wc"], ["rm", "wc/f"], + ["propset", "p", "v", "wc/f"]): + with self.subTest(args=args): + self.assertTrue(self.refused("svn", *args, remote_wc=True)) + self.assertFalse(self.refused("svn", *args, remote_wc=False)) + + def test_reads_and_local_operations_pass(self): + for args in (["cat", REMOTE + "/trunk/readme.txt"], ["list", "--xml", REMOTE + "/tags"], + ["info", "--show-item", "revision", REMOTE], ["log", "-v", REMOTE], + ["export", REMOTE + "/tags/1.0", "/tmp/x"], ["checkout", REMOTE + "/trunk", "/tmp/wc"], + ["diff", "--summarize", REMOTE + "/a", REMOTE + "/b"], ["proplist", "-R", REMOTE], + ["add", "wc/new"], ["status", "--xml", "wc"], ["update", "wc"], ["--version", "--quiet"]): + with self.subTest(args=args): + self.assertFalse(self.refused("svn", *args, remote_wc=True)) + + +class ShimTest(unittest.TestCase): + """The shims on PATH really stop the command before it reaches svn.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.tmp) + self.env = support.base_env(self.tmp) + + def run_shim(self, *argv): + return subprocess.run(list(argv), env=self.env, stdin=subprocess.DEVNULL, capture_output=True, text=True, + timeout=60) + + def test_remote_writes_never_reach_svn(self): + for argv in (["svnmucc", "--non-interactive", "-m", "x", "mkdir", REMOTE + "/tags/9"], + ["svn", "--non-interactive", "mkdir", "-m", "x", REMOTE + "/tags/9"], + ["svnrdump", "--non-interactive", "load", REMOTE]): + with self.subTest(argv=argv): + result = self.run_shim(*argv) + self.assertEqual(result.returncode, 99, result.stderr) + self.assertIn("write-guard: refused", result.stderr) + + def test_local_writes_and_reads_pass_through(self): + repo = os.path.join(self.tmp, "repo") + subprocess.run(["svnadmin", "create", repo], check=True) + url = "file://" + repo + self.assertEqual(self.run_shim("svnmucc", "-m", "x", "mkdir", url + "/a").returncode, 0) + self.assertEqual(self.run_shim("svn", "mkdir", "-m", "x", url + "/b").returncode, 0) + listing = self.run_shim("svn", "list", url) + self.assertEqual(listing.stdout.split(), ["a/", "b/"]) + + def test_shims_are_first_on_the_test_path(self): + for tool in ("svn", "svnmucc", "svnrdump"): + found = subprocess.run(["sh", "-c", f"command -v {tool}"], env=self.env, capture_output=True, text=True) + self.assertEqual(os.path.dirname(found.stdout.strip()), support.GUARD_BIN) + + +if __name__ == "__main__": + unittest.main()