From fc543e5d28a57637c5bcff2ab329f949545e1a55 Mon Sep 17 00:00:00 2001 From: Kyle King Date: Tue, 1 Sep 2026 20:30:23 -0600 Subject: [PATCH] ci: Add canary testing --- README.md | 11 ++ pyproject.toml | 2 +- scripts/canary.py | 341 ++++++++++++++++++++++++++++++++++++++ scripts/canary_repos.json | 44 +++++ tox.ini | 11 ++ 5 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 scripts/canary.py create mode 100644 scripts/canary_repos.json diff --git a/README.md b/README.md index 8f37780..cc2404e 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,17 @@ To run the pre-commit hook test: tox -e py310-hook ``` +### Canary Testing + +The canary env clones real downstream repositories that pin `mdformat-footnote` and checks that formatting their docs is idempotent, never crashes, never drops a referenced footnote definition, and does not introduce new markup escapes. It needs network access and is not part of the default `tox` run: + +```bash +tox -e canary # all repos +tox -e canary -- ruff # a single repo, to isolate a failure +``` + +Clones are cached under `.tox/canary/cache/`; `tox -e canary --recreate` clears them. Repositories are configured in [`scripts/canary_repos.json`](scripts/canary_repos.json), and [`scripts/canary.py`](scripts/canary.py) documents how to add one. Each entry mirrors the mdformat arguments and plugin set that repository actually runs, so a canary failure means a real downstream break rather than a difference in configuration. + ## Publish to PyPi Publishing is handled using a trusted action as part of the release process. Authentication is via OIDC diff --git a/pyproject.toml b/pyproject.toml index 4ce065b..edf865b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ footnote = "mdformat_footnote" [tool.flit.sdist] include = [] -exclude = [".github/", "tests/"] +exclude = [".github/", "scripts/", "tests/"] [tool.isort] # Force imports to be sorted by module, independent of import type diff --git a/scripts/canary.py b/scripts/canary.py new file mode 100644 index 0000000..c279d39 --- /dev/null +++ b/scripts/canary.py @@ -0,0 +1,341 @@ +"""Run mdformat idempotency checks against real downstream repos (canary testing). + +Adapted from https://github.com/KyleKing/mdformat-plugin-template. + +Repos to check are configured in 'scripts/canary_repos.json'. Every entry is a +repository that pins 'mdformat-footnote' in its own tooling and has markdown +containing footnote definitions, so a regression here is a regression someone +downstream would hit. JSON (not a Python module of 'Repo(...)' calls) so the +'Repo' shape can change without a migration; unknown or missing fields are +ignored or defaulted rather than raising. + +To add or update an entry, run +`git -C .tox/canary/cache/ show HEAD:.pre-commit-config.yaml` and check +for a mdformat hook plus its args/excludes, then mirror them in the JSON entry +so canary tracks what the downstream repo actually formats. Only 'name' and +'url' are required. + +Example entry, appended to the 'repos' array in 'canary_repos.json':: + + { + "name": "some-project", + "url": "https://github.com/some-org/some-project", + "patterns": ["docs/**/*.md"], + "excludes": ["docs/changelog.md"], + "options": {"wrap": 120}, + "extensions": ["mkdocs"] + } + +'options' is passed straight to 'mdformat.text', so it carries this plugin's own +settings too, under 'plugin.footnote':: + + "options": {"plugin": {"footnote": {"keep_orphans": true}}} + +'extensions' names other mdformat plugins the repo's own docs build depends on +(e.g. 'mkdocs' for MkDocs admonitions), on top of the 'footnote' this project +always tests. Add the matching PyPI package to the canary tox env's deps in +tox.ini. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +import difflib +import json +from pathlib import Path +import re +import subprocess +import sys +from typing import Any + +import mdformat + +# Idempotency misses escapes mdformat adds to the original (e.g. autorefs +# [`pkg`][] -> \[`pkg`\][]); the original-vs-pass-1 diff catches them. +_ESCAPE_RE = re.compile(r"\\([\[\]<>])") + +_DEFINITION_RE = re.compile(r"^ {0,3}\[\^([^\]\s]+)\]:", re.MULTILINE) +_REFERENCE_RE = re.compile(r"\[\^([^\]\s]+)\](?!:)") + + +@dataclass(frozen=True) +class Repo: + """A downstream repository to check for idempotent mdformat output.""" + + name: str + url: str + patterns: tuple[str, ...] + excludes: tuple[str, ...] = () + options: dict[str, Any] = field(default_factory=dict) + extensions: tuple[str, ...] = () + + @property + def display(self) -> str: + """Derive 'org/repo' from URL for display.""" + return "/".join(self.url.rstrip("/").split("/")[-2:]) + + +def _new_escapes(original: str, formatted: str) -> int: + """Count markup escapes formatting introduced that were not in the original.""" + return max( + 0, len(_ESCAPE_RE.findall(formatted)) - len(_ESCAPE_RE.findall(original)) + ) + + +def _lost_footnotes(original: str, formatted: str) -> tuple[str, ...]: + """Labels that were referenced and defined in the original, but lost a definition. + + Unreferenced definitions are excluded because dropping those is the + documented default behavior (see '--keep-footnote-orphans'). + """ + expected = set(_REFERENCE_RE.findall(original)) & set( + _DEFINITION_RE.findall(original) + ) + return tuple(sorted(expected - set(_DEFINITION_RE.findall(formatted)))) + + +@dataclass(frozen=True) +class FileResult: + """Result of running mdformat idempotency check on a single file.""" + + path: Path + error: str | None = None + diff: str | None = None + new_escapes: int = 0 + lost_footnotes: tuple[str, ...] = () + + @property + def passed(self) -> bool: + """True if the file produced no errors, no diff, and no lost footnotes. + + 'new_escapes' is a warning surfaced separately, not a failure. + """ + return self.error is None and self.diff is None and not self.lost_footnotes + + +@dataclass(frozen=True) +class CheckResult: + """Aggregated idempotency check results for a single repository.""" + + repo: Repo + file_results: tuple[FileResult, ...] + + @property + def passed(self) -> bool: + """True if all file results passed.""" + return all(r.passed for r in self.file_results) + + @property + def escape_warnings(self) -> tuple[tuple[Path, int], ...]: + """(path, count) for files where formatting introduced markup escapes.""" + return tuple( + (r.path, r.new_escapes) for r in self.file_results if r.new_escapes + ) + + @property + def output(self) -> str: + """Format failure details for display.""" + lines: list[str] = [] + for result in self.file_results: + if result.error: + lines.extend((f"Error: {result.path}", f" {result.error}")) + elif result.lost_footnotes: + labels = ", ".join(f"[^{lbl}]" for lbl in result.lost_footnotes) + lines.extend((f"Lost footnotes: {result.path}", f" {labels}")) + elif result.diff: + lines.append(f"Not idempotent: {result.path}") + lines.extend(f" {line}" for line in result.diff.splitlines()[:40]) + return "\n".join(lines) + + +# Not "tmp" — tox wipes the env_tmp_dir at the start of every run, which would +# defeat clone caching. "cache" persists until `tox -e canary --recreate`. +_CANARY_DIR = Path(__file__).parent.parent / ".tox" / "canary" / "cache" + +_REPOS_PATH = Path(__file__).parent / "canary_repos.json" + +_EXTENSIONS = {"footnote"} + + +def _load_repos(path: Path) -> list[Repo]: + """Parse 'canary_repos.json', defaulting fields this version doesn't know.""" + data = json.loads(path.read_text(encoding="utf-8")) + return [ + Repo( + name=entry["name"], + url=entry["url"], + patterns=tuple(entry.get("patterns", ())), + excludes=tuple(entry.get("excludes", ())), + options=entry.get("options", {}), + extensions=tuple(entry.get("extensions", ())), + ) + for entry in data.get("repos", []) + ] + + +def _clone_or_pull(repo: Repo, target_dir: Path) -> None: + if not target_dir.exists(): + subprocess.run( + [ + "git", + "clone", + "--depth", + "1", + "--filter=blob:none", + "--sparse", + repo.url, + str(target_dir), + ], + check=True, + ) + else: + subprocess.run( + ["git", "fetch", "--depth", "1", "origin"], + cwd=target_dir, + check=True, + ) + subprocess.run( + ["git", "reset", "--hard", "FETCH_HEAD"], + cwd=target_dir, + check=True, + ) + # Leading slash anchors to the repo root; without it git warns that a + # bare 'README.md' matches at any depth. + anchored = [f"/{pattern}" for pattern in repo.patterns] + subprocess.run( + ["git", "sparse-checkout", "set", "--no-cone", *anchored], + cwd=target_dir, + check=True, + ) + + +def _collect_files(repo: Repo, target_dir: Path) -> list[Path]: + """Expand glob patterns and filter excludes, returning sorted file list.""" + included: set[Path] = set() + for pattern in repo.patterns: + included.update(target_dir.glob(pattern)) + + excluded: set[Path] = set() + for pattern in repo.excludes: + excluded.update(target_dir.glob(pattern)) + + # Sparse checkout leaves symlinks pointing at paths it did not materialize + return sorted(path for path in included - excluded if path.is_file()) + + +def _check_file( + path: Path, options: dict[str, Any], extensions: set[str] +) -> FileResult: + """Verify mdformat produces idempotent output for a single file.""" + try: + original = path.read_text(encoding="utf-8") + except Exception as err: + return FileResult(path=path, error=f"read error: {err}") + + try: + pass1 = mdformat.text(original, options=options, extensions=extensions) + pass2 = mdformat.text(pass1, options=options, extensions=extensions) + except Exception as err: + return FileResult(path=path, error=f"mdformat error: {err}") + + shared = { + "new_escapes": _new_escapes(original, pass1), + "lost_footnotes": _lost_footnotes(original, pass1), + } + if pass1 == pass2: + return FileResult(path=path, **shared) + + diff = "".join( + difflib.unified_diff( + pass1.splitlines(keepends=True), + pass2.splitlines(keepends=True), + fromfile=f"{path} (pass 1)", + tofile=f"{path} (pass 2)", + n=3, + ) + ) + return FileResult(path=path, diff=diff, **shared) + + +def _check_repo(repo: Repo, target_dir: Path) -> CheckResult: + files = _collect_files(repo, target_dir) + if not files: + no_match = FileResult( + path=target_dir, + error=f"no files matched patterns {repo.patterns}", + ) + return CheckResult(repo=repo, file_results=(no_match,)) + extensions = _EXTENSIONS | set(repo.extensions) + return CheckResult( + repo=repo, + file_results=tuple(_check_file(f, repo.options, extensions) for f in files), + ) + + +def _resolve_repos(argv: list[str], all_repos: list[Repo]) -> list[Repo]: + if not argv: + return list(all_repos) + valid = {r.name for r in all_repos} + unknown = [name for name in argv if name not in valid] + if unknown: + print( + f"Unknown repo(s): {', '.join(unknown)}. Valid: {', '.join(sorted(valid))}" + ) + sys.exit(1) + return [r for r in all_repos if r.name in argv] + + +def _print_results(results: list[CheckResult]) -> None: + print("--- Canary Results ---") + for result in results: + label = "PASS" if result.passed else "FAIL" + failures = [r for r in result.file_results if not r.passed] + suffix = f" ({len(failures)} file(s))" if failures else "" + print(f"{label} {result.repo.display}{suffix}") + if not result.passed: + output = result.output + if output: + for line in output.splitlines()[:30]: + print(f" {line}") + else: + print(" (no output)") + warnings = result.escape_warnings + if warnings: + total = sum(count for _, count in warnings) + print( + f" WARN formatting introduced {total} markup escape(s) in " + f"{len(warnings)} file(s) — review for broken autorefs/links:" + ) + for path, count in warnings[:10]: + print(f" {count:>4} {path}") + + +def main(argv: list[str]) -> None: + """Run canary checks against all or a named subset of repos.""" + all_repos = _load_repos(_REPOS_PATH) + + if not all_repos: + print("No canary repos configured in scripts/canary_repos.json. Skipping.") + return + + repos = _resolve_repos(argv, all_repos) + + _CANARY_DIR.mkdir(parents=True, exist_ok=True) + with ThreadPoolExecutor(max_workers=len(repos)) as pool: + list(pool.map(lambda r: _clone_or_pull(r, _CANARY_DIR / r.name), repos)) + results = [_check_repo(repo, _CANARY_DIR / repo.name) for repo in repos] + + _print_results(results) + + failures = [r for r in results if not r.passed] + count = len(failures) + if count: + noun = "failure" if count == 1 else "failures" + names = " ".join(r.repo.name for r in failures) + print(f"\n{count} {noun}. Run: tox -e canary -- {names} to isolate.") + sys.exit(1) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/scripts/canary_repos.json b/scripts/canary_repos.json new file mode 100644 index 0000000..33dee4a --- /dev/null +++ b/scripts/canary_repos.json @@ -0,0 +1,44 @@ +{ + "$comment": "Downstream repos that pin mdformat-footnote and have markdown with footnote definitions. See canary.py's module docstring for how to add an entry.", + "repos": [ + { + "name": "free-threaded-compatibility", + "url": "https://github.com/Quansight-Labs/free-threaded-compatibility", + "patterns": ["docs/**/*.md"], + "extensions": ["frontmatter", "mkdocs", "tables"] + }, + { + "name": "headscale", + "url": "https://github.com/juanfont/headscale", + "patterns": ["docs/**/*.md"], + "extensions": ["frontmatter", "mkdocs"] + }, + { + "name": "ruff", + "url": "https://github.com/astral-sh/ruff", + "patterns": ["README.md", "crates/*/README.md"], + "extensions": ["mkdocs"] + }, + { + "name": "specifications", + "url": "https://github.com/mongodb/specifications", + "patterns": ["docs/**/*.md", "source/**/*.md"], + "excludes": ["source/extended-json/extended-json.md"], + "options": {"wrap": 120, "number": true}, + "extensions": ["frontmatter", "gfm", "gfm_alerts", "mkdocs"] + }, + { + "name": "spyglass", + "url": "https://github.com/LorenFrankLab/spyglass", + "patterns": ["docs/**/*.md", "maintenance_scripts/*.md"], + "options": {"wrap": 80, "number": true}, + "extensions": ["gfm", "mkdocs"] + }, + { + "name": "xls", + "url": "https://github.com/google/xls", + "patterns": ["docs_src/**/*.md"], + "extensions": ["gfm"] + } + ] +} diff --git a/tox.ini b/tox.ini index 2efc5d8..dcafed7 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,17 @@ commands = pre-commit run {posargs} extras = dev commands = pre-commit run --config .pre-commit-test.yaml {posargs:--all-files --verbose --show-diff-on-failure} +[testenv:canary] +description = Check mdformat idempotency against real downstream repos (opt-in, requires network) +extras = test +deps = + mdformat-frontmatter + mdformat-gfm + mdformat-gfm-alerts + mdformat-mkdocs + mdformat-tables +commands = python scripts/canary.py {posargs} + [flake8] max-line-length = 88 max-complexity = 10