diff --git a/CHANGELOG.md b/CHANGELOG.md index fc36ffed..fab084d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project are documented here. ## Unreleased +### Changed + +- `scripts/gen-llms-txt.py` runs `scripts/llms_leakguard.py` before it + writes `llms.txt` or `llms-full.txt`. A link to `AGENTS.md`, + `CLAUDE.md`, or `GEMINI.md` fails that check unless the URL is in + `scripts/leakguard.toml`. The manual index link + `https://github.com/HaoZeke/eb-stack/blob/main/AGENTS.md` is allowlisted. + `uvx prek run -a` runs the same check. + ### Fixed - `stack shard` plans a build-list line it cannot resolve as an opaque diff --git a/prek.toml b/prek.toml index ee0cfb95..54d43304 100644 --- a/prek.toml +++ b/prek.toml @@ -34,6 +34,19 @@ hooks = [ { id = "detect-private-key" }, ] +[[repos]] +repo = "local" +hooks = [ + { + id = "llms-leakguard", + name = "llms leakguard", + entry = "python3 scripts/llms_leakguard.py", + language = "system", + pass_filenames = false, + always_run = true, + }, +] + [[repos]] repo = "https://github.com/codespell-project/codespell" rev = "v2.4.1" diff --git a/scripts/gen-llms-txt.py b/scripts/gen-llms-txt.py index c8eedbd5..b3509b58 100644 --- a/scripts/gen-llms-txt.py +++ b/scripts/gen-llms-txt.py @@ -48,6 +48,11 @@ def main() -> int: args = parser.parse_args() pages = load_pages() files = render(pages) + blocked = leakguard_violations(files) + if blocked: + for line in blocked: + print(line, file=sys.stderr) + return 1 if args.check and args.pages_only: print("pass only one of --check and --pages-only", file=sys.stderr) return 2 @@ -436,6 +441,20 @@ def project_entries() -> list[str]: return [f"- [{title}]({url}): {summary}" for title, url, summary in entries] +def leakguard_violations(files: dict[str, str]) -> list[str]: + """The same check `scripts/llms_leakguard.py` runs for prek.""" + import importlib.util + + path = ROOT / "scripts" / "llms_leakguard.py" + spec = importlib.util.spec_from_file_location("llms_leakguard", path) + if spec is None or spec.loader is None: + print(f"leakguard: cannot load {path}", file=sys.stderr) + return [f"leakguard: cannot load {path}"] + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.violations_in(files) + + def write(files: dict[str, str]) -> None: for relative, text in files.items(): path = ROOT / relative diff --git a/scripts/leakguard.toml b/scripts/leakguard.toml new file mode 100644 index 00000000..da73edca --- /dev/null +++ b/scripts/leakguard.toml @@ -0,0 +1,6 @@ +# URLs a generated public file may link to even when the path is an +# agent instruction file. The manual index (docs/orgmode/index.org, +# Discovery) links AGENTS.md, and llms.txt repeats that URL. +allow_urls = [ + "https://github.com/HaoZeke/eb-stack/blob/main/AGENTS.md", +] diff --git a/scripts/llms_leakguard.py b/scripts/llms_leakguard.py new file mode 100644 index 00000000..5874594c --- /dev/null +++ b/scripts/llms_leakguard.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Refuse agent-instruction links in the public llms files unless allowlisted. + +`scripts/gen-llms-txt.py` runs this before it writes. `prek` runs it on the +committed `llms.txt` and `llms-full.txt`. Both read `scripts/leakguard.toml`. +""" + +from __future__ import annotations + +import re +import sys +import tomllib +from pathlib import Path +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = ROOT / "scripts" / "leakguard.toml" +CHECKED = ("llms.txt", "llms-full.txt") + +# The manual index publishes this name. Other instruction files are the same +# class of link, so a later page cannot add one without an allowlist entry. +INSTRUCTION_FILES = ("AGENTS.md", "CLAUDE.md", "GEMINI.md") + +LINK = re.compile(r"\[[^\]]*\]\((https?://[^)\s]+)\)") + + +def allow_urls() -> set[str]: + data = tomllib.loads(CONFIG.read_text(encoding="utf-8")) + urls = data.get("allow_urls", []) + if not isinstance(urls, list) or not all(isinstance(url, str) for url in urls): + raise SystemExit("scripts/leakguard.toml: allow_urls must be a list of strings") + return set(urls) + + +def instruction_file(url: str) -> str | None: + name = Path(urlparse(url).path).name + if name in INSTRUCTION_FILES: + return name + return None + + +def violations(name: str, text: str, allow: set[str]) -> list[str]: + found = [] + for url in LINK.findall(text): + filename = instruction_file(url) + if filename is None or url in allow: + continue + found.append( + f"leakguard: {name} links to {filename} ({url}); " + "allowlist the URL in scripts/leakguard.toml or drop the link" + ) + return found + + +def violations_in(files: dict[str, str], allow: set[str] | None = None) -> list[str]: + allowed = allow_urls() if allow is None else allow + found = [] + for name, text in files.items(): + found.extend(violations(name, text, allowed)) + return found + + +def main() -> int: + allow = allow_urls() + found = [] + for name in CHECKED: + path = ROOT / name + text = path.read_text(encoding="utf-8") if path.is_file() else "" + found.extend(violations(name, text, allow)) + if found: + for line in found: + print(line, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/llms_txt.rs b/tests/llms_txt.rs index ba43d904..3ec1ddf8 100644 --- a/tests/llms_txt.rs +++ b/tests/llms_txt.rs @@ -97,3 +97,27 @@ fn llms_txt_matches_the_org_manual() { ); assert!(root.join("docs/llm/reference/cli.md").is_file()); } + +/// The public manual links AGENTS.md. The leakguard allowlists that URL, and +/// `scripts/llms_leakguard.py` is the pre-check `gen-llms-txt.py` runs. +#[test] +fn the_agents_md_link_is_allowlisted() { + let root = repo(); + let config = + std::fs::read_to_string(root.join("scripts/leakguard.toml")).expect("leakguard.toml"); + assert!( + config.contains("https://github.com/HaoZeke/eb-stack/blob/main/AGENTS.md"), + "{config}" + ); + let output = Command::new("python3") + .arg(root.join("scripts/llms_leakguard.py")) + .current_dir(&root) + .output() + .expect("python3 scripts/llms_leakguard.py"); + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +}