diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..638dac1 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,6 @@ +paths: + .github/workflows/dependency-scan.yml: + ignore: + # GitHub documents job.workflow_repository and job.workflow_sha (the repository and + # commit of the workflow file), but actionlint doesn't know them yet + - 'property "workflow_(repository|sha)" is not defined in object type' diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..811996a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + # a new release waits a week, so a compromised one is likely found and pulled first + cooldown: + default-days: 7 + groups: + github-actions: + patterns: ["*"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a3e051d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,56 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +# nothing by default: each job asks for the permissions it needs +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + lint: + name: Lint + runs-on: ubuntu-slim + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: false + + - name: Check the workflows with actionlint + run: uvx --from actionlint-py==1.7.12.25 actionlint + + - name: Audit the workflows with zizmor + run: uvx zizmor@1.32.0 --offline .github + + - name: Lint and check the format of the Python code with ruff + run: | + uvx ruff@0.17.5 check scripts tests + uvx ruff@0.17.5 format --check scripts tests + + test: + name: Test + runs-on: ubuntu-slim + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: false + + - name: Run the tests + run: uv run --no-project --python 3.13 --with pytest==9.* --with pyyaml==6.* pytest -v diff --git a/.github/workflows/dependency-scan.yml b/.github/workflows/dependency-scan.yml new file mode 100644 index 0000000..3b45f59 --- /dev/null +++ b/.github/workflows/dependency-scan.yml @@ -0,0 +1,52 @@ +name: Dependency scan + +# Checks the Python packages locked in uv.lock against the OSV database of known +# vulnerabilities, on every pull request and in the merge queue. An organization ruleset requires it ("Require workflows to +# pass before merging"), so it runs in the repository of the pull request, with that +# repository's files and token: see README.md. +# +# A ruleset ignores the filters of the triggers (branches, paths, types), so the job checks +# by itself whether there is anything to scan. The scan is scripts/dependency_scan.py of this +# repository, taken from the same commit as this file. +on: + pull_request: + merge_group: + +permissions: + contents: read + +jobs: + scan: + name: Dependency scan + runs-on: ubuntu-slim + timeout-minutes: 15 + steps: + # the repository of the pull request, which the scan reads + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # The script of the scan. With a ruleset, this workflow runs in another repository: + # `job.workflow_repository` and `job.workflow_sha` name this repository and the commit + # of this file, which the ruleset pins, so the script can't differ from the reviewed one. + # It goes next to the files of the pull request, untracked, so the scan doesn't see it. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: .dependency-scan + sparse-checkout: scripts + persist-credentials: false + + # runs the script with the Python of its inline metadata; no cache: the job must not + # restore files that another run could have written + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: false + + - name: Check uv.lock against OSV + env: + EXCEPTIONS_FILE: .github/vulnerability-exceptions.toml + # --no-config: the uv settings of the repository (uv.toml, [tool.uv]) are not for the + # script, and --no-project: the repository's project isn't needed to run it + run: uv run --no-config --no-project .dependency-scan/scripts/dependency_scan.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b267296 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +.pytest_cache/ +.ruff_cache/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..22ea0e6 --- /dev/null +++ b/README.md @@ -0,0 +1,93 @@ +# security-workflows + +GitHub Actions workflows that organization rulesets require in Algorithmiq's repositories. +They are public so that one ruleset can run them in every repository, public, internal or +private: a ruleset can only run a workflow whose repository is at least as visible as the +repositories it targets. + +This repository holds generic workflow code only. It must never contain secrets, names of +private repositories or packages, or other organization-specific details: those come from the +repository the workflow runs in. + +## Dependency scan + +[`dependency-scan.yml`](.github/workflows/dependency-scan.yml) checks the Python packages +locked in `uv.lock` against the [OSV](https://osv.dev) database of known vulnerabilities, on +every pull request and in the merge queue. + +### How it works + +A ruleset runs the workflow in the repository of the pull request, with that repository's +files and `GITHUB_TOKEN` (read-only), and ignores the filters of its triggers. The scan is +[`scripts/dependency_scan.py`](scripts/dependency_scan.py) of this repository: the job checks +it out from the commit of the workflow file (`job.workflow_repository`, `job.workflow_sha`), +which the ruleset pins, and runs it with `uv run` (Python 3.13, standard library only, as its +inline metadata says). The script: + +1. finds the `uv.lock` files that git tracks. With none, it passes with a notice; +2. reads the packages of each one that come from a package index, with their versions: + every extra and dependency group, whatever the index. The project itself and packages from + git, a URL or a local path are left out: they have no version on PyPI to look up; +3. asks OSV which vulnerabilities affect each name and version, and reads their records. OSV + lists a vulnerability once per database (GHSA, PYSEC, …): the GHSA record is kept, as it + carries GitHub's severity; +4. writes a table of the findings to the job summary, and an error annotation for each one + that blocks. + +A finding **blocks the merge** when its severity is **high** or **critical**, or unknown (no +GHSA record): an unknown severity never lets a vulnerability through. Low and moderate +findings are listed without failing the job. A withdrawn advisory never blocks. If OSV can't +be read, the job fails. + +### What leaves the repository + +Only the name, the ecosystem (`PyPI`) and the version of each locked package, sent to +[OSV's API](https://google.github.io/osv.dev/post-v1-querybatch/) with Python's default +User-Agent; then the identifiers of the vulnerabilities found, to read their records. That +includes packages from private indexes: OSV matches by name, whatever the index. If a private +package shares its name with a public one that has advisories, the findings of the public one +are reported: record them as exceptions. + +### Exceptions + +A finding that is acceptable for a while (no fix yet, code that is never reached, a false +positive) goes in `.github/vulnerability-exceptions.toml` of the repository, one table per +vulnerability: + +```toml +[[exception]] +id = "GHSA-xxxx-xxxx-xxxx" # the GHSA identifier, or any alias that the job summary shows +issue = "https://github.com///issues/" # the issue that approves it +reason = "The vulnerable function is only used by the test suite." +``` + +All three keys are required; any other key is an error. Exceptions are reviewed in pull +requests like any other change, and the history of the file shows who approved what. An +exception that no longer matches a finding gives a warning: remove it. + +### Setting it up (organization owners) + +1. **Settings → Repository → Rulesets → New branch ruleset**, enforcement **Evaluate** at first. +2. **Target repositories**: by custom property, for example the repositories that must have a + dependency scan. **Target branches**: the default branch. The rule blocks direct pushes, so + only target branches that change through pull requests. +3. Rule **Require workflows to pass before merging**: repository + `Algorithmiq/security-workflows`, workflow `.github/workflows/dependency-scan.yml`, and the + **full commit SHA** of a reviewed version as the reference, not a branch: a change here + reaches every targeted repository only when the ruleset moves to it. +4. Once the runs in Evaluate mode look right, switch the enforcement to **Active**. Pull + requests opened before the ruleset existed run the workflow on their next push. + +To move to a new version: review the change here, then update the SHA in the ruleset. + +## Development + +The tests run `scripts/dependency_scan.py` as the workflow does: + +```sh +uv run --no-project --with pytest --with pyyaml pytest +``` + +The end-to-end tests lock small projects with known vulnerable packages and need the network +(PyPI and OSV); set `OFFLINE=1` to skip them. CI also runs ruff, actionlint and zizmor. Changes need a pull request and a review: +every targeted repository runs this code. diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..71de868 --- /dev/null +++ b/ruff.toml @@ -0,0 +1 @@ +line-length = 100 diff --git a/scripts/dependency_scan.py b/scripts/dependency_scan.py new file mode 100644 index 0000000..2f6d909 --- /dev/null +++ b/scripts/dependency_scan.py @@ -0,0 +1,359 @@ +# /// script +# requires-python = ">=3.13" +# dependencies = [] +# /// +"""Check the packages locked in the uv.lock files of the repository against OSV. + +Each package from a package index is looked up by name and version in the OSV database +(https://osv.dev), which is all that leaves the runner. A finding blocks the merge when its +severity is high or critical, or unknown, unless .github/vulnerability-exceptions.toml lists +it. The severity is GitHub's, from the GHSA records that OSV mirrors. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tomllib +import urllib.error +import urllib.request +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from pathlib import Path + +OSV = "https://api.osv.dev/v1" +# OSV answers at most this many queries per batch +BATCH = 1000 + + +class ScanError(Exception): + """A problem that stops the scan, as opposed to a finding.""" + + +@dataclass +class Approval: + id: str + issue: str + reason: str + used: bool = False + + +@dataclass +class Finding: + lock: str + package: str + version: str + id: str + aliases: list[str] + summary: str + fix_versions: list[str] + severity: str = "unknown" + withdrawn: bool = False + exception: Approval | None = None + + @property + def ids(self) -> set[str]: + return {self.id, *self.aliases} + + @property + def ghsa(self) -> str | None: + return next((i for i in [self.id, *self.aliases] if i.startswith("GHSA-")), None) + + @property + def blocking(self) -> bool: + if self.withdrawn or self.exception is not None: + return False + return self.severity not in {"low", "moderate", "medium"} + + +@dataclass(frozen=True) +class Package: + lock: str + name: str + version: str + + +@dataclass +class Osv: + """OSV's API: the batch query, and the full records, read once each.""" + + post: Callable[[str, dict], dict] + get: Callable[[str], dict] + records: dict[str, dict] = field(default_factory=dict) + + def vulnerabilities(self, packages: list[Package]) -> list[list[str]]: + """The ids of the vulnerabilities of each package, in the order of PACKAGES.""" + ids: list[list[str]] = [[] for _ in packages] + pending = [(index, None) for index in range(len(packages))] + while pending: + batch, pending = pending[:BATCH], pending[BATCH:] + queries = [ + { + "package": {"name": packages[i].name, "ecosystem": "PyPI"}, + "version": packages[i].version, + **({"page_token": token} if token else {}), + } + for i, token in batch + ] + results = self.post(f"{OSV}/querybatch", {"queries": queries})["results"] + for (index, _), result in zip(batch, results, strict=True): + ids[index] += [vuln["id"] for vuln in result.get("vulns", [])] + if result.get("next_page_token"): + pending.append((index, result["next_page_token"])) + return ids + + def record(self, id: str) -> dict: + if id not in self.records: + self.records[id] = self.get(f"{OSV}/vulns/{id}") + return self.records[id] + + def prefetch(self, ids: set[str]) -> None: + missing = sorted(ids - set(self.records)) + with ThreadPoolExecutor(max_workers=8) as pool: + for id, record in zip(missing, pool.map(self.record, missing), strict=True): + self.records[id] = record + + +def http_get(url: str) -> dict: + with urllib.request.urlopen(url, timeout=30) as response: + return json.load(response) + + +def http_post(url: str, body: dict) -> dict: + request = urllib.request.Request( + url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"} + ) + with urllib.request.urlopen(request, timeout=60) as response: + return json.load(response) + + +def load_exceptions(path: Path) -> list[Approval]: + """Read the approved exceptions: an `[[exception]]` table per vulnerability.""" + if not path.is_file(): + return [] + try: + data = tomllib.loads(path.read_text(encoding="utf-8")) + except tomllib.TOMLDecodeError as error: + raise ScanError(f"{path} is not valid TOML: {error}") from error + unknown = set(data) - {"exception"} + if unknown: + raise ScanError(f"{path}: unknown keys {sorted(unknown)}, only [[exception]] tables") + exceptions = [] + for number, entry in enumerate(data.get("exception", []), start=1): + where = f"{path}, exception {number}" + if not isinstance(entry, dict): + raise ScanError(f"{where}: must be an [[exception]] table") + extra = set(entry) - {"id", "issue", "reason"} + if extra: + raise ScanError(f"{where}: unknown keys {sorted(extra)}") + for key in ("id", "issue", "reason"): + value = entry.get(key) + if not isinstance(value, str) or not value.strip(): + raise ScanError(f"{where}: `{key}` is required and can't be empty") + if not entry["issue"].startswith("https://"): + raise ScanError(f"{where}: `issue` must be the link to the issue that approves it") + exceptions.append(Approval(entry["id"].strip(), entry["issue"], entry["reason"])) + return exceptions + + +def lock_files(root: Path) -> list[Path]: + """The uv.lock files tracked by git, so generated or ignored ones don't count.""" + output = subprocess.run( + ["git", "ls-files", "-z", "--", "uv.lock", "*/uv.lock"], + cwd=root, + check=True, + capture_output=True, + text=True, + ).stdout + return sorted(root / name for name in output.split("\0") if name) + + +def locked_packages(lock: Path, root: Path) -> list[Package]: + """The packages of LOCK that come from a package index, with their versions. + + The project itself and the packages from git, a URL or a local path have no version on + PyPI to look up, so they're left out. + """ + try: + data = tomllib.loads(lock.read_text(encoding="utf-8")) + except tomllib.TOMLDecodeError as error: + raise ScanError(f"{lock.relative_to(root)} is not valid TOML: {error}") from error + name = str(lock.relative_to(root)) + return sorted( + { + Package(name, package["name"], package["version"]) + for package in data.get("package", []) + if "registry" in package.get("source", {}) and "version" in package + }, + key=lambda p: (p.name, p.version), + ) + + +def normalize(name: str) -> str: + return name.lower().replace("_", "-").replace(".", "-") + + +def fix_versions(record: dict, package: str) -> list[str]: + """The versions that fix RECORD for PACKAGE, from its affected ranges.""" + return sorted( + { + event["fixed"] + for affected in record.get("affected", []) + if normalize(affected.get("package", {}).get("name", "")) == normalize(package) + for range_ in affected.get("ranges", []) + for event in range_.get("events", []) + if "fixed" in event + } + ) + + +def find(packages: list[Package], osv: Osv) -> list[Finding]: + """The vulnerabilities of PACKAGES, one finding per vulnerability and package. + + OSV lists a vulnerability once per database (GHSA, PYSEC, …), the records naming each + other as aliases: the GHSA record is kept, as it has the severity. + """ + ids = osv.vulnerabilities(packages) + osv.prefetch({i for found in ids for i in found}) + findings = [] + for package, found in zip(packages, ids, strict=True): + seen: set[str] = set() + for id in sorted(found, key=lambda i: (not i.startswith("GHSA-"), i)): + record = osv.record(id) + aliases = [a for a in record.get("aliases", []) if a != id] + if seen & {id, *aliases}: + continue + seen |= {id, *aliases} + findings.append( + Finding( + lock=package.lock, + package=package.name, + version=package.version, + id=id, + aliases=aliases, + summary=record.get("summary") or record.get("details", "").split("\n")[0], + fix_versions=fix_versions(record, package.name), + withdrawn=record.get("withdrawn") is not None, + ) + ) + return findings + + +def classify(findings: list[Finding], exceptions: list[Approval], osv: Osv) -> None: + """Fill in the severity and the exception of each finding.""" + for finding in findings: + if finding.ghsa is not None: + try: + record = osv.record(finding.ghsa) + except (urllib.error.URLError, TimeoutError, ValueError) as error: + print(f"::warning::Could not read {finding.ghsa}: {error}") + else: + severity = (record.get("database_specific") or {}).get("severity") + finding.severity = (severity or "unknown").lower() + for exception in exceptions: + if exception.id in finding.ids: + finding.exception = exception + exception.used = True + break + + +def command(text: str) -> str: + """Escape TEXT for the message of a workflow command (::error:: and the like).""" + return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def cell(text: str) -> str: + return text.replace("|", "\\|").replace("\n", " ") + + +def report(findings: list[Finding], exceptions: list[Approval], locks: list[str]) -> str: + """The job summary, in Markdown.""" + blocking = [f for f in findings if f.blocking] + lines = ["## Dependency scan", ""] + lines.append(f"Checked {', '.join(f'`{lock}`' for lock in locks)} against OSV.") + lines.append("") + if not findings: + lines.append("No known vulnerabilities.") + else: + lines.append( + f"{len(findings)} finding(s), {len(blocking)} blocking: high, critical or of " + "unknown severity, and not in `.github/vulnerability-exceptions.toml`." + ) + lines += [ + "", + "| | Package | Advisory | Severity | Fixed in | Summary |", + "|---|---|---|---|---|---|", + ] + order = {"critical": 0, "high": 1, "unknown": 2, "moderate": 3, "medium": 3, "low": 4} + + def key(f: Finding) -> tuple: + return (not f.blocking, order.get(f.severity, 2), f.package) + + for f in sorted(findings, key=key): + if f.blocking: + status = "❌ blocking" + elif f.withdrawn: + status = "withdrawn" + elif f.exception is not None: + status = f"[excepted]({f.exception.issue})" + else: + status = "reported" + link = f"[{f.id}](https://osv.dev/vulnerability/{f.id})" + fixed = cell(", ".join(f.fix_versions) or "no fix yet") + lines.append( + f"| {status} | {cell(f.package)} {cell(f.version)} (`{f.lock}`) | {link} | " + f"{f.severity} | {fixed} | {cell(f.summary)} |" + ) + unused = [e for e in exceptions if not e.used] + if unused: + lines += ["", "Exceptions that match no finding any more (remove them):", ""] + lines += [f"- `{e.id}` ({e.issue})" for e in unused] + return "\n".join(lines) + "\n" + + +def main(osv: Osv | None = None) -> int: + root = Path.cwd() + summary = os.environ.get("GITHUB_STEP_SUMMARY") + locks = lock_files(root) + if not locks: + print("::notice title=Dependency scan::No uv.lock in this repository: nothing to scan.") + if summary: + Path(summary).write_text("## Dependency scan\n\nNo `uv.lock`: nothing to scan.\n") + return 0 + osv = osv or Osv(post=http_post, get=http_get) + exceptions_file = os.environ.get("EXCEPTIONS_FILE", ".github/vulnerability-exceptions.toml") + try: + exceptions = load_exceptions(root / exceptions_file) + packages = [package for lock in locks for package in locked_packages(lock, root)] + findings = find(packages, osv) + except ScanError as error: + print(f"::error title=Dependency scan::{command(str(error))}") + return 1 + except (urllib.error.URLError, TimeoutError, ValueError, KeyError) as error: + print(f"::error title=Dependency scan::OSV could not be read: {command(str(error))}") + return 1 + classify(findings, exceptions, osv) + text = report(findings, exceptions, [str(lock.relative_to(root)) for lock in locks]) + print(text) + if summary: + Path(summary).write_text(text) + for f in findings: + if f.blocking: + fix = f"fixed in {', '.join(f.fix_versions)}" if f.fix_versions else "no fix yet" + print( + f"::error file={f.lock},title={f.severity} vulnerability in {f.package}::" + + command(f"{f.package} {f.version}: {f.id} ({fix}). {f.summary}") + ) + for e in exceptions: + if not e.used: + print( + f"::warning title=Unused exception::{e.id} matches no finding: remove it ({e.issue})" + ) + return 1 if any(f.blocking for f in findings) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_dependency_scan.py b/tests/test_dependency_scan.py new file mode 100644 index 0000000..41d1e38 --- /dev/null +++ b/tests/test_dependency_scan.py @@ -0,0 +1,362 @@ +"""Tests of scripts/dependency_scan.py, the scan of .github/workflows/dependency-scan.yml. + +Run with: uv run --no-project --with pytest --with pyyaml pytest +The end-to-end tests need the network (PyPI and OSV). +""" + +from __future__ import annotations + +import importlib.util +import os +import shutil +import subprocess +import sys +import textwrap +import urllib.error +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github" / "workflows" / "dependency-scan.yml" +SCRIPT = ROOT / "scripts" / "dependency_scan.py" + + +@pytest.fixture(scope="session") +def script() -> Path: + return SCRIPT + + +def test_the_workflow_runs_the_script() -> None: + """The workflow checks out this repository's scripts/ and runs the scan from there.""" + steps = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"]["scan"]["steps"] + own = next( + s for s in steps if s.get("with", {}).get("repository") == "${{ job.workflow_repository }}" + ) + assert own["with"]["ref"] == "${{ job.workflow_sha }}" + path = own["with"]["path"] + run = steps[-1]["run"] + assert run.endswith(f"{path}/{SCRIPT.relative_to(ROOT)}") + + +@pytest.fixture(scope="session") +def scan(script: Path): + spec = importlib.util.spec_from_file_location("dependency_scan", script) + module = importlib.util.module_from_spec(spec) + sys.modules["dependency_scan"] = module + spec.loader.exec_module(module) + return module + + +# --- the exceptions file --------------------------------------------------------------- + + +def write(path: Path, text: str) -> Path: + path.write_text(textwrap.dedent(text), encoding="utf-8") + return path + + +def test_no_exceptions_file(scan, tmp_path: Path) -> None: + assert scan.load_exceptions(tmp_path / "missing.toml") == [] + + +def test_exceptions_are_read(scan, tmp_path: Path) -> None: + path = write( + tmp_path / "e.toml", + """ + [[exception]] + id = "GHSA-aaaa-bbbb-cccc" + issue = "https://github.com/example/project/issues/1" + reason = "Only the test suite imports it." + """, + ) + [approval] = scan.load_exceptions(path) + assert approval.id == "GHSA-aaaa-bbbb-cccc" + assert approval.issue.endswith("/issues/1") + + +@pytest.mark.parametrize( + ("text", "message"), + [ + ('[[exception]]\nid = "X"\nissue = "https://e/1"\n', "`reason` is required"), + ( + '[[exception]]\nid = ""\nissue = "https://e/1"\nreason = "r"\n', + "`id` is required", + ), + ('[[exception]]\nid = "X"\nissue = "#1"\nreason = "r"\n', "link to the issue"), + ( + '[[exception]]\nid = "X"\nissue = "https://e/1"\nreason = "r"\nuntil = "x"\n', + "unknown keys", + ), + ('[[exceptions]]\nid = "X"\n', "unknown keys"), + ("[[exception]\n", "not valid TOML"), + ], +) +def test_invalid_exceptions(scan, tmp_path: Path, text: str, message: str) -> None: + with pytest.raises(scan.ScanError, match=message): + scan.load_exceptions(write(tmp_path / "e.toml", text)) + + +# --- which findings block --------------------------------------------------------------- + + +def osv_record(id: str, aliases: list[str], severity: str | None = None, **extra) -> dict: + record = { + "id": id, + "aliases": aliases, + "summary": f"Summary of {id}", + "affected": [ + { + "package": {"name": "Pkg", "ecosystem": "PyPI"}, + "ranges": [ + {"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": "1.1"}]} + ], + } + ], + **extra, + } + if severity: + record["database_specific"] = {"severity": severity} + return record + + +RECORDS = { + r["id"]: r + for r in [ + osv_record("GHSA-crit-crit-crit", [], "CRITICAL"), + osv_record("GHSA-high-high-high", ["PYSEC-2020-1"], "HIGH"), + osv_record("PYSEC-2020-1", ["GHSA-high-high-high"]), + osv_record("GHSA-medi-medi-medi", [], "MODERATE"), + osv_record("GHSA-loww-loww-loww", [], "LOW"), + osv_record("PYSEC-2021-2", ["CVE-2021-2"]), + osv_record("PYSEC-2021-3", ["GHSA-only-alia-sed1"]), + osv_record("GHSA-only-alia-sed1", ["PYSEC-2021-3"], "MODERATE"), + osv_record("GHSA-gone-gone-gone", [], "CRITICAL", withdrawn="2025-01-01T00:00:00Z"), + ] +} + + +def fake_osv(scan, found: dict[tuple[str, str], list[str]]): + """An OSV whose batch query answers FOUND, by (name, version), and with RECORDS.""" + queries = [] + + def post(url: str, body: dict) -> dict: + assert url.endswith("/querybatch") + queries.extend(body["queries"]) + return { + "results": [ + {"vulns": [{"id": i} for i in found.get((q["package"]["name"], q["version"]), [])]} + for q in body["queries"] + ] + } + + def get(url: str) -> dict: + id = url.rsplit("/", 1)[1] + if id not in RECORDS: + raise urllib.error.URLError("not found") + return RECORDS[id] + + osv = scan.Osv(post=post, get=get) + osv.queries = queries + return osv + + +def scan_one(scan, ids: list[str], exceptions=()) -> list: + package = scan.Package("uv.lock", "pkg", "1.0") + osv = fake_osv(scan, {("pkg", "1.0"): ids}) + findings = scan.find([package], osv) + scan.classify(findings, list(exceptions), osv) + return findings + + +@pytest.mark.parametrize( + ("ids", "severity", "blocking"), + [ + (["GHSA-crit-crit-crit"], "critical", True), + (["GHSA-high-high-high"], "high", True), + (["GHSA-medi-medi-medi"], "moderate", False), + (["GHSA-loww-loww-loww"], "low", False), + # only a PYSEC id, whose GHSA alias has the severity + (["PYSEC-2021-3"], "moderate", False), + # no GHSA: the severity is unknown, which blocks + (["PYSEC-2021-2"], "unknown", True), + # a withdrawn advisory never blocks + (["GHSA-gone-gone-gone"], "critical", False), + ], +) +def test_blocking(scan, ids: list[str], severity: str, blocking: bool) -> None: + [f] = scan_one(scan, ids) + assert (f.severity, f.blocking) == (severity, blocking) + + +def test_aliases_are_one_finding(scan) -> None: + """OSV lists a vulnerability once per database: the GHSA record is kept.""" + [f] = scan_one(scan, ["PYSEC-2020-1", "GHSA-high-high-high"]) + assert f.id == "GHSA-high-high-high" + assert f.fix_versions == ["1.1"] + + +def test_an_unreadable_record_stops_the_scan(scan) -> None: + with pytest.raises(urllib.error.URLError): + scan_one(scan, ["GHSA-miss-miss-miss"]) + + +def test_an_exception_matches_an_alias(scan) -> None: + approval = scan.Approval("CVE-2021-2", "https://e/1", "reason") + [f] = scan_one(scan, ["PYSEC-2021-2"], [approval]) + assert f.exception is approval + assert approval.used + assert not f.blocking + + +def test_the_report_lists_unused_exceptions(scan) -> None: + approval = scan.Approval("GHSA-none-none-none", "https://e/1", "reason") + findings = scan_one(scan, ["GHSA-high-high-high"], [approval]) + text = scan.report(findings, [approval], ["uv.lock"]) + assert "❌ blocking" in text + assert "match no finding" in text + assert "GHSA-none-none-none" in text + + +def test_only_packages_from_an_index_are_looked_up(scan, tmp_path: Path) -> None: + lock = write( + tmp_path / "uv.lock", + """ + version = 1 + [[package]] + name = "demo" + version = "0.1.0" + source = { editable = "." } + [[package]] + name = "pkg" + version = "1.0" + source = { registry = "https://pypi.org/simple" } + [[package]] + name = "private" + version = "2.0" + source = { registry = "https://packages.example.com/simple/" } + [[package]] + name = "fromgit" + version = "0.3" + source = { git = "https://github.com/example/fromgit?rev=abc#abc" } + """, + ) + packages = scan.locked_packages(lock, tmp_path) + assert [(p.name, p.version) for p in packages] == [("pkg", "1.0"), ("private", "2.0")] + osv = fake_osv(scan, {}) + scan.find(packages, osv) + # only the name, the ecosystem and the version are sent + assert osv.queries == [ + {"package": {"name": "pkg", "ecosystem": "PyPI"}, "version": "1.0"}, + {"package": {"name": "private", "ecosystem": "PyPI"}, "version": "2.0"}, + ] + + +def test_workflow_command_messages_are_escaped(scan) -> None: + assert scan.command("50% done\nnext") == "50%25 done%0Anext" + + +# --- end to end: real lock files and OSV -------------------------------------------------- + +needs_network = pytest.mark.skipif( + shutil.which("uv") is None or os.environ.get("OFFLINE") == "1", + reason="needs uv (to lock the projects) and the network", +) + + +def git(cwd: Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True) + + +def project(path: Path, dependencies: list[str]) -> Path: + """A git repository with a locked project that depends on DEPENDENCIES.""" + path.mkdir(parents=True) + quoted = ", ".join(f'"{d}"' for d in dependencies) + (path / "pyproject.toml").write_text( + f'[project]\nname = "demo"\nversion = "0"\nrequires-python = ">=3.11"\n' + f"dependencies = [{quoted}]\n", + encoding="utf-8", + ) + subprocess.run(["uv", "lock", "--quiet"], cwd=path, check=True) + git(path, "init", "--quiet") + git(path, "add", "--all") + return path + + +def run(script: Path, cwd: Path) -> subprocess.CompletedProcess: + summary = cwd.parent / f"{cwd.name}-summary.md" + env = { + **os.environ, + "GITHUB_STEP_SUMMARY": str(summary), + "EXCEPTIONS_FILE": ".github/vulnerability-exceptions.toml", + } + result = subprocess.run( + ["uv", "run", "--no-config", "--no-project", str(script)], + cwd=cwd, + env=env, + capture_output=True, + text=True, + check=False, + ) + result.summary = summary.read_text(encoding="utf-8") if summary.exists() else "" + return result + + +def test_nothing_to_scan(script: Path, tmp_path: Path) -> None: + repository = tmp_path / "empty" + repository.mkdir() + git(repository, "init", "--quiet") + result = run(script, repository) + assert result.returncode == 0, result.stdout + result.stderr + assert "nothing to scan" in result.stdout + + +@needs_network +def test_a_clean_lock_passes(script: Path, tmp_path: Path) -> None: + result = run(script, project(tmp_path / "clean", ["packaging>=25"])) + assert result.returncode == 0, result.stdout + result.stderr + assert "No known vulnerabilities." in result.summary + + +@needs_network +def test_a_vulnerable_lock_fails_until_excepted(script: Path, tmp_path: Path) -> None: + # jinja2 2.10 has high-severity advisories, e.g. a sandbox escape + repository = project(tmp_path / "vulnerable", ["jinja2==2.10"]) + result = run(script, repository) + assert result.returncode == 1, result.stdout + result.stderr + assert "::error file=uv.lock,title=high vulnerability in jinja2::" in result.stdout + assert "❌ blocking" in result.summary + + # approving every blocking advisory makes the scan pass, and lists them as excepted + ids = sorted( + { + line.split(": ", 1)[1].split(" ", 1)[0] + for line in result.stdout.splitlines() + if line.startswith("::error file=") + } + ) + assert ids + exceptions = repository / ".github" / "vulnerability-exceptions.toml" + exceptions.parent.mkdir() + exceptions.write_text( + "".join( + f'[[exception]]\nid = "{i}"\nissue = "https://github.com/example/demo/issues/1"\nreason = "Test."\n\n' + for i in ids + ), + encoding="utf-8", + ) + result = run(script, repository) + assert result.returncode == 0, result.stdout + result.stderr + assert "excepted" in result.summary + + +@needs_network +def test_every_tracked_lock_is_scanned(script: Path, tmp_path: Path) -> None: + repository = project(tmp_path / "monorepo", ["packaging>=25"]) + project(repository / "tools", ["jinja2==2.10"]) + shutil.rmtree(repository / "tools" / ".git") + git(repository, "add", "--all") + result = run(script, repository) + assert result.returncode == 1, result.stdout + result.stderr + assert "tools/uv.lock" in result.summary