From 72a08b1b122f21fe7de0fc43ccd67569f79baba0 Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Thu, 19 Nov 2026 15:27:50 -0500 Subject: [PATCH 1/3] feat(hackbot-runtime): start checkouts from the full-history source snapshot --- .../hackbot-runtime/hackbot_runtime/source.py | 74 ++++++++++++++++++- libs/hackbot-runtime/tests/test_source.py | 71 +++++++++++++++++- 2 files changed, 142 insertions(+), 3 deletions(-) diff --git a/libs/hackbot-runtime/hackbot_runtime/source.py b/libs/hackbot-runtime/hackbot_runtime/source.py index 7ce5d271354..cee4dd9a21c 100644 --- a/libs/hackbot-runtime/hackbot_runtime/source.py +++ b/libs/hackbot-runtime/hackbot_runtime/source.py @@ -1,17 +1,75 @@ """Prepare a source checkout for agents that operate on a code repository.""" import logging +import os +import re +import shutil import subprocess import sys from pathlib import Path +from urllib.parse import quote log = logging.getLogger("hackbot_runtime.source") +_GITHUB_REPO = re.compile(r"https://github\.com/([^/]+/[^/]+?)(?:\.git)?") + + +def _download(bucket: str, name: str): + """Yield the bytes of ``gs://bucket/name`` as the agent's own identity.""" + import google.auth + from google.auth.transport.requests import AuthorizedSession + + credentials, _ = google.auth.default( + scopes=["https://www.googleapis.com/auth/devstorage.read_only"] + ) + url = ( + f"https://storage.googleapis.com/storage/v1/b/{bucket}" + f"/o/{quote(name, safe='')}?alt=media" + ) + with AuthorizedSession(credentials).get(url, stream=True, timeout=60) as resp: + resp.raise_for_status() + yield from resp.iter_content(chunk_size=1 << 20) + + +def _restore_snapshot(source_repo: Path, repo_url: str) -> bool: + """Unpack the full-history snapshot of a GitHub repository, if one is set up. + + The snapshot is a daily ``.git`` tar (``SOURCE_SNAPSHOT_BUCKET``). Returns + False, leaving nothing behind, when there is none or it cannot be read. + """ + bucket = os.environ.get("SOURCE_SNAPSHOT_BUCKET") + match = _GITHUB_REPO.fullmatch(repo_url) + if not bucket or not match: + return False + name = f"github/{match[1]}.tar" + log.info("restoring %s from gs://%s/%s", source_repo, bucket, name) + try: + source_repo.mkdir(parents=True, exist_ok=True) + tar = subprocess.Popen( + ["tar", "-x", "-C", str(source_repo)], stdin=subprocess.PIPE + ) + try: + for chunk in _download(bucket, name): + tar.stdin.write(chunk) + finally: + tar.stdin.close() + returncode = tar.wait() + if returncode: + raise RuntimeError(f"tar exited with {returncode}") + except Exception: + log.warning("snapshot restore failed; cloning instead", exc_info=True) + shutil.rmtree(source_repo / ".git", ignore_errors=True) + return False + return True + def ensure_source_repo( source_repo: Path, repo_url: str, ref: str | None = None, depth: int | None = None ) -> None: - """Ensure a shallow checkout of ``repo_url`` exists at ``source_repo``. + """Ensure a checkout of ``repo_url`` exists at ``source_repo``. + + A fresh checkout starts from the full-history snapshot when one is set up + (``_restore_snapshot``) and is otherwise a shallow clone. Idempotent: clones if absent, otherwise shallow-fetches and hard-resets to the requested ``ref`` (``origin/HEAD`` when ``ref`` is None). Recovers from a @@ -83,6 +141,20 @@ def ensure_source_repo( stderr=sys.stderr, ) return + if _restore_snapshot(source_repo, repo_url): + # The snapshot has full history, so fetching only adds newer commits. + for args in ( + ["fetch", "origin", fetch_target], + ["checkout", "-q", "-f", "FETCH_HEAD"], + ): + subprocess.run( + ["git", "-C", str(source_repo), *args], + check=True, + stdout=sys.stderr, + stderr=sys.stderr, + ) + log.info("snapshot checkout complete") + return source_repo.mkdir(parents=True, exist_ok=True) if ref: # A bare clone can't fetch an arbitrary commit directly, so init an empty diff --git a/libs/hackbot-runtime/tests/test_source.py b/libs/hackbot-runtime/tests/test_source.py index d38efa3cfac..14a6157e251 100644 --- a/libs/hackbot-runtime/tests/test_source.py +++ b/libs/hackbot-runtime/tests/test_source.py @@ -1,9 +1,10 @@ -"""Tests for ensure_source_repo (shallow git checkout helper).""" +"""Tests for ensure_source_repo (snapshot or shallow git checkout).""" import subprocess from pathlib import Path -from hackbot_runtime import ensure_source_repo +import pytest +from hackbot_runtime import ensure_source_repo, source def _commit(path: Path, message: str) -> str: @@ -130,3 +131,69 @@ def test_depth_applies_to_fresh_clone_without_ref(tmp_path): text=True, ) assert int(count.stdout.strip()) == 3 + + +GITHUB_URL = "https://github.com/org/repo.git" + + +def _snapshot(tmp_path: Path, remote: Path) -> bytes: + """A .git tar of a full clone of ``remote``, as the daily snapshot Job makes.""" + clone = tmp_path / "snapshot-clone" + subprocess.run( + ["git", "clone", "-q", "--no-checkout", f"file://{remote}", str(clone)], + check=True, + ) + tar = subprocess.run( + ["tar", "-c", "-C", str(clone), ".git"], check=True, capture_output=True + ) + return tar.stdout + + +def test_fresh_checkout_uses_snapshot_and_fetches_newer_commits(tmp_path, monkeypatch): + remote = tmp_path / "remote" + _make_remote(remote) + snapshot = _snapshot(tmp_path, remote) + (remote / "README.md").write_text("newer") + newer = _commit(remote, "after the snapshot") + requested = [] + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + monkeypatch.setattr( + source, + "_download", + lambda bucket, name: requested.append((bucket, name)) or iter([snapshot]), + ) + + dest = tmp_path / "dest" + ensure_source_repo(dest, GITHUB_URL, depth=1) + + assert requested == [("snapshots", "github/org/repo.tar")] + rev = subprocess.run( + ["git", "-C", str(dest), "rev-list", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.split() + assert rev[0] == newer + assert len(rev) == 2 # full history, not the requested depth + assert (dest / "README.md").read_text() == "newer" + + +def test_failed_snapshot_restore_leaves_no_git_dir(tmp_path, monkeypatch): + def fail(bucket, name): + yield b"not a tar" + raise OSError("connection reset") + + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + monkeypatch.setattr(source, "_download", fail) + dest = tmp_path / "dest" + + assert not source._restore_snapshot(dest, GITHUB_URL) + assert not (dest / ".git").exists() + + +def test_snapshot_is_skipped_without_bucket_or_for_other_hosts(monkeypatch, tmp_path): + monkeypatch.setattr(source, "_download", lambda *a: pytest.fail("downloaded")) + monkeypatch.delenv("SOURCE_SNAPSHOT_BUCKET", raising=False) + assert not source._restore_snapshot(tmp_path / "a", GITHUB_URL) + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + assert not source._restore_snapshot(tmp_path / "b", "https://hg.mozilla.org/x") From ba0e8c91aee74cf2540d3e5b5599363a4a26967e Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Thu, 19 Nov 2026 15:46:42 -0500 Subject: [PATCH 2/3] fix(hackbot-runtime): fall back to cloning when a snapshot fetch fails --- .../hackbot-runtime/hackbot_runtime/source.py | 40 ++++++------ libs/hackbot-runtime/tests/test_source.py | 64 +++++++++++++++++-- 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/libs/hackbot-runtime/hackbot_runtime/source.py b/libs/hackbot-runtime/hackbot_runtime/source.py index cee4dd9a21c..2b3708f186b 100644 --- a/libs/hackbot-runtime/hackbot_runtime/source.py +++ b/libs/hackbot-runtime/hackbot_runtime/source.py @@ -31,15 +31,17 @@ def _download(bucket: str, name: str): yield from resp.iter_content(chunk_size=1 << 20) -def _restore_snapshot(source_repo: Path, repo_url: str) -> bool: - """Unpack the full-history snapshot of a GitHub repository, if one is set up. +def _restore_snapshot(source_repo: Path, repo_url: str, fetch_target: str) -> bool: + """Check out ``fetch_target`` from the full-history snapshot, if one is set up. - The snapshot is a daily ``.git`` tar (``SOURCE_SNAPSHOT_BUCKET``). Returns - False, leaving nothing behind, when there is none or it cannot be read. + The snapshot is a daily ``.git`` tar (``SOURCE_SNAPSHOT_BUCKET``); fetching + then adds only newer commits. Returns False, leaving nothing behind, when + there is none or any step fails, so the caller can clone instead. """ bucket = os.environ.get("SOURCE_SNAPSHOT_BUCKET") match = _GITHUB_REPO.fullmatch(repo_url) - if not bucket or not match: + # Only into an empty directory, so a failure can remove all of it. + if not bucket or not match or (source_repo.exists() and any(source_repo.iterdir())): return False name = f"github/{match[1]}.tar" log.info("restoring %s from gs://%s/%s", source_repo, bucket, name) @@ -56,10 +58,22 @@ def _restore_snapshot(source_repo: Path, repo_url: str) -> bool: returncode = tar.wait() if returncode: raise RuntimeError(f"tar exited with {returncode}") + for args in ( + ["remote", "set-url", "origin", repo_url], + ["fetch", "origin", fetch_target], + ["checkout", "-q", "-f", "FETCH_HEAD"], + ): + subprocess.run( + ["git", "-C", str(source_repo), *args], + check=True, + stdout=sys.stderr, + stderr=sys.stderr, + ) except Exception: log.warning("snapshot restore failed; cloning instead", exc_info=True) - shutil.rmtree(source_repo / ".git", ignore_errors=True) + shutil.rmtree(source_repo, ignore_errors=True) return False + log.info("snapshot checkout complete") return True @@ -141,19 +155,7 @@ def ensure_source_repo( stderr=sys.stderr, ) return - if _restore_snapshot(source_repo, repo_url): - # The snapshot has full history, so fetching only adds newer commits. - for args in ( - ["fetch", "origin", fetch_target], - ["checkout", "-q", "-f", "FETCH_HEAD"], - ): - subprocess.run( - ["git", "-C", str(source_repo), *args], - check=True, - stdout=sys.stderr, - stderr=sys.stderr, - ) - log.info("snapshot checkout complete") + if _restore_snapshot(source_repo, repo_url, fetch_target): return source_repo.mkdir(parents=True, exist_ok=True) if ref: diff --git a/libs/hackbot-runtime/tests/test_source.py b/libs/hackbot-runtime/tests/test_source.py index 14a6157e251..48f37556072 100644 --- a/libs/hackbot-runtime/tests/test_source.py +++ b/libs/hackbot-runtime/tests/test_source.py @@ -136,13 +136,27 @@ def test_depth_applies_to_fresh_clone_without_ref(tmp_path): GITHUB_URL = "https://github.com/org/repo.git" +def _redirect_github(monkeypatch, remote: Path) -> None: + """Send git's requests for GITHUB_URL to the local ``remote``.""" + monkeypatch.setenv("GIT_CONFIG_COUNT", "1") + monkeypatch.setenv("GIT_CONFIG_KEY_0", f"url.file://{remote}.insteadOf") + monkeypatch.setenv("GIT_CONFIG_VALUE_0", GITHUB_URL) + + def _snapshot(tmp_path: Path, remote: Path) -> bytes: - """A .git tar of a full clone of ``remote``, as the daily snapshot Job makes.""" + """A .git tar of a full clone of ``remote``, as the daily snapshot Job makes. + + Its origin is a path that no longer exists, so a restore must fetch from the + requested URL instead. + """ clone = tmp_path / "snapshot-clone" subprocess.run( ["git", "clone", "-q", "--no-checkout", f"file://{remote}", str(clone)], check=True, ) + subprocess.run( + ["git", "-C", str(clone), "remote", "set-url", "origin", "/gone"], check=True + ) tar = subprocess.run( ["tar", "-c", "-C", str(clone), ".git"], check=True, capture_output=True ) @@ -156,6 +170,7 @@ def test_fresh_checkout_uses_snapshot_and_fetches_newer_commits(tmp_path, monkey (remote / "README.md").write_text("newer") newer = _commit(remote, "after the snapshot") requested = [] + _redirect_github(monkeypatch, remote) monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") monkeypatch.setattr( source, @@ -187,13 +202,52 @@ def fail(bucket, name): monkeypatch.setattr(source, "_download", fail) dest = tmp_path / "dest" - assert not source._restore_snapshot(dest, GITHUB_URL) - assert not (dest / ".git").exists() + assert not source._restore_snapshot(dest, GITHUB_URL, "HEAD") + assert not dest.exists() + + +def test_failed_fetch_after_restore_falls_back_to_clone(tmp_path, monkeypatch): + remote = tmp_path / "remote" + _make_remote(remote) + snapshot = _snapshot(tmp_path, remote) + _redirect_github(monkeypatch, remote) + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + monkeypatch.setattr(source, "_download", lambda bucket, name: iter([snapshot])) + real_run = subprocess.run + + def fail_snapshot_fetch(cmd, **kwargs): + if cmd[-2:] == ["origin", "HEAD"] and "--depth=1" not in cmd: + raise subprocess.CalledProcessError(128, cmd) + return real_run(cmd, **kwargs) + + monkeypatch.setattr(source.subprocess, "run", fail_snapshot_fetch) + dest = tmp_path / "dest" + ensure_source_repo(dest, GITHUB_URL) + + assert (dest / "README.md").read_text() == "hello" + shallow = real_run( + ["git", "-C", str(dest), "rev-parse", "--is-shallow-repository"], + check=True, + capture_output=True, + text=True, + ) + assert shallow.stdout.strip() == "true" # the fallback clone, not the snapshot + + +def test_snapshot_is_skipped_for_a_non_empty_directory(tmp_path, monkeypatch): + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + monkeypatch.setattr(source, "_download", lambda *a: pytest.fail("downloaded")) + (tmp_path / "keep.txt").write_text("not ours") + + assert not source._restore_snapshot(tmp_path, GITHUB_URL, "HEAD") + assert (tmp_path / "keep.txt").exists() def test_snapshot_is_skipped_without_bucket_or_for_other_hosts(monkeypatch, tmp_path): monkeypatch.setattr(source, "_download", lambda *a: pytest.fail("downloaded")) monkeypatch.delenv("SOURCE_SNAPSHOT_BUCKET", raising=False) - assert not source._restore_snapshot(tmp_path / "a", GITHUB_URL) + assert not source._restore_snapshot(tmp_path / "a", GITHUB_URL, "HEAD") monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") - assert not source._restore_snapshot(tmp_path / "b", "https://hg.mozilla.org/x") + assert not source._restore_snapshot( + tmp_path / "b", "https://hg.mozilla.org/x", "HEAD" + ) From 14651c85da27a44b2f4909f2d1c94e0e59f3d82c Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Thu, 19 Nov 2026 15:51:54 -0500 Subject: [PATCH 3/3] fix(hackbot-runtime): run snapshot git commands against the unpacked repository --- .../hackbot-runtime/hackbot_runtime/source.py | 11 ++++++-- libs/hackbot-runtime/tests/test_source.py | 27 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/libs/hackbot-runtime/hackbot_runtime/source.py b/libs/hackbot-runtime/hackbot_runtime/source.py index 2b3708f186b..98d0c2565a5 100644 --- a/libs/hackbot-runtime/hackbot_runtime/source.py +++ b/libs/hackbot-runtime/hackbot_runtime/source.py @@ -48,7 +48,7 @@ def _restore_snapshot(source_repo: Path, repo_url: str, fetch_target: str) -> bo try: source_repo.mkdir(parents=True, exist_ok=True) tar = subprocess.Popen( - ["tar", "-x", "-C", str(source_repo)], stdin=subprocess.PIPE + ["tar", "-x", "-f", "-", "-C", str(source_repo)], stdin=subprocess.PIPE ) try: for chunk in _download(bucket, name): @@ -63,8 +63,15 @@ def _restore_snapshot(source_repo: Path, repo_url: str, fetch_target: str) -> bo ["fetch", "origin", fetch_target], ["checkout", "-q", "-f", "FETCH_HEAD"], ): + # Name the unpacked repository, so a tar without .git fails here + # instead of git finding an enclosing repository. subprocess.run( - ["git", "-C", str(source_repo), *args], + [ + "git", + f"--git-dir={source_repo / '.git'}", + f"--work-tree={source_repo}", + *args, + ], check=True, stdout=sys.stderr, stderr=sys.stderr, diff --git a/libs/hackbot-runtime/tests/test_source.py b/libs/hackbot-runtime/tests/test_source.py index 48f37556072..541d8f13c12 100644 --- a/libs/hackbot-runtime/tests/test_source.py +++ b/libs/hackbot-runtime/tests/test_source.py @@ -251,3 +251,30 @@ def test_snapshot_is_skipped_without_bucket_or_for_other_hosts(monkeypatch, tmp_ assert not source._restore_snapshot( tmp_path / "b", "https://hg.mozilla.org/x", "HEAD" ) + + +def test_snapshot_without_git_dir_leaves_an_enclosing_repo_alone(tmp_path, monkeypatch): + outer = tmp_path / "outer" + _make_remote(outer) + subprocess.run( + ["git", "-C", str(outer), "remote", "add", "origin", "/elsewhere"], check=True + ) + (tmp_path / "junk").mkdir() + (tmp_path / "junk" / "file").write_text("x") + no_git = subprocess.run( + ["tar", "-c", "-C", str(tmp_path / "junk"), "file"], + check=True, + capture_output=True, + ).stdout + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + monkeypatch.setattr(source, "_download", lambda bucket, name: iter([no_git])) + + assert not source._restore_snapshot(outer / "dest", GITHUB_URL, "HEAD") + origin = subprocess.run( + ["git", "-C", str(outer), "remote", "get-url", "origin"], + check=True, + capture_output=True, + text=True, + ) + assert origin.stdout.strip() == "/elsewhere" + assert (outer / "README.md").read_text() == "hello"