diff --git a/libs/hackbot-runtime/hackbot_runtime/source.py b/libs/hackbot-runtime/hackbot_runtime/source.py index 7ce5d27135..98d0c2565a 100644 --- a/libs/hackbot-runtime/hackbot_runtime/source.py +++ b/libs/hackbot-runtime/hackbot_runtime/source.py @@ -1,17 +1,96 @@ """Prepare a source checkout for agents that operate on a code repository.""" import logging +import os +import re +import shutil import subprocess import sys from pathlib import Path +from urllib.parse import quote log = logging.getLogger("hackbot_runtime.source") +_GITHUB_REPO = re.compile(r"https://github\.com/([^/]+/[^/]+?)(?:\.git)?") + + +def _download(bucket: str, name: str): + """Yield the bytes of ``gs://bucket/name`` as the agent's own identity.""" + import google.auth + from google.auth.transport.requests import AuthorizedSession + + credentials, _ = google.auth.default( + scopes=["https://www.googleapis.com/auth/devstorage.read_only"] + ) + url = ( + f"https://storage.googleapis.com/storage/v1/b/{bucket}" + f"/o/{quote(name, safe='')}?alt=media" + ) + with AuthorizedSession(credentials).get(url, stream=True, timeout=60) as resp: + resp.raise_for_status() + yield from resp.iter_content(chunk_size=1 << 20) + + +def _restore_snapshot(source_repo: Path, repo_url: str, fetch_target: str) -> bool: + """Check out ``fetch_target`` from the full-history snapshot, if one is set up. + + The snapshot is a daily ``.git`` tar (``SOURCE_SNAPSHOT_BUCKET``); fetching + then adds only newer commits. Returns False, leaving nothing behind, when + there is none or any step fails, so the caller can clone instead. + """ + bucket = os.environ.get("SOURCE_SNAPSHOT_BUCKET") + match = _GITHUB_REPO.fullmatch(repo_url) + # Only into an empty directory, so a failure can remove all of it. + if not bucket or not match or (source_repo.exists() and any(source_repo.iterdir())): + return False + name = f"github/{match[1]}.tar" + log.info("restoring %s from gs://%s/%s", source_repo, bucket, name) + try: + source_repo.mkdir(parents=True, exist_ok=True) + tar = subprocess.Popen( + ["tar", "-x", "-f", "-", "-C", str(source_repo)], stdin=subprocess.PIPE + ) + try: + for chunk in _download(bucket, name): + tar.stdin.write(chunk) + finally: + tar.stdin.close() + returncode = tar.wait() + if returncode: + raise RuntimeError(f"tar exited with {returncode}") + for args in ( + ["remote", "set-url", "origin", repo_url], + ["fetch", "origin", fetch_target], + ["checkout", "-q", "-f", "FETCH_HEAD"], + ): + # Name the unpacked repository, so a tar without .git fails here + # instead of git finding an enclosing repository. + subprocess.run( + [ + "git", + f"--git-dir={source_repo / '.git'}", + f"--work-tree={source_repo}", + *args, + ], + check=True, + stdout=sys.stderr, + stderr=sys.stderr, + ) + except Exception: + log.warning("snapshot restore failed; cloning instead", exc_info=True) + shutil.rmtree(source_repo, ignore_errors=True) + return False + log.info("snapshot checkout complete") + return True + def ensure_source_repo( source_repo: Path, repo_url: str, ref: str | None = None, depth: int | None = None ) -> None: - """Ensure a shallow checkout of ``repo_url`` exists at ``source_repo``. + """Ensure a checkout of ``repo_url`` exists at ``source_repo``. + + A fresh checkout starts from the full-history snapshot when one is set up + (``_restore_snapshot``) and is otherwise a shallow clone. Idempotent: clones if absent, otherwise shallow-fetches and hard-resets to the requested ``ref`` (``origin/HEAD`` when ``ref`` is None). Recovers from a @@ -83,6 +162,8 @@ def ensure_source_repo( stderr=sys.stderr, ) return + if _restore_snapshot(source_repo, repo_url, fetch_target): + return source_repo.mkdir(parents=True, exist_ok=True) if ref: # A bare clone can't fetch an arbitrary commit directly, so init an empty diff --git a/libs/hackbot-runtime/tests/test_source.py b/libs/hackbot-runtime/tests/test_source.py index d38efa3cfa..541d8f13c1 100644 --- a/libs/hackbot-runtime/tests/test_source.py +++ b/libs/hackbot-runtime/tests/test_source.py @@ -1,9 +1,10 @@ -"""Tests for ensure_source_repo (shallow git checkout helper).""" +"""Tests for ensure_source_repo (snapshot or shallow git checkout).""" import subprocess from pathlib import Path -from hackbot_runtime import ensure_source_repo +import pytest +from hackbot_runtime import ensure_source_repo, source def _commit(path: Path, message: str) -> str: @@ -130,3 +131,150 @@ def test_depth_applies_to_fresh_clone_without_ref(tmp_path): text=True, ) assert int(count.stdout.strip()) == 3 + + +GITHUB_URL = "https://github.com/org/repo.git" + + +def _redirect_github(monkeypatch, remote: Path) -> None: + """Send git's requests for GITHUB_URL to the local ``remote``.""" + monkeypatch.setenv("GIT_CONFIG_COUNT", "1") + monkeypatch.setenv("GIT_CONFIG_KEY_0", f"url.file://{remote}.insteadOf") + monkeypatch.setenv("GIT_CONFIG_VALUE_0", GITHUB_URL) + + +def _snapshot(tmp_path: Path, remote: Path) -> bytes: + """A .git tar of a full clone of ``remote``, as the daily snapshot Job makes. + + Its origin is a path that no longer exists, so a restore must fetch from the + requested URL instead. + """ + clone = tmp_path / "snapshot-clone" + subprocess.run( + ["git", "clone", "-q", "--no-checkout", f"file://{remote}", str(clone)], + check=True, + ) + subprocess.run( + ["git", "-C", str(clone), "remote", "set-url", "origin", "/gone"], check=True + ) + tar = subprocess.run( + ["tar", "-c", "-C", str(clone), ".git"], check=True, capture_output=True + ) + return tar.stdout + + +def test_fresh_checkout_uses_snapshot_and_fetches_newer_commits(tmp_path, monkeypatch): + remote = tmp_path / "remote" + _make_remote(remote) + snapshot = _snapshot(tmp_path, remote) + (remote / "README.md").write_text("newer") + newer = _commit(remote, "after the snapshot") + requested = [] + _redirect_github(monkeypatch, remote) + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + monkeypatch.setattr( + source, + "_download", + lambda bucket, name: requested.append((bucket, name)) or iter([snapshot]), + ) + + dest = tmp_path / "dest" + ensure_source_repo(dest, GITHUB_URL, depth=1) + + assert requested == [("snapshots", "github/org/repo.tar")] + rev = subprocess.run( + ["git", "-C", str(dest), "rev-list", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.split() + assert rev[0] == newer + assert len(rev) == 2 # full history, not the requested depth + assert (dest / "README.md").read_text() == "newer" + + +def test_failed_snapshot_restore_leaves_no_git_dir(tmp_path, monkeypatch): + def fail(bucket, name): + yield b"not a tar" + raise OSError("connection reset") + + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + monkeypatch.setattr(source, "_download", fail) + dest = tmp_path / "dest" + + assert not source._restore_snapshot(dest, GITHUB_URL, "HEAD") + assert not dest.exists() + + +def test_failed_fetch_after_restore_falls_back_to_clone(tmp_path, monkeypatch): + remote = tmp_path / "remote" + _make_remote(remote) + snapshot = _snapshot(tmp_path, remote) + _redirect_github(monkeypatch, remote) + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + monkeypatch.setattr(source, "_download", lambda bucket, name: iter([snapshot])) + real_run = subprocess.run + + def fail_snapshot_fetch(cmd, **kwargs): + if cmd[-2:] == ["origin", "HEAD"] and "--depth=1" not in cmd: + raise subprocess.CalledProcessError(128, cmd) + return real_run(cmd, **kwargs) + + monkeypatch.setattr(source.subprocess, "run", fail_snapshot_fetch) + dest = tmp_path / "dest" + ensure_source_repo(dest, GITHUB_URL) + + assert (dest / "README.md").read_text() == "hello" + shallow = real_run( + ["git", "-C", str(dest), "rev-parse", "--is-shallow-repository"], + check=True, + capture_output=True, + text=True, + ) + assert shallow.stdout.strip() == "true" # the fallback clone, not the snapshot + + +def test_snapshot_is_skipped_for_a_non_empty_directory(tmp_path, monkeypatch): + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + monkeypatch.setattr(source, "_download", lambda *a: pytest.fail("downloaded")) + (tmp_path / "keep.txt").write_text("not ours") + + assert not source._restore_snapshot(tmp_path, GITHUB_URL, "HEAD") + assert (tmp_path / "keep.txt").exists() + + +def test_snapshot_is_skipped_without_bucket_or_for_other_hosts(monkeypatch, tmp_path): + monkeypatch.setattr(source, "_download", lambda *a: pytest.fail("downloaded")) + monkeypatch.delenv("SOURCE_SNAPSHOT_BUCKET", raising=False) + assert not source._restore_snapshot(tmp_path / "a", GITHUB_URL, "HEAD") + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + assert not source._restore_snapshot( + tmp_path / "b", "https://hg.mozilla.org/x", "HEAD" + ) + + +def test_snapshot_without_git_dir_leaves_an_enclosing_repo_alone(tmp_path, monkeypatch): + outer = tmp_path / "outer" + _make_remote(outer) + subprocess.run( + ["git", "-C", str(outer), "remote", "add", "origin", "/elsewhere"], check=True + ) + (tmp_path / "junk").mkdir() + (tmp_path / "junk" / "file").write_text("x") + no_git = subprocess.run( + ["tar", "-c", "-C", str(tmp_path / "junk"), "file"], + check=True, + capture_output=True, + ).stdout + monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots") + monkeypatch.setattr(source, "_download", lambda bucket, name: iter([no_git])) + + assert not source._restore_snapshot(outer / "dest", GITHUB_URL, "HEAD") + origin = subprocess.run( + ["git", "-C", str(outer), "remote", "get-url", "origin"], + check=True, + capture_output=True, + text=True, + ) + assert origin.stdout.strip() == "/elsewhere" + assert (outer / "README.md").read_text() == "hello"