Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 82 additions & 1 deletion libs/hackbot-runtime/hackbot_runtime/source.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,96 @@
"""Prepare a source checkout for agents that operate on a code repository."""

import logging
import os
import re
import shutil
import subprocess
import sys
from pathlib import Path
from urllib.parse import quote

log = logging.getLogger("hackbot_runtime.source")

_GITHUB_REPO = re.compile(r"https://github\.com/([^/]+/[^/]+?)(?:\.git)?")


def _download(bucket: str, name: str):
"""Yield the bytes of ``gs://bucket/name`` as the agent's own identity."""
import google.auth
from google.auth.transport.requests import AuthorizedSession

credentials, _ = google.auth.default(
scopes=["https://www.googleapis.com/auth/devstorage.read_only"]
)
url = (
f"https://storage.googleapis.com/storage/v1/b/{bucket}"
f"/o/{quote(name, safe='')}?alt=media"
)
with AuthorizedSession(credentials).get(url, stream=True, timeout=60) as resp:
resp.raise_for_status()
yield from resp.iter_content(chunk_size=1 << 20)


def _restore_snapshot(source_repo: Path, repo_url: str, fetch_target: str) -> bool:
"""Check out ``fetch_target`` from the full-history snapshot, if one is set up.

The snapshot is a daily ``.git`` tar (``SOURCE_SNAPSHOT_BUCKET``); fetching
then adds only newer commits. Returns False, leaving nothing behind, when
there is none or any step fails, so the caller can clone instead.
"""
bucket = os.environ.get("SOURCE_SNAPSHOT_BUCKET")
match = _GITHUB_REPO.fullmatch(repo_url)
# Only into an empty directory, so a failure can remove all of it.
if not bucket or not match or (source_repo.exists() and any(source_repo.iterdir())):
return False
name = f"github/{match[1]}.tar"
log.info("restoring %s from gs://%s/%s", source_repo, bucket, name)
try:
source_repo.mkdir(parents=True, exist_ok=True)
tar = subprocess.Popen(
["tar", "-x", "-f", "-", "-C", str(source_repo)], stdin=subprocess.PIPE
)
try:
for chunk in _download(bucket, name):
tar.stdin.write(chunk)
finally:
tar.stdin.close()
returncode = tar.wait()
if returncode:
raise RuntimeError(f"tar exited with {returncode}")
for args in (
["remote", "set-url", "origin", repo_url],
["fetch", "origin", fetch_target],
["checkout", "-q", "-f", "FETCH_HEAD"],
):
# Name the unpacked repository, so a tar without .git fails here
# instead of git finding an enclosing repository.
subprocess.run(
[
"git",
f"--git-dir={source_repo / '.git'}",
f"--work-tree={source_repo}",
*args,
],
check=True,
stdout=sys.stderr,
stderr=sys.stderr,
)
except Exception:
log.warning("snapshot restore failed; cloning instead", exc_info=True)
shutil.rmtree(source_repo, ignore_errors=True)
return False
log.info("snapshot checkout complete")
return True


def ensure_source_repo(
source_repo: Path, repo_url: str, ref: str | None = None, depth: int | None = None
) -> None:
"""Ensure a shallow checkout of ``repo_url`` exists at ``source_repo``.
"""Ensure a checkout of ``repo_url`` exists at ``source_repo``.

A fresh checkout starts from the full-history snapshot when one is set up
(``_restore_snapshot``) and is otherwise a shallow clone.

Idempotent: clones if absent, otherwise shallow-fetches and hard-resets to
the requested ``ref`` (``origin/HEAD`` when ``ref`` is None). Recovers from a
Expand Down Expand Up @@ -83,6 +162,8 @@ def ensure_source_repo(
stderr=sys.stderr,
)
return
if _restore_snapshot(source_repo, repo_url, fetch_target):
return
source_repo.mkdir(parents=True, exist_ok=True)
if ref:
# A bare clone can't fetch an arbitrary commit directly, so init an empty
Expand Down
152 changes: 150 additions & 2 deletions libs/hackbot-runtime/tests/test_source.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
"""Tests for ensure_source_repo (shallow git checkout helper)."""
"""Tests for ensure_source_repo (snapshot or shallow git checkout)."""

import subprocess
from pathlib import Path

from hackbot_runtime import ensure_source_repo
import pytest
from hackbot_runtime import ensure_source_repo, source


def _commit(path: Path, message: str) -> str:
Expand Down Expand Up @@ -130,3 +131,150 @@ def test_depth_applies_to_fresh_clone_without_ref(tmp_path):
text=True,
)
assert int(count.stdout.strip()) == 3


GITHUB_URL = "https://github.com/org/repo.git"


def _redirect_github(monkeypatch, remote: Path) -> None:
"""Send git's requests for GITHUB_URL to the local ``remote``."""
monkeypatch.setenv("GIT_CONFIG_COUNT", "1")
monkeypatch.setenv("GIT_CONFIG_KEY_0", f"url.file://{remote}.insteadOf")
monkeypatch.setenv("GIT_CONFIG_VALUE_0", GITHUB_URL)


def _snapshot(tmp_path: Path, remote: Path) -> bytes:
"""A .git tar of a full clone of ``remote``, as the daily snapshot Job makes.

Its origin is a path that no longer exists, so a restore must fetch from the
requested URL instead.
"""
clone = tmp_path / "snapshot-clone"
subprocess.run(
["git", "clone", "-q", "--no-checkout", f"file://{remote}", str(clone)],
check=True,
)
subprocess.run(
["git", "-C", str(clone), "remote", "set-url", "origin", "/gone"], check=True
)
tar = subprocess.run(
["tar", "-c", "-C", str(clone), ".git"], check=True, capture_output=True
)
return tar.stdout


def test_fresh_checkout_uses_snapshot_and_fetches_newer_commits(tmp_path, monkeypatch):
remote = tmp_path / "remote"
_make_remote(remote)
snapshot = _snapshot(tmp_path, remote)
(remote / "README.md").write_text("newer")
newer = _commit(remote, "after the snapshot")
requested = []
_redirect_github(monkeypatch, remote)
monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots")
monkeypatch.setattr(
source,
"_download",
lambda bucket, name: requested.append((bucket, name)) or iter([snapshot]),
)

dest = tmp_path / "dest"
ensure_source_repo(dest, GITHUB_URL, depth=1)

assert requested == [("snapshots", "github/org/repo.tar")]
rev = subprocess.run(
["git", "-C", str(dest), "rev-list", "HEAD"],
check=True,
capture_output=True,
text=True,
).stdout.split()
assert rev[0] == newer
assert len(rev) == 2 # full history, not the requested depth
assert (dest / "README.md").read_text() == "newer"


def test_failed_snapshot_restore_leaves_no_git_dir(tmp_path, monkeypatch):
def fail(bucket, name):
yield b"not a tar"
raise OSError("connection reset")

monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots")
monkeypatch.setattr(source, "_download", fail)
dest = tmp_path / "dest"

assert not source._restore_snapshot(dest, GITHUB_URL, "HEAD")
assert not dest.exists()


def test_failed_fetch_after_restore_falls_back_to_clone(tmp_path, monkeypatch):
remote = tmp_path / "remote"
_make_remote(remote)
snapshot = _snapshot(tmp_path, remote)
_redirect_github(monkeypatch, remote)
monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots")
monkeypatch.setattr(source, "_download", lambda bucket, name: iter([snapshot]))
real_run = subprocess.run

def fail_snapshot_fetch(cmd, **kwargs):
if cmd[-2:] == ["origin", "HEAD"] and "--depth=1" not in cmd:
raise subprocess.CalledProcessError(128, cmd)
return real_run(cmd, **kwargs)

monkeypatch.setattr(source.subprocess, "run", fail_snapshot_fetch)
dest = tmp_path / "dest"
ensure_source_repo(dest, GITHUB_URL)

assert (dest / "README.md").read_text() == "hello"
shallow = real_run(
["git", "-C", str(dest), "rev-parse", "--is-shallow-repository"],
check=True,
capture_output=True,
text=True,
)
assert shallow.stdout.strip() == "true" # the fallback clone, not the snapshot


def test_snapshot_is_skipped_for_a_non_empty_directory(tmp_path, monkeypatch):
monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots")
monkeypatch.setattr(source, "_download", lambda *a: pytest.fail("downloaded"))
(tmp_path / "keep.txt").write_text("not ours")

assert not source._restore_snapshot(tmp_path, GITHUB_URL, "HEAD")
assert (tmp_path / "keep.txt").exists()


def test_snapshot_is_skipped_without_bucket_or_for_other_hosts(monkeypatch, tmp_path):
monkeypatch.setattr(source, "_download", lambda *a: pytest.fail("downloaded"))
monkeypatch.delenv("SOURCE_SNAPSHOT_BUCKET", raising=False)
assert not source._restore_snapshot(tmp_path / "a", GITHUB_URL, "HEAD")
monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots")
assert not source._restore_snapshot(
tmp_path / "b", "https://hg.mozilla.org/x", "HEAD"
)


def test_snapshot_without_git_dir_leaves_an_enclosing_repo_alone(tmp_path, monkeypatch):
outer = tmp_path / "outer"
_make_remote(outer)
subprocess.run(
["git", "-C", str(outer), "remote", "add", "origin", "/elsewhere"], check=True
)
(tmp_path / "junk").mkdir()
(tmp_path / "junk" / "file").write_text("x")
no_git = subprocess.run(
["tar", "-c", "-C", str(tmp_path / "junk"), "file"],
check=True,
capture_output=True,
).stdout
monkeypatch.setenv("SOURCE_SNAPSHOT_BUCKET", "snapshots")
monkeypatch.setattr(source, "_download", lambda bucket, name: iter([no_git]))

assert not source._restore_snapshot(outer / "dest", GITHUB_URL, "HEAD")
origin = subprocess.run(
["git", "-C", str(outer), "remote", "get-url", "origin"],
check=True,
capture_output=True,
text=True,
)
assert origin.stdout.strip() == "/elsewhere"
assert (outer / "README.md").read_text() == "hello"