diff --git a/README.md b/README.md index ccc7755..1f5310f 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,19 @@ anything git-receive-pack -> 403 (read-only) Concurrent clients for the same repo are serialized so a burst triggers a single upstream fetch; a short TTL coalesces repeated requests. +If a client `want`s a commit by SHA that the mirror never captured - typically a +GitHub pull-request merge commit, which lives only under the unadvertised +`refs/pull//merge` and so is what `actions/checkout` fetches on a PR build - the +proxy fetches that SHA from upstream on demand and serves it, instead of failing +with `not our ref`. This relies on the upstream serving arbitrary SHAs (GitHub's +`uploadpack.allowAnySHA1InWant`, which is on); an upstream that refuses simply +leaves the request to fail as it would without the proxy. Ordinary branch/tag +clones are unaffected - they pay only a cheap local object check, never an extra +upstream call. Each fetched SHA is pinned under a reserved ref so the mirror can +keep serving it; `--max-wants` bounds how many such pins a mirror retains, pruning +the oldest so they cannot accumulate without bound (like the mirror-level LRU, but +scoped to a single mirror's pins). + ### git-LFS LFS objects use a different HTTP API from the git protocol, so they are cached @@ -173,6 +186,7 @@ Every flag has an environment-variable equivalent. | `--max-concurrent-requests` | `GITCACHEPROXY_MAX_CONCURRENT_REQUESTS` | `64` | Max concurrent in-flight requests; excess queue (`0` = unlimited) | | `--max-decoded-body-mb` | `GITCACHEPROXY_MAX_DECODED_BODY_MB` | `512` | Cap on a decoded upload-pack request body, in MiB (bounds memory / gzip bombs) | | `--cache-max-mb` | `GITCACHEPROXY_CACHE_MAX_MB` | `0` | Cap on total on-disk mirror cache, in MiB; evicts least-recently-used idle mirrors when exceeded (`0` = unlimited, no eviction) | +| `--max-wants` | `GITCACHEPROXY_MAX_WANTS` | `100` | Cap on want-by-SHA pins retained per mirror; oldest pruned beyond it (`0` = unlimited) | | `--git-binary` | `GITCACHEPROXY_GIT_BINARY` | `git` | Path to git | Endpoints: `/healthz`, `/readyz`, `/metrics` (Prometheus - per-repo request and diff --git a/src/config.rs b/src/config.rs index f0fe357..e6ca313 100644 --- a/src/config.rs +++ b/src/config.rs @@ -89,6 +89,14 @@ pub struct Config { #[arg(long, env = "GITCACHEPROXY_CACHE_MAX_MB", default_value_t = 0)] pub cache_max_mb: u64, + /// Maximum number of want-by-SHA pins a mirror retains. A client may `want` a + /// commit that lives only under an unadvertised upstream ref (e.g. a GitHub PR + /// merge commit); the proxy fetches it on demand and pins it so it can keep + /// serving it. Beyond this cap the oldest pins are pruned so they cannot + /// accumulate without bound. `0` = unlimited. + #[arg(long, env = "GITCACHEPROXY_MAX_WANTS", default_value_t = 100)] + pub max_wants: usize, + /// Path to the git binary. #[arg(long, env = "GITCACHEPROXY_GIT_BINARY", default_value = "git")] pub git_binary: String, diff --git a/src/evict.rs b/src/evict.rs index 9f935b3..a0f6f2f 100644 --- a/src/evict.rs +++ b/src/evict.rs @@ -611,6 +611,7 @@ mod tests { git_binary: "git".into(), upstream_auth_header: None, fetch_ttl: Duration::from_secs(10), + max_wants: 100, } } diff --git a/src/git.rs b/src/git.rs index 9627f46..6d5970b 100644 --- a/src/git.rs +++ b/src/git.rs @@ -12,8 +12,13 @@ //! upstream is only ever *pulled* from - nothing is pushed or replicated //! proactively. A miss transparently pulls from upstream, so the cache is never //! stale for the ref the client actually asked for. +//! +//! Want-by-SHA: a client can `want` a commit that lives only under an unadvertised +//! upstream ref (e.g. a GitHub PR merge commit), which `clone --mirror` never +//! captured. Such wants are fetched from upstream on demand and pinned under +//! `PROXY_WANTS_NS` so the mirror can serve them. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::Path; use std::pin::Pin; use std::process::Stdio; @@ -31,6 +36,9 @@ use tokio_util::io::ReaderStream; use crate::metrics::{Metrics, ServeKind, Status, UpstreamOp}; use crate::repo::RepoRef; +/// Reserved ref namespace for objects fetched by bare SHA on a want-miss. +const PROXY_WANTS_NS: &str = "refs/proxy-wants"; + /// What `ensure_fresh` did - for metrics. #[derive(Debug, Clone, Copy)] pub enum CacheOutcome { @@ -50,6 +58,8 @@ pub struct GitConfig { pub upstream_auth_header: Option, /// Skip the upstream fetch if the mirror was refreshed within this window. pub fetch_ttl: Duration, + /// Maximum want-by-SHA pins a mirror retains; oldest pruned beyond it (`0` = unlimited). + pub max_wants: usize, } /// Per-repo serialization point. `fetch_lock` is a `Mutex`, not an `RwLock`, on @@ -141,10 +151,17 @@ impl GitCache { Ok(()) } - /// Ensure the mirror exists and (when `want_fetch`) is fresh. Concurrent - /// callers for the same repo are serialized; the first does the work, the - /// rest see it already fresh. - pub async fn ensure_fresh(&self, repo: &RepoRef, want_fetch: bool) -> Result { + /// Ensure the mirror exists and (when `want_fetch`) is fresh, then satisfy any + /// `want` in `body` for a SHA the mirror never captured. Concurrent callers for + /// the same repo are serialized; the first does the work, the rest see it already + /// fresh. `body` is the client's upload-pack request (empty for `info/refs`), + /// scanned for want-by-SHA misses - see `ensure_wanted_oids`. + pub async fn ensure_fresh( + &self, + repo: &RepoRef, + want_fetch: bool, + body: &[u8], + ) -> Result { let slot = self.slot(&repo.name).await; // Mark the repo used on every request - a served cache hit counts as much as // a fetch - so the eviction index keeps a truthful last-access ordering. @@ -153,23 +170,29 @@ impl GitCache { } let mut last = slot.fetch_lock.lock().await; - if !repo.cache_dir.join("HEAD").exists() { + let outcome = if !repo.cache_dir.join("HEAD").exists() { self.clone_mirror(repo).await?; *last = Some(Instant::now()); - return Ok(CacheOutcome::Cloned); - } - if want_fetch { - let stale = match *last { + CacheOutcome::Cloned + } else if want_fetch + && match *last { None => true, Some(t) => self.cfg.fetch_ttl.is_zero() || t.elapsed() >= self.cfg.fetch_ttl, - }; - if stale { - self.fetch(repo).await?; - *last = Some(Instant::now()); - return Ok(CacheOutcome::Fetched); } + { + self.fetch(repo).await?; + *last = Some(Instant::now()); + CacheOutcome::Fetched + } else { + CacheOutcome::Cached + }; + + // Best-effort: on failure, fall through and let upload-pack surface the + // normal "not our ref" error for any want it still cannot satisfy. + if let Err(e) = self.ensure_wanted_oids(repo, body).await { + tracing::warn!(repo = %repo.name, error = %e, "ensure wanted oids failed"); } - Ok(CacheOutcome::Cached) + Ok(outcome) } /// `git upload-pack --advertise-refs`, wrapped with the smart-HTTP service @@ -333,6 +356,10 @@ impl GitCache { .arg("--prune") .arg("--quiet") .arg("origin") + // Mirror refspec, minus the local-only pin namespace: `--prune` would + // otherwise drop those refs as "not on origin" (see `prune_wants`). + .arg("+refs/*:refs/*") + .arg(format!("^{PROXY_WANTS_NS}/*")) .status() .await .context("spawn git fetch")?; @@ -352,6 +379,175 @@ impl GitCache { Ok(()) } + /// Fetch and pin under `PROXY_WANTS_NS` any `want`ed object the mirror lacks - a + /// commit reachable only by bare SHA, e.g. a GitHub PR merge commit. A no-op + /// unless the body names a missing object, so an ordinary clone pays only a cheap + /// `cat-file` check. Relies on the upstream serving arbitrary SHAs + /// (`uploadpack.allowAnySHA1InWant`). + async fn ensure_wanted_oids(&self, repo: &RepoRef, body: &[u8]) -> Result<()> { + let wants = parse_wants(body); + if wants.is_empty() { + return Ok(()); + } + let mut missing = self.missing_oids(repo, &wants).await?; + if missing.is_empty() { + return Ok(()); + } + // Never fetch more than the mirror will retain: anything past the pin cap + // would be pruned straight away (see `prune_wants`), so fetching it is pure + // waste. The excess is dropped and upload-pack rejects those wants. + if self.cfg.max_wants > 0 && missing.len() > self.cfg.max_wants { + tracing::warn!( + repo = %repo.name, + requested = missing.len(), + cap = self.cfg.max_wants, + "capping want-by-sha fetch; excess wants left unserved" + ); + missing.truncate(self.cfg.max_wants); + } + tracing::info!( + repo = %repo.name, + count = missing.len(), + "fetching want-by-sha objects missing from mirror (e.g. PR merge refs)" + ); + let started = Instant::now(); + let mut cmd = self.fetch_cmd(); + cmd.current_dir(&repo.cache_dir) + .arg("fetch") + .arg("--no-tags") + .arg("--quiet") + .arg("origin"); + for oid in &missing { + // Pin under a reserved ref (force: the ref name is the SHA and may + // already exist), making the object a `want` tip safe from `git gc`. + cmd.arg(format!("+{oid}:{PROXY_WANTS_NS}/{oid}")); + } + let status = cmd + .status() + .await + .context("spawn git fetch (want-by-sha)")?; + if !status.success() { + // Upstream would not serve one of these SHAs (e.g. arbitrary-SHA fetches + // are disabled). Leave it: upload-pack will reject the want with "not + // our ref". + self.metrics + .record_upstream(UpstreamOp::WantFetch, Status::Error, "-"); + tracing::warn!(repo = %repo.name, "want-by-sha fetch from upstream failed"); + return Ok(()); + } + self.metrics + .record_upstream(UpstreamOp::WantFetch, Status::Ok, &repo.name); + self.metrics.observe_upstream( + UpstreamOp::WantFetch, + &repo.name, + started.elapsed().as_secs_f64(), + ); + self.mark_changed(repo); + // Best-effort: a failed prune is not worth failing the request over. + if let Err(e) = self.prune_wants(repo).await { + tracing::warn!(repo = %repo.name, error = %e, "pruning want-by-sha pins failed"); + } + Ok(()) + } + + /// Cap how many want-by-SHA pins a mirror keeps, deleting the oldest by creation + /// date beyond `max_wants`. Pins are excluded from `fetch --prune` and `git gc`, + /// so this is what bounds their growth - the cache-size LRU scoped to a mirror's + /// pins. A dropped pin is re-fetched on the next want-miss for its SHA. + async fn prune_wants(&self, repo: &RepoRef) -> Result<()> { + if self.cfg.max_wants == 0 { + return Ok(()); + } + let out = Command::new(&self.cfg.git_binary) + .current_dir(&repo.cache_dir) + .arg("for-each-ref") + .arg("--sort=-creatordate") + .arg("--format=%(refname)") + .arg(format!("{PROXY_WANTS_NS}/")) + .output() + .await + .context("spawn git for-each-ref (proxy-wants)")?; + if !out.status.success() { + bail!("git for-each-ref failed for {}", repo.name); + } + let listing = String::from_utf8_lossy(&out.stdout); + let refs: Vec<&str> = listing.lines().collect(); + if refs.len() <= self.cfg.max_wants { + return Ok(()); + } + // `update-ref --stdin` deletes the stale tail in one atomic transaction. + let mut deletions = String::new(); + for r in &refs[self.cfg.max_wants..] { + deletions.push_str("delete "); + deletions.push_str(r); + deletions.push('\n'); + } + let mut child = Command::new(&self.cfg.git_binary) + .current_dir(&repo.cache_dir) + .arg("update-ref") + .arg("--stdin") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .context("spawn git update-ref --stdin")?; + child + .stdin + .take() + .context("update-ref: no stdin")? + .write_all(deletions.as_bytes()) + .await + .context("write update-ref deletions")?; + let status = child.wait().await.context("git update-ref --stdin")?; + if !status.success() { + bail!("git update-ref --stdin failed for {}", repo.name); + } + tracing::info!( + repo = %repo.name, + pruned = refs.len() - self.cfg.max_wants, + "pruned oldest want-by-sha pins over cap" + ); + Ok(()) + } + + /// Which of `oids` are absent from the mirror's object store, via a single + /// `git cat-file --batch-check` (prints ` missing` for an absent object). + /// A present object is serveable: it entered the mirror via a real ref or a pin. + async fn missing_oids(&self, repo: &RepoRef, oids: &HashSet) -> Result> { + let mut child = Command::new(&self.cfg.git_binary) + .current_dir(&repo.cache_dir) + .arg("cat-file") + .arg("--batch-check") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .context("spawn git cat-file --batch-check")?; + let mut stdin = child.stdin.take().context("cat-file: no stdin")?; + let mut query = String::with_capacity(oids.len() * 41); + for oid in oids { + query.push_str(oid); + query.push('\n'); + } + stdin + .write_all(query.as_bytes()) + .await + .context("write cat-file query")?; + drop(stdin); // EOF so cat-file finishes and exits + let out = child + .wait_with_output() + .await + .context("git cat-file --batch-check")?; + if !out.status.success() { + bail!("git cat-file --batch-check failed for {}", repo.name); + } + Ok(String::from_utf8_lossy(&out.stdout) + .lines() + .filter_map(|l| l.strip_suffix(" missing")) + .map(str::to_string) + .collect()) + } + /// Command for upstream operations (clone/fetch): injects the auth header via /// env-based git config so the token stays out of argv. fn fetch_cmd(&self) -> Command { @@ -369,6 +565,10 @@ impl GitCache { /// the client's protocol version so v2 clients get a v2 advertisement. fn local_cmd(&self, git_protocol: Option<&str>) -> Command { let mut c = Command::new(&self.cfg.git_binary); + // Hide the pin namespace from the advertisement so it never leaks into a + // client's ref list; a hidden ref is still honored as a `want` tip. + c.arg("-c") + .arg(format!("uploadpack.hideRefs={PROXY_WANTS_NS}")); if let Some(p) = git_protocol { c.env("GIT_PROTOCOL", p); } @@ -432,16 +632,114 @@ fn pkt_line(s: &str) -> Vec { v } +/// Extract the unique object IDs from the `want` lines of an `upload-pack` request +/// body (protocol v0/v1: `want [capabilities]`; v2: `want ` among the +/// fetch-command args). The body is pkt-line framed, so we walk the frames rather +/// than scanning raw bytes - a `want ` byte sequence inside packed negotiation data +/// must not be mistaken for a request line. Malformed framing stops the walk, +/// missing at worst a want that upload-pack then rejects. +fn parse_wants(body: &[u8]) -> HashSet { + let mut oids: HashSet = HashSet::new(); + let mut pos = 0; + while pos + 4 <= body.len() { + let Some(len) = std::str::from_utf8(&body[pos..pos + 4]) + .ok() + .and_then(|h| usize::from_str_radix(h, 16).ok()) + else { + break; // not a hex length prefix: malformed, stop + }; + // 0000 flush / 0001 delim / 0002 response-end carry no payload. + if len < 4 { + pos += 4; + continue; + } + let end = pos + len; + if end > body.len() { + break; // truncated frame + } + if let Some(rest) = body[pos + 4..end].strip_prefix(b"want ") { + let oid: Vec = rest + .iter() + .copied() + .take_while(u8::is_ascii_hexdigit) + .collect(); + // sha1 (40) or sha256 (64); ignore anything else (e.g. a stray token). + if oid.len() == 40 || oid.len() == 64 { + oids.insert(String::from_utf8(oid).expect("ascii hex is valid utf-8")); + } + } + pos = end; + } + oids +} + #[cfg(test)] mod tests { use super::*; + /// Collect an iterator of oids into a `HashSet` for order-insensitive assertions + /// against `parse_wants`. + fn want_set>(oids: I) -> HashSet { + oids.into_iter().collect() + } + #[test] fn pkt_line_encodes_length() { assert_eq!(pkt_line("a"), b"0005a"); assert_eq!(&pkt_line("# service=git-upload-pack\n")[..4], b"001e"); } + #[test] + fn parse_wants_extracts_oids_from_both_protocol_versions() { + let sha_a = "a".repeat(40); + let sha_b = "b".repeat(40); + // Protocol v0/v1: first want carries capabilities after the oid. + let mut v1 = Vec::new(); + v1.extend_from_slice(&pkt_line(&format!( + "want {sha_a} multi_ack side-band-64k\n" + ))); + v1.extend_from_slice(&pkt_line(&format!("want {sha_b}\n"))); + v1.extend_from_slice(b"0000"); + assert_eq!(parse_wants(&v1), want_set([sha_a.clone(), sha_b.clone()])); + + // Protocol v2: a fetch command with want args framed by a delim-pkt. + let mut v2 = Vec::new(); + v2.extend_from_slice(&pkt_line("command=fetch\n")); + v2.extend_from_slice(b"0001"); + v2.extend_from_slice(&pkt_line(&format!("want {sha_a}\n"))); + v2.extend_from_slice(&pkt_line("have cccccccccccccccccccccccccccccccccccccccc\n")); + v2.extend_from_slice(&pkt_line("done\n")); + v2.extend_from_slice(b"0000"); + assert_eq!(parse_wants(&v2), want_set([sha_a.clone()])); + } + + #[test] + fn parse_wants_dedups_and_ignores_non_wants() { + let sha = "0123456789abcdef0123456789abcdef01234567".to_string(); + let mut body = Vec::new(); + body.extend_from_slice(&pkt_line(&format!("want {sha}\n"))); + body.extend_from_slice(&pkt_line(&format!("want {sha}\n"))); // duplicate + body.extend_from_slice(&pkt_line("command=ls-refs\n")); // not a want + body.extend_from_slice(b"0000"); + assert_eq!(parse_wants(&body), want_set([sha])); + + // A request with no want lines (e.g. a bare ls-refs) yields nothing, so the + // caller skips the cat-file check and any upstream call entirely. + let mut ls = Vec::new(); + ls.extend_from_slice(&pkt_line("command=ls-refs\n")); + ls.extend_from_slice(b"0000"); + assert!(parse_wants(&ls).is_empty()); + } + + #[test] + fn parse_wants_tolerates_malformed_framing() { + assert!(parse_wants(b"").is_empty()); + assert!(parse_wants(b"xyz").is_empty()); // too short for a length prefix + assert!(parse_wants(b"zzzz").is_empty()); // non-hex length prefix + // A length that runs past the buffer is ignored rather than panicking. + assert!(parse_wants(b"0099want ").is_empty()); + } + #[tokio::test] async fn timed_reader_records_serve_duration_at_eof() { use tokio::io::AsyncReadExt; diff --git a/src/main.rs b/src/main.rs index 2e21204..99ee4e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -51,6 +51,7 @@ async fn main() -> Result<()> { git_binary: cfg.git_binary.clone(), upstream_auth_header: upstream_auth_header.clone(), fetch_ttl: Duration::from_secs(cfg.fetch_ttl_seconds), + max_wants: cfg.max_wants, }; let metrics = Arc::new(metrics::Metrics::new()); diff --git a/src/metrics.rs b/src/metrics.rs index 618cb27..1ea516e 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -48,6 +48,8 @@ pub enum Status { pub enum UpstreamOp { Clone, Fetch, + /// On-demand fetch of a bare SHA the mirror lacks (e.g. a PR merge commit). + WantFetch, } /// The `kind` label on `serve_duration_seconds`. @@ -95,6 +97,7 @@ impl UpstreamOp { match self { Self::Clone => "clone", Self::Fetch => "fetch", + Self::WantFetch => "want_fetch", } } } @@ -125,8 +128,8 @@ pub struct Metrics { /// upstream_error | unauthorized | rejected; repo = the served repo path when /// result = ok, else `-`. requests: IntCounterVec, - /// `upstream_ops_total{op, result, repo}` - op = clone | fetch; result = ok | - /// error; repo = the repo path when result = ok, else `-`. + /// `upstream_ops_total{op, result, repo}` - op = clone | fetch | want_fetch; + /// result = ok | error; repo = the repo path when result = ok, else `-`. upstream: IntCounterVec, /// `cache_bytes` - total size of the on-disk mirror cache, maintained /// incrementally as mirrors are added, refreshed, and evicted. Populated only diff --git a/src/server.rs b/src/server.rs index 3db5dd1..704c532 100644 --- a/src/server.rs +++ b/src/server.rs @@ -336,7 +336,7 @@ async fn info_refs(st: AppState, path: &str, git_protocol: Option<&str>) -> Resp // here we only account for the client request. The `repo` label is emitted only // once a request is served; failures use `-` so a flood of distinct but doomed // repo paths cannot inflate label cardinality (see `metrics`). - if let Err(e) = st.cache.ensure_fresh(&repo, true).await { + if let Err(e) = st.cache.ensure_fresh(&repo, true, &[]).await { st.metrics .record_request(RequestKind::InfoRefs, Status::UpstreamError, "-"); tracing::warn!(repo = %name, error = %e, "ensure_fresh failed"); @@ -386,8 +386,9 @@ async fn upload_pack( }; // The preceding info/refs already refreshed; here just ensure the mirror is - // present (a client could POST against a not-yet-cloned repo). - if let Err(e) = st.cache.ensure_fresh(&repo, false).await { + // present (a client could POST against a not-yet-cloned repo) and that any + // want-by-SHA the body asks for is fetched and pinned before serving. + if let Err(e) = st.cache.ensure_fresh(&repo, false, &body).await { st.metrics .record_request(RequestKind::UploadPack, Status::UpstreamError, "-"); tracing::warn!(repo = %name, error = %e, "ensure mirror exists failed"); diff --git a/tests/e2e.rs b/tests/e2e.rs index f17235f..b0d60c4 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -87,6 +87,7 @@ async fn upload_pack_decodes_gzip_encoded_request() { git_binary: "git".into(), upstream_auth_header: None, fetch_ttl: Duration::from_secs(0), + max_wants: 100, }; let state = AppState { cache: Arc::new(GitCache::new(cfg, metrics.clone(), None)), @@ -145,6 +146,514 @@ async fn upload_pack_decodes_gzip_encoded_request() { ); } +/// Regression test for GitHub PR checkouts: `actions/checkout` fetches the merge +/// commit by bare SHA, but that commit lives only under the unadvertised +/// `refs/pull//merge`, so `clone --mirror` never captured it and upload-pack +/// used to reject the `want` with "not our ref". The proxy must fetch the missing +/// SHA from upstream on demand and serve it. +/// +/// The `file://` upstream here mimics GitHub: the pull ref is hidden from the +/// advertisement (`uploadpack.hideRefs`) yet fetchable by bare SHA +/// (`uploadpack.allowAnySHA1InWant`). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn serves_want_by_sha_for_unadvertised_pull_ref() { + // --- Upstream: a main branch, plus a merge commit reachable from no branch, + // stored only under the (soon-to-be-hidden) refs/pull/1/merge. --- + let work = tempfile::tempdir().unwrap(); + git(work.path(), &["init", "-q", "-b", "main", "."]); + std::fs::write(work.path().join("README.md"), "base\n").unwrap(); + git(work.path(), &["add", "."]); + git(work.path(), &["commit", "-q", "-m", "base"]); + + let up = tempfile::tempdir().unwrap(); + let upstream_repo = up.path().join("repo.git"); + git( + work.path(), + &[ + "clone", + "-q", + "--mirror", + ".", + upstream_repo.to_str().unwrap(), + ], + ); + + // A distinct merge-only commit, pushed to refs/pull/1/merge, then rewound off + // main so it is reachable from no advertised ref. + std::fs::write(work.path().join("pr.txt"), "merge\n").unwrap(); + git(work.path(), &["add", "."]); + git(work.path(), &["commit", "-q", "-m", "pr-merge"]); + let merge_sha = git(work.path(), &["rev-parse", "HEAD"]).trim().to_string(); + git( + work.path(), + &[ + "push", + "-q", + upstream_repo.to_str().unwrap(), + "HEAD:refs/pull/1/merge", + ], + ); + git(work.path(), &["reset", "-q", "--hard", "HEAD~1"]); + + // Make the bare upstream behave like GitHub for pull refs. + git( + &upstream_repo, + &["config", "uploadpack.hideRefs", "refs/pull"], + ); + git( + &upstream_repo, + &["config", "uploadpack.allowAnySHA1InWant", "true"], + ); + + // --- Proxy over the file:// upstream. --- + let cache = tempfile::tempdir().unwrap(); + let metrics = Arc::new(Metrics::new()); + let cfg = GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + fetch_ttl: Duration::from_secs(0), + max_wants: 100, + }; + let state = AppState { + cache: Arc::new(GitCache::new(cfg, metrics.clone(), None)), + lfs: Arc::new(Lfs::new( + LfsConfig { + upstream_base: format!("file://{}", up.path().display()), + cache_root: cache.path().to_path_buf(), + upstream_auth_header: None, + serve_token: None, + }, + None, + )), + upstream_base: format!("file://{}", up.path().display()), + cache_root: cache.path().to_path_buf(), + serve_token: None, + max_decoded_body: 512 * 1024 * 1024, + max_concurrent: 8, + metrics: metrics.clone(), + }; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, router(state)).await.unwrap(); + }); + + // Prime the mirror (a normal clone) and confirm the advertisement never leaks + // the merge commit - it lives only under the hidden pull ref upstream. + let dest = tempfile::tempdir().unwrap(); + let checkout = dest.path().join("checkout"); + let out = tokio::process::Command::new("git") + .arg("clone") + .arg("-q") + .arg(format!("http://{addr}/repo.git")) + .arg(&checkout) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .await + .expect("spawn git clone"); + assert!( + out.status.success(), + "initial clone failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Now fetch the merge commit by bare SHA, exactly as actions/checkout does on a + // PR build. Before the fix this failed with "not our ref". + let out = tokio::process::Command::new("git") + .current_dir(&checkout) + .arg("fetch") + .arg("-q") + .arg(format!("http://{addr}/repo.git")) + .arg(&merge_sha) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .await + .expect("spawn git fetch by sha"); + assert!( + out.status.success(), + "fetch of PR merge commit by sha failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // The fetched object is really present in the client now. + let ty = git(&checkout, &["cat-file", "-t", &merge_sha]); + assert_eq!(ty.trim(), "commit", "merge commit should be fetched"); + + // The on-demand fetch was recorded as a distinct upstream op for the repo. + let scraped = metrics.gather(); + assert!( + scraped.contains(r#"op="want_fetch",repo="repo.git",result="ok""#), + "missing want_fetch metric:\n{scraped}" + ); + + // A second bare-SHA fetch is served straight from the pinned mirror ref: no new + // want_fetch op is recorded (the object is already present). + let before = want_fetch_ok_count(&scraped); + let out = tokio::process::Command::new("git") + .current_dir(&checkout) + .arg("fetch") + .arg("-q") + .arg(format!("http://{addr}/repo.git")) + .arg(&merge_sha) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .await + .expect("spawn second git fetch by sha"); + assert!(out.status.success(), "second bare-sha fetch failed"); + assert_eq!( + want_fetch_ok_count(&metrics.gather()), + before, + "a cached want should not trigger another upstream want_fetch" + ); + + server.abort(); +} + +/// The value of the `want_fetch` success counter in a metrics scrape, or 0. +fn want_fetch_ok_count(scraped: &str) -> u64 { + scraped + .lines() + .find_map(|l| { + l.strip_prefix( + r#"gitcacheproxy_upstream_ops_total{op="want_fetch",repo="repo.git",result="ok"} "#, + ) + }) + .and_then(|n| n.trim().parse().ok()) + .unwrap_or(0) +} + +/// When the wanted SHA cannot be obtained from upstream (here: it does not exist +/// at all), the on-demand fetch fails: the proxy records a `want_fetch` error and +/// leaves upload-pack to reject the want, exactly as it would without this feature. +/// This is the graceful-degradation path - no crash, no hang, no regression. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn want_by_sha_fetch_failure_is_recorded_and_degrades_gracefully() { + let work = tempfile::tempdir().unwrap(); + git(work.path(), &["init", "-q", "-b", "main", "."]); + std::fs::write(work.path().join("README.md"), "base\n").unwrap(); + git(work.path(), &["add", "."]); + git(work.path(), &["commit", "-q", "-m", "base"]); + + let up = tempfile::tempdir().unwrap(); + let upstream_repo = up.path().join("repo.git"); + git( + work.path(), + &[ + "clone", + "-q", + "--mirror", + ".", + upstream_repo.to_str().unwrap(), + ], + ); + + // A well-formed sha that no upstream object matches: the on-demand fetch for it + // must fail rather than invent it. + let bogus_sha = "0123456789abcdef0123456789abcdef01234567".to_string(); + + let cache = tempfile::tempdir().unwrap(); + let metrics = Arc::new(Metrics::new()); + let cfg = GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + fetch_ttl: Duration::from_secs(0), + max_wants: 100, + }; + let state = AppState { + cache: Arc::new(GitCache::new(cfg, metrics.clone(), None)), + lfs: Arc::new(Lfs::new( + LfsConfig { + upstream_base: format!("file://{}", up.path().display()), + cache_root: cache.path().to_path_buf(), + upstream_auth_header: None, + serve_token: None, + }, + None, + )), + upstream_base: format!("file://{}", up.path().display()), + cache_root: cache.path().to_path_buf(), + serve_token: None, + max_decoded_body: 512 * 1024 * 1024, + max_concurrent: 8, + metrics: metrics.clone(), + }; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, router(state)).await.unwrap(); + }); + + let dest = tempfile::tempdir().unwrap(); + let checkout = dest.path().join("checkout"); + let out = tokio::process::Command::new("git") + .arg("clone") + .arg("-q") + .arg(format!("http://{addr}/repo.git")) + .arg(&checkout) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .await + .expect("spawn git clone"); + assert!(out.status.success(), "initial clone failed"); + + // The bare-SHA fetch fails (the proxy could not obtain the object), rather than + // hanging or 500-ing. + let out = tokio::process::Command::new("git") + .current_dir(&checkout) + .arg("fetch") + .arg("-q") + .arg(format!("http://{addr}/repo.git")) + .arg(&bogus_sha) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .await + .expect("spawn git fetch by sha"); + assert!( + !out.status.success(), + "fetch of an unobtainable sha should fail, not succeed" + ); + + // The failed on-demand fetch was recorded as a want_fetch error. + let scraped = metrics.gather(); + assert!( + scraped.contains(r#"op="want_fetch",repo="-",result="error""#), + "missing want_fetch error metric:\n{scraped}" + ); + + server.abort(); +} + +/// Want-by-SHA pins are excluded from `fetch --prune` and `git gc`, so left +/// unchecked they would grow without bound. `max_wants` caps how many a mirror +/// keeps: fetching a second merge SHA past a cap of 1 must evict the older pin, so +/// the reserved namespace never accumulates beyond the cap. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn want_by_sha_pins_are_capped_per_mirror() { + let work = tempfile::tempdir().unwrap(); + git(work.path(), &["init", "-q", "-b", "main", "."]); + std::fs::write(work.path().join("README.md"), "base\n").unwrap(); + git(work.path(), &["add", "."]); + git(work.path(), &["commit", "-q", "-m", "base"]); + + let up = tempfile::tempdir().unwrap(); + let upstream_repo = up.path().join("repo.git"); + git( + work.path(), + &[ + "clone", + "-q", + "--mirror", + ".", + upstream_repo.to_str().unwrap(), + ], + ); + + // Two distinct merge-only commits under unadvertised pull refs, each reachable + // from no branch - the shape `actions/checkout` fetches by bare SHA. + let mut merge_shas = Vec::new(); + for n in 1..=2 { + std::fs::write(work.path().join("pr.txt"), format!("merge {n}\n")).unwrap(); + git(work.path(), &["add", "."]); + git( + work.path(), + &["commit", "-q", "-m", &format!("pr-merge-{n}")], + ); + merge_shas.push(git(work.path(), &["rev-parse", "HEAD"]).trim().to_string()); + git( + work.path(), + &[ + "push", + "-q", + upstream_repo.to_str().unwrap(), + &format!("HEAD:refs/pull/{n}/merge"), + ], + ); + git(work.path(), &["reset", "-q", "--hard", "HEAD~1"]); + } + git( + &upstream_repo, + &["config", "uploadpack.hideRefs", "refs/pull"], + ); + git( + &upstream_repo, + &["config", "uploadpack.allowAnySHA1InWant", "true"], + ); + + let cache = tempfile::tempdir().unwrap(); + let metrics = Arc::new(Metrics::new()); + let cfg = GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + fetch_ttl: Duration::from_secs(0), + max_wants: 1, // keep at most one pin per mirror + }; + let state = AppState { + cache: Arc::new(GitCache::new(cfg, metrics.clone(), None)), + lfs: Arc::new(Lfs::new( + LfsConfig { + upstream_base: format!("file://{}", up.path().display()), + cache_root: cache.path().to_path_buf(), + upstream_auth_header: None, + serve_token: None, + }, + None, + )), + upstream_base: format!("file://{}", up.path().display()), + cache_root: cache.path().to_path_buf(), + serve_token: None, + max_decoded_body: 512 * 1024 * 1024, + max_concurrent: 8, + metrics: metrics.clone(), + }; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, router(state)).await.unwrap(); + }); + + let dest = tempfile::tempdir().unwrap(); + let checkout = dest.path().join("checkout"); + let out = tokio::process::Command::new("git") + .arg("clone") + .arg("-q") + .arg(format!("http://{addr}/repo.git")) + .arg(&checkout) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .await + .expect("spawn git clone"); + assert!(out.status.success(), "initial clone failed"); + + // Fetch both merge commits by bare SHA, in order. Each pins a reserved ref; the + // second push over the cap must evict the first. + for sha in &merge_shas { + let out = tokio::process::Command::new("git") + .current_dir(&checkout) + .arg("fetch") + .arg("-q") + .arg(format!("http://{addr}/repo.git")) + .arg(sha) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .await + .expect("spawn git fetch by sha"); + assert!( + out.status.success(), + "bare-sha fetch failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + + // The mirror retains exactly one pin - the cap - not one per fetched SHA. + let mirror = cache.path().join("repo.git"); + let pins = git( + &mirror, + &["for-each-ref", "--format=%(refname)", "refs/proxy-wants/"], + ); + let pins: Vec<&str> = pins.lines().collect(); + assert_eq!( + pins.len(), + 1, + "pins should be capped at max_wants, got: {pins:?}" + ); + + server.abort(); +} + +/// A single request that `want`s more missing SHAs than `max_wants` must fetch and +/// pin only up to the cap, never the whole set - so one client cannot make a mirror +/// grow without bound in one go. Drives `ensure_fresh` directly with a two-`want` +/// body against a cap of one. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn caps_want_by_sha_fetch_within_a_single_request() { + use git_cache_proxy::repo; + + let work = tempfile::tempdir().unwrap(); + git(work.path(), &["init", "-q", "-b", "main", "."]); + std::fs::write(work.path().join("README.md"), "base\n").unwrap(); + git(work.path(), &["add", "."]); + git(work.path(), &["commit", "-q", "-m", "base"]); + + let up = tempfile::tempdir().unwrap(); + let upstream_repo = up.path().join("repo.git"); + git( + work.path(), + &[ + "clone", + "-q", + "--mirror", + ".", + upstream_repo.to_str().unwrap(), + ], + ); + + // Two merge-only commits under unadvertised pull refs. + let mut merge_shas = Vec::new(); + for n in 1..=2 { + std::fs::write(work.path().join("pr.txt"), format!("merge {n}\n")).unwrap(); + git(work.path(), &["add", "."]); + git( + work.path(), + &["commit", "-q", "-m", &format!("pr-merge-{n}")], + ); + merge_shas.push(git(work.path(), &["rev-parse", "HEAD"]).trim().to_string()); + git( + work.path(), + &[ + "push", + "-q", + upstream_repo.to_str().unwrap(), + &format!("HEAD:refs/pull/{n}/merge"), + ], + ); + git(work.path(), &["reset", "-q", "--hard", "HEAD~1"]); + } + // Hide the pull refs so the proxy's mirror clone never captures them - only then + // are the merge commits missing and reachable solely by bare SHA. + git( + &upstream_repo, + &["config", "uploadpack.hideRefs", "refs/pull"], + ); + git( + &upstream_repo, + &["config", "uploadpack.allowAnySHA1InWant", "true"], + ); + + let cache = tempfile::tempdir().unwrap(); + let metrics = Arc::new(Metrics::new()); + let cfg = GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + fetch_ttl: Duration::from_secs(0), + max_wants: 1, + }; + let gitcache = GitCache::new(cfg, metrics, None); + let upstream_base = format!("file://{}", up.path().display()); + let repo = repo::resolve("repo.git", &upstream_base, cache.path()).unwrap(); + + // A single upload-pack body wanting both missing merge commits at once. + let mut body = Vec::new(); + body.extend_from_slice(&pkt(&format!("want {}\n", merge_shas[0]))); + body.extend_from_slice(&pkt(&format!("want {}\n", merge_shas[1]))); + body.extend_from_slice(b"0000"); + gitcache.ensure_fresh(&repo, false, &body).await.unwrap(); + + // Only one of the two wanted SHAs was pinned - the per-request cap held. + let mirror = cache.path().join("repo.git"); + let pins = git( + &mirror, + &["for-each-ref", "--format=%(refname)", "refs/proxy-wants/"], + ); + assert_eq!( + pins.lines().count(), + 1, + "one request must not pin more than max_wants, got: {pins:?}" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn clones_through_proxy_serves_all_refs_and_rejects_push() { // --- Build an upstream bare repo with a branch and a tag, not just HEAD. --- @@ -177,6 +686,7 @@ async fn clones_through_proxy_serves_all_refs_and_rejects_push() { git_binary: "git".into(), upstream_auth_header: None, fetch_ttl: Duration::from_secs(0), + max_wants: 100, }; // Eviction enabled with an effectively unbounded cap: no mirror is ever // evicted, but the on-request index bookkeeping (touch on serve, mark-changed diff --git a/tests/http.rs b/tests/http.rs index 98f1a6a..00dd2bd 100644 --- a/tests/http.rs +++ b/tests/http.rs @@ -25,6 +25,7 @@ fn state(serve_token: Option) -> AppState { git_binary: "git".into(), upstream_auth_header: None, fetch_ttl: Duration::from_secs(10), + max_wants: 100, }; let lfs = Arc::new(Lfs::new( LfsConfig { @@ -212,6 +213,7 @@ async fn upstream_failure_returns_bad_gateway_and_records_error() { git_binary: "git".into(), upstream_auth_header: None, fetch_ttl: Duration::from_secs(10), + max_wants: 100, }; let lfs = Arc::new(Lfs::new( LfsConfig { diff --git a/tests/lfs.rs b/tests/lfs.rs index 7f9380c..46e4b3d 100644 --- a/tests/lfs.rs +++ b/tests/lfs.rs @@ -337,6 +337,7 @@ fn proxy_state(addr: SocketAddr, cache: &std::path::Path, metrics: Arc) git_binary: "git".into(), upstream_auth_header: None, fetch_ttl: Duration::from_secs(10), + max_wants: 100, }; AppState { cache: Arc::new(GitCache::new(cfg, metrics.clone(), Some(idx.clone()))),