From 808d1504306d7d72c94518baddc7d8e4d9475ae6 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Sat, 7 Nov 2026 17:48:43 +0800 Subject: [PATCH 1/2] feat(cache): read several remote cache entries per key Vite+'s Cloudflare remote cache now keeps a key's newest entries instead of only the last one (voidzero-dev/vite-plus-remote-cache-cloudflare#47). An exact fetch returns them newest first: { kind: "exact", entries: [{ value, blob_id }] } - The client decodes the list. An empty list, or the single entry that earlier servers sent, is a malformed response. - A lookup checks the entries newest first and restores the first one that passes validation. If none does, the miss is the newest entry's: its fingerprint mismatch, or the read failure if it doesn't decode. - The e2e backend is that pull request's head. The tap's request lines list the entries of an exact fetch by blob number, newest first. Co-Authored-By: Claude Opus 5.5 --- crates/vt/src/session/cache/remote.rs | 152 +++++++++++++++--- .../remote_cache/snapshots/corrupt_archive.md | 2 +- .../snapshots/local_and_remote_hits.md | 2 +- .../fixtures/remote_cache/snapshots/read.md | 2 +- .../remote_cache/snapshots/restore.md | 4 +- .../remote_cache/snapshots/restore_failure.md | 4 +- crates/vt_remote_cache/README.md | 2 +- crates/vt_remote_cache/src/lib.rs | 97 ++++++----- packages/tools/README.md | 4 +- packages/tools/package.json | 2 +- packages/tools/src/remote-cache/daemon.ts | 26 ++- pnpm-lock.yaml | 20 +-- 12 files changed, 233 insertions(+), 84 deletions(-) diff --git a/crates/vt/src/session/cache/remote.rs b/crates/vt/src/session/cache/remote.rs index 301c288c0..852fae067 100644 --- a/crates/vt/src/session/cache/remote.rs +++ b/crates/vt/src/session/cache/remote.rs @@ -29,7 +29,7 @@ use vt_plan::{ remote_cache::{RemoteCacheAuth, ResolvedRemoteCacheConfig}, }; use vt_remote_cache::{ - Client, Download, Fetched, + Client, Download, Entry, Fetched, auth::{Anonymous, Auth, GithubOidc}, }; use vt_str::Str; @@ -100,11 +100,14 @@ pub(super) struct Restore { pub blob_id: Option, } -/// Turn the result of a fetch into an entry to restore or a miss. `validate` -/// checks an exact entry against the current execution. A fallback's miss -/// reason compares its key with `cache_key`, and no match is a miss without -/// an entry. A failed fetch, an entry that doesn't decode or whose blob -/// doesn't match its value, or a validation error is a read failure. +/// Turn the result of a fetch into an entry to restore or a miss. An exact +/// match's entries are checked newest first, and the first that passes is +/// restored. `validate` checks an entry against the current execution. If no +/// entry passes, the miss is the newest entry's: a fingerprint mismatch, or a +/// read failure if it doesn't decode, its blob doesn't match its value, or it +/// can't be validated. A fallback's miss reason compares its key with +/// `cache_key`, and no match is a miss without an entry. A failed fetch is a +/// read failure. #[expect( clippy::result_large_err, reason = "`CacheMiss` is intentionally large, and a lookup returns it once" @@ -112,23 +115,43 @@ pub(super) struct Restore { pub(super) fn resolve( fetched: Result, ReadError>, cache_key: &CacheEntryKey, - validate: impl FnOnce(&CacheEntryValue) -> anyhow::Result>, + mut validate: impl FnMut(&CacheEntryValue) -> anyhow::Result>, ) -> Result { - let (value, blob_id) = match fetched.map_err(ReadError::into_miss)? { - Some(Fetched::Exact { value, blob_id }) => { - let value: CacheEntryValue = deserialize_cache(&value) - .map_err(|err| ReadError::CorruptValue(err).into_miss())?; - if value.output_archive.is_some() != blob_id.is_some() { - return Err(ReadError::MismatchedBlob.into_miss()); - } - (value, blob_id) - } + let entries = match fetched.map_err(ReadError::into_miss)? { + Some(Fetched::Exact { entries }) => entries, Some(Fetched::Fallback { key }) => { let key = decode_key(&key).map_err(ReadError::into_miss)?; return Err(CacheMiss::FingerprintMismatch(key.into_mismatch(cache_key))); } None => return Err(CacheMiss::NotFound), }; + let mut newest_miss = None; + for entry in entries { + match check_entry(entry, &mut validate) { + Ok(restore) => return Ok(restore), + Err(miss) => { + newest_miss.get_or_insert(miss); + } + } + } + // An exact match has at least one entry, so there is a newest miss. + Err(newest_miss.unwrap_or(CacheMiss::NotFound)) +} + +/// Check one entry of an exact match, as [`resolve`] describes. +#[expect( + clippy::result_large_err, + reason = "`CacheMiss` is intentionally large, and a lookup checks only a few entries" +)] +fn check_entry( + Entry { value, blob_id }: Entry, + validate: &mut impl FnMut(&CacheEntryValue) -> anyhow::Result>, +) -> Result { + let value: CacheEntryValue = + deserialize_cache(&value).map_err(|err| ReadError::CorruptValue(err).into_miss())?; + if value.output_archive.is_some() != blob_id.is_some() { + return Err(ReadError::MismatchedBlob.into_miss()); + } match validate(&value) { Ok(None) => Ok(Restore { value, blob_id }), Ok(Some(mismatch)) => Err(CacheMiss::FingerprintMismatch(mismatch)), @@ -167,8 +190,8 @@ impl RemoteClients { Ok(client) } - /// Fetch the entry stored under `cache_key` in the remote cache that - /// `remote_config` configures, falling back to the entry last stored for + /// Fetch the entries stored under `cache_key` in the remote cache that + /// `remote_config` configures, falling back to the key last stored for /// `execution_cache_key`. Returns `None` if neither key matched. Stops /// when `cancel_token` is cancelled. pub(super) async fn fetch( @@ -492,15 +515,42 @@ mod tests { } } + /// An entry with `value` and the blob `blob_id`. + fn entry(value: &CacheEntryValue, blob_id: &str) -> Entry { + Entry { value: serialize_cache(value).unwrap(), blob_id: Some(Str::from(blob_id)) } + } + fn exact(value: &CacheEntryValue) -> Fetched { - Fetched::Exact { value: serialize_cache(value).unwrap(), blob_id: Some(Str::from("1")) } + Fetched::Exact { entries: vec![entry(value, "1")] } + } + + /// `cache_value()` with `inputs` as its globbed inputs, each with hash 1. + fn with_inputs(inputs: &[&str]) -> CacheEntryValue { + let globbed_inputs = + inputs.iter().map(|path| (RelativePathBuf::new(*path).unwrap(), 1)).collect(); + CacheEntryValue { globbed_inputs, ..cache_value() } + } + + fn inputs(paths: &[&str]) -> BTreeMap { + with_inputs(paths).globbed_inputs + } + + /// The path of an added input, which `miss` must be. + fn added_input(miss: &CacheMiss) -> &str { + match miss { + CacheMiss::FingerprintMismatch(FingerprintMismatch::InputChanged { + kind: InputChangeKind::Added, + path, + }) => path.as_str(), + miss => panic!("expected an added input, got {miss:?}"), + } } /// Validate as `try_hit_remote` does, with no envs and `globbed_inputs` as /// the current inputs. fn validate_against( globbed_inputs: BTreeMap, - ) -> impl FnOnce(&CacheEntryValue) -> anyhow::Result> { + ) -> impl FnMut(&CacheEntryValue) -> anyhow::Result> { move |value| { let workspace_root = vt_path::current_dir().unwrap(); value.validate( @@ -555,6 +605,63 @@ mod tests { ); } + #[test] + fn newest_entry_that_validates_is_restored() { + let key = cache_key(ResolvedGlobConfig::default_auto()); + let entries = vec![entry(&cache_value(), "2"), entry(&cache_value(), "1")]; + let restore = + resolve(Ok(Some(Fetched::Exact { entries })), &key, validate_against(BTreeMap::new())) + .unwrap(); + assert_eq!(restore.blob_id.as_deref(), Some("2")); + } + + #[test] + fn older_entry_that_validates_is_restored_after_newer_ones_fail() { + let key = cache_key(ResolvedGlobConfig::default_auto()); + let entries = vec![ + entry(&with_inputs(&["src/c.txt"]), "4"), + Entry { value: b"not a cache value".to_vec(), blob_id: Some(Str::from("3")) }, + entry(&with_inputs(&["src/a.txt"]), "2"), + entry(&with_inputs(&["src/a.txt"]), "1"), + ]; + let mut validated = 0; + let mut validate = validate_against(inputs(&["src/a.txt"])); + let restore = resolve(Ok(Some(Fetched::Exact { entries })), &key, |value| { + validated += 1; + validate(value) + }) + .unwrap(); + assert_eq!(restore.blob_id.as_deref(), Some("2")); + // The corrupt entry isn't validated, and the entry after the restored + // one isn't checked. + assert_eq!(validated, 2); + } + + #[test] + fn newest_entry_explains_the_miss_when_none_validates() { + let key = cache_key(ResolvedGlobConfig::default_auto()); + let entries = vec![entry(&cache_value(), "2"), entry(&with_inputs(&["src/b.txt"]), "1")]; + let miss = resolve( + Ok(Some(Fetched::Exact { entries })), + &key, + validate_against(inputs(&["src/a.txt"])), + ) + .unwrap_err(); + assert_eq!(added_input(&miss), "src/a.txt"); + + let entries = vec![ + Entry { value: b"not a cache value".to_vec(), blob_id: None }, + entry(&cache_value(), "1"), + ]; + let miss = resolve( + Ok(Some(Fetched::Exact { entries })), + &key, + validate_against(inputs(&["src/a.txt"])), + ) + .unwrap_err(); + assert_eq!(read_failure(miss), "remote cache value is corrupt"); + } + #[test] fn exact_entry_that_cannot_be_validated_is_a_read_failure() { // The stored env query isn't a valid glob, so validating it errors. @@ -610,7 +717,8 @@ mod tests { let mut trailing = serialize_cache(&cache_value()).unwrap(); trailing.push(0); for value in [b"not a cache value".to_vec(), trailing] { - let fetched = Ok(Some(Fetched::Exact { value, blob_id: None })); + let fetched = + Ok(Some(Fetched::Exact { entries: vec![Entry { value, blob_id: None }] })); let miss = resolve(fetched, &key, not_validated).unwrap_err(); assert_eq!(read_failure(miss), "remote cache value is corrupt"); } @@ -622,7 +730,7 @@ mod tests { let without_archive = CacheEntryValue { output_archive: None, ..cache_value() }; for (value, blob_id) in [(cache_value(), None), (without_archive, Some(Str::from("1")))] { let value = serialize_cache(&value).unwrap(); - let fetched = Ok(Some(Fetched::Exact { value, blob_id })); + let fetched = Ok(Some(Fetched::Exact { entries: vec![Entry { value, blob_id }] })); let miss = resolve(fetched, &key, not_validated).unwrap_err(); assert_eq!(read_failure(miss), "remote cache entry's blob doesn't match its value"); } diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md index 2ee9ceaf5..512ff79ab 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md @@ -33,7 +33,7 @@ The downloaded archive doesn't decode, so the task reruns. ``` $ vtt write-file dist/output.txt built ○ cache miss: downloaded archive is corrupt, executing -[remote-cache] POST /fetch 200 exact +[remote-cache] POST /fetch 200 exact [1] [remote-cache] GET /blob/1 200 ``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md index 711285526..97e1d862f 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md @@ -40,7 +40,7 @@ checked --- vt run: 2/2 cache hit (100%, 1 remote), 2/2 successful. (Run `vt run --last-details` for full details) -[remote-cache] POST /fetch 200 exact +[remote-cache] POST /fetch 200 exact [1] [remote-cache] GET /blob/1 200 ``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md index 9d5a33314..92ba2d60f 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md @@ -26,7 +26,7 @@ An endpoint without a mode selects read. The task reruns without uploading. ``` $ vtt write-file dist/output.txt built ○ cache miss: 'src/a.txt' modified, executing -[remote-cache] POST /fetch 200 exact +[remote-cache] POST /fetch 200 exact [1] ``` ## `remote-cache-server stop` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md index aabd036b3..378d13372 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md @@ -33,7 +33,7 @@ $ vtt write-file dist/output.txt built ◉ remote cache hit, replaying --- vt run: remote cache hit. -[remote-cache] POST /fetch 200 exact +[remote-cache] POST /fetch 200 exact [1] [remote-cache] GET /blob/1 200 ``` @@ -73,7 +73,7 @@ The exact entry fails validation, so the archive isn't downloaded. ``` $ vtt write-file dist/output.txt built ○ cache miss: 'src/a.txt' modified, executing -[remote-cache] POST /fetch 200 exact +[remote-cache] POST /fetch 200 exact [1] ``` ## `remote-cache-server stop` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md index 4f8aaf0f3..16ac8d2c2 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md @@ -41,7 +41,7 @@ The remote hit can't be restored, so the task fails. $ vtt write-file dist/output.txt built ◉ remote cache hit, replaying ✗ Cache restore failed: failed to extract the output archive: failed to unpack `/dist/output.txt`: failed to unpack `dist/output.txt` into `/dist/output.txt`: -[remote-cache] POST /fetch 200 exact +[remote-cache] POST /fetch 200 exact [1] [remote-cache] GET /blob/1 200 ``` @@ -93,7 +93,7 @@ $ vtt write-file dist/output.txt built ◉ remote cache hit, replaying --- vt run: remote cache hit. -[remote-cache] POST /fetch 200 exact +[remote-cache] POST /fetch 200 exact [1] [remote-cache] GET /blob/1 200 ``` diff --git a/crates/vt_remote_cache/README.md b/crates/vt_remote_cache/README.md index 86ddb1816..29c1ab722 100644 --- a/crates/vt_remote_cache/README.md +++ b/crates/vt_remote_cache/README.md @@ -8,7 +8,7 @@ Client for the [remote cache server API](https://github.com/voidzero-dev/vite-ta `GithubOidc` authenticates stores with a GitHub Actions OIDC token, sent as `Authorization: Bearer `. Fetches and downloads carry no credentials. It takes the job's token request URL and request token, the values of `ACTIONS_ID_TOKEN_REQUEST_URL` and `ACTIONS_ID_TOKEN_REQUEST_TOKEN`, and the token's audience, but reads no envs itself. The first store requests a token from the request URL with the audience added to its query, and later stores reuse it until two minutes before the expiry in its `exp` claim, which isn't verified. The margin covers the upload, since a server can check the token only once the whole store has arrived, as Cloudflare Workers do. A token without an `exp` claim is a malformed response. Concurrent stores wait for the same token request. Once a request fails, every later store fails the same way without another request. If the request URL isn't an HTTP or HTTPS URL, or the request token can't be sent in a header, every store fails. Neither token appears in its debug output or errors. -`Client::fetch` sends a key and a secondary key as a CBOR map of byte strings. A 404 response means neither key matched, so `fetch` returns `None` without decoding the body. The body of a 200 response is a CBOR map tagged by `kind`, and `Fetched` decodes only two kinds: `exact`, an exact match with its value and blob ID, and `fallback`, a fallback match with the key it's stored under. Any other kind is a malformed response. An exact match must include `blob_id`, which is null when there's no blob. The fallback's value and blob ID aren't decoded. +`Client::fetch` sends a key and a secondary key as a CBOR map of byte strings. A 404 response means neither key matched, so `fetch` returns `None` without decoding the body. The body of a 200 response is a CBOR map tagged by `kind`, and `Fetched` decodes only two kinds: `exact`, an exact match with the entries stored under the key, and `fallback`, a fallback match with the key it's stored under. Any other kind is a malformed response. An exact match's `entries` is a list of at least one entry, newest first, each with its value and blob ID. Each entry must include `blob_id`, which is null when there's no blob. The fallback's entries aren't decoded. `Client::download` gets a blob by its ID. After a 200 response, it returns a `Download`, from which the caller reads the blob's chunks as they arrive. diff --git a/crates/vt_remote_cache/src/lib.rs b/crates/vt_remote_cache/src/lib.rs index 759792bb0..72f402ee8 100644 --- a/crates/vt_remote_cache/src/lib.rs +++ b/crates/vt_remote_cache/src/lib.rs @@ -13,7 +13,7 @@ use reqwest::{ header::{CONTENT_TYPE, HeaderMap}, multipart::{Form, Part}, }; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use url::{ParseError, Url}; use vt_path::AbsolutePath; use vt_str::Str; @@ -106,23 +106,31 @@ struct FetchRequest<'a> { secondary_key: &'a [u8], } +/// One of the entries stored under a key. +#[derive(Debug, PartialEq, Eq, Deserialize)] +pub struct Entry { + /// The stored value. + #[serde(with = "serde_bytes")] + pub value: Vec, + /// The ID to download the entry's blob with, if it has one. The field + /// must be present, with null for no blob. + #[serde(deserialize_with = "Option::deserialize")] + pub blob_id: Option, +} + /// The body of a 200 fetch response: an exact or fallback match. #[derive(Debug, PartialEq, Eq, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum Fetched { - /// The entry stored under the requested key. + /// The entries stored under the requested key, newest first. Exact { - /// The stored value. - #[serde(with = "serde_bytes")] - value: Vec, - /// The ID to download the entry's blob with, if it has one. The field - /// must be present, with null for no blob. - #[serde(deserialize_with = "Option::deserialize")] - blob_id: Option, + /// At least one entry. An empty list is a malformed response. + #[serde(deserialize_with = "non_empty")] + entries: Vec, }, /// No entry is stored under the requested key, but the secondary key is - /// associated with the entry stored under `key`. The fallback entry's - /// value and blob ID aren't decoded. + /// associated with the entries stored under `key`. Their values and blob + /// IDs aren't decoded. Fallback { /// The key the fallback entry is stored under. #[serde(with = "serde_bytes")] @@ -175,8 +183,8 @@ impl Client { self.auth.headers(operation, &self.http).await.map_err(Error::Auth) } - /// Fetch the entry stored under `key` with `POST {endpoint}/fetch`, - /// falling back to the entry associated with `secondary_key`. Returns + /// Fetch the entries stored under `key` with `POST {endpoint}/fetch`, + /// falling back to the key associated with `secondary_key`. Returns /// `None` for a 404 response, which means neither key matched. /// /// # Errors @@ -277,6 +285,14 @@ fn network_error(err: reqwest::Error) -> Error { Error::Network(err.without_url()) } +fn non_empty<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let entries = Vec::::deserialize(deserializer)?; + if entries.is_empty() { + return Err(serde::de::Error::invalid_length(0, &"at least one entry")); + } + Ok(entries) +} + fn decode_fetched(body: &[u8]) -> Result { ciborium::from_reader(body).map_err(Error::MalformedResponse) } @@ -367,27 +383,30 @@ mod tests { encode_cbor(&ciborium::Value::Map(map)) } + fn cbor_value(fields: Vec<(&str, ciborium::Value)>) -> ciborium::Value { + ciborium::Value::Map(fields.into_iter().map(|(name, value)| (name.into(), value)).collect()) + } + + /// A fetch response with an exact match of `entries`. + fn exact(entries: Vec) -> Vec { + cbor_map(vec![("kind", "exact".into()), ("entries", ciborium::Value::Array(entries))]) + } + #[test] fn decodes_each_kind_of_fetch_response() { let bytes = |bytes: &[u8]| ciborium::Value::Bytes(bytes.to_vec()); - let exact = cbor_map(vec![ - ("kind", "exact".into()), - ("value", bytes(b"\x00value")), - ("blob_id", "1".into()), - ]); - assert_eq!( - decode_fetched(&exact).unwrap(), - Fetched::Exact { value: b"\x00value".to_vec(), blob_id: Some(Str::from("1")) } - ); - - let without_blob = cbor_map(vec![ - ("kind", "exact".into()), - ("value", bytes(b"value")), - ("blob_id", ciborium::Value::Null), + let entries = exact(vec![ + cbor_value(vec![("value", bytes(b"\x00newest")), ("blob_id", "2".into())]), + cbor_value(vec![("blob_id", ciborium::Value::Null), ("value", bytes(b"older"))]), ]); assert_eq!( - decode_fetched(&without_blob).unwrap(), - Fetched::Exact { value: b"value".to_vec(), blob_id: None } + decode_fetched(&entries).unwrap(), + Fetched::Exact { + entries: vec![ + Entry { value: b"\x00newest".to_vec(), blob_id: Some(Str::from("2")) }, + Entry { value: b"older".to_vec(), blob_id: None }, + ] + } ); let fallback = cbor_map(vec![ @@ -409,17 +428,18 @@ mod tests { cbor_map(vec![("kind", "unknown".into())]), // A miss is a 404 response, not a kind. cbor_map(vec![("kind", "not_found".into())]), - // The value must be a byte string. + // An exact match has at least one entry. + exact(vec![]), + // The entries must be a list, not the single entry that earlier servers sent. cbor_map(vec![ ("kind", "exact".into()), - ("value", 1.into()), + ("value", ciborium::Value::Bytes(b"value".to_vec())), ("blob_id", ciborium::Value::Null), ]), + // The value must be a byte string. + exact(vec![cbor_value(vec![("value", 1.into()), ("blob_id", ciborium::Value::Null)])]), // `blob_id` is nullable, but it must be present. - cbor_map(vec![ - ("kind", "exact".into()), - ("value", ciborium::Value::Bytes(b"value".to_vec())), - ]), + exact(vec![cbor_value(vec![("value", ciborium::Value::Bytes(b"value".to_vec()))])]), ] { let error = decode_fetched(&body).unwrap_err(); assert!(matches!(error, Error::MalformedResponse(_)), "{error:?}"); @@ -452,16 +472,15 @@ mod tests { async fn fetch_posts_the_keys_as_cbor() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let client = client_for(&listener); - let body = cbor_map(vec![ - ("kind", "exact".into()), + let body = exact(vec![cbor_value(vec![ ("value", ciborium::Value::Bytes(b"v".to_vec())), ("blob_id", ciborium::Value::Null), - ]); + ])]); let server = std::thread::spawn(move || serve_once(&listener, "HTTP/1.1 200 OK", &body)); assert_eq!( client.fetch(b"k", b"s").await.unwrap(), - Some(Fetched::Exact { value: b"v".to_vec(), blob_id: None }) + Some(Fetched::Exact { entries: vec![Entry { value: b"v".to_vec(), blob_id: None }] }) ); let request = server.join().unwrap(); diff --git a/packages/tools/README.md b/packages/tools/README.md index 60882249e..e1953a276 100644 --- a/packages/tools/README.md +++ b/packages/tools/README.md @@ -19,12 +19,12 @@ An E2E case starts its own backend in its first step and stops it in its last, s A backend whose case directory disappears stops by itself, as does one that has had no requests for ten minutes, e.g. because its case timed out before the stop step. -The endpoint is a tap in front of the backend. It forwards every request and response unchanged and records a line for each response, in the order of the responses. Each line has the method, the path below the base path, and the status. Successful fetch responses add their kind. Blob IDs are random, so blob paths show the blob's number in upload order instead, keeping snapshots deterministic. After the command exits, `run` prints the lines for the requests it caused to stderr, and `stop` prints any that are left: +The endpoint is a tap in front of the backend. It forwards every request and response unchanged and records a line for each response, in the order of the responses. Each line has the method, the path below the base path, and the status. Successful fetch responses add their kind, and exact matches add their entries, newest first, by the number of each entry's blob, or `-` for an entry without one. Blob IDs are random, so blob paths and entries show the blob's number in upload order instead, keeping snapshots deterministic. After the command exits, `run` prints the lines for the requests it caused to stderr, and `stop` prints any that are left: ```text [remote-cache] POST /fetch 404 [remote-cache] POST /store 200 -[remote-cache] POST /fetch 200 exact +[remote-cache] POST /fetch 200 exact [1] [remote-cache] GET /blob/1 200 ``` diff --git a/packages/tools/package.json b/packages/tools/package.json index 969df3117..8eadb31af 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -14,7 +14,7 @@ "@openai/codex": "catalog:", "@playwright/browser-chromium": "catalog:", "@vitest/browser-playwright": "catalog:", - "@voidzero-dev/vite-plus-remote-cache-cloudflare": "github:voidzero-dev/vite-plus-remote-cache-cloudflare#43603a62673dd61619e72dfec4990fe7ac5da110", + "@voidzero-dev/vite-plus-remote-cache-cloudflare": "github:voidzero-dev/vite-plus-remote-cache-cloudflare#78764c66432639b5215171e3186cec2120d55cc2", "@voidzero-dev/vite-task-client": "workspace:*", "astro": "catalog:", "bun": "catalog:", diff --git a/packages/tools/src/remote-cache/daemon.ts b/packages/tools/src/remote-cache/daemon.ts index 44666c9b3..02c678570 100644 --- a/packages/tools/src/remote-cache/daemon.ts +++ b/packages/tools/src/remote-cache/daemon.ts @@ -58,6 +58,24 @@ function cborFields(body: Buffer): Record { return {}; } +/** + * The blob numbers of an exact fetch's entries, newest first, e.g. `[3, 1]`. + * An entry without a blob shows `-`. + */ +function entryBlobs(entries: unknown): string | undefined { + if (!Array.isArray(entries)) return undefined; + const numbers = entries.map((entry: unknown) => { + const id = + typeof entry === 'object' && entry !== null + ? (entry as Record)['blob_id'] + : undefined; + if (id === null) return '-'; + const index = typeof id === 'string' ? blobs.indexOf(id) : -1; + return index === -1 ? '?' : String(index + 1); + }); + return `[${numbers.join(', ')}]`; +} + /** `route` with a stored blob's ID replaced by its number. */ function numbered(route: string): string { const id = /^\/blob\/(.+)$/.exec(route)?.[1]; @@ -80,7 +98,8 @@ const tap = createServer((request, response) => { const status = reply.statusCode!; if (status >= 500) anomalies.push(`${label} got ${status} from the backend`); response.writeHead(status, endToEnd(reply.headers)); - // Fetch and store responses are small CBOR maps with the kind or blob ID. + // Fetch and store responses are CBOR maps with the kind and entries, or + // the blob ID. const chunks: Buffer[] = []; if ((route === '/fetch' || route === '/store') && status === 200) { reply.on('data', (chunk: Buffer) => chunks.push(chunk)); @@ -91,7 +110,10 @@ const tap = createServer((request, response) => { const blobId = fields['blob_id']; if (route === '/store' && typeof blobId === 'string') blobs.push(blobId); const kind = route === '/fetch' ? fields['kind'] : undefined; - requests.push([label, status, kind].filter((part) => part !== undefined).join(' ')); + const entries = route === '/fetch' ? entryBlobs(fields['entries']) : undefined; + requests.push( + [label, status, kind, entries].filter((part) => part !== undefined).join(' '), + ); }); }, ); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 837bc2217..c1067ead2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -129,8 +129,8 @@ importers: specifier: 'catalog:' version: 5.0.3(playwright@1.63.0)(vite@8.3.2(@types/node@25.0.3)(esbuild@0.28.1)(yaml@2.9.1))(vitest@5.0.3) '@voidzero-dev/vite-plus-remote-cache-cloudflare': - specifier: github:voidzero-dev/vite-plus-remote-cache-cloudflare#43603a62673dd61619e72dfec4990fe7ac5da110 - version: https://codeload.github.com/voidzero-dev/vite-plus-remote-cache-cloudflare/tar.gz/43603a62673dd61619e72dfec4990fe7ac5da110 + specifier: github:voidzero-dev/vite-plus-remote-cache-cloudflare#78764c66432639b5215171e3186cec2120d55cc2 + version: https://codeload.github.com/voidzero-dev/vite-plus-remote-cache-cloudflare/tar.gz/78764c66432639b5215171e3186cec2120d55cc2 '@voidzero-dev/vite-task-client': specifier: workspace:* version: link:../vite-task-client @@ -2127,10 +2127,10 @@ packages: os: [linux] libc: [musl] - '@voidzero-dev/vite-plus-remote-cache-cloudflare@https://codeload.github.com/voidzero-dev/vite-plus-remote-cache-cloudflare/tar.gz/43603a62673dd61619e72dfec4990fe7ac5da110': - resolution: {gitHosted: true, integrity: sha512-hw7Lh08g7mF2TpQvO8cLBozopX5YRtE5wPTNHTX18oMfGwbzK+0wR15dmxgSaWJkbmo/zvz+75XTcvNdlYmhsg==, tarball: https://codeload.github.com/voidzero-dev/vite-plus-remote-cache-cloudflare/tar.gz/43603a62673dd61619e72dfec4990fe7ac5da110} + '@voidzero-dev/vite-plus-remote-cache-cloudflare@https://codeload.github.com/voidzero-dev/vite-plus-remote-cache-cloudflare/tar.gz/78764c66432639b5215171e3186cec2120d55cc2': + resolution: {gitHosted: true, integrity: sha512-XuUNsHBk1X8+yohToh4u4wkf95Lybz18RuKSb9rvvW6texENy72j8lxj1DNVjEPvZ1Qr+tAlB+CMskWdNRArGA==, tarball: https://codeload.github.com/voidzero-dev/vite-plus-remote-cache-cloudflare/tar.gz/78764c66432639b5215171e3186cec2120d55cc2} version: 0.0.0 - engines: {node: '>=22.12.0'} + engines: {node: ^22.18.0 || ^24.11.0 || >=26.0.0} '@voidzero-dev/vite-plus-win32-arm64-msvc@1.1.0': resolution: {integrity: sha512-EXWz8Tt7skmI0fLMSq5zYcuHo6SCF0reG55qDB/NMvWVIAca6Dborq7D1HyKA8v6p39zOl8r07gR7yXuYv7eeQ==} @@ -2752,8 +2752,8 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} - jose@6.2.12: - resolution: {integrity: sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==} + jose@6.2.15: + resolution: {integrity: sha512-qdF6ewcZARtef63fJ9AKKZPgxw+t7USH6HLXYcD4CrdmSvDp79rpJSPFEKc90QBeGS63u1KQtN2MZQHC0yDIzA==} js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -5067,9 +5067,9 @@ snapshots: '@voidzero-dev/vite-plus-linux-x64-musl@1.1.0': optional: true - '@voidzero-dev/vite-plus-remote-cache-cloudflare@https://codeload.github.com/voidzero-dev/vite-plus-remote-cache-cloudflare/tar.gz/43603a62673dd61619e72dfec4990fe7ac5da110': + '@voidzero-dev/vite-plus-remote-cache-cloudflare@https://codeload.github.com/voidzero-dev/vite-plus-remote-cache-cloudflare/tar.gz/78764c66432639b5215171e3186cec2120d55cc2': dependencies: - jose: 6.2.12 + jose: 6.2.15 '@voidzero-dev/vite-plus-win32-arm64-msvc@1.1.0': optional: true @@ -5615,7 +5615,7 @@ snapshots: isexe@2.0.0: {} - jose@6.2.12: {} + jose@6.2.15: {} js-tokens@4.0.0: {} From 20fa9768076fcc6dbdd0ce16779b6346196da065 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Sat, 7 Nov 2026 17:54:35 +0800 Subject: [PATCH 2/2] test(cache): cover keeping several remote cache entries per key New e2e cases run against the Worker, which now keeps a key's newest entries: - older_entries: a lookup checks the entries newest first and restores the first one that matches, here the oldest. - newest_entry_explains_miss: when none matches, the miss reason comes from the newest entry. - entries_per_key: a store retires the key's oldest entry past the limit. - entries_per_key_isolation: stores to one key don't retire another key's entries, even when the fetch falls back to it. - expired_entries: expired entries aren't fetched, and a key whose entries have all expired is a miss without a fallback. - expired_entry_takes_no_slot: an expired entry doesn't count against the limit, so a store keeps the older live entry. To set them up, `remote-cache-server start` takes `--entries-per-key N` for the namespace, and `remote-cache-server expire-entry NUMBER` makes the entry with that blob expire now. Co-Authored-By: Claude Opus 5.5 --- .../remote_cache_entries/package.json | 4 + .../remote_cache_entries/snapshots.toml | 610 ++++++++++++++++++ .../snapshots/entries_per_key.md | 90 +++ .../snapshots/entries_per_key_isolation.md | 84 +++ .../snapshots/expired_entries.md | 92 +++ .../snapshots/expired_entry_takes_no_slot.md | 75 +++ .../snapshots/newest_entry_explains_miss.md | 56 ++ .../snapshots/older_entries.md | 82 +++ .../fixtures/remote_cache_entries/src/a.txt | 1 + .../remote_cache_entries/vite-task.json | 12 + packages/tools/README.md | 3 +- packages/tools/src/remote-cache/backend.ts | 37 +- packages/tools/src/remote-cache/cli.ts | 34 +- packages/tools/src/remote-cache/daemon.ts | 19 +- 14 files changed, 1172 insertions(+), 27 deletions(-) create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/package.json create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots.toml create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/entries_per_key.md create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/entries_per_key_isolation.md create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/expired_entries.md create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/expired_entry_takes_no_slot.md create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/newest_entry_explains_miss.md create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/older_entries.md create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/src/a.txt create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/vite-task.json diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/package.json b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/package.json new file mode 100644 index 000000000..42dc07a3e --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/package.json @@ -0,0 +1,4 @@ +{ + "name": "remote-cache-entries", + "private": true +} diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots.toml new file mode 100644 index 000000000..e7ddb904e --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots.toml @@ -0,0 +1,610 @@ +# These cases use the remote cache backend, so they are ignored because it +# runs on Node.js, and skipped on musl because workerd's prebuilt binaries +# require glibc. Each starts its own backend first and stops it last. Steps +# that upload run with `--github-actions`, as a job of a push to the main +# branch, and hide the message about pending uploads, whose timing varies. +# Request lines list an exact fetch's entries by blob number, newest first. +[[e2e]] +name = "older_entries" +comment = "A key keeps its newest entries. A lookup checks them newest first and restores the first one that matches." +cfg = 'not(target_env = "musl")' +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ], comment = "The first entry, for `src/a.txt` with 1, has blob 1." }, + [ + "vtt", + "write-file", + "src/a.txt", + "2", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ], comment = "The fetch returns the key's only entry, which doesn't match. The entry for 2 is added." }, + [ + "vtt", + "write-file", + "src/a.txt", + "3", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ], comment = "The entries come newest first." }, + [ + "vtt", + "write-file", + "src/a.txt", + "1", + ], + { argv = [ + "remote-cache-server", + "run", + "vt", + "run", + "build", + ], comment = "The local entry is for 3. Of the remote entries, only the oldest matches, so its blob is downloaded." }, + { argv = [ + "vtt", + "print-file", + "output.txt", + ], comment = "The outputs of the run for 1 are restored." }, + [ + "remote-cache-server", + "stop", + ], +] + +[[e2e]] +name = "newest_entry_explains_miss" +cfg = 'not(target_env = "musl")' +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ] }, + [ + "vtt", + "write-file", + "src/b.txt", + "x", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ], comment = "The entry for `src/b.txt` with x is added." }, + [ + "vtt", + "write-file", + "src/b.txt", + "y", + ], + [ + "vt", + "cache", + "clean", + ], + { argv = [ + "remote-cache-server", + "run", + "vt", + "run", + "build", + ], comment = "No entry matches. The miss is explained against the newest entry, in which `src/b.txt` had other contents. The older entry had no `src/b.txt`." }, + [ + "remote-cache-server", + "stop", + ], +] + +[[e2e]] +name = "entries_per_key" +comment = "The namespace keeps two entries per key." +cfg = 'not(target_env = "musl")' +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + "--entries-per-key", + "2", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ] }, + [ + "vtt", + "write-file", + "src/a.txt", + "2", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ] }, + [ + "vtt", + "write-file", + "src/a.txt", + "3", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ], comment = "The third entry retires the oldest one." }, + [ + "vtt", + "write-file", + "src/a.txt", + "1", + ], + [ + "vt", + "cache", + "clean", + ], + { argv = [ + "remote-cache-server", + "run", + "vt", + "run", + "build", + ], comment = "The entry for 1 was retired, so the run misses." }, + [ + "vtt", + "write-file", + "src/a.txt", + "2", + ], + { argv = [ + "remote-cache-server", + "run", + "vt", + "run", + "build", + ], comment = "The entry for 2 is still kept." }, + [ + "remote-cache-server", + "stop", + ], +] + +[[e2e]] +name = "entries_per_key_isolation" +comment = "The namespace keeps one entry per key. Tasks with other values of `MODE` have other keys." +cfg = 'not(target_env = "musl")' +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + "--entries-per-key", + "1", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + [ + "MODE", + "a", + ], + ], comment = "The entry for `MODE` a and `src/a.txt` with 1 has blob 1." }, + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + [ + "MODE", + "b", + ], + ], comment = "Key b has no entry yet, so the fetch falls back to key a." }, + [ + "vtt", + "write-file", + "src/a.txt", + "2", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + [ + "MODE", + "b", + ], + ], comment = "The entry for 2 replaces key b's entry for 1." }, + [ + "vtt", + "write-file", + "src/a.txt", + "1", + ], + [ + "vt", + "cache", + "clean", + ], + { argv = [ + "remote-cache-server", + "run", + "vt", + "run", + "build", + ], envs = [ + [ + "MODE", + "b", + ], + ], comment = "Key b only has its entry for 2, so the run misses." }, + { argv = [ + "remote-cache-server", + "run", + "vt", + "run", + "build", + ], envs = [ + [ + "MODE", + "a", + ], + ], comment = "Key a still has its entry for 1: entries of other keys don't take its place." }, + [ + "remote-cache-server", + "stop", + ], +] + +[[e2e]] +name = "expired_entries" +cfg = 'not(target_env = "musl")' +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ] }, + [ + "vtt", + "write-file", + "src/a.txt", + "2", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ] }, + [ + "remote-cache-server", + "expire-entry", + "2", + ], + [ + "vt", + "cache", + "clean", + ], + { argv = [ + "remote-cache-server", + "run", + "vt", + "run", + "build", + ], comment = "The expired entry isn't fetched, and the miss is explained against the newest entry that hasn't expired." }, + [ + "vtt", + "write-file", + "src/a.txt", + "1", + ], + { argv = [ + "remote-cache-server", + "run", + "vt", + "run", + "build", + ], comment = "The entry for 1 hasn't expired." }, + [ + "remote-cache-server", + "expire-entry", + "1", + ], + [ + "vt", + "cache", + "clean", + ], + { argv = [ + "remote-cache-server", + "run", + "vt", + "run", + "build", + ], comment = "Once every entry of the key has expired, the fetch finds nothing, not even a fallback." }, + [ + "remote-cache-server", + "stop", + ], +] + +[[e2e]] +name = "expired_entry_takes_no_slot" +comment = "The namespace keeps two entries per key." +cfg = 'not(target_env = "musl")' +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + "--entries-per-key", + "2", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ] }, + [ + "vtt", + "write-file", + "src/a.txt", + "2", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ] }, + [ + "remote-cache-server", + "expire-entry", + "2", + ], + [ + "vtt", + "write-file", + "src/a.txt", + "3", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ], comment = "Only the entry for 1 hasn't expired, so the entry for 3 retires nothing, although the key has two entries." }, + [ + "vtt", + "write-file", + "src/a.txt", + "1", + ], + { argv = [ + "remote-cache-server", + "run", + "vt", + "run", + "build", + ], comment = "The entry for 1 is still kept." }, + [ + "remote-cache-server", + "stop", + ], +] diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/entries_per_key.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/entries_per_key.md new file mode 100644 index 000000000..52f77ce81 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/entries_per_key.md @@ -0,0 +1,90 @@ +# entries_per_key + +The namespace keeps two entries per key. + +## `remote-cache-server start --entries-per-key 2` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +``` +$ vtt cp src/a.txt output.txt + +[remote-cache] POST /fetch 404 +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/a.txt 2` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/a.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [1] +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/a.txt 3` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +The third entry retires the oldest one. + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/a.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [2, 1] +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/a.txt 1` + +``` +``` + +## `vt cache clean` + +``` +``` + +## `remote-cache-server run vt run build` + +The entry for 1 was retired, so the run misses. + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/a.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [3, 2] +``` + +## `vtt write-file src/a.txt 2` + +``` +``` + +## `remote-cache-server run vt run build` + +The entry for 2 is still kept. + +``` +$ vtt cp src/a.txt output.txt ◉ remote cache hit, replaying + +--- +vt run: remote cache hit. +[remote-cache] POST /fetch 200 exact [3, 2] +[remote-cache] GET /blob/2 200 +``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/entries_per_key_isolation.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/entries_per_key_isolation.md new file mode 100644 index 000000000..3129ab700 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/entries_per_key_isolation.md @@ -0,0 +1,84 @@ +# entries_per_key_isolation + +The namespace keeps one entry per key. Tasks with other values of `MODE` have other keys. + +## `remote-cache-server start --entries-per-key 1` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 MODE=a remote-cache-server run --github-actions vt run build` + +The entry for `MODE` a and `src/a.txt` with 1 has blob 1. + +``` +$ vtt cp src/a.txt output.txt + +[remote-cache] POST /fetch 404 +[remote-cache] POST /store 200 +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 MODE=b remote-cache-server run --github-actions vt run build` + +Key b has no entry yet, so the fetch falls back to key a. + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: env 'MODE' changed, executing + +[remote-cache] POST /fetch 200 fallback +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/a.txt 2` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 MODE=b remote-cache-server run --github-actions vt run build` + +The entry for 2 replaces key b's entry for 1. + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/a.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [2] +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/a.txt 1` + +``` +``` + +## `vt cache clean` + +``` +``` + +## `MODE=b remote-cache-server run vt run build` + +Key b only has its entry for 2, so the run misses. + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/a.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [3] +``` + +## `MODE=a remote-cache-server run vt run build` + +Key a still has its entry for 1: entries of other keys don't take its place. + +``` +$ vtt cp src/a.txt output.txt ◉ remote cache hit, replaying + +--- +vt run: remote cache hit. +[remote-cache] POST /fetch 200 exact [1] +[remote-cache] GET /blob/1 200 +``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/expired_entries.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/expired_entries.md new file mode 100644 index 000000000..78096a58c --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/expired_entries.md @@ -0,0 +1,92 @@ +# expired_entries + +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +``` +$ vtt cp src/a.txt output.txt + +[remote-cache] POST /fetch 404 +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/a.txt 2` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/a.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [1] +[remote-cache] POST /store 200 +``` + +## `remote-cache-server expire-entry 2` + +``` +``` + +## `vt cache clean` + +``` +``` + +## `remote-cache-server run vt run build` + +The expired entry isn't fetched, and the miss is explained against the newest entry that hasn't expired. + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/a.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [1] +``` + +## `vtt write-file src/a.txt 1` + +``` +``` + +## `remote-cache-server run vt run build` + +The entry for 1 hasn't expired. + +``` +$ vtt cp src/a.txt output.txt ◉ remote cache hit, replaying + +--- +vt run: remote cache hit. +[remote-cache] POST /fetch 200 exact [1] +[remote-cache] GET /blob/1 200 +``` + +## `remote-cache-server expire-entry 1` + +``` +``` + +## `vt cache clean` + +``` +``` + +## `remote-cache-server run vt run build` + +Once every entry of the key has expired, the fetch finds nothing, not even a fallback. + +``` +$ vtt cp src/a.txt output.txt + +[remote-cache] POST /fetch 404 +``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/expired_entry_takes_no_slot.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/expired_entry_takes_no_slot.md new file mode 100644 index 000000000..3df0b1069 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/expired_entry_takes_no_slot.md @@ -0,0 +1,75 @@ +# expired_entry_takes_no_slot + +The namespace keeps two entries per key. + +## `remote-cache-server start --entries-per-key 2` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +``` +$ vtt cp src/a.txt output.txt + +[remote-cache] POST /fetch 404 +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/a.txt 2` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/a.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [1] +[remote-cache] POST /store 200 +``` + +## `remote-cache-server expire-entry 2` + +``` +``` + +## `vtt write-file src/a.txt 3` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +Only the entry for 1 hasn't expired, so the entry for 3 retires nothing, although the key has two entries. + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/a.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [1] +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/a.txt 1` + +``` +``` + +## `remote-cache-server run vt run build` + +The entry for 1 is still kept. + +``` +$ vtt cp src/a.txt output.txt ◉ remote cache hit, replaying + +--- +vt run: remote cache hit. +[remote-cache] POST /fetch 200 exact [3, 1] +[remote-cache] GET /blob/1 200 +``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/newest_entry_explains_miss.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/newest_entry_explains_miss.md new file mode 100644 index 000000000..baad6b91d --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/newest_entry_explains_miss.md @@ -0,0 +1,56 @@ +# newest_entry_explains_miss + +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +``` +$ vtt cp src/a.txt output.txt + +[remote-cache] POST /fetch 404 +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/b.txt x` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +The entry for `src/b.txt` with x is added. + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'b.txt' added in 'src', executing + +[remote-cache] POST /fetch 200 exact [1] +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/b.txt y` + +``` +``` + +## `vt cache clean` + +``` +``` + +## `remote-cache-server run vt run build` + +No entry matches. The miss is explained against the newest entry, in which `src/b.txt` had other contents. The older entry had no `src/b.txt`. + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/b.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [2, 1] +``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/older_entries.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/older_entries.md new file mode 100644 index 000000000..bd3e92976 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/snapshots/older_entries.md @@ -0,0 +1,82 @@ +# older_entries + +A key keeps its newest entries. A lookup checks them newest first and restores the first one that matches. + +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +The first entry, for `src/a.txt` with 1, has blob 1. + +``` +$ vtt cp src/a.txt output.txt + +[remote-cache] POST /fetch 404 +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/a.txt 2` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +The fetch returns the key's only entry, which doesn't match. The entry for 2 is added. + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/a.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [1] +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/a.txt 3` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +The entries come newest first. + +``` +$ vtt cp src/a.txt output.txt ○ cache miss: 'src/a.txt' modified, executing + +[remote-cache] POST /fetch 200 exact [2, 1] +[remote-cache] POST /store 200 +``` + +## `vtt write-file src/a.txt 1` + +``` +``` + +## `remote-cache-server run vt run build` + +The local entry is for 3. Of the remote entries, only the oldest matches, so its blob is downloaded. + +``` +$ vtt cp src/a.txt output.txt ◉ remote cache hit, replaying + +--- +vt run: remote cache hit. +[remote-cache] POST /fetch 200 exact [3, 2, 1] +[remote-cache] GET /blob/1 200 +``` + +## `vtt print-file output.txt` + +The outputs of the run for 1 are restored. + +``` +1 +``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/src/a.txt b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/src/a.txt new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/src/a.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/vite-task.json b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/vite-task.json new file mode 100644 index 000000000..89142de09 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_entries/vite-task.json @@ -0,0 +1,12 @@ +{ + "tasks": { + "build": { + "command": "vtt cp src/a.txt output.txt", + "cache": { + "env": ["MODE"], + "input": ["src/**"], + "output": ["output.txt"] + } + } + } +} diff --git a/packages/tools/README.md b/packages/tools/README.md index e1953a276..31d3775f1 100644 --- a/packages/tools/README.md +++ b/packages/tools/README.md @@ -12,9 +12,10 @@ remote-cache-server stop An E2E case starts its own backend in its first step and stops it in its last, so the backend keeps its state for the whole case. Each subcommand works in the current directory, the case's directory: -- `remote-cache-server start` starts the backend in the background on free loopback ports and returns once it's ready. The backend runs in its own session and doesn't use the terminal, so the step can finish and Ctrl-C in later steps doesn't reach it. It writes its endpoint, `http://127.0.0.1:/projects/test`, to `remote-cache/server.json`, and its output to `remote-cache/server.log`. +- `remote-cache-server start [--entries-per-key N]` starts the backend in the background on free loopback ports and returns once it's ready. The namespace keeps the newest `N` entries of each key, or the Worker's default of 8. The backend runs in its own session and doesn't use the terminal, so the step can finish and Ctrl-C in later steps doesn't reach it. It writes its endpoint, `http://127.0.0.1:/projects/test`, to `remote-cache/server.json`, and its output to `remote-cache/server.log`. - `remote-cache-server run [--github-actions] COMMAND [ARGS...]` runs the command with `VP_REMOTE_CACHE_URL` set to the endpoint. The fixed base path gives the endpoint a namespace path. With `--github-actions`, the command runs as if in a GitHub Actions job with `id-token: write`: `ACTIONS_ID_TOKEN_REQUEST_URL` points to the backend's stand-in for GitHub's token service, and `ACTIONS_ID_TOKEN_REQUEST_TOKEN` defaults to `main-push`. The command inherits stdio and handles Ctrl-C, which `run` ignores. `run` exits with the command's exit code. - `remote-cache-server corrupt-blob NUMBER` overwrites a stored blob, numbered as in the request lines, with other bytes. +- `remote-cache-server expire-entry NUMBER` makes the entry whose blob has that number expire now, as if its retention had passed. Nothing removes it, as before the Worker's scheduled cleanup. - `remote-cache-server stop` stops the backend. It fails if the backend answered a request with a 5xx status or couldn't be reached. A backend whose case directory disappears stops by itself, as does one that has had no requests for ten minutes, e.g. because its case timed out before the stop step. diff --git a/packages/tools/src/remote-cache/backend.ts b/packages/tools/src/remote-cache/backend.ts index 70acd75ed..cf7560ffc 100644 --- a/packages/tools/src/remote-cache/backend.ts +++ b/packages/tools/src/remote-cache/backend.ts @@ -19,6 +19,8 @@ export interface Backend { origin: string; /** Replace the contents of the blob with ID `id`. */ writeBlob(id: string, contents: Uint8Array): Promise; + /** Make the entry whose blob has ID `id` expire now. */ + expireEntry(id: string): Promise; close(): Promise; } @@ -45,7 +47,9 @@ async function bundle(main: string): Promise { * `wrangler.jsonc`. Its namespaces are only the one in `basePath`. * * Like a deployment, the database has every migration applied, and the - * namespace is registered for `endpoint` and the repository `registered`. The + * namespace is registered for `endpoint` and the repository `registered`. It + * keeps the newest `entriesPerKey` entries of each key, or the Worker's + * default number. The * Worker fetches GitHub's key set to verify upload tokens, and gets one with * `key` instead. Any other outbound request fails. */ @@ -53,12 +57,14 @@ export async function startBackend({ basePath, directory, endpoint, + entriesPerKey, key, registered, }: { basePath: string; directory: string; endpoint: string; + entriesPerKey: number | undefined; key: TrustedKey; registered: typeof repository; }): Promise { @@ -104,19 +110,21 @@ export async function startBackend({ const statements = unstable_splitSqlQuery(readFileSync(join(migrations, file), 'utf8')); await database.batch(statements.map((sql) => database.prepare(sql))); } + const scope: Record = { + scope_id: namespace, + endpoint, + repository: registered.name, + repository_id: registered.id, + repository_owner_id: registered.ownerId, + branch: registered.branch, + ...(entriesPerKey === undefined ? {} : { entries_per_key: entriesPerKey }), + }; + const columns = Object.keys(scope); await database .prepare( - `INSERT INTO scopes (scope_id, endpoint, repository, repository_id, repository_owner_id, branch) - VALUES (?, ?, ?, ?, ?, ?)`, - ) - .bind( - namespace, - endpoint, - registered.name, - registered.id, - registered.ownerId, - registered.branch, + `INSERT INTO scopes (${columns.join(', ')}) VALUES (${columns.map(() => '?').join(', ')})`, ) + .bind(...Object.values(scope)) .run(); const bucket = await miniflare.getR2Bucket('ARTIFACTS'); @@ -135,6 +143,13 @@ export async function startBackend({ .bind(contents.length, id) .run(); }, + async expireEntry(id) { + const { meta } = await database + .prepare('UPDATE generations SET expires_at = unixepoch() WHERE blob_id = ?') + .bind(id) + .run(); + if (meta.changes === 0) throw new Error(`No blob ${id} is stored`); + }, close: () => miniflare.dispose(), }; } catch (error) { diff --git a/packages/tools/src/remote-cache/cli.ts b/packages/tools/src/remote-cache/cli.ts index 6294065a9..1a8692c03 100755 --- a/packages/tools/src/remote-cache/cli.ts +++ b/packages/tools/src/remote-cache/cli.ts @@ -11,7 +11,7 @@ import { defaultRequestToken } from './github.ts'; import { logFile, serverFile, stateDirectory, type ServerInfo } from './state.ts'; const usage = - 'Usage: remote-cache-server start | run [--github-actions] COMMAND [ARGS...] | corrupt-blob NUMBER | stop'; + 'Usage: remote-cache-server start [--entries-per-key N] | run [--github-actions] COMMAND [ARGS...] | corrupt-blob NUMBER | expire-entry NUMBER | stop'; const directory = resolve(stateDirectory); function fail(message: string): never { @@ -53,7 +53,16 @@ function printRequests(lines: string[]): void { for (const line of lines) console.error(`[remote-cache] ${line}`); } -async function start(): Promise { +async function start(argv: string[]): Promise { + // The daemon takes the number of entries per key after the directory. + const [option, entriesPerKey] = argv; + if ( + argv.length > 0 && + (argv.length !== 2 || option !== '--entries-per-key' || !/^[1-9]\d*$/.test(entriesPerKey!)) + ) { + fail(usage); + } + const options = argv.length > 0 ? [entriesPerKey!] : []; if (existsSync(serverFile(directory))) fail('a backend is already running here.'); mkdirSync(directory, { recursive: true }); const log = openSync(logFile(directory), 'a'); @@ -62,7 +71,12 @@ async function start(): Promise { // exits, and Ctrl-C in later steps doesn't reach the backend. const daemon = spawn( process.execPath, - [...process.execArgv, fileURLToPath(new URL('daemon.ts', import.meta.url)), directory], + [ + ...process.execArgv, + fileURLToPath(new URL('daemon.ts', import.meta.url)), + directory, + ...options, + ], { cwd: tmpdir(), detached: true, stdio: ['ignore', log, log, 'ipc'] }, ); closeSync(log); @@ -95,9 +109,10 @@ async function run(argv: string[]): Promise { process.exitCode = code; } -async function corruptBlob([number]: string[]): Promise { - if (number === undefined || !/^[1-9]\d*$/.test(number)) fail(usage); - await control(readInfo(), `/corrupt-blob/${number}`, `no blob ${number} has been stored.`); +/** Change what the backend stores for blob `number`, numbered as in the request lines. */ +async function changeBlob(change: string, [number, ...rest]: string[]): Promise { + if (number === undefined || !/^[1-9]\d*$/.test(number) || rest.length > 0) fail(usage); + await control(readInfo(), `/${change}/${number}`, `no blob ${number} has been stored.`); } async function stop(): Promise { @@ -111,8 +126,9 @@ async function stop(): Promise { } const [subcommand, ...args] = process.argv.slice(2); -if (subcommand === 'start') await start(); +if (subcommand === 'start') await start(args); else if (subcommand === 'run') await run(args); -else if (subcommand === 'corrupt-blob') await corruptBlob(args); -else if (subcommand === 'stop') await stop(); +else if (subcommand === 'corrupt-blob' || subcommand === 'expire-entry') { + await changeBlob(subcommand, args); +} else if (subcommand === 'stop') await stop(); else fail(usage); diff --git a/packages/tools/src/remote-cache/daemon.ts b/packages/tools/src/remote-cache/daemon.ts index 02c678570..16d8bfe17 100644 --- a/packages/tools/src/remote-cache/daemon.ts +++ b/packages/tools/src/remote-cache/daemon.ts @@ -18,10 +18,12 @@ import { basePath, serverFile, type ServerInfo } from './state.ts'; // and responses unchanged and records a line for each response. Blob IDs in // the lines become numbers in upload order, so they're the same on every run. // A control server hands the lines to `remote-cache-server run` and `stop`, -// changes blobs for `remote-cache-server corrupt-blob`, and stands in for -// GitHub Actions' token service. +// changes blobs and entries for `remote-cache-server corrupt-blob` and +// `expire-entry`, and stands in for GitHub Actions' token service. const directory = process.argv[2]!; +/** The number of entries the namespace keeps per key, if not the Worker's default. */ +const entriesPerKey = process.argv[3] === undefined ? undefined : Number(process.argv[3]); /** Stop after this long without requests, e.g. when a case timed out before its stop step. */ const idleTimeout = 10 * 60 * 1000; /** Headers about a single connection. Node sets them for each hop itself. */ @@ -154,16 +156,20 @@ const control = createServer((request, response) => { else reply({ count: 1, value }); return; } - const corrupt = /^\/corrupt-blob\/(\d+)$/.exec(request.url ?? '')?.[1]; + const [, change, number] = /^\/(corrupt-blob|expire-entry)\/(\d+)$/.exec(request.url ?? '') ?? []; if (request.method === 'POST' && request.url === '/take') { reply(requests.splice(0)); - } else if (request.method === 'POST' && corrupt !== undefined) { - const id = blobs[Number(corrupt) - 1]; + } else if (request.method === 'POST' && number !== undefined) { + const id = blobs[Number(number) - 1]; if (id === undefined) { response.writeHead(404).end(); return; } - backend.writeBlob(id, Buffer.from('corrupt')).then( + const changed = + change === 'corrupt-blob' + ? backend.writeBlob(id, Buffer.from('corrupt')) + : backend.expireEntry(id); + changed.then( () => reply(null), (error: unknown) => { console.error(error); @@ -208,6 +214,7 @@ const backend = await startBackend({ basePath, directory, endpoint: info.url, + entriesPerKey, key: signingKey, registered: repository, });