diff --git a/crates/db/src/oci_dao.rs b/crates/db/src/oci_dao.rs index 46eefd9c..af2e942f 100644 --- a/crates/db/src/oci_dao.rs +++ b/crates/db/src/oci_dao.rs @@ -194,7 +194,10 @@ pub async fn find_link( } /// Record a use of a link, unless one was recorded less than `every` ago -/// (so pulls do not write a row on every request). +/// (so requests do not write a row each). A use keeps a link no manifest +/// names from the sweeper ([`unreferenced_links`]), so only a client that +/// may push the image records one: a pull by anyone else must not keep the +/// blobs of a deleted image pullable. pub async fn touch_link( pool: &PgPool, repository_id: i64, diff --git a/crates/server/src/admin/images.rs b/crates/server/src/admin/images.rs index b1b6358f..8e2e6cd2 100644 --- a/crates/server/src/admin/images.rs +++ b/crates/server/src/admin/images.rs @@ -27,8 +27,10 @@ //! nothing matched. Each delete is in the audit log (`forge::audit`, "image //! delete") with the image and reference, which the admin change log's path //! does not name. As after a registry delete, the manifests' configs and -//! layers stay linked to the image until the registry sweeper finds them -//! unreferenced, and the blob outbox deletes a blob no image links any more. +//! layers stay linked to the image, pullable by digest, until the registry +//! sweeper finds them unreferenced and not pushed, mounted or confirmed by a +//! pusher for `docker.unreferenced_blob_ttl_seconds`; a read does not +//! prolong that. The blob outbox deletes a blob no image links any more. //! //! `GET /admin/repositories/{name}/images/delete-preview?image=[&reference=]` //! says what that delete would remove, and deletes nothing ([`preview`]): diff --git a/crates/server/src/images/dispatch.rs b/crates/server/src/images/dispatch.rs index d99f497c..2afc72dc 100644 --- a/crates/server/src/images/dispatch.rs +++ b/crates/server/src/images/dispatch.rs @@ -955,11 +955,13 @@ impl Conversation<'_> { .get(&digest) .cloned(); match found { - Some(hit) if self.permission == Permission::Read => self - .reg - .serve_blob(&hit, self.head, &self.headers) - .await - .unwrap_or_else(|failure| self.reg.failure_response(failure)), + Some(hit) if self.permission == Permission::Read => { + let confirms = self.reg.confirms_link(self.ctx, &hit).await; + self.reg + .serve_blob(&hit, confirms, self.head, &self.headers) + .await + .unwrap_or_else(|failure| self.reg.failure_response(failure)) + } _ => self.misused("serving a blob no query found"), } } diff --git a/crates/server/src/images/mod.rs b/crates/server/src/images/mod.rs index f34ea8d0..7733984f 100644 --- a/crates/server/src/images/mod.rs +++ b/crates/server/src/images/mod.rs @@ -79,7 +79,8 @@ pub struct RegistryConfig { } impl RegistryConfig { - /// How often a pull (or `HEAD`) of a blob records a use of its link: a + /// How often a `HEAD` (or pull) of a blob by a caller who may push its + /// image records a use of its link (`Registry::confirms_link`): a /// tenth of the unreferenced-blob TTL, at most hourly. A link a client /// confirmed then stays at least nine tenths of the TTL, whereas an /// hourly record with the smallest TTL (an hour) could leave it none. diff --git a/crates/server/src/images/pull.rs b/crates/server/src/images/pull.rs index b07bf16c..ff05f4bb 100644 --- a/crates/server/src/images/pull.rs +++ b/crates/server/src/images/pull.rs @@ -8,6 +8,7 @@ use axum::body::Body; use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; use axum::response::{IntoResponse, Response}; use bytes::Bytes; +use forge_core::authz::Level; use forge_db::model::{RepoKind, Repository}; use forge_db::oci_dao::{self, ManifestRow}; use forge_format_api::images::{BlobInfo, ManifestInfo, Reference, TagPage, MAX_MANIFEST_BYTES}; @@ -15,7 +16,7 @@ use forge_storage::BlobRef; use futures_util::future::BoxFuture; use super::digest::ContentDigest; -use super::dispatch::Caller; +use super::dispatch::{mutation_credential, Caller, RequestCtx}; use super::download::Download; use super::{header_value, Failure, ImgResult, Registry}; @@ -88,8 +89,9 @@ pub(crate) struct BlobHit { pub size: i64, pub blob_sha256: String, pub s3_key: String, - /// Whether it is an `oci_blob_link` (touched on use) rather than a - /// manifest read as a blob. + /// Whether it is an `oci_blob_link` (whose use a caller who may push + /// the image records, [`Registry::confirms_link`]) rather than a manifest + /// read as a blob. pub linked: bool, /// The download it is served from while a proxy fetches it; `None` once /// it is in the store. @@ -375,15 +377,55 @@ impl Registry { Ok(response) } - /// Serve a blob the format asked for, with one `Range`, and record the - /// pull of its link. + /// Whether serving `hit` records a use of its link, which keeps a link + /// no manifest names from the sweeper for another + /// `docker.unreferenced_blob_ttl_seconds`: only to a caller who may push + /// the image the link is in, with a credential (and a registry token's + /// scope) a push is accepted from. Such a caller is confirming a blob it + /// is about to name in a manifest (`docker push` asks with a `HEAD` + /// whether the image has a layer and skips its upload), and could push + /// the blob again anyway. Anyone else's fetch records nothing: the grace + /// period is for uploads before their manifest, and a reader who knows + /// the digest of a deleted image's layer must not keep it pullable by + /// fetching it (#576). + pub(crate) async fn confirms_link(&self, ctx: &RequestCtx, hit: &BlobHit) -> bool { + if !hit.linked || !mutation_credential(&ctx.caller.principal) { + return false; + } + if let Some(claims) = &ctx.caller.claims { + if !claims.grants(&ctx.target.name, "push") { + return false; + } + } + // Where a push to the name the client used lands: only a link there + // is one this caller's manifest could name. + let landing = match ctx.target.repo.kind() { + RepoKind::Hosted => Some(ctx.target.repo.clone()), + RepoKind::Group => self + .writable_member(&ctx.target.repo) + .await + .unwrap_or_default(), + RepoKind::Proxy => None, + }; + match landing { + Some(repo) if repo.id == hit.repository_id => { + self.rbac_allows(&ctx.caller.principal, &repo, &hit.image, Level::Write) + .await + } + _ => false, + } + } + + /// Serve a blob the format asked for, with one `Range`. `confirms` + /// ([`Self::confirms_link`]): record the use of its link. pub(crate) async fn serve_blob( &self, hit: &BlobHit, + confirms: bool, head: bool, headers: &HeaderMap, ) -> ImgResult { - if hit.linked { + if confirms && hit.linked { if let Err(e) = oci_dao::touch_link( &self.inner.pool, hit.repository_id, diff --git a/crates/server/tests/registry.rs b/crates/server/tests/registry.rs index 54bec4ea..751bbc3c 100644 --- a/crates/server/tests/registry.rs +++ b/crates/server/tests/registry.rs @@ -225,6 +225,21 @@ impl Oci { .unwrap() } + /// A `HEAD` with a push token, as `docker push` asks whether the image + /// has a layer already. + async fn head_to_push(&self, name: &str, path: &str) -> reqwest::Response { + self.request( + Method::HEAD, + &format!("/v2/{name}/{path}"), + "pull,push", + name, + ) + .await + .send() + .await + .unwrap() + } + async fn delete(&self, name: &str, path: &str) -> reqwest::Response { self.request( Method::DELETE, @@ -4329,9 +4344,10 @@ async fn ref_count(srv: &TestServer, digest: &str) -> Option { /// A client confirming a blob it is about to name in a manifest records the /// use of its link, so the sweeper does not release it first (#218): a push -/// of a blob the image already has, a mount of it, and a `HEAD` of it, -/// recorded every tenth of `docker.unreferenced_blob_ttl_seconds` (six -/// minutes for the smallest TTL, an hour) instead of hourly. +/// of a blob the image already has, a mount of it, and a `HEAD` of it with a +/// push token, recorded every tenth of `docker.unreferenced_blob_ttl_seconds` +/// (six minutes for the smallest TTL, an hour) instead of hourly. A pull +/// records nothing (#576). #[tokio::test] async fn confirming_a_blob_records_the_use_of_its_link() { let srv = common::start_with( @@ -4407,18 +4423,225 @@ async fn confirming_a_blob_records_the_use_of_its_link() { assert!(used_recently().await, "mounted"); unused_for(7).await; + let blob = format!("blobs/{digest}"); + assert_eq!(oci.head(&name, &blob).await.status(), 200); + assert_eq!(oci.get(&name, &blob).await.status(), 200); + assert!( + !used_recently().await, + "a pull token's HEAD and GET are no push in progress" + ); + assert_eq!(oci.head_to_push(&name, &blob).await.status(), 200); + assert!( + used_recently().await, + "a pusher's HEAD after a tenth of the TTL" + ); + + unused_for(2).await; + assert_eq!(oci.head_to_push(&name, &blob).await.status(), 200); + assert!(!used_recently().await, "not recorded on every request"); + + // Through a group, a push lands in its writable member: a pusher's HEAD + // there confirms the member's link. + let group = repo_name("group"); + create_repo( + &srv, + json!({"name": group, "kind": "group", "format": "docker", "writable_member": repo}), + ) + .await; + let resp = admin_post( + &srv, + &format!("/admin/repositories/{group}/members"), + json!({"member": repo, "priority": 1}), + ) + .await; + assert_eq!(resp.status(), 201); + let via = format!("{group}/app"); + unused_for(7).await; + assert_eq!(oci.head(&via, &blob).await.status(), 200); + assert!(!used_recently().await, "a pull through the group"); + assert_eq!(oci.head_to_push(&via, &blob).await.status(), 200); + assert!(used_recently().await, "a pusher's HEAD through the group"); +} + +/// The blobs of a deleted manifest stay linked for the grace period an +/// upload needs, and no longer: fetching them does not start it again, so a +/// reader who knows a layer's digest cannot keep it pullable (#576). A blob +/// another manifest of the image names, and a push in progress, are left +/// alone. +#[tokio::test] +async fn fetching_the_blobs_of_a_deleted_image_does_not_keep_them() { + // The sweep releases every unreferenced link of the database that is old + // enough, so this test has one of its own. + let srv = common::start_with( + common::fresh_pool().await, + common::ServerOpts { + bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), + extra_toml: Some("[docker]\nunreferenced_blob_ttl_seconds = 3600\n".into()), + ..Default::default() + }, + ) + .await; + let repo = repo_name("dock"); + create_repo( + &srv, + json!({"name": repo, "kind": "hosted", "format": "docker"}), + ) + .await; + let oci = admin_oci(&srv).await; + let (user, secret, user_id) = user_with_token(&srv).await; + grant(&srv, user_id, &repo, "read").await; + let reader = Oci::new(&srv, Some((user, secret))); + let (user, secret, user_id) = user_with_token(&srv).await; + grant(&srv, user_id, &repo, "write").await; + let writer = Oci::new(&srv, Some((user, secret))); + + // Two manifests share a layer; each has a layer and a config of its own. + let name = format!("{repo}/app"); + let shared = format!("shared layer {}", unique()).into_bytes(); + let secret_layer = format!("layer with a secret {}", unique()).into_bytes(); + let (leaked_cfg, kept_cfg) = (config("amd64"), config("arm64")); + for blob in [&shared, &secret_layer, &leaked_cfg, &kept_cfg] { + oci.push_blob(&name, blob).await; + } + let leaked = image(&leaked_cfg, &[&shared, &secret_layer]); + let kept = image(&kept_cfg, &[&shared]); + for (tag, manifest) in [("leaked", &leaked), ("kept", &kept)] { + let resp = oci.put_manifest(&name, tag, OCI_MANIFEST, manifest).await; + assert_eq!(resp.status(), 201); + } + // A push in progress: a layer whose manifest is still to come, and the + // shared layer mounted into another image. + let pending = format!("pending layer {}", unique()).into_bytes(); + writer.push_blob(&name, &pending).await; + let other = format!("{repo}/other"); + let token = writer + .token(&[ + format!("repository:{other}:pull,push"), + format!("repository:{name}:pull"), + ]) + .await; + let resp = writer + .client + .post(format!("{}/v2/{other}/blobs/uploads/", writer.base)) + .bearer_auth(token) + .query(&[("mount", sha(&shared).as_str()), ("from", name.as_str())]) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "mounted"); + + // Time passes for every link of the database. + let pass = |minutes: i32| { + let pool = srv.pool.clone(); + async move { + sqlx::query( + "UPDATE oci_blob_link SET last_used_at = last_used_at - make_interval(mins => $1)", + ) + .bind(minutes) + .execute(&pool) + .await + .unwrap(); + } + }; + let sweep = || { + forge_server::images::gc::sweep_once( + &srv.pool, + chrono::Duration::hours(1), + chrono::Duration::hours(1), + chrono::Duration::zero(), + ) + }; + let blob = |bytes: &[u8]| format!("blobs/{}", sha(bytes)); + // The image was pushed long ago. + pass(600).await; + + assert_eq!( + oci.delete(&name, &format!("manifests/{}", sha(&leaked))) + .await + .status(), + 202 + ); + // What only the deleted manifest named goes with the next sweep, however + // it is fetched before: by a reader, by a writer's pull, with a HEAD. + for client in [&reader, &writer] { + assert_eq!(client.get(&name, &blob(&secret_layer)).await.status(), 200); + assert_eq!(client.head(&name, &blob(&leaked_cfg)).await.status(), 200); + } + // A reader is granted no push scope, so its "push" HEAD confirms nothing. assert_eq!( - oci.head(&name, &format!("blobs/{digest}")).await.status(), + reader + .head_to_push(&name, &blob(&secret_layer)) + .await + .status(), + 200 + ); + // The writer goes on with its push: the pending layer, and the mount. + assert_eq!( + writer.head_to_push(&name, &blob(&pending)).await.status(), + 200 + ); + assert_eq!( + writer.head_to_push(&other, &blob(&shared)).await.status(), 200 ); - assert!(used_recently().await, "a HEAD after a tenth of the TTL"); + assert_eq!( + sweep().await.unwrap().links, + 2, + "the secret layer, its config" + ); + for client in [&oci, &reader, &writer] { + assert_eq!(client.get(&name, &blob(&secret_layer)).await.status(), 404); + assert_eq!(client.get(&name, &blob(&leaked_cfg)).await.status(), 404); + } + assert_eq!( + ref_count(&srv, &sha(&secret_layer)).await, + Some(0), + "nothing links the layer any more: the blob outbox deletes it" + ); - unused_for(2).await; + // The other manifest pulls whole, and the push in progress completes. + assert_eq!(reader.get(&name, "manifests/kept").await.status(), 200); + for bytes in [&shared, &kept_cfg] { + let resp = reader.get(&name, &blob(bytes)).await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.bytes().await.unwrap().as_ref(), bytes.as_slice()); + } + pass(59).await; + assert_eq!(sweep().await.unwrap().links, 0, "confirmed 59 minutes ago"); + let cfg = config("riscv64"); + writer.push_blob(&name, &cfg).await; + let resp = writer + .put_manifest(&name, "next", OCI_MANIFEST, &image(&cfg, &[&pending])) + .await; + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); + writer.push_blob(&other, &cfg).await; + let resp = writer + .put_manifest(&other, "1", OCI_MANIFEST, &image(&cfg, &[&shared])) + .await; + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); + + // A layer nobody confirms or names within the grace period goes, read + // or not. + let abandoned = writer + .push_blob(&name, format!("abandoned layer {}", unique()).as_bytes()) + .await; + pass(59).await; assert_eq!( - oci.head(&name, &format!("blobs/{digest}")).await.status(), + writer + .get(&name, &format!("blobs/{abandoned}")) + .await + .status(), 200 ); - assert!(!used_recently().await, "not recorded on every request"); + pass(2).await; + assert_eq!(sweep().await.unwrap().links, 1); + assert_eq!( + writer + .get(&name, &format!("blobs/{abandoned}")) + .await + .status(), + 404 + ); } #[tokio::test] diff --git a/docs/changes/2.2.0/576.security.md b/docs/changes/2.2.0/576.security.md new file mode 100644 index 00000000..90dd86fb --- /dev/null +++ b/docs/changes/2.2.0/576.security.md @@ -0,0 +1,11 @@ +Docker: the config and layers of a deleted image, manifest or tag stayed +pullable by digest for as long as someone fetched them once within +`docker.unreferenced_blob_ttl_seconds` (24 h), since every `GET` or `HEAD` +of a blob no manifest references renewed its grace period: anyone who could +read the repository and knew a digest kept the content of a deleted image +alive. A fetch renews it only for a client that may push the image (a `HEAD` +with a push token, as `docker push` sends before it uploads a layer), so the +registry sweeper releases such a blob at most that TTL after it was last +pushed or mounted, however often it is pulled. The image page's delete +dialog and [docs/docker.md](docs/docker.md#retention-and-garbage-collection) +say what stays pullable and for how long. diff --git a/docs/docker.md b/docs/docker.md index 4b1b9961..ab8d885e 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -389,10 +389,23 @@ releases the blobs no manifest of the image references any more once they have been unused for `docker.unreferenced_blob_ttl_seconds` (24 h by default: the time a client has between pushing layers and pushing their manifest). Pushing or mounting a blob the image already has counts as a -use, and so does a pull or `HEAD` of it, recorded at most every tenth of -that TTL (at most hourly). The blob outbox deletes a blob after its grace period once no -image of any repository links it, so deleting an image never removes a layer -another image uses. +use, and so does a `HEAD` or pull of it by a client that may push the image +(with a registry token, one whose scope has `push`: `docker push` asks with a +`HEAD` whether the image has a layer before it uploads it), recorded at most +every tenth of that TTL (at most hourly). The blob outbox deletes a blob +after its grace period once no image of any repository links it, so deleting +an image never removes a layer another image uses. + +So a deleted image, manifest or tag's config and layers do not go at once. +Until the sweeper releases them, whoever may read the repository and knows a +blob's digest can still pull it (`GET /v2//blobs/`): for up +to `docker.unreferenced_blob_ttl_seconds` after the blob was last pushed, +mounted or confirmed by a pusher, plus one sweep interval. For an image +pushed more than that TTL ago, that is the next sweep. Pulling a blob does +not prolong this: no reader keeps a deleted image's content alive by +fetching it. Someone who may push the image can, and could push the content +again as well. When an image is deleted because a layer leaked a secret, +rotate the secret: clients that pulled the image keep their copy. A docker repository's retention policy (`PUT /admin/repositories//retention`) has image rules: diff --git a/forge.toml.example b/forge.toml.example index d91fe9a5..03f1f41b 100644 --- a/forge.toml.example +++ b/forge.toml.example @@ -248,7 +248,7 @@ token_ttl_seconds = 300 # (default: 300) registry token life # and kept in the database, sealed by the secret encryption. # token_secret = "…" upload_ttl_seconds = 86400 # (default: 24 h) abandoned upload sessions expire -unreferenced_blob_ttl_seconds = 86400 # (default: 24 h, at least 1 h) unused blobs no manifest references +unreferenced_blob_ttl_seconds = 86400 # (default: 24 h, at least 1 h) blobs no manifest references, since their last push sweep_interval_seconds = 300 # (default: 300) [cache] diff --git a/web/src/components/ImageDeleteDialog.tsx b/web/src/components/ImageDeleteDialog.tsx index d63d5e3e..5a2e63c9 100644 --- a/web/src/components/ImageDeleteDialog.tsx +++ b/web/src/components/ImageDeleteDialog.tsx @@ -231,6 +231,13 @@ export function ImageDeleteDialog({ } > {warning} + {target && target.kind !== 'tag' && ( + + The config and layers no other manifest of {image} uses can still be pulled by digest + until the registry releases them: with the default settings, up to a day after they were + last pushed. Pulling them does not prolong that. + + )} {del.isError && ( {del.error.body || del.error.message} diff --git a/web/src/routes/ImageDetail.test.tsx b/web/src/routes/ImageDetail.test.tsx index 00c8a9ec..c0bb060c 100644 --- a/web/src/routes/ImageDetail.test.tsx +++ b/web/src/routes/ImageDetail.test.tsx @@ -820,6 +820,7 @@ describe('ImageDetail', () => { const dialog = screen.getByRole('dialog') expect(dialog).toHaveTextContent('Delete stable/web 1.9.0?') expect(dialog).toHaveTextContent('This cannot be undone') + expect(dialog).toHaveTextContent('can still be pulled by digest until the registry releases them') await user.click(await confirmButton(dialog, 'Delete version')) // Its own tag is the version: nothing else is said to go. expect(dialog).toHaveTextContent( @@ -963,6 +964,7 @@ describe('ImageDetail', () => { const dialog = screen.getByRole('dialog') expect(dialog).toHaveTextContent('Delete tag 1.36?') expect(dialog).toHaveTextContent('can still be pulled by digest') + expect(dialog).not.toHaveTextContent('until the registry releases them') await user.click(within(dialog).getByRole('button', { name: 'Cancel' })) expect(asked).toEqual([]) @@ -994,6 +996,7 @@ describe('ImageDetail', () => { expect(dialog).toHaveTextContent('Delete manifest 1111111111aa?') const confirm = await confirmButton(dialog, 'Delete manifest') expect(dialog).toHaveTextContent('with its tag 1.36') + expect(dialog).toHaveTextContent('can still be pulled by digest until the registry releases them') expect(dialog).toHaveTextContent( 'With it go the 1 manifest it lists and 1 referrer (signatures, SBOMs, attestations); what a tag or another manifest still needs stays.', ) @@ -1036,6 +1039,7 @@ describe('ImageDetail', () => { await user.click(await screen.findByRole('button', { name: 'Delete image…' })) const dialog = screen.getByRole('dialog') expect(dialog).toHaveTextContent('Delete team/app?') + expect(dialog).toHaveTextContent('can still be pulled by digest until the registry releases them') expect(dialog).toHaveTextContent('2 tags and 4 manifests. Clients') await user.click(within(dialog).getByRole('button', { name: 'Delete image' }))