diff --git a/crates/server/src/admin/images.rs b/crates/server/src/admin/images.rs index b1b6358f..a192ebbf 100644 --- a/crates/server/src/admin/images.rs +++ b/crates/server/src/admin/images.rs @@ -10,7 +10,9 @@ //! * `reference` a digest: deletes the manifest with its tags, and what only //! it kept ([`taken_along`]): the manifests an index lists that nothing //! else lists or tags, and its referrers (signatures, SBOMs, -//! attestations), as image retention would take them along. +//! attestations), as image retention would take them along. A manifest +//! a push adds while the delete waits for the image's locks stays, with +//! what it lists. //! * no `reference`: deletes the image, every manifest and tag of it, in //! batches of [`IMAGE_DELETE_BATCH`] manifests, the newest first, each in //! a transaction of its own: an image of tens of thousands of manifests @@ -23,6 +25,7 @@ //! sends it) is refused (**400**): only its absence deletes the whole image. //! So is a query key other than `image` and `reference` (`tag=`, `digest=`, //! `ref=`): a misspelt reference is never taken for its absence. +//! A delete that deadlocks with a push is run once more. //! Answers `{"manifests": , "tags": }`, what was deleted; **404** when //! nothing matched. Each delete is in the audit log (`forge::audit`, "image //! delete") with the image and reference, which the admin change log's path @@ -126,7 +129,10 @@ pub async fn delete( let grace = state.config.outbox.grace(); let (result, what) = match &target { Target::Manifest(digest) => ( - delete_manifest(&state, repo.id, &query.image, digest.as_str(), grace).await, + retrying_a_deadlock(&query.image, || { + delete_manifest(&state.pool, repo.id, &query.image, digest.as_str(), grace) + }) + .await, "manifest not found", ), Target::Tag(tag) => ( @@ -134,7 +140,7 @@ pub async fn delete( "tag not found", ), Target::Image => ( - delete_image(&state, repo.id, &query.image, grace).await, + delete_image(&state.pool, repo.id, &query.image, grace).await, "image not found", ), }; @@ -243,7 +249,10 @@ fn manifest_preview( tags: &[RetentionTag], target: i64, ) -> DeletePreview { - let doomed: HashSet = taken_along(manifests, tags, target).into_iter().collect(); + // Nothing arrives while a preview waits: it takes no lock. + let doomed: HashSet = taken_along(manifests, tags, target, &HashSet::new()) + .into_iter() + .collect(); let doomed_digests: HashSet<&str> = manifests .iter() .filter(|m| doomed.contains(&m.id)) @@ -316,17 +325,34 @@ const IMAGE_DELETE_BATCH: usize = 100; /// Delete the manifest `digest` with what only it kept. The image's /// manifests and tags are locked first, so a push cannot tag or list a /// manifest this decided to delete before it is gone. +/// +/// The locking statement alone does not show what to decide on: it waits for +/// a push that holds one of the manifests (an index listing it holds it +/// `FOR SHARE`), and its snapshot predates what that push committed +/// meanwhile, the index included (#560). So the image is read a second time, +/// under the locks: whatever lists or tags a manifest the first statement +/// locked had to commit before the lock was granted, and the second read +/// sees it. A manifest only the second read returns arrived in between; what +/// may list or tag it by now is not known, so it stays, and keeps what it +/// lists and what refers to it. async fn delete_manifest( - state: &AppState, + pool: &sqlx::PgPool, repo_id: i64, image: &str, digest: &str, grace: chrono::Duration, ) -> sqlx::Result { - let mut tx = forge_db::begin(&state.pool).await?; + let mut tx = forge_db::begin(pool).await?; + let (_, before) = oci_dao::lock_image_for_retention(&mut tx, repo_id, image).await?; let (tags, manifests) = oci_dao::lock_image_for_retention(&mut tx, repo_id, image).await?; + let held: HashSet = before.iter().map(|m| m.id).collect(); + let arrived: HashSet = manifests + .iter() + .map(|m| m.id) + .filter(|id| !held.contains(id)) + .collect(); let doomed = match manifests.iter().find(|m| m.digest == digest) { - Some(target) => taken_along(&manifests, &tags, target.id), + Some(target) => taken_along(&manifests, &tags, target.id, &arrived), None => Vec::new(), }; let deleted = oci_dao::delete_locked_manifests(&mut tx, &doomed, grace).await?; @@ -338,29 +364,57 @@ async fn delete_manifest( /// batch per transaction, and then what a push added meanwhile, until none /// is left. async fn delete_image( - state: &AppState, + pool: &sqlx::PgPool, repo_id: i64, image: &str, grace: chrono::Duration, ) -> sqlx::Result { let mut deleted = Deleted::default(); loop { - let ids = oci_dao::image_manifest_ids(&state.pool, repo_id, image).await?; + let ids = oci_dao::image_manifest_ids(pool, repo_id, image).await?; if ids.is_empty() { return Ok(deleted); } for batch in ids.chunks(IMAGE_DELETE_BATCH) { - let mut tx = forge_db::begin(&state.pool).await?; - let locked = oci_dao::lock_manifests(&mut tx, batch).await?; - let done = oci_dao::delete_locked_manifests(&mut tx, &locked, grace).await?; - tx.commit().await?; - let done = Deleted::from(done); + let done = retrying_a_deadlock(image, || delete_batch(pool, batch, grace)).await?; deleted.manifests += done.manifests; deleted.tags += done.tags; } } } +/// Lock and delete one batch of [`delete_image`], in a transaction. +async fn delete_batch( + pool: &sqlx::PgPool, + batch: &[i64], + grace: chrono::Duration, +) -> sqlx::Result { + let mut tx = forge_db::begin(pool).await?; + let locked = oci_dao::lock_manifests(&mut tx, batch).await?; + let done = oci_dao::delete_locked_manifests(&mut tx, &locked, grace).await?; + tx.commit().await?; + Ok(done.into()) +} + +/// Run `transaction`, and once more when it deadlocked (`40P01`). A delete +/// locks the manifests in id order and a push the manifests its index lists +/// in digest order, so the two can wait for each other; Postgres then aborts +/// one side. The delete's transaction rolled back whole, so running it again +/// is safe, as for a repository delete. +async fn retrying_a_deadlock(image: &str, transaction: F) -> sqlx::Result +where + F: Fn() -> Fut, + Fut: std::future::Future>, +{ + match transaction().await { + Err(sqlx::Error::Database(db)) if db.code().as_deref() == Some("40P01") => { + tracing::warn!(image, "image delete deadlocked; retrying"); + transaction().await + } + other => other, + } +} + impl From for Deleted { fn from(deleted: oci_dao::DeletedManifests) -> Self { Deleted { @@ -375,11 +429,14 @@ impl From for Deleted { /// whose subject it is, and theirs) that nothing kept still needs. A /// manifest is kept when it has a tag or is not reachable from `target`; a /// kept manifest keeps its children and its referrers, as in image -/// retention. `target` itself goes whatever lists it. +/// retention. So is a manifest of `arrived`, one a push added while the +/// delete waited for the image's locks. `target` itself goes whatever lists +/// it. pub(crate) fn taken_along( manifests: &[RetentionManifest], tags: &[RetentionTag], target: i64, + arrived: &HashSet, ) -> Vec { let by_digest: HashMap<&str, &RetentionManifest> = manifests.iter().map(|m| (m.digest.as_str(), m)).collect(); @@ -420,7 +477,9 @@ pub(crate) fn taken_along( let mut kept: Vec = manifests .iter() .map(|m| m.id) - .filter(|id| *id != target && (!seen.contains(id) || tagged.contains(id))) + .filter(|id| { + *id != target && (!seen.contains(id) || tagged.contains(id) || arrived.contains(id)) + }) .collect(); let mut keeping: HashSet = kept.iter().copied().collect(); let mut i = 0; @@ -464,6 +523,11 @@ mod tests { } } + /// No manifest arrived while the delete waited. + fn none() -> HashSet { + HashSet::new() + } + fn sorted(mut ids: Vec) -> Vec { ids.sort(); ids @@ -472,7 +536,10 @@ mod tests { #[test] fn an_image_manifest_goes_alone() { let manifests = [manifest(1, &[], None), manifest(2, &[], None)]; - assert_eq!(taken_along(&manifests, &[tag("v1", 1)], 1), vec![1]); + assert_eq!( + taken_along(&manifests, &[tag("v1", 1)], 1, &none()), + vec![1] + ); } #[test] @@ -486,7 +553,7 @@ mod tests { manifest(5, &[], Some(2)), manifest(6, &[], None), ]; - let doomed = taken_along(&manifests, &[tag("v1", 1), tag("v2", 6)], 1); + let doomed = taken_along(&manifests, &[tag("v1", 1), tag("v2", 6)], 1, &none()); assert_eq!(doomed[0], 1, "the target first"); assert_eq!(sorted(doomed), vec![1, 2, 3, 4, 5]); } @@ -504,7 +571,7 @@ mod tests { manifest(8, &[], None), ]; let tags = [tag("v1", 1), tag("v1-arm", 3), tag("sig", 4)]; - assert_eq!(taken_along(&manifests, &tags, 1), vec![1]); + assert_eq!(taken_along(&manifests, &tags, 1, &none()), vec![1]); } #[test] @@ -517,7 +584,7 @@ mod tests { manifest(3, &[], Some(2)), ]; assert_eq!( - sorted(taken_along(&manifests, &[tag("v1", 1)], 2)), + sorted(taken_along(&manifests, &[tag("v1", 1)], 2, &none())), vec![2, 3] ); } @@ -580,6 +647,34 @@ mod tests { #[test] fn a_referrer_goes_without_its_subject() { let manifests = [manifest(1, &[], None), manifest(2, &[], Some(1))]; - assert_eq!(taken_along(&manifests, &[tag("v1", 1)], 2), vec![2]); + assert_eq!( + taken_along(&manifests, &[tag("v1", 1)], 2, &none()), + vec![2] + ); + } + + #[test] + fn what_arrived_while_the_delete_waited_stays_with_what_it_keeps() { + // Index 1 lists 2 and 3, and 4 signs 3. While the delete of 1 + // waited, a push added the index 5, which lists 2, and 6, a + // signature of 1: both stay, untagged as they are, and 5 keeps 2. + let manifests = [ + manifest(1, &[2, 3], None), + manifest(2, &[], None), + manifest(3, &[], None), + manifest(4, &[], Some(3)), + manifest(5, &[2], None), + manifest(6, &[], Some(1)), + ]; + let arrived = HashSet::from([5, 6]); + assert_eq!( + sorted(taken_along(&manifests, &[tag("v1", 1)], 1, &arrived)), + vec![1, 3, 4] + ); + // The manifest asked for goes even when it only just arrived. + assert_eq!( + taken_along(&manifests, &[tag("v1", 1)], 6, &arrived), + vec![6] + ); } } diff --git a/crates/server/tests/registry.rs b/crates/server/tests/registry.rs index 54bec4ea..d18992e7 100644 --- a/crates/server/tests/registry.rs +++ b/crates/server/tests/registry.rs @@ -1527,6 +1527,223 @@ async fn admins_delete_an_image_of_many_manifests_in_batches() { assert_eq!(oci.head(&other, "manifests/v0").await.status(), 200); } +/// An index listing the manifests `children` as platform manifests. +fn index_of(children: &[&str], note: &str) -> Vec { + json!({ + "schemaVersion": 2, + "mediaType": OCI_INDEX, + "manifests": children + .iter() + .map(|digest| json!({"mediaType": OCI_MANIFEST, "digest": digest, "size": 1})) + .collect::>(), + "annotations": {"note": note}, + }) + .to_string() + .into_bytes() +} + +async fn manifest_id(srv: &TestServer, repo_id: i64, image: &str, digest: &str) -> i64 { + sqlx::query_scalar( + "SELECT id FROM oci_manifest WHERE repository_id = $1 AND image = $2 AND digest = $3", + ) + .bind(repo_id) + .bind(image) + .bind(digest) + .fetch_one(&srv.pool) + .await + .unwrap() +} + +/// An administrator deletes `v1`'s index while CI pushes `v2`, an index +/// listing the same platform manifest (#560). The push holds the platform +/// manifest `FOR SHARE` and commits its index while the delete waits for +/// that manifest's lock; the delete's locking statement, whose snapshot +/// predates the push, never returns the new index. The delete used to decide +/// on that statement's rows and deleted the platform manifest `v2` lists; it +/// reads the image again under the locks and keeps it. +#[tokio::test] +async fn an_admin_delete_keeps_a_platform_manifest_an_index_pushed_meanwhile_lists() { + let srv = common::start().await; + let repo = repo_name("dock"); + let created = create_repo( + &srv, + json!({"name": repo, "kind": "hosted", "format": "docker"}), + ) + .await; + let repo_id = created["id"].as_i64().unwrap(); + let oci = admin_oci(&srv).await; + let name = format!("{repo}/app"); + let layer = format!("shared platform {}", unique()).into_bytes(); + // The platform manifest, kept by the indexes alone, and v1's index. + let child = oci.push_image(&name, "amd64", "amd64", &layer).await; + assert_eq!(oci.delete(&name, "manifests/amd64").await.status(), 202); + let old_index = index_of(&[&child], "v1"); + assert_eq!( + oci.put_manifest(&name, "v1", OCI_INDEX, &old_index) + .await + .status(), + 201 + ); + let child_id = manifest_id(&srv, repo_id, "app", &child).await; + + // The push of v2 stops where it creates its tag, behind another + // transaction's uncommitted `v2`: by then it holds the platform manifest + // FOR SHARE and has inserted its index. + let mut blocker = srv.pool.begin().await.unwrap(); + sqlx::query( + "INSERT INTO oci_tag (repository_id, image, tag, manifest_id) VALUES ($1, 'app', 'v2', $2)", + ) + .bind(repo_id) + .bind(child_id) + .execute(&mut *blocker) + .await + .unwrap(); + let new_index = index_of(&[&child], "v2"); + let push = tokio::spawn({ + let request = srv + .client() + .put(format!("{}/v2/{name}/manifests/v2", srv.base_url)) + .header(AUTHORIZATION, oci.bearer(&name, "pull,push").await) + .header(CONTENT_TYPE, OCI_INDEX) + .body(new_index.clone()); + async move { request.send().await.unwrap().status() } + }); + common::wait_for_lock_wait(&srv.pool, &["%INSERT INTO oci_tag%ON CONFLICT%"]).await; + + // The delete of v1's index waits for the platform manifest. + let (u, p) = srv.admin_basic(); + let delete = tokio::spawn({ + let request = srv + .client() + .delete(format!("{}/admin/repositories/{repo}/images", srv.base_url)) + .basic_auth(u, Some(p)) + .query(&[("image", "app".to_string()), ("reference", sha(&old_index))]); + async move { request.send().await.unwrap() } + }); + common::wait_for_lock_wait(&srv.pool, &["%FROM oci_manifest m%FOR UPDATE OF m%"]).await; + + // The push commits its index and its tag; only then is the delete + // granted the lock. + blocker.rollback().await.unwrap(); + assert_eq!(push.await.unwrap(), 201); + let resp = delete.await.unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await.unwrap(), + json!({"manifests": 1, "tags": 1}) + ); + + assert_eq!(oci.head(&name, "manifests/v1").await.status(), 404); + assert_eq!( + oci.head(&name, &format!("manifests/{}", sha(&old_index))) + .await + .status(), + 404 + ); + // v2 pulls whole: its index, and the platform manifest it lists. + assert_eq!(oci.head(&name, "manifests/v2").await.status(), 200); + assert_eq!( + oci.get(&name, &format!("manifests/{child}")).await.status(), + 200, + "the platform manifest v2 lists" + ); +} + +/// A delete locks the manifests in id order, a push the manifests its index +/// lists in digest order: the two can wait for each other, and Postgres +/// aborts one of them (`40P01`). The delete, of a manifest or of the whole +/// image, then runs once more instead of answering 500 (#560). +#[tokio::test] +async fn an_admin_delete_that_deadlocks_with_a_push_runs_again() { + let srv = common::start().await; + let repo = repo_name("dock"); + let created = create_repo( + &srv, + json!({"name": repo, "kind": "hosted", "format": "docker"}), + ) + .await; + let repo_id = created["id"].as_i64().unwrap(); + let oci = admin_oci(&srv).await; + for (image, whole) in [("one", false), ("all", true)] { + let name = format!("{repo}/{image}"); + let layer = format!("deadlock {image} {}", unique()).into_bytes(); + let first = oci.push_image(&name, "amd64", "amd64", &layer).await; + let second = oci.push_image(&name, "arm64", "arm64", &layer).await; + for tag in ["amd64", "arm64"] { + let resp = oci.delete(&name, &format!("manifests/{tag}")).await; + assert_eq!(resp.status(), 202); + } + let index = index_of(&[&first, &second], "v1"); + assert_eq!( + oci.put_manifest(&name, "v1", OCI_INDEX, &index) + .await + .status(), + 201 + ); + let first_id = manifest_id(&srv, repo_id, image, &first).await; + let second_id = manifest_id(&srv, repo_id, image, &second).await; + assert!(first_id < second_id); + + // A push of an index: it holds the manifest the delete locks last... + let mut push = srv.pool.begin().await.unwrap(); + let share = "SELECT id FROM oci_manifest WHERE id = $1 FOR SHARE"; + sqlx::query(share) + .bind(second_id) + .execute(&mut *push) + .await + .unwrap(); + // ...while the delete starts, locks the other and waits. + let (u, p) = srv.admin_basic(); + let delete = tokio::spawn({ + let mut query = vec![("image", image.to_string())]; + if !whole { + query.push(("reference", sha(&index))); + } + let request = srv + .client() + .delete(format!("{}/admin/repositories/{repo}/images", srv.base_url)) + .basic_auth(u, Some(p)) + .query(&query); + async move { request.send().await.unwrap() } + }); + common::wait_for_lock_wait( + &srv.pool, + &[ + "%FROM oci_manifest m%FOR UPDATE OF m%", + "%FROM oci_manifest WHERE id = ANY%FOR UPDATE%", + ], + ) + .await; + // The push asks for the manifest the delete holds: a cycle. The + // delete waited first, so its deadlock check (after 1 s) runs first + // and aborts it, and the push is granted the lock. + sqlx::query(share) + .bind(first_id) + .execute(&mut *push) + .await + .expect("the delete is the transaction Postgres aborts"); + // The delete runs again and waits for the push, which commits. + common::wait_for_lock_wait( + &srv.pool, + &[ + "%FROM oci_manifest m%FOR UPDATE OF m%", + "%FROM oci_manifest WHERE id = ANY%FOR UPDATE%", + ], + ) + .await; + push.commit().await.unwrap(); + + let resp = delete.await.unwrap(); + assert_eq!(resp.status(), 200, "{image}"); + assert_eq!( + resp.json::().await.unwrap(), + json!({"manifests": 3, "tags": 1}), + "{image}" + ); + assert_eq!(oci.head(&name, "manifests/v1").await.status(), 404); + } +} + const HELM_CONFIG: &str = "application/vnd.cncf.helm.config.v1+json"; const HELM_CHART: &str = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"; diff --git a/docs/changes/2.2.0/560.fixed.md b/docs/changes/2.2.0/560.fixed.md new file mode 100644 index 00000000..8386df5f --- /dev/null +++ b/docs/changes/2.2.0/560.fixed.md @@ -0,0 +1,7 @@ +Deleting a manifest of an image from the web UI's image page or with +`DELETE /admin/repositories/{name}/images` while a push added an index +listing one of its platform manifests could delete that platform manifest: +the new tag stayed, and pulling it failed for that platform. The delete now +decides on the image as it is once it holds the image's locks, and a +manifest pushed meanwhile stays with what it lists. A delete of a manifest +or an image that deadlocked with a push answered 500; it is run once more. diff --git a/docs/docker.md b/docs/docker.md index 4b1b9961..476bdc66 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -341,10 +341,12 @@ curl -u admin@example.com: -X DELETE \ A manifest goes with its tags and with what only it kept: the manifests an index lists and the referrers (signatures, SBOMs, attestations) of it, unless a tag or another manifest of the image still needs them, as image retention -decides. A manifest an index lists is deleted when asked for, even though -pulling that platform through the index then fails. The answer counts the -manifests and tags deleted (404 when nothing matched). A hosted repository is -deleted only once it holds no image, so emptying one is deleting its images. +decides. A manifest pushed while the delete waits for a push to the image +stays, with the manifests it lists. A manifest an index lists is deleted when +asked for, even though pulling that platform through the index then fails. +The answer counts the manifests and tags deleted (404 when nothing matched). +A hosted repository is deleted only once it holds no image, so emptying one +is deleting its images. To see what a delete would remove before running it, ask `GET /admin/repositories//images/delete-preview` with the same `image`