diff --git a/.sqlx/query-14a89a45dfd88b0445de5759e5dcd6eca6ba43acadf742ac66a743606a5d332d.json b/.sqlx/query-14a89a45dfd88b0445de5759e5dcd6eca6ba43acadf742ac66a743606a5d332d.json new file mode 100644 index 00000000..fd0603c8 --- /dev/null +++ b/.sqlx/query-14a89a45dfd88b0445de5759e5dcd6eca6ba43acadf742ac66a743606a5d332d.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (SELECT count(*) FROM oci_manifest\n WHERE repository_id = $1 AND image = $2) AS \"manifests!\",\n (SELECT count(*) FROM oci_tag\n WHERE repository_id = $1 AND image = $2) AS \"tags!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifests!", + "type_info": "Int8", + "origin": "Expression" + }, + { + "ordinal": 1, + "name": "tags!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "14a89a45dfd88b0445de5759e5dcd6eca6ba43acadf742ac66a743606a5d332d" +} diff --git a/.sqlx/query-8c2603f0e4ec470f7d6774f03a8eb01652b0de2346cb411eded082f59a4ae70d.json b/.sqlx/query-8c2603f0e4ec470f7d6774f03a8eb01652b0de2346cb411eded082f59a4ae70d.json new file mode 100644 index 00000000..73e2002b --- /dev/null +++ b/.sqlx/query-8c2603f0e4ec470f7d6774f03a8eb01652b0de2346cb411eded082f59a4ae70d.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (SELECT count(*) FROM oci_manifest m\n WHERE m.repository_id = ($1::bigint[])[$3::int] AND m.image = $2\n AND NOT EXISTS (\n SELECT FROM oci_manifest h\n WHERE h.repository_id = ANY(($1::bigint[])[1:$3::int - 1])\n AND h.image = $2 AND h.digest = m.digest)) AS \"manifests!\",\n (SELECT count(*) FROM oci_tag t\n WHERE t.repository_id = ($1::bigint[])[$3::int] AND t.image = $2\n AND NOT EXISTS (\n SELECT FROM oci_tag h\n WHERE h.repository_id = ANY(($1::bigint[])[1:$3::int - 1])\n AND h.image = $2 AND h.tag = t.tag)) AS \"tags!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifests!", + "type_info": "Int8", + "origin": "Expression" + }, + { + "ordinal": 1, + "name": "tags!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Int8Array", + "Text", + "Int4" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "8c2603f0e4ec470f7d6774f03a8eb01652b0de2346cb411eded082f59a4ae70d" +} diff --git a/.sqlx/query-c6dd56d4c99e21704ae5c6ca33814d02c0e21a19d79aefc4e950f9cd63b23bee.json b/.sqlx/query-c6dd56d4c99e21704ae5c6ca33814d02c0e21a19d79aefc4e950f9cd63b23bee.json deleted file mode 100644 index 28704076..00000000 --- a/.sqlx/query-c6dd56d4c99e21704ae5c6ca33814d02c0e21a19d79aefc4e950f9cd63b23bee.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT m.digest, m.media_type, m.artifact_type, m.subject_digest, m.size,\n m.summary, m.created_at, m.last_pulled_at,\n coalesce(array_agg(t.tag ORDER BY t.tag) FILTER (WHERE t.tag IS NOT NULL),\n '{}') AS \"tags!\",\n ARRAY(SELECT r.digest FROM oci_manifest_ref r\n WHERE r.manifest_id = m.id AND r.kind = 'manifest'\n ORDER BY r.digest) AS \"children!\"\n FROM oci_manifest m\n LEFT JOIN oci_tag t ON t.manifest_id = m.id\n WHERE m.repository_id = $1 AND m.image = $2\n GROUP BY m.id\n ORDER BY m.created_at DESC, m.digest\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "digest", - "type_info": "Text", - "origin": { - "Table": { - "table": "oci_manifest", - "name": "digest" - } - } - }, - { - "ordinal": 1, - "name": "media_type", - "type_info": "Text", - "origin": { - "Table": { - "table": "oci_manifest", - "name": "media_type" - } - } - }, - { - "ordinal": 2, - "name": "artifact_type", - "type_info": "Text", - "origin": { - "Table": { - "table": "oci_manifest", - "name": "artifact_type" - } - } - }, - { - "ordinal": 3, - "name": "subject_digest", - "type_info": "Text", - "origin": { - "Table": { - "table": "oci_manifest", - "name": "subject_digest" - } - } - }, - { - "ordinal": 4, - "name": "size", - "type_info": "Int8", - "origin": { - "Table": { - "table": "oci_manifest", - "name": "size" - } - } - }, - { - "ordinal": 5, - "name": "summary", - "type_info": "Jsonb", - "origin": { - "Table": { - "table": "oci_manifest", - "name": "summary" - } - } - }, - { - "ordinal": 6, - "name": "created_at", - "type_info": "Timestamptz", - "origin": { - "Table": { - "table": "oci_manifest", - "name": "created_at" - } - } - }, - { - "ordinal": 7, - "name": "last_pulled_at", - "type_info": "Timestamptz", - "origin": { - "Table": { - "table": "oci_manifest", - "name": "last_pulled_at" - } - } - }, - { - "ordinal": 8, - "name": "tags!", - "type_info": "TextArray", - "origin": "Expression" - }, - { - "ordinal": 9, - "name": "children!", - "type_info": "TextArray", - "origin": "Expression" - } - ], - "parameters": { - "Left": [ - "Int8", - "Text" - ] - }, - "nullable": [ - false, - false, - true, - true, - false, - false, - false, - true, - null, - null - ] - }, - "hash": "c6dd56d4c99e21704ae5c6ca33814d02c0e21a19d79aefc4e950f9cd63b23bee" -} diff --git a/.sqlx/query-edc8c59bd5d1f33414e34a9d826d26dfbfd141d2487c26b4fedb5da3b60dee99.json b/.sqlx/query-edc8c59bd5d1f33414e34a9d826d26dfbfd141d2487c26b4fedb5da3b60dee99.json new file mode 100644 index 00000000..4be9b6c8 --- /dev/null +++ b/.sqlx/query-edc8c59bd5d1f33414e34a9d826d26dfbfd141d2487c26b4fedb5da3b60dee99.json @@ -0,0 +1,104 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH tagged AS MATERIALIZED (\n -- `untagged_since` is NULL exactly while a tag points at the\n -- manifest (`set_tag`, `delete_tag`).\n SELECT m.*\n FROM oci_manifest m\n WHERE m.repository_id = ($1::bigint[])[$3::int] AND m.image = $2\n AND m.untagged_since IS NULL\n ORDER BY m.created_at DESC, m.digest\n LIMIT $4\n ),\n newest AS MATERIALIZED (\n SELECT m.*\n FROM oci_manifest m\n WHERE m.repository_id = ($1::bigint[])[$3::int] AND m.image = $2\n ORDER BY m.created_at DESC, m.digest\n LIMIT $5\n ),\n listed AS (\n SELECT * FROM tagged\n UNION ALL\n SELECT * FROM newest WHERE id NOT IN (SELECT id FROM tagged)\n )\n -- The manifest shown for the digest is the first repository's (`s`),\n -- the listed one (`m`) when none before it holds the digest. Both\n -- lateral subqueries run once per listed manifest, by key: a join\n -- would be free to read the table instead.\n SELECT coalesce(s.repository_id, m.repository_id) AS \"repository_id!\",\n m.digest AS \"digest!\",\n coalesce(s.media_type, m.media_type) AS \"media_type!\",\n CASE WHEN s.id IS NULL THEN m.artifact_type ELSE s.artifact_type END\n AS artifact_type,\n CASE WHEN s.id IS NULL THEN m.subject_digest ELSE s.subject_digest END\n AS subject_digest,\n coalesce(s.size, m.size) AS \"size!\",\n coalesce(s.summary, m.summary) AS \"summary!\",\n coalesce(s.created_at, m.created_at) AS \"created_at!\",\n CASE WHEN s.id IS NULL THEN m.last_pulled_at ELSE s.last_pulled_at END\n AS last_pulled_at,\n t.tags AS \"tags!\",\n ARRAY(SELECT r.digest FROM oci_manifest_ref r\n WHERE r.manifest_id = coalesce(s.id, m.id) AND r.kind = 'manifest'\n ORDER BY r.digest) AS \"children!\",\n ARRAY(SELECT (SELECT c.summary FROM oci_manifest c\n WHERE c.repository_id = ANY($1) AND c.image = $2\n AND c.digest = r.digest\n ORDER BY array_position($1, c.repository_id)\n LIMIT 1)\n FROM oci_manifest_ref r\n WHERE r.manifest_id = coalesce(s.id, m.id) AND r.kind = 'manifest'\n ORDER BY r.digest) AS \"child_summaries!: Vec>\"\n FROM listed m\n LEFT JOIN LATERAL (\n SELECT h.* FROM oci_manifest h\n WHERE h.repository_id = ANY(($1::bigint[])[1:$3::int - 1])\n AND h.image = $2 AND h.digest = m.digest\n ORDER BY array_position($1, h.repository_id)\n LIMIT 1\n ) s ON true\n CROSS JOIN LATERAL (\n SELECT ARRAY(SELECT t.tag FROM oci_tag t\n WHERE t.manifest_id = m.id\n AND NOT EXISTS (\n SELECT FROM oci_tag h\n WHERE h.repository_id = ANY(($1::bigint[])[1:$3::int - 1])\n AND h.image = $2 AND h.tag = t.tag)) AS tags\n ) t\n WHERE s.id IS NULL OR cardinality(t.tags) > 0\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "repository_id!", + "type_info": "Int8", + "origin": "Expression" + }, + { + "ordinal": 1, + "name": "digest!", + "type_info": "Text", + "origin": "Expression" + }, + { + "ordinal": 2, + "name": "media_type!", + "type_info": "Text", + "origin": "Expression" + }, + { + "ordinal": 3, + "name": "artifact_type", + "type_info": "Text", + "origin": "Expression" + }, + { + "ordinal": 4, + "name": "subject_digest", + "type_info": "Text", + "origin": "Expression" + }, + { + "ordinal": 5, + "name": "size!", + "type_info": "Int8", + "origin": "Expression" + }, + { + "ordinal": 6, + "name": "summary!", + "type_info": "Jsonb", + "origin": "Expression" + }, + { + "ordinal": 7, + "name": "created_at!", + "type_info": "Timestamptz", + "origin": "Expression" + }, + { + "ordinal": 8, + "name": "last_pulled_at", + "type_info": "Timestamptz", + "origin": "Expression" + }, + { + "ordinal": 9, + "name": "tags!", + "type_info": "TextArray", + "origin": "Expression" + }, + { + "ordinal": 10, + "name": "children!", + "type_info": "TextArray", + "origin": "Expression" + }, + { + "ordinal": 11, + "name": "child_summaries!: Vec>", + "type_info": "JsonbArray", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Int8Array", + "Text", + "Int4", + "Int8", + "Int8" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "edc8c59bd5d1f33414e34a9d826d26dfbfd141d2487c26b4fedb5da3b60dee99" +} diff --git a/crates/db/src/oci_dao.rs b/crates/db/src/oci_dao.rs index 46eefd9c..e42d543a 100644 --- a/crates/db/src/oci_dao.rs +++ b/crates/db/src/oci_dao.rs @@ -1414,10 +1414,11 @@ pub async fn count_images(pool: &PgPool, repository_id: i64, query: &str) -> sql .await } -/// A manifest of an image with the tags pointing at it, for the detail -/// view. +/// A manifest of an image's page ([`image_listing`]). #[derive(Debug, Clone)] pub struct ManifestListing { + /// The repository holding it. + pub repository_id: i64, pub digest: String, pub media_type: String, pub artifact_type: Option, @@ -1426,40 +1427,177 @@ pub struct ManifestListing { pub summary: JsonValue, pub created_at: DateTime, pub last_pulled_at: Option>, + /// The listed repository's tags of this digest. pub tags: Vec, /// An index's child manifests. pub children: Vec, -} - -/// Every manifest of an image, newest first, with its tags. -pub async fn image_manifests( - pool: &PgPool, - repository_id: i64, + /// The summary of each of `children`, `None` for one the repositories + /// do not hold. + pub child_summaries: Vec>, +} + +/// The newest of an image in `repositories[position]`, for its page: the +/// repository's `max_tagged` newest tagged manifests with their tags and its +/// `max_manifests` newest manifests, in no order (#561: the page shows a +/// bounded number of both, so it must not read an image's tens of +/// thousands; [`image_totals`] counts them). +/// +/// `repositories` are the repositories serving the image, the highest +/// priority first: a group's members, or the repository alone. A pull +/// resolves a tag in the first repository that has it and a digest is shown +/// once, so the ones before `position` hide what they also hold: their tags +/// are left out, and a digest they hold comes as their manifest (the +/// first's), only when a tag listed here points at it. A manifest's +/// `child_summaries` are read in all of them. +/// +/// The newest manifests are read from `oci_manifest_newest_idx` and the rest +/// by key from them. +pub async fn image_listing<'e, E>( + executor: E, + repositories: &[i64], + position: usize, image: &str, -) -> sqlx::Result> { + max_tagged: i64, + max_manifests: i64, +) -> sqlx::Result> +where + E: sqlx::PgExecutor<'e>, +{ + // SQL arrays count from 1: `$1[$3]` is the repository, `$1[1:$3 - 1]` + // the ones before it. + let position = i32::try_from(position + 1).unwrap_or(i32::MAX); sqlx::query_as!( ManifestListing, r#" - SELECT m.digest, m.media_type, m.artifact_type, m.subject_digest, m.size, - m.summary, m.created_at, m.last_pulled_at, - coalesce(array_agg(t.tag ORDER BY t.tag) FILTER (WHERE t.tag IS NOT NULL), - '{}') AS "tags!", + WITH tagged AS MATERIALIZED ( + -- `untagged_since` is NULL exactly while a tag points at the + -- manifest (`set_tag`, `delete_tag`). + SELECT m.* + FROM oci_manifest m + WHERE m.repository_id = ($1::bigint[])[$3::int] AND m.image = $2 + AND m.untagged_since IS NULL + ORDER BY m.created_at DESC, m.digest + LIMIT $4 + ), + newest AS MATERIALIZED ( + SELECT m.* + FROM oci_manifest m + WHERE m.repository_id = ($1::bigint[])[$3::int] AND m.image = $2 + ORDER BY m.created_at DESC, m.digest + LIMIT $5 + ), + listed AS ( + SELECT * FROM tagged + UNION ALL + SELECT * FROM newest WHERE id NOT IN (SELECT id FROM tagged) + ) + -- The manifest shown for the digest is the first repository's (`s`), + -- the listed one (`m`) when none before it holds the digest. Both + -- lateral subqueries run once per listed manifest, by key: a join + -- would be free to read the table instead. + SELECT coalesce(s.repository_id, m.repository_id) AS "repository_id!", + m.digest AS "digest!", + coalesce(s.media_type, m.media_type) AS "media_type!", + CASE WHEN s.id IS NULL THEN m.artifact_type ELSE s.artifact_type END + AS artifact_type, + CASE WHEN s.id IS NULL THEN m.subject_digest ELSE s.subject_digest END + AS subject_digest, + coalesce(s.size, m.size) AS "size!", + coalesce(s.summary, m.summary) AS "summary!", + coalesce(s.created_at, m.created_at) AS "created_at!", + CASE WHEN s.id IS NULL THEN m.last_pulled_at ELSE s.last_pulled_at END + AS last_pulled_at, + t.tags AS "tags!", ARRAY(SELECT r.digest FROM oci_manifest_ref r - WHERE r.manifest_id = m.id AND r.kind = 'manifest' - ORDER BY r.digest) AS "children!" - FROM oci_manifest m - LEFT JOIN oci_tag t ON t.manifest_id = m.id - WHERE m.repository_id = $1 AND m.image = $2 - GROUP BY m.id - ORDER BY m.created_at DESC, m.digest + WHERE r.manifest_id = coalesce(s.id, m.id) AND r.kind = 'manifest' + ORDER BY r.digest) AS "children!", + ARRAY(SELECT (SELECT c.summary FROM oci_manifest c + WHERE c.repository_id = ANY($1) AND c.image = $2 + AND c.digest = r.digest + ORDER BY array_position($1, c.repository_id) + LIMIT 1) + FROM oci_manifest_ref r + WHERE r.manifest_id = coalesce(s.id, m.id) AND r.kind = 'manifest' + ORDER BY r.digest) AS "child_summaries!: Vec>" + FROM listed m + LEFT JOIN LATERAL ( + SELECT h.* FROM oci_manifest h + WHERE h.repository_id = ANY(($1::bigint[])[1:$3::int - 1]) + AND h.image = $2 AND h.digest = m.digest + ORDER BY array_position($1, h.repository_id) + LIMIT 1 + ) s ON true + CROSS JOIN LATERAL ( + SELECT ARRAY(SELECT t.tag FROM oci_tag t + WHERE t.manifest_id = m.id + AND NOT EXISTS ( + SELECT FROM oci_tag h + WHERE h.repository_id = ANY(($1::bigint[])[1:$3::int - 1]) + AND h.image = $2 AND h.tag = t.tag)) AS tags + ) t + WHERE s.id IS NULL OR cardinality(t.tags) > 0 "#, - repository_id, + repositories, image, + position, + max_tagged, + max_manifests, ) - .fetch_all(pool) + .fetch_all(executor) .await } +/// How many manifests and how many tags of an image `repositories[position]` +/// adds to those of the repositories before it (see [`image_listing`]): a +/// digest or a tag one of them also holds is theirs. It reads the image's +/// index entries, not its manifests. +pub async fn image_totals( + pool: &PgPool, + repositories: &[i64], + position: usize, + image: &str, +) -> sqlx::Result<(i64, i64)> { + if position == 0 { + // Nothing comes before the first: all of its own. + let totals = sqlx::query!( + r#" + SELECT (SELECT count(*) FROM oci_manifest + WHERE repository_id = $1 AND image = $2) AS "manifests!", + (SELECT count(*) FROM oci_tag + WHERE repository_id = $1 AND image = $2) AS "tags!" + "#, + repositories.first().copied(), + image, + ) + .fetch_one(pool) + .await?; + return Ok((totals.manifests, totals.tags)); + } + let position = i32::try_from(position + 1).unwrap_or(i32::MAX); + let totals = sqlx::query!( + r#" + SELECT (SELECT count(*) FROM oci_manifest m + WHERE m.repository_id = ($1::bigint[])[$3::int] AND m.image = $2 + AND NOT EXISTS ( + SELECT FROM oci_manifest h + WHERE h.repository_id = ANY(($1::bigint[])[1:$3::int - 1]) + AND h.image = $2 AND h.digest = m.digest)) AS "manifests!", + (SELECT count(*) FROM oci_tag t + WHERE t.repository_id = ($1::bigint[])[$3::int] AND t.image = $2 + AND NOT EXISTS ( + SELECT FROM oci_tag h + WHERE h.repository_id = ANY(($1::bigint[])[1:$3::int - 1]) + AND h.image = $2 AND h.tag = t.tag)) AS "tags!" + "#, + repositories, + image, + position, + ) + .fetch_one(pool) + .await?; + Ok((totals.manifests, totals.tags)) +} + // ── Upload sessions ───────────────────────────────────────────────────────── /// A blob upload session. diff --git a/crates/db/tests/oci_image_page.rs b/crates/db/tests/oci_image_page.rs new file mode 100644 index 00000000..33beff45 --- /dev/null +++ b/crates/db/tests/oci_image_page.rs @@ -0,0 +1,260 @@ +//! `oci_dao`'s reads for an image's page against a real Postgres: the newest +//! of an image without reading the whole image, and what a group's lower +//! priority member adds to the members before it (#561). +//! +//! In a test binary of its own, and so in a database of its own: a plan +//! follows the table's statistics, and the `oci_images` test counts the rows +//! of plans that an image of 30,000 manifests in the same table changes. + +mod common; + +use forge_db::oci_dao; +use forge_db::repository_dao::{self, NewRepository}; + +use common::{pool, unique}; + +/// Index entries and table rows of `oci_manifest` this transaction read. +async fn rows_read(conn: &mut sqlx::PgConnection) -> i64 { + sqlx::query_scalar( + r#" + SELECT coalesce(sum(pg_stat_get_xact_tuples_returned(i.indexrelid)), 0)::bigint + + coalesce(pg_stat_get_xact_tuples_returned('oci_manifest'::regclass), 0) + FROM pg_index i + WHERE i.indrelid = 'oci_manifest'::regclass + "#, + ) + .fetch_one(conn) + .await + .unwrap() +} + +/// A blob for seeded manifests to name. +async fn seeded_blob(pool: &sqlx::PgPool) -> String { + let sha = format!("{:064x}", (u128::from(unique()) << 64) | 0x561); + sqlx::query( + "INSERT INTO blob (sha256, size, s3_key, state) VALUES ($1, 1, $1, 'committed') \ + ON CONFLICT DO NOTHING", + ) + .bind(&sha) + .execute(pool) + .await + .unwrap(); + sha +} + +/// Manifests `from .. to` of the image `app` in `repo`, each tagged +/// ``: manifest `n` has the digest `sha256:` and was +/// pushed `n` minutes after `pushed_from`. +async fn seed_tagged( + pool: &sqlx::PgPool, + repo: i64, + blob: &str, + (from, to): (i64, i64), + pushed_from: &str, + prefix: &str, +) { + sqlx::query( + r#" + WITH pushed AS ( + INSERT INTO oci_manifest + (repository_id, image, digest, blob_sha256, media_type, size, summary, created_at) + SELECT $1, 'app', 'sha256:' || lpad(to_hex(n), 64, '0'), $2, + 'application/vnd.oci.image.manifest.v1+json', 1, + jsonb_build_object('layers_size', n), + $5::timestamptz + n * interval '1 minute' + FROM generate_series($3::bigint, $4::bigint - 1) AS n + RETURNING id, digest + ) + INSERT INTO oci_tag (repository_id, image, tag, manifest_id) + SELECT $1, 'app', $6 || ('x' || right(digest, 8))::bit(32)::int, id FROM pushed + "#, + ) + .bind(repo) + .bind(blob) + .bind(from) + .bind(to) + .bind(pushed_from) + .bind(prefix) + .execute(pool) + .await + .unwrap(); +} + +fn digest(n: i64) -> String { + format!("sha256:{n:064x}") +} + +/// The page of an image with 30,000 tagged manifests reads the 1,000 it +/// lists through `oci_manifest_newest_idx`, not the image: the query before +/// read (and returned) every manifest of it on each request (#561). The +/// totals count the image from its index entries. +#[tokio::test] +async fn an_image_page_reads_the_manifests_it_lists_not_the_image() { + let pool = pool().await; + let repo = repository_dao::insert_repository( + &pool, + NewRepository::hosted(format!("page-{}", unique())), + ) + .await + .unwrap() + .id; + let blob = seeded_blob(&pool).await; + const MANIFESTS: i64 = 30_000; + seed_tagged(&pool, repo, &blob, (0, MANIFESTS), "2025-01-01", "t").await; + sqlx::query("ANALYZE oci_manifest") + .execute(&pool) + .await + .unwrap(); + sqlx::query("ANALYZE oci_tag").execute(&pool).await.unwrap(); + + let mut tx = pool.begin().await.unwrap(); + let before = rows_read(&mut tx).await; + let listed = oci_dao::image_listing(&mut *tx, &[repo], 0, "app", 1000, 1000) + .await + .unwrap(); + let read = rows_read(&mut tx).await - before; + drop(tx); + // The newest 1,000, each with its tag. + let mut digests: Vec<&str> = listed.iter().map(|m| m.digest.as_str()).collect(); + digests.sort_unstable(); + let newest: Vec = (MANIFESTS - 1000..MANIFESTS).map(digest).collect(); + assert_eq!(digests, newest); + assert!(listed + .iter() + .all(|m| m.repository_id == repo && m.tags.len() == 1)); + let last = listed.iter().find(|m| m.digest == newest[999]).unwrap(); + assert_eq!(last.tags, [format!("t{}", MANIFESTS - 1)]); + assert!( + read < 10_000, + "listing 1,000 of {MANIFESTS} manifests read {read} manifest rows" + ); + + assert_eq!( + oci_dao::image_totals(&pool, &[repo], 0, "app") + .await + .unwrap(), + (MANIFESTS, MANIFESTS) + ); + + // With `FORGE_TEST_DATABASE_URL` the suites share a database: leave no + // large image in it, nor the size it gave the indexes, for the plans + // of their tests. + sqlx::query("DELETE FROM repository WHERE id = $1") + .bind(repo) + .execute(&pool) + .await + .unwrap(); + for tidy in [ + "REINDEX TABLE oci_manifest", + "REINDEX TABLE oci_tag", + "VACUUM ANALYZE oci_manifest", + "VACUUM ANALYZE oci_tag", + ] { + sqlx::query(tidy).execute(&pool).await.unwrap(); + } +} + +/// A repository after another one (a group's lower priority member) lists +/// and counts what it adds to it: not a tag the first one has, and a digest +/// the first one holds only as the first one's manifest, when one of its +/// own tags points at it. +#[tokio::test] +async fn a_lower_priority_repository_lists_what_it_adds_to_the_first() { + let pool = pool().await; + let mut repos = Vec::new(); + for _ in 0..2 { + let repo = repository_dao::insert_repository( + &pool, + NewRepository::hosted(format!("member-{}", unique())), + ) + .await + .unwrap(); + repos.push(repo.id); + } + let (first, second) = (repos[0], repos[1]); + let blob = seeded_blob(&pool).await; + // The first holds 0 … 9 (tagged a); the second 5 … 14, tagged b + // and, for 5 … 9, also a: tags of the first. + seed_tagged(&pool, first, &blob, (0, 10), "2025-03-01", "a").await; + seed_tagged(&pool, second, &blob, (5, 15), "2025-02-01", "b").await; + sqlx::query( + r#" + INSERT INTO oci_tag (repository_id, image, tag, manifest_id) + SELECT repository_id, image, 'a' || n - 5, id + FROM oci_manifest + JOIN generate_series(5, 9) AS n ON digest = 'sha256:' || lpad(to_hex(n), 64, '0') + WHERE repository_id = $1 AND image = 'app' + "#, + ) + .bind(second) + .execute(&pool) + .await + .unwrap(); + // An index of the second: a child only the first holds, one both hold + // (the first's counts) and one neither holds. + sqlx::query( + r#" + INSERT INTO oci_manifest_ref (manifest_id, kind, digest) + SELECT id, 'manifest', unnest($2::text[]) + FROM oci_manifest + WHERE repository_id = $1 AND image = 'app' AND digest = $3 + "#, + ) + .bind(second) + .bind([digest(1), digest(7), digest(99)]) + .bind(digest(14)) + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "UPDATE oci_manifest SET summary = '{\"layers_size\": 700}' \ + WHERE repository_id = $1 AND image = 'app' AND digest = $2", + ) + .bind(second) + .bind(digest(7)) + .execute(&pool) + .await + .unwrap(); + + let listed = oci_dao::image_listing(&pool, &repos, 1, "app", 1000, 1000) + .await + .unwrap(); + let mut rows: Vec<(String, i64, Vec)> = listed + .iter() + .map(|m| { + let mut tags = m.tags.clone(); + tags.sort(); + (m.digest.clone(), m.repository_id, tags) + }) + .collect(); + rows.sort(); + let mut expected: Vec<(String, i64, Vec)> = (5..10) + .map(|n| (digest(n), first, vec![format!("b{n}")])) + .chain((10..15).map(|n| (digest(n), second, vec![format!("b{n}")]))) + .collect(); + expected.sort(); + assert_eq!(rows, expected); + let index = listed.iter().find(|m| m.digest == digest(14)).unwrap(); + assert_eq!(index.children, [digest(1), digest(7), digest(99)]); + let sizes: Vec> = index + .child_summaries + .iter() + .map(|child| child.as_ref().map(|c| c["layers_size"].as_i64().unwrap())) + .collect(); + assert_eq!(sizes, [Some(1), Some(7), None]); + // Five manifests and ten tags more than the first's. + assert_eq!( + oci_dao::image_totals(&pool, &repos, 1, "app") + .await + .unwrap(), + (5, 10) + ); + + // The bounds: the newest tagged manifests, and the newest of all. + let listed = oci_dao::image_listing(&pool, &repos, 0, "app", 2, 3) + .await + .unwrap(); + let mut digests: Vec<&str> = listed.iter().map(|m| m.digest.as_str()).collect(); + digests.sort_unstable(); + assert_eq!(digests, [digest(7), digest(8), digest(9)]); +} diff --git a/crates/server/src/browse_images.rs b/crates/server/src/browse_images.rs index b4b7ca65..520b98da 100644 --- a/crates/server/src/browse_images.rs +++ b/crates/server/src/browse_images.rs @@ -61,6 +61,15 @@ const MAX_TAKE: usize = 200; /// push). A CI that tags each build, or a proxy of an image with thousands /// of tags upstream, makes images with tens of thousands; the page counts /// them all ([`ImageDetail::tags_total`]). +/// +/// The store is asked for no more either (#561, +/// [`oci_dao::image_listing`]): of each repository serving the image, the +/// tags of its newest [`MAX_DETAIL_TAGS`] tagged manifests and its newest +/// [`MAX_DETAIL_MANIFESTS`] manifests. That is the whole image while it is +/// within the bounds, and the newest of a larger one. Only a group whose +/// members hold the same digest pushed at different times can list, past +/// the bounds, another tag than the newest by the listed manifest's push: +/// a member's newest are chosen by its own pushes. const MAX_DETAIL_TAGS: usize = 1000; /// The most manifests an image's page lists: those of the tags it lists, /// then the newest others ([`ImageDetail::manifests_total`]). @@ -454,14 +463,36 @@ pub async fn image_detail( Err(resp) => return resp, }; let group = repo.kind() == RepoKind::Group; - let mut manifests: Vec = Vec::new(); + let ids: Vec = sources.iter().map(|source| source.id).collect(); + // A manifest with its source's position: the highest priority member's + // manifests come first. + let mut manifests: Vec<(usize, ManifestItem)> = Vec::new(); let mut seen_digests = HashSet::new(); let mut tags: BTreeMap = BTreeMap::new(); - for source in &sources { - let rows = match oci_dao::image_manifests(&state.pool, source.id, &image).await { + let (mut tags_total, mut manifests_total) = (0, 0); + for position in 0..sources.len() { + // The newest of the member, not its tens of thousands (#561). + let rows = match oci_dao::image_listing( + &state.pool, + &ids, + position, + &image, + MAX_DETAIL_TAGS as i64, + MAX_DETAIL_MANIFESTS as i64, + ) + .await + { Ok(rows) => rows, Err(e) => return db_error(e), }; + // The whole image is counted, not read. + match oci_dao::image_totals(&state.pool, &ids, position, &image).await { + Ok((member_manifests, member_tags)) => { + manifests_total += usize::try_from(member_manifests).unwrap_or(0); + tags_total += usize::try_from(member_tags).unwrap_or(0); + } + Err(e) => return db_error(e), + } for row in rows { for tag in &row.tags { // The highest-priority member's tag wins, as in a pull. @@ -471,50 +502,63 @@ pub async fn image_detail( if !seen_digests.insert(row.digest.clone()) { continue; } - manifests.push(ManifestItem { - digest: row.digest, - media_type: row.media_type, - artifact_type: row.artifact_type, - subject_digest: row.subject_digest, - manifest_size: row.size, - size: row.summary.get("layers_size").and_then(|v| v.as_i64()), - platforms: platforms(&row.summary), - // The tags pulls resolve to it, below: a lower priority - // member's tag that a higher one also has is not its. - tags: Vec::new(), - children: row.children, - created_at: row.created_at, - last_pulled_at: row.last_pulled_at, - source: group.then(|| source.name.clone()), - }); + let Some(source) = sources.iter().position(|s| s.id == row.repository_id) else { + continue; + }; + let mut size = row.summary.get("layers_size").and_then(|v| v.as_i64()); + let mut platforms = platforms(&row.summary); + // An index is as large as the images it lists (that Forge + // holds), and runs on their platforms when it names none itself. + let children: Vec<&serde_json::Value> = row.child_summaries.iter().flatten().collect(); + let total: i64 = children + .iter() + .filter_map(|child| child.get("layers_size").and_then(|v| v.as_i64())) + .sum(); + if total > 0 { + size = Some(total); + } + if platforms.is_empty() { + platforms = children + .iter() + .flat_map(|child| self::platforms(child)) + .collect(); + platforms.dedup(); + } + manifests.push(( + source, + ManifestItem { + digest: row.digest, + media_type: row.media_type, + artifact_type: row.artifact_type, + subject_digest: row.subject_digest, + manifest_size: row.size, + size, + platforms, + // The tags pulls resolve to it, below: a lower priority + // member's tag that a higher one also has is not its. + tags: Vec::new(), + children: row.children, + created_at: row.created_at, + last_pulled_at: row.last_pulled_at, + source: group.then(|| sources[source].name.clone()), + }, + )); } } if manifests.is_empty() { return (StatusCode::NOT_FOUND, "image not found").into_response(); } - // An index is as large as the images it lists (that Forge holds), and - // runs on their platforms when it names none itself. - let by_digest: HashMap, Vec)> = manifests - .iter() - .map(|m| (m.digest.clone(), (m.size, m.platforms.clone()))) - .collect(); - for m in manifests.iter_mut().filter(|m| !m.children.is_empty()) { - let children: Vec<&(Option, Vec)> = - m.children.iter().filter_map(|c| by_digest.get(c)).collect(); - let total: i64 = children.iter().filter_map(|(size, _)| *size).sum(); - if total > 0 { - m.size = Some(total); - } - if m.platforms.is_empty() { - let mut platforms: Vec = children - .iter() - .flat_map(|(_, p)| p.iter().cloned()) - .collect(); - platforms.dedup(); - m.platforms = platforms; - } - } - let (tags, tags_total, manifests, manifests_total) = list_tags(tags, manifests); + // Each member's newest first, as the store listed a whole image. + manifests.sort_by(|(p, a), (q, b)| { + p.cmp(q) + .then_with(|| b.created_at.cmp(&a.created_at)) + .then_with(|| a.digest.cmp(&b.digest)) + }); + let manifests = manifests.into_iter().map(|(_, m)| m).collect(); + let (tags, listed_tags, manifests, listed_manifests) = list_tags(tags, manifests); + // The totals count the whole image; what was read of it is its newest. + let tags_total = tags_total.max(listed_tags); + let manifests_total = manifests_total.max(listed_manifests); let (registry, prefix) = pull_prefix(&state, &repo); Json(ImageDetail { repo: repo.name.clone(), @@ -531,8 +575,8 @@ pub async fn image_detail( } /// The tags (tag → the digest pulls resolve) and manifests of an image's -/// page, each manifest badged with the tags pointing at it, with the totals -/// of both. The lists are bounded ([`MAX_DETAIL_TAGS`], +/// page, each manifest badged with the tags pointing at it, with how many +/// of both it was given. The lists are bounded ([`MAX_DETAIL_TAGS`], /// [`MAX_DETAIL_MANIFESTS`]; past the tag bound, a manifest is badged with /// the listed tags only), and every lookup goes through an index by digest: /// comparing every tag with every manifest took seconds per request for an diff --git a/crates/server/tests/registry.rs b/crates/server/tests/registry.rs index 54bec4ea..c02120bb 100644 --- a/crates/server/tests/registry.rs +++ b/crates/server/tests/registry.rs @@ -1911,6 +1911,352 @@ async fn a_group_lists_a_large_proxys_images_in_batches() { /// Helm charts pushed as artifacts: the image page lists them by their /// config type, and `/browse/chart` reads their metadata, README and values /// from the blobs Forge holds, through a group too. +/// The digest of the seeded manifest `n` ([`seed_manifests`]). +fn seeded_digest(n: i64) -> String { + format!("sha256:{n:064x}") +} + +/// Manifests `range` of `image` in repository `repo_id`, as pushes left +/// them: manifest `n` has the digest [`seeded_digest`], was pushed `n` +/// minutes after `pushed_from`, is `n + 1` bytes of layers for linux/amd64 +/// and, with `tag_prefix`, is tagged ``. They +/// share the blob of the image `template` of repository `template_repo`. +async fn seed_manifests( + srv: &TestServer, + (template_repo, template): (i64, &str), + repo_id: i64, + image: &str, + range: std::ops::Range, + pushed_from: &str, + tag_prefix: Option<&str>, +) { + sqlx::query( + r#" + INSERT INTO oci_manifest + (repository_id, image, digest, blob_sha256, media_type, size, summary, + created_at, untagged_since) + SELECT $3, $4, 'sha256:' || lpad(to_hex(n), 64, '0'), t.blob_sha256, + 'application/vnd.oci.image.manifest.v1+json', 100, + jsonb_build_object('layers_size', n + 1, 'platform', 'linux/amd64'), + $7::timestamptz + n * interval '1 minute', + CASE WHEN $8::text IS NULL THEN now() END + FROM (SELECT blob_sha256 FROM oci_manifest + WHERE repository_id = $1 AND image = $2 LIMIT 1) t, + generate_series($5::bigint, $6::bigint - 1) AS n + "#, + ) + .bind(template_repo) + .bind(template) + .bind(repo_id) + .bind(image) + .bind(range.start) + .bind(range.end) + .bind(pushed_from) + .bind(tag_prefix) + .execute(&srv.pool) + .await + .unwrap(); + let Some(prefix) = tag_prefix else { + return; + }; + sqlx::query( + r#" + INSERT INTO oci_tag (repository_id, image, tag, manifest_id) + SELECT m.repository_id, m.image, $5 || lpad(n::text, 5, '0'), m.id + FROM generate_series($3::bigint, $4::bigint - 1) AS n + JOIN oci_manifest m ON m.repository_id = $1 AND m.image = $2 + AND m.digest = 'sha256:' || lpad(to_hex(n), 64, '0') + "#, + ) + .bind(repo_id) + .bind(image) + .bind(range.start) + .bind(range.end) + .bind(prefix) + .execute(&srv.pool) + .await + .unwrap(); +} + +/// Point `tag` of `image` at its seeded manifest `n`, as a push does. +async fn seed_tag(srv: &TestServer, repo_id: i64, image: &str, tag: &str, n: i64) { + sqlx::query( + r#" + WITH tagged AS ( + UPDATE oci_manifest SET untagged_since = NULL + WHERE repository_id = $1 AND image = $2 AND digest = $4 + RETURNING id + ) + INSERT INTO oci_tag (repository_id, image, tag, manifest_id) + SELECT $1, $2, $3, id FROM tagged + "#, + ) + .bind(repo_id) + .bind(image) + .bind(tag) + .bind(seeded_digest(n)) + .execute(&srv.pool) + .await + .unwrap(); +} + +/// The image page of `repo`'s `image`, as an administrator's browser asks. +async fn image_page(srv: &TestServer, repo: &str, image: &str) -> Value { + let session = srv + .login(srv.admin_user.as_deref().unwrap(), "adminpw") + .await; + let resp = session + .apply( + srv.client() + .get(format!("{}/browse/images/{repo}/{image}", srv.base_url)), + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "{}", resp.text().await.unwrap()); + resp.json().await.unwrap() +} + +/// The strings of `items[..][key]`, in the page's order. +fn strings(items: &Value, key: &str) -> Vec { + items + .as_array() + .unwrap() + .iter() + .map(|item| item[key].as_str().unwrap().to_string()) + .collect() +} + +/// An image past the page's bounds (#561): the page lists its newest 1,000 +/// tags by name with their manifests, newest first, counts all of both, and +/// still sizes an index by children the listing has no room for. An image +/// with few tags lists their manifests and fills up with its newest others. +#[tokio::test] +async fn the_image_page_lists_the_newest_of_an_image_past_its_bounds() { + let srv = common::start().await; + let repo = repo_name("big"); + let repo_id = create_repo( + &srv, + json!({"name": repo, "kind": "hosted", "format": "docker"}), + ) + .await["id"] + .as_i64() + .unwrap(); + // A pushed image, the newest of all: its blob is the seeded manifests'. + let pushed = admin_oci(&srv) + .await + .push_image(&format!("{repo}/app"), "seed", "amd64", b"layer") + .await; + let template = (repo_id, "app"); + // 2,500 tagged manifests, then 1,500 older ones without a tag. + seed_manifests( + &srv, + template, + repo_id, + "app", + 0..2_500, + "2025-01-01", + Some("t-"), + ) + .await; + seed_manifests( + &srv, + template, + repo_id, + "app", + 2_500..4_000, + "2024-01-01", + None, + ) + .await; + // An index among the newest, of two of the old manifests and one the + // repository does not hold. + sqlx::query( + r#" + WITH index AS ( + UPDATE oci_manifest + SET media_type = 'application/vnd.oci.image.index.v1+json', summary = '{}' + WHERE repository_id = $1 AND image = 'app' AND digest = $2 + RETURNING id + ) + INSERT INTO oci_manifest_ref (manifest_id, kind, digest) + SELECT id, 'manifest', unnest($3::text[]) FROM index + "#, + ) + .bind(repo_id) + .bind(seeded_digest(2_400)) + .bind([ + seeded_digest(2_500), + seeded_digest(2_501), + seeded_digest(9_999), + ]) + .execute(&srv.pool) + .await + .unwrap(); + + let page = image_page(&srv, &repo, "app").await; + assert_eq!(page["tags_total"], 2_501); + assert_eq!(page["manifests_total"], 4_001); + // The newest 1,000 tags: the pushed one and t-01501 … t-02499, by name. + let mut expected: Vec = (1_501..2_500).map(|n| format!("t-{n:05}")).collect(); + expected.insert(0, "seed".to_string()); + assert_eq!(strings(&page["tags"], "tag"), expected); + // Their manifests and no other (the bound is reached), newest first, + // each badged with its tag. + let mut digests = vec![pushed.clone()]; + digests.extend((1_501..2_500).rev().map(seeded_digest)); + assert_eq!(strings(&page["manifests"], "digest"), digests); + for tag in page["tags"].as_array().unwrap() { + let manifest = page["manifests"] + .as_array() + .unwrap() + .iter() + .find(|m| m["digest"] == tag["digest"]) + .unwrap_or_else(|| panic!("{tag}: its manifest is listed")); + assert_eq!(manifest["tags"], json!([tag["tag"]])); + } + assert_eq!(page["tags"][0]["digest"], pushed.as_str()); + assert_eq!(page["tags"][1]["digest"], seeded_digest(1_501).as_str()); + // The index: as large as its two children Forge holds, on their + // platform, though the page lists neither. + let index = &page["manifests"][100]; + assert_eq!(index["digest"], seeded_digest(2_400).as_str()); + assert_eq!(index["children"].as_array().unwrap().len(), 3); + assert_eq!(index["size"], 2_501 + 2_502); + assert_eq!(index["platforms"], json!(["linux/amd64"])); + let tag = &page["tags"][900]; + assert_eq!(tag["tag"], "t-02400"); + assert_eq!(tag["size"], 2_501 + 2_502); + assert_eq!(tag["platforms"], json!(["linux/amd64"])); + + // Three old tags and 1,500 newer manifests without one: the tags' + // manifests, then the newest others up to the bound. + seed_manifests(&srv, template, repo_id, "ci", 0..1_500, "2025-01-01", None).await; + for n in 0..3 { + seed_tag(&srv, repo_id, "ci", &format!("v{n}"), n).await; + } + let page = image_page(&srv, &repo, "ci").await; + assert_eq!(page["tags_total"], 3); + assert_eq!(page["manifests_total"], 1_500); + assert_eq!(strings(&page["tags"], "tag"), ["v0", "v1", "v2"]); + let mut digests: Vec = (503..1_500).rev().map(seeded_digest).collect(); + digests.extend((0..3).rev().map(seeded_digest)); + assert_eq!(strings(&page["manifests"], "digest"), digests); + assert_eq!(page["manifests"][999]["tags"], json!(["v0"])); +} + +/// A group's image page past the bounds (#561) reads the newest of each +/// member serving the image and lists what a pull through the group gets: +/// a tag two members have is the higher priority member's, a digest two +/// members hold is listed once as that member's, and both are counted once. +#[tokio::test] +async fn a_groups_image_page_lists_the_newest_of_its_members_past_the_bounds() { + let srv = common::start().await; + let (high, low, group) = (repo_name("high"), repo_name("low"), repo_name("grp")); + let mut ids = Vec::new(); + for member in [&high, &low] { + let row = create_repo( + &srv, + json!({"name": member, "kind": "hosted", "format": "docker"}), + ) + .await; + ids.push(row["id"].as_i64().unwrap()); + } + create_repo( + &srv, + json!({"name": group, "kind": "group", "format": "docker"}), + ) + .await; + for (member, priority) in [(&high, 10), (&low, 5)] { + let resp = admin_post( + &srv, + &format!("/admin/repositories/{group}/members"), + json!({"member": member, "priority": priority}), + ) + .await; + assert_eq!(resp.status(), 201); + } + let (high_id, low_id) = (ids[0], ids[1]); + admin_oci(&srv) + .await + .push_image(&format!("{high}/seed"), "1", "amd64", b"layer") + .await; + let template = (high_id, "seed"); + // The higher member: manifests 0 … 1199, tagged a-, the newer pushes. + seed_manifests( + &srv, + template, + high_id, + "app", + 0..1_200, + "2025-03-01", + Some("a-"), + ) + .await; + // The lower member: manifests 1000 … 2599 (200 of them the higher + // member's too), tagged b-, pushed a month earlier. + seed_manifests( + &srv, + template, + low_id, + "app", + 1_000..2_600, + "2025-02-01", + Some("b-"), + ) + .await; + // It also has the higher member's newest tag, on another manifest, and + // its own newest push is a manifest the higher member holds. + seed_tag(&srv, low_id, "app", "a-01199", 2_599).await; + sqlx::query( + "UPDATE oci_manifest SET created_at = '2025-02-20' \ + WHERE repository_id = $1 AND image = 'app' AND digest = $2", + ) + .bind(low_id) + .bind(seeded_digest(1_100)) + .execute(&srv.pool) + .await + .unwrap(); + + let page = image_page(&srv, &group, "app").await; + // 1,200 tags and as many manifests of the higher member, and what the + // lower one adds: 1,600 tags (not `a-01199`) and 1,400 manifests. + assert_eq!(page["tags_total"], 2_800); + assert_eq!(page["manifests_total"], 2_600); + // The newest 1,000 tags, by the push of the manifest a pull gets: the + // higher member's a-00201 … a-01199 and the lower member's tag of the + // manifest 1100, which the group serves from the higher member. + let mut expected: Vec = (201..1_200).map(|n| format!("a-{n:05}")).collect(); + expected.push("b-01100".to_string()); + assert_eq!(strings(&page["tags"], "tag"), expected); + let tags = page["tags"].as_array().unwrap(); + assert_eq!(tags[998]["tag"], "a-01199"); + assert_eq!(tags[998]["digest"], seeded_digest(1_199).as_str()); + assert_eq!(tags[999]["digest"], seeded_digest(1_100).as_str()); + // Their manifests, then the newest other one: all the higher member's. + let digests: Vec = (200..1_200).rev().map(seeded_digest).collect(); + assert_eq!(strings(&page["manifests"], "digest"), digests); + let manifests = page["manifests"].as_array().unwrap(); + assert!(manifests.iter().all(|m| m["source"] == high.as_str())); + assert_eq!(manifests[99]["tags"], json!(["a-01100", "b-01100"])); + assert_eq!(manifests[999]["tags"], json!([])); + for tag in tags { + assert!( + manifests.iter().any(|m| m["digest"] == tag["digest"]), + "{tag}: its manifest is listed" + ); + } + + // Each member's own page still lists its own. + let page = image_page(&srv, &low, "app").await; + assert_eq!(page["tags_total"], 1_601); + assert_eq!(page["manifests_total"], 1_600); + let tags = page["tags"].as_array().unwrap(); + let shared = tags.iter().find(|t| t["tag"] == "a-01199").unwrap(); + assert_eq!(shared["digest"], seeded_digest(2_599).as_str()); + assert!(page["manifests"][0].get("source").is_none()); +} + #[tokio::test] async fn helm_charts_are_browsable() { let srv = common::start().await; diff --git a/docs/changes/2.2.0/561.fixed.md b/docs/changes/2.2.0/561.fixed.md new file mode 100644 index 00000000..32bdd9ca --- /dev/null +++ b/docs/changes/2.2.0/561.fixed.md @@ -0,0 +1,8 @@ +The page of an image in the web UI (`GET /browse/images/{repo}/{image}`) +lists at most its newest 1,000 tags and 1,000 manifests, but every request +still read all of the image's manifests and tags from the database, with +the tags and the children of each, before it kept those: work that grew +with the image, for any reader of the repository, once per member of a +group. The page now reads the manifests it lists, through a new index on +`oci_manifest` (repository, image, push time) that the upgrade's migration +builds, and counts the others. What it shows is unchanged. diff --git a/migrations/0020_oci_manifest_newest.sql b/migrations/0020_oci_manifest_newest.sql new file mode 100644 index 00000000..7c6aa5ef --- /dev/null +++ b/migrations/0020_oci_manifest_newest.sql @@ -0,0 +1,10 @@ +-- --------------------------------------------------------------------------- +-- An image's newest manifests, without reading the whole image (#561) +-- +-- The image page lists at most the newest 1,000 tags and 1,000 manifests of +-- an image (#240), but found them by reading every manifest and tag of it: +-- no index knows a manifest's age. This one lists an image's manifests +-- newest first, so the page reads the ones it shows. +-- --------------------------------------------------------------------------- +CREATE INDEX oci_manifest_newest_idx + ON oci_manifest (repository_id, image, created_at DESC, digest);