Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions crates/core/src/format/descriptor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ pub fn check_one(descriptor: &Descriptor) -> Result<(), String> {
"format {id:?}: only a package format keeps indexes"
));
}
if descriptor.readme && descriptor.storage != Storage::Packages {
return Err(format!(
"format {id:?}: only a package format's manifests carry a README"
));
}
validate_options(descriptor).map_err(|e| format!("format {id:?}: {e}"))?;
validate_assets(&descriptor.assets).map_err(|e| format!("format {id:?}: {e}"))
}
Expand Down Expand Up @@ -166,6 +171,7 @@ mod tests {
options: &[],
prereleases: forge_format_api::Prereleases::Dash,
indexes: false,
readme: false,
};

#[test]
Expand Down Expand Up @@ -252,6 +258,17 @@ mod tests {
..GOOD
};
assert!(check(&[&files_with_indexes]).is_err());
let files_with_readme = Descriptor {
storage: Storage::Files,
readme: true,
..GOOD
};
assert!(check(&[&files_with_readme]).is_err());
let with_readme = Descriptor {
readme: true,
..GOOD
};
assert_eq!(check(&[&with_readme]), Ok(()));
}

#[test]
Expand Down
7 changes: 7 additions & 0 deletions crates/core/src/format/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,13 @@ pub trait Format: Send + Sync {
false
}

/// Whether the format's manifests carry a README ([`Self::readme`]).
/// The web UI's package page resolves the latest version's manifest
/// only when they do: on a proxy that read can ask the upstream.
fn has_readme(&self) -> bool {
false
}

/// The README a manifest carries, for the web UI's package page.
fn readme(&self, manifest: &[u8]) -> Option<String> {
let _ = manifest;
Expand Down
7 changes: 7 additions & 0 deletions crates/core/src/format/spec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ pub struct SpecFormat {
assets: Assets,
storage: Storage,
upstream_token: UpstreamToken,
readme: bool,
}

impl SpecFormat {
Expand All @@ -48,6 +49,7 @@ impl SpecFormat {
assets: descriptor.assets,
storage: descriptor.storage,
upstream_token: descriptor.upstream_token,
readme: descriptor.readme,
spec,
}
}
Expand Down Expand Up @@ -340,6 +342,10 @@ impl Format for SpecFormat {
self.upstream_token == UpstreamToken::Bearer
}

fn has_readme(&self) -> bool {
self.readme
}

fn readme(&self, manifest: &[u8]) -> Option<String> {
self.spec.readme(manifest)
}
Expand Down Expand Up @@ -390,6 +396,7 @@ mod tests {
options: &[],
prereleases: forge_format_api::Prereleases::Dash,
indexes: false,
readme: false,
};

impl FormatSpec for Unlimited {
Expand Down
28 changes: 26 additions & 2 deletions crates/core/src/repo/group.rs
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,10 @@ pub trait MemberOpsFactory: Send + Sync {
/// Identity-aware group read resolver.
///
/// Constructed per request with the [`Principal`] in hand so visibility
/// filtering is baked into every method. Implements [`RepoReadOps`] keyed by the
/// filtering is baked into every method. It reads a group's memberships and
/// the principal's grants once, on its first read of the group, and answers
/// every later read from that graph: make a new resolver for each request,
/// since one kept longer sees no change to either. Implements [`RepoReadOps`] keyed by the
/// **group** repository name; the per-member ops are keyed by each member's own
/// name (resolved from the `group_member` join).
pub struct GroupRepo {
Expand All @@ -229,6 +232,11 @@ pub struct GroupRepo {
max_depth: u32,
/// Version precedence for merged results. `None` ⇒ string order (tests).
format: Option<Arc<dyn Format>>,
/// The graph of each group this resolver was asked about, by the
/// group's name, built on its first read ([`Self::group_graph`]) and
/// kept for the resolver's lifetime: a request's reads walk one graph
/// (#537).
graphs: tokio::sync::Mutex<HashMap<String, Arc<GroupGraph>>>,
}

/// A visible member: its repository row (carrying its own `name`) and the ops to
Expand Down Expand Up @@ -599,6 +607,7 @@ impl GroupRepo {
authz,
max_depth,
format: None,
graphs: tokio::sync::Mutex::new(HashMap::new()),
}
}

Expand Down Expand Up @@ -630,6 +639,21 @@ impl GroupRepo {
Ok(row)
}

/// The compact readable graph of `group` for this principal, built
/// ([`Self::build_group_graph`]) on the resolver's first read of the
/// group and shared by the later ones, with the per-id walks it
/// memoizes. Reads that come in together wait for the one build. A
/// failed build is not kept: the next read tries again.
async fn group_graph(&self, group: &str) -> Result<Arc<GroupGraph>, RepoError> {
let mut graphs = self.graphs.lock().await;
if let Some(graph) = graphs.get(group) {
return Ok(graph.clone());
}
let graph = self.build_group_graph(group).await?;
graphs.insert(group.to_string(), graph.clone());
Ok(graph)
}

/// Build the compact readable group graph for this principal.
///
/// Two breadth-first passes over the membership DAG below `group`. Both
Expand All @@ -646,7 +670,7 @@ impl GroupRepo {
/// 2. **Visibility**: the same walk over the loaded adjacency lists, keeping
/// only edges whose member the principal may read, each narrowed to
/// the membership's scope.
async fn group_graph(&self, group: &str) -> Result<Arc<GroupGraph>, RepoError> {
async fn build_group_graph(&self, group: &str) -> Result<Arc<GroupGraph>, RepoError> {
let group_row = self.group_row(group).await?;

// Pass 1. A group past the depth bound is not expanded: no caller can
Expand Down
98 changes: 95 additions & 3 deletions crates/core/tests/group_read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,74 @@ async fn highest_priority_member_serves_overlapping_version() {
);
}

/// A resolver builds a group's graph once, on its first read, and answers
/// its later reads from it (#537): the package page's four reads
/// (`list_versions`, `registration`, `tags`, `resolve_nuspec`) load the
/// memberships and the caller's grants once, not once each. A member added
/// after the first read is therefore not in the graph of that resolver,
/// while the next request's resolver has it.
#[tokio::test]
async fn a_resolver_builds_its_graph_once() {
let pool = common::pool().await;
let mut blobs = StubBlobs::default();

let first = seed_member(&pool, false).await;
let later = seed_member(&pool, false).await;
let group = seed_group(&pool).await;
seed_version(&pool, &mut blobs, first.id, "Sample", "1.0.0", "f").await;
seed_version(&pool, &mut blobs, later.id, "Sample", "2.0.0", "l").await;
repository_dao::add_group_member(&pool, group.id, first.id, 100)
.await
.unwrap();

let principal = user(&pool).await;
let g = group_repo(&pool, blobs.clone(), principal.clone());
let versions = |entries: Vec<forge_core::repo::VersionEntry>| -> Vec<String> {
entries.into_iter().map(|v| v.version).collect()
};
let listed = g.list_versions(&group.name, "sample").await.unwrap();
assert_eq!(versions(listed), ["1.0.0"]);

// A graph built again for a read would have this member.
repository_dao::add_group_member(&pool, group.id, later.id, 10)
.await
.unwrap();
let listed = g.list_versions(&group.name, "sample").await.unwrap();
assert_eq!(versions(listed), ["1.0.0"], "the listing built a graph");
let registration = g.registration(&group.name, "sample").await.unwrap();
assert_eq!(
registration
.iter()
.map(|e| e.version.as_str())
.collect::<Vec<_>>(),
["1.0.0"],
"the registration built a graph"
);
g.tags(&group.name, "sample").await.unwrap();
assert!(
g.resolve_nuspec(&group.name, "sample", "2.0.0")
.await
.unwrap()
.is_none(),
"the manifest read built a graph"
);
assert!(g
.resolve_nuspec(&group.name, "sample", "1.0.0")
.await
.unwrap()
.is_some());

// The next request's resolver reads the memberships as they are now.
let next = group_repo(&pool, blobs, principal);
let listed = next.list_versions(&group.name, "sample").await.unwrap();
assert_eq!(versions(listed), ["1.0.0", "2.0.0"]);
assert!(next
.resolve_nuspec(&group.name, "sample", "2.0.0")
.await
.unwrap()
.is_some());
}

#[tokio::test]
async fn list_versions_is_union_across_members() {
let pool = common::pool().await;
Expand Down Expand Up @@ -3590,10 +3658,11 @@ async fn a_scoped_member_is_never_asked_about_other_ids() {
scope(&pool, &group, &internal, &["Envisia.*"]).await;
let internal_ops = PassThrough::new(&["Envisia.Lib", "Internal.Other"]);
let public_ops = PassThrough::new(&["Newtonsoft.Json", "Serilog"]);
let principal = user(&pool).await;
let g = pass_through_group_repo(
&pool,
StubBlobs::default(),
user(&pool).await,
principal.clone(),
&[(&internal, &internal_ops), (&public, &public_ops)],
);

Expand Down Expand Up @@ -3685,6 +3754,13 @@ async fn a_scoped_member_is_never_asked_about_other_ids() {

// Without patterns the member is asked about every id again.
scope(&pool, &group, &internal, &[]).await;
// The next request's resolver: one keeps the graph it built.
let g = pass_through_group_repo(
&pool,
StubBlobs::default(),
principal,
&[(&internal, &internal_ops), (&public, &public_ops)],
);
g.list_versions(&group.name, "Serilog").await.unwrap();
assert!(internal_ops.asked().contains(&"Serilog".to_string()));
}
Expand Down Expand Up @@ -3821,10 +3897,11 @@ async fn a_scope_on_a_nested_group_holds_for_every_member_below_it() {
scope(&pool, &outer, &inner, &["Envisia.*"]).await;
let internal_ops = PassThrough::new(&["Envisia.Lib", "Internal.Other"]);
let public_ops = PassThrough::new(&["Newtonsoft.Json"]);
let principal = user(&pool).await;
let g = pass_through_group_repo(
&pool,
StubBlobs::default(),
user(&pool).await,
principal.clone(),
&[(&internal, &internal_ops), (&public, &public_ops)],
);

Expand All @@ -3847,6 +3924,13 @@ async fn a_scope_on_a_nested_group_holds_for_every_member_below_it() {
// A scope inside the nested group narrows further: an id must match
// the scope of every membership on the way.
scope(&pool, &inner, &internal, &["Envisia.Tools.*"]).await;
// The next request's resolver: one keeps the graph it built.
let g = pass_through_group_repo(
&pool,
StubBlobs::default(),
principal,
&[(&internal, &internal_ops), (&public, &public_ops)],
);
g.list_versions(&outer.name, "Envisia.Data").await.unwrap();
g.list_versions(&outer.name, "Envisia.Tools.Cli")
.await
Expand Down Expand Up @@ -3887,7 +3971,13 @@ async fn a_scope_never_widens_what_the_guards_allow() {
scope(&pool, &group, &hosted, &["Envisia.*"]).await;
scope(&pool, &group, &public, &["Envisia.*", "Sample", "Serilog"]).await;
let public_ops = PassThrough::new(&["Envisia.Fake", "Sample", "Serilog"]);
let g = pass_through_group_repo(&pool, blobs, user(&pool).await, &[(&public, &public_ops)]);
let principal = user(&pool).await;
let g = pass_through_group_repo(
&pool,
blobs.clone(),
principal.clone(),
&[(&public, &public_ops)],
);

// The ids as NuGet clients ask for them: lowercase.
// A reserved id: served by the owner alone, the proxy is not asked
Expand Down Expand Up @@ -3922,5 +4012,7 @@ async fn a_scope_never_widens_what_the_guards_allow() {

// With the owner's scope gone, it serves its id again.
scope(&pool, &group, &hosted, &[]).await;
// The next request's resolver: one keeps the graph it built.
let g = pass_through_group_repo(&pool, blobs, principal, &[(&public, &public_ops)]);
assert_eq!(sorted_versions(&g, &group.name, "sample").await, ["2.0.0"]);
}
39 changes: 22 additions & 17 deletions crates/core/tests/proxy_read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2993,24 +2993,29 @@ async fn group_search_asks_upstreams_for_the_page_window() {
forge_db::authz_dao::assign_role(&pool, user.id, role)
.await
.unwrap();
let group_ops = GroupRepo::new(
pool.clone(),
Principal::User {
user_id: user.id,
token_id: None,
method: forge_core::auth::AuthMethod::Password,
},
Arc::new(ProxySearchFactory(Arc::new(proxy_repo(
&pool,
cas,
Cache::l1_only(128),
)))),
authz,
20,
)
.with_format(Arc::new(nuget()));
let principal = Principal::User {
user_id: user.id,
token_id: None,
method: forge_core::auth::AuthMethod::Password,
};
let factory: Arc<dyn forge_core::repo::group::MemberOpsFactory> = Arc::new(ProxySearchFactory(
Arc::new(proxy_repo(&pool, cas, Cache::l1_only(128))),
));
// A resolver per request, as the server makes one: it keeps the group's
// graph, so one made earlier does not see a later change to the group.
let new_group_ops = || {
GroupRepo::new(
pool.clone(),
principal.clone(),
factory.clone(),
authz.clone(),
20,
)
.with_format(Arc::new(nuget()))
};
let group_ops = new_group_ops();
let search = |q: &str, skip: usize, take: usize| {
let (group_ops, group) = (&group_ops, &group);
let (group_ops, group) = (new_group_ops(), &group);
let params = SearchParams {
q: q.into(),
skip,
Expand Down
1 change: 1 addition & 0 deletions crates/format-api/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ pub trait FormatSpec: Send + Sync + 'static {
}

/// The README a manifest carries, shown on the web UI's package page.
/// Asked only of a format whose descriptor says [`Descriptor::readme`].
/// `None` by default.
fn readme(&self, manifest: &[u8]) -> Option<String> {
let _ = manifest;
Expand Down
9 changes: 9 additions & 0 deletions crates/format-api/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ impl RepositoryKind {
/// options: &[],
/// prereleases: Prereleases::Dash,
/// indexes: false,
/// readme: false,
/// };
/// ```
///
Expand Down Expand Up @@ -167,6 +168,14 @@ pub struct Descriptor {
///
/// [`FormatSpec::index`]: crate::FormatSpec::index
pub indexes: bool,
/// The format's manifests carry a README ([`FormatSpec::readme`], npm):
/// the web UI's package page reads the latest version's manifest for
/// it. Without it the page reads no manifest, which on a proxy can be a
/// request to the upstream, and `readme` is never asked. A package
/// format only.
///
/// [`FormatSpec::readme`]: crate::FormatSpec::readme
pub readme: bool,
}

/// Which versions of a format are prereleases ([`Descriptor::prereleases`]).
Expand Down
1 change: 1 addition & 0 deletions crates/formats/apt/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ pub const DESCRIPTOR: Descriptor = Descriptor {
// A `-` starts the Debian revision (`1.0-1`), it marks no prerelease.
prereleases: Prereleases::None,
indexes: true,
readme: false,
};

/// The apt format.
Expand Down
1 change: 1 addition & 0 deletions crates/formats/docker/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ pub const DESCRIPTOR: Descriptor = Descriptor {
options: &[],
prereleases: forge_format_api::Prereleases::Dash,
indexes: false,
readme: false,
};

/// The Docker / OCI format.
Expand Down
Loading
Loading