Skip to content

Commit b0500b0

Browse files
authored
feat(ui): providers page with a published providers document (#1452)
* feat(ui): providers page with a published providers document * fix(providers): load pydantic only when reading provider settings The CLI imports project_modules at startup, so a module-level pydantic import broke the startup import footprint.
1 parent a047997 commit b0500b0

35 files changed

Lines changed: 1071 additions & 79 deletions

File tree

‎clients/typescript/src/schema.d.ts‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -610,6 +610,22 @@ export interface paths {
610610
patch?: never;
611611
trace?: never;
612612
};
613+
"/api/control/locations/{location_id}/targets/{target}/providers": {
614+
parameters: {
615+
query?: never;
616+
header?: never;
617+
path?: never;
618+
cookie?: never;
619+
};
620+
get: operations["readPersonProviders"];
621+
put?: never;
622+
post?: never;
623+
delete?: never;
624+
options?: never;
625+
head?: never;
626+
patch?: never;
627+
trace?: never;
628+
};
613629
"/api/control/locations/{location_id}/targets/{target}/runs": {
614630
parameters: {
615631
query?: never;
@@ -2384,6 +2400,46 @@ export interface components {
23842400
state_database?: string | null;
23852401
state_schema: string;
23862402
};
2403+
/** @description Code that injects a provider: a node (`loader`, `task`, `asset`, `check`, `hook` or `materialization`, keyed `kind:name`), an `event_exporter` or a `sink`. */
2404+
ProviderConsumer: {
2405+
kind: string;
2406+
name: string;
2407+
};
2408+
/** @description A provider as its location published it, from the class alone: never a setting value. */
2409+
ProviderRecord: {
2410+
/** @description The class as `module.qualname`. */
2411+
class: string;
2412+
/** @description The class docstring. */
2413+
description: string;
2414+
/** @description Jobs whose selection includes one of those nodes. */
2415+
jobs?: string[];
2416+
name: string;
2417+
/** @description Snapshot nodes that run that code: Python nodes, models calling a hook or using a materialization, and sources a loader fills. */
2418+
nodes?: string[];
2419+
/** @description The file declaring the class, relative to the project. */
2420+
path: string;
2421+
settings: components["schemas"]["ProviderSetting"][];
2422+
/** @description Code that injects the provider, in discovery order. */
2423+
used_by: components["schemas"]["ProviderConsumer"][];
2424+
};
2425+
/** @description One declared setting field; its value stays in the location's environment. */
2426+
ProviderSetting: {
2427+
description?: string | null;
2428+
/** @description The environment variable the setting is read from. */
2429+
env: string;
2430+
name: string;
2431+
required: boolean;
2432+
/** @description Whether the annotation is a pydantic secret type. */
2433+
secret: boolean;
2434+
/** @description The annotation as written, such as `str` or `SecretStr | None`. */
2435+
type: string;
2436+
};
2437+
/** @description Providers the current snapshot declares; empty before a publish or when the project has none. */
2438+
ProvidersRead: {
2439+
providers: components["schemas"]["ProviderRecord"][];
2440+
/** Format: uuid */
2441+
snapshot_id?: string | null;
2442+
};
23872443
/** @description A changed subject, such as `model:orders`, or `role:analyst` (`role:` for those who can't assign roles). */
23882444
PublishedChange: {
23892445
change_kind: string;
@@ -4867,6 +4923,44 @@ export interface operations {
48674923
};
48684924
};
48694925
};
4926+
readPersonProviders: {
4927+
parameters: {
4928+
query?: never;
4929+
header?: never;
4930+
path: {
4931+
location_id: string;
4932+
target: string;
4933+
};
4934+
cookie?: never;
4935+
};
4936+
requestBody?: never;
4937+
responses: {
4938+
/** @description Declared providers without setting values */
4939+
200: {
4940+
headers: {
4941+
[name: string]: unknown;
4942+
};
4943+
content: {
4944+
"application/json": components["schemas"]["ProvidersRead"];
4945+
};
4946+
};
4947+
/** @description Session rejected */
4948+
401: {
4949+
headers: {
4950+
[name: string]: unknown;
4951+
};
4952+
content?: never;
4953+
};
4954+
403: {
4955+
headers: {
4956+
[name: string]: unknown;
4957+
};
4958+
content: {
4959+
"application/json": components["schemas"]["ApiErrorBody"];
4960+
};
4961+
};
4962+
};
4963+
};
48704964
listPersonRuns: {
48714965
parameters: {
48724966
query?: {

‎crates/sqlbuild-host/src/entries/_helpers/user_code.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[
8989
kind: HostKind::Function,
9090
edge: HostEdge::UserCode,
9191
},
92+
HostEntry {
93+
name: "provider_settings",
94+
kind: HostKind::Function,
95+
edge: HostEdge::UserCode,
96+
},
9297
HostEntry {
9398
name: "requires_arguments",
9499
kind: HostKind::Function,

‎crates/sqlbuild-host/src/entries/models.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ pub enum UserCodeHost {
145145
ModuleProviderClasses,
146146
ParameterAnnotations,
147147
ProviderUsages,
148+
ProviderSettings,
148149
RequiresArguments,
149150
RestorePythonRootModules,
150151
SourceLine,

‎crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,10 @@ pub(crate) fn person_routes() -> Router<ControlStore> {
151151
"/api/control/locations/{location_id}/targets/{target}/jobs/{job_key}",
152152
get(crate::api::_helpers::resources::person_reads::job),
153153
)
154+
.route(
155+
"/api/control/locations/{location_id}/targets/{target}/providers",
156+
get(crate::api::_helpers::resources::person_reads::providers),
157+
)
154158
.route(
155159
"/api/control/locations/{location_id}/targets/{target}/triggers",
156160
get(crate::api::_helpers::resources::person_reads::triggers),

‎crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,9 @@ use crate::declarations::main::list_person_jobs::list_person_jobs;
1414
use crate::declarations::main::list_person_trigger_changes::list_person_trigger_changes;
1515
use crate::declarations::main::list_person_triggers::list_person_triggers;
1616
use crate::declarations::main::read_person_job::read_person_job;
17+
use crate::declarations::main::read_person_providers::read_person_providers;
1718
use crate::declarations::models::{
18-
JobDetailRecord, JobRecord, PersonTriggerChange, PersonTriggerRecord,
19+
JobDetailRecord, JobRecord, PersonTriggerChange, PersonTriggerRecord, ProvidersRead,
1920
};
2021
use crate::platform::main::list_person_platform_events::list_person_platform_events;
2122
use crate::platform::main::read_person_location_health::read_person_location_health;
@@ -292,6 +293,27 @@ pub(crate) async fn jobs(
292293
))
293294
}
294295

296+
#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/providers", operation_id = "readPersonProviders",
297+
params(("location_id" = Uuid, Path), ("target" = String, Path)),
298+
responses((status = 200, body = ProvidersRead, description = "Declared providers without setting values"), (status = 401, description = "Session rejected"),
299+
(status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))]
300+
pub(crate) async fn providers(
301+
(State(store), Extension(context)): (State<ControlStore>, Extension<AuditContext>),
302+
Extension(default_role): Extension<DefaultRole>,
303+
Path((location_id, target)): Path<(Uuid, String)>,
304+
Extension(AuthenticatedPerson(principal, _)): Extension<AuthenticatedPerson>,
305+
) -> Result<Json<ProvidersRead>, ApiError> {
306+
Ok(Json(
307+
read_person_providers(
308+
&store,
309+
(&context, principal),
310+
(location_id, &target),
311+
default_role,
312+
)
313+
.await?,
314+
))
315+
}
316+
295317
#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/triggers", operation_id = "listPersonTriggers",
296318
params(("location_id" = Uuid, Path), ("target" = String, Path)),
297319
responses((status = 200, body = Vec<PersonTriggerRecord>, description = "Published triggers with their newest tick"), (status = 401, description = "Session rejected"),

‎crates/sqlbuild-orchestration/src/api/models.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ use crate::api::_helpers::resources::{
3838
person_reads::run_node_error,
3939
person_reads::jobs,
4040
person_reads::job,
41+
person_reads::providers,
4142
person_reads::triggers,
4243
person_reads::ticks,
4344
person_reads::trigger_changes,

‎crates/sqlbuild-orchestration/src/declarations/main/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,5 @@ pub mod list_triggers;
1010
pub mod pause_development_triggers;
1111
pub(crate) mod publish_definitions;
1212
pub mod read_person_job;
13+
pub mod read_person_providers;
1314
pub mod snapshot_definitions;
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
//! A person reads the providers of the current snapshot with view on every current node, like jobs.
2+
3+
use std::collections::BTreeSet;
4+
5+
use serde::Deserialize;
6+
use uuid::Uuid;
7+
8+
use crate::authorization::main::authorize_listing::authorize_listing;
9+
use crate::credentials::models::{AuditContext, DefaultRole};
10+
use crate::declarations::_helpers::reads::jobs_in;
11+
use crate::declarations::models::{ProviderRecord, ProvidersRead};
12+
use crate::snapshots::main::read_current_record::read_current_record;
13+
use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents;
14+
use crate::store::errors::ControlError;
15+
use crate::store::main::{begin::begin, commit::commit};
16+
use crate::store::models::{ControlStore, TransactionMode};
17+
18+
pub async fn read_person_providers(
19+
store: &ControlStore,
20+
credentials: (&AuditContext, Uuid),
21+
(location_id, target): (Uuid, &str),
22+
default_role: DefaultRole,
23+
) -> Result<ProvidersRead, ControlError> {
24+
let (transaction, _) = authorize_listing(
25+
begin(store, TransactionMode::Write).await?,
26+
credentials,
27+
(location_id, target),
28+
default_role,
29+
)
30+
.await?;
31+
let (transaction, snapshot) = read_current_record(transaction, (location_id, target)).await?;
32+
let Some(snapshot) = snapshot else {
33+
commit(transaction).await?;
34+
return Ok(ProvidersRead {
35+
snapshot_id: None,
36+
providers: Vec::new(),
37+
});
38+
};
39+
let (transaction, documents) =
40+
read_snapshot_documents(transaction, snapshot.snapshot_id, Some(&["providers"])).await?;
41+
let (transaction, jobs) = jobs_in(transaction, (location_id, target)).await?;
42+
commit(transaction).await?;
43+
let mut providers = match documents.get("providers") {
44+
Some(document) => Vec::<ProviderRecord>::deserialize(&document["providers"])
45+
.map_err(|error| ControlError::Invalid(format!("providers document: {error}")))?,
46+
None => Vec::new(),
47+
};
48+
for provider in &mut providers {
49+
provider.jobs.clear();
50+
let nodes: BTreeSet<&String> = provider.nodes.iter().collect();
51+
for job in &jobs {
52+
if job.selection.iter().any(|key| nodes.contains(key)) {
53+
provider.jobs.push(job.job_key.clone());
54+
}
55+
}
56+
}
57+
Ok(ProvidersRead {
58+
snapshot_id: Some(snapshot.snapshot_id),
59+
providers,
60+
})
61+
}

‎crates/sqlbuild-orchestration/src/declarations/models.rs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,3 +256,53 @@ pub(crate) struct GroupWaits {
256256
pub(crate) edges: BTreeMap<String, BTreeSet<String>>,
257257
pub(crate) reasons: BTreeMap<(String, String), String>,
258258
}
259+
260+
/// Providers the current snapshot declares; empty before a publish or when the project has none.
261+
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
262+
pub struct ProvidersRead {
263+
pub snapshot_id: Option<uuid::Uuid>,
264+
pub providers: Vec<ProviderRecord>,
265+
}
266+
267+
/// A provider as its location published it, from the class alone: never a setting value.
268+
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
269+
pub struct ProviderRecord {
270+
pub name: String,
271+
/// The class as `module.qualname`.
272+
pub class: String,
273+
/// The class docstring.
274+
pub description: String,
275+
/// The file declaring the class, relative to the project.
276+
pub path: String,
277+
pub settings: Vec<ProviderSetting>,
278+
/// Code that injects the provider, in discovery order.
279+
pub used_by: Vec<ProviderConsumer>,
280+
/// Snapshot nodes that run that code: Python nodes, models calling a hook or using a materialization, and sources a loader fills.
281+
#[serde(default)]
282+
pub nodes: Vec<String>,
283+
/// Jobs whose selection includes one of those nodes.
284+
#[serde(default)]
285+
pub jobs: Vec<String>,
286+
}
287+
288+
/// One declared setting field; its value stays in the location's environment.
289+
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
290+
pub struct ProviderSetting {
291+
pub name: String,
292+
/// The annotation as written, such as `str` or `SecretStr | None`.
293+
#[serde(rename = "type")]
294+
pub type_name: String,
295+
pub required: bool,
296+
/// Whether the annotation is a pydantic secret type.
297+
pub secret: bool,
298+
/// The environment variable the setting is read from.
299+
pub env: String,
300+
pub description: Option<String>,
301+
}
302+
303+
/// Code that injects a provider: a node (`loader`, `task`, `asset`, `check`, `hook` or `materialization`, keyed `kind:name`), an `event_exporter` or a `sink`.
304+
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
305+
pub struct ProviderConsumer {
306+
pub kind: String,
307+
pub name: String,
308+
}

‎crates/sqlbuild-orchestration/tests/api/test_contracts.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@ mod test_person_location_work;
7878
mod test_person_plans;
7979
#[path = "test_person_sessions.rs"]
8080
mod test_person_sessions;
81+
#[path = "test_providers.rs"]
82+
mod test_providers;
8183
#[path = "test_proxy.rs"]
8284
mod test_proxy;
8385
#[path = "test_publish_encoding.rs"]

0 commit comments

Comments
 (0)