From 24c8efe4851c1535fcaf1038ef8480594c28bed5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:59:31 +0000 Subject: [PATCH 1/2] feat(ui): providers page with a published providers document --- clients/typescript/src/schema.d.ts | 94 +++++++++ .../src/entries/_helpers/user_code.rs | 5 + crates/sqlbuild-host/src/entries/models.rs | 1 + .../authentication/password_sessions.rs | 4 + .../api/_helpers/resources/person_reads.rs | 24 ++- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/declarations/main/mod.rs | 1 + .../main/read_person_providers.rs | 61 ++++++ .../src/declarations/models.rs | 50 +++++ .../tests/api/test_contracts.rs | 2 + .../tests/api/test_providers.rs | 141 +++++++++++++ .../tests/api/test_types.rs | 6 + .../_helpers/location_command/constants.rs | 8 + .../bindings/_helpers/location_command/mod.rs | 1 + .../_helpers/location_command/snapshot.rs | 4 + .../location_command/snapshot_providers.rs | 169 +++++++++++++++ fensu.toml | 2 +- python-boundary.toml | 2 +- .../sqlbuild/references/docs/cli/server.md | 2 +- .../docs/concepts/python-nodes/providers.md | 13 ++ src/sqlbuild/_native_host.py | 4 + src/sqlbuild/_native_host.pyi | 3 + .../_helpers/filesystem/project_modules.py | 67 +++++- .../compiler/compiled_project/_test_types.py | 8 + .../test_orchestration_snapshot.py | 196 ++++++++++++++++++ ui/src/lib/api/http-client.ts | 11 +- ui/src/lib/api/http-providers.test.ts | 33 +++ ui/src/lib/api/http-providers.ts | 16 ++ ui/src/lib/api/mock/platform.ts | 67 +++--- ui/src/lib/api/types/platform.ts | 31 ++- ui/src/routes/providers/+page.svelte | 6 +- ui/src/routes/providers/[name]/+page.svelte | 76 ++++--- ui/src/routes/providers/utils.ts | 25 ++- website/src/content/docs/docs/cli/server.mdx | 2 +- .../docs/concepts/python-nodes/providers.mdx | 12 ++ 35 files changed, 1069 insertions(+), 79 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/read_person_providers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_providers.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs create mode 100644 ui/src/lib/api/http-providers.test.ts create mode 100644 ui/src/lib/api/http-providers.ts diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ff3e5842bc..bec1363178 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -610,6 +610,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/providers": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonProviders"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/runs": { parameters: { query?: never; @@ -2374,6 +2390,46 @@ export interface components { state_database?: string | null; state_schema: string; }; + /** @description Code that injects a provider: a node (`loader`, `task`, `asset`, `check`, `hook` or `materialization`, keyed `kind:name`), an `event_exporter` or a `sink`. */ + ProviderConsumer: { + kind: string; + name: string; + }; + /** @description A provider as its location published it, from the class alone: never a setting value. */ + ProviderRecord: { + /** @description The class as `module.qualname`. */ + class: string; + /** @description The class docstring. */ + description: string; + /** @description Jobs whose selection includes one of those nodes. */ + jobs?: string[]; + name: string; + /** @description Snapshot nodes that run that code: Python nodes, models calling a hook or using a materialization, and sources a loader fills. */ + nodes?: string[]; + /** @description The file declaring the class, relative to the project. */ + path: string; + settings: components["schemas"]["ProviderSetting"][]; + /** @description Code that injects the provider, in discovery order. */ + used_by: components["schemas"]["ProviderConsumer"][]; + }; + /** @description One declared setting field; its value stays in the location's environment. */ + ProviderSetting: { + description?: string | null; + /** @description The environment variable the setting is read from. */ + env: string; + name: string; + required: boolean; + /** @description Whether the annotation is a pydantic secret type. */ + secret: boolean; + /** @description The annotation as written, such as `str` or `SecretStr | None`. */ + type: string; + }; + /** @description Providers the current snapshot declares; empty before a publish or when the project has none. */ + ProvidersRead: { + providers: components["schemas"]["ProviderRecord"][]; + /** Format: uuid */ + snapshot_id?: string | null; + }; /** @description A changed subject, such as `model:orders`, or `role:analyst` (`role:` for those who can't assign roles). */ PublishedChange: { change_kind: string; @@ -4857,6 +4913,44 @@ export interface operations { }; }; }; + readPersonProviders: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Declared providers without setting values */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ProvidersRead"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; listPersonRuns: { parameters: { query?: { diff --git a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs index 51d567c4e1..a3bf48bb33 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs @@ -89,6 +89,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::UserCode, }, + HostEntry { + name: "provider_settings", + kind: HostKind::Function, + edge: HostEdge::UserCode, + }, HostEntry { name: "requires_arguments", kind: HostKind::Function, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index ff74332945..cfbffa3472 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -145,6 +145,7 @@ pub enum UserCodeHost { ModuleProviderClasses, ParameterAnnotations, ProviderUsages, + ProviderSettings, RequiresArguments, RestorePythonRootModules, SourceLine, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index f048bc2f1b..e457021dee 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -151,6 +151,10 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/jobs/{job_key}", get(crate::api::_helpers::resources::person_reads::job), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/providers", + get(crate::api::_helpers::resources::person_reads::providers), + ) .route( "/api/control/locations/{location_id}/targets/{target}/triggers", get(crate::api::_helpers::resources::person_reads::triggers), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs index 1e2b6c3f05..07b107d06c 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs @@ -14,8 +14,9 @@ use crate::declarations::main::list_person_jobs::list_person_jobs; use crate::declarations::main::list_person_trigger_changes::list_person_trigger_changes; use crate::declarations::main::list_person_triggers::list_person_triggers; use crate::declarations::main::read_person_job::read_person_job; +use crate::declarations::main::read_person_providers::read_person_providers; use crate::declarations::models::{ - JobDetailRecord, JobRecord, PersonTriggerChange, PersonTriggerRecord, + JobDetailRecord, JobRecord, PersonTriggerChange, PersonTriggerRecord, ProvidersRead, }; use crate::platform::main::list_person_platform_events::list_person_platform_events; use crate::platform::main::read_person_location_health::read_person_location_health; @@ -292,6 +293,27 @@ pub(crate) async fn jobs( )) } +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/providers", operation_id = "readPersonProviders", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = ProvidersRead, description = "Declared providers without setting values"), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn providers( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + Ok(Json( + read_person_providers( + &store, + (&context, principal), + (location_id, &target), + default_role, + ) + .await?, + )) +} + #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/triggers", operation_id = "listPersonTriggers", params(("location_id" = Uuid, Path), ("target" = String, Path)), responses((status = 200, body = Vec, description = "Published triggers with their newest tick"), (status = 401, description = "Session rejected"), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index e892515a8a..fe53b255bf 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -38,6 +38,7 @@ use crate::api::_helpers::resources::{ person_reads::run_node_error, person_reads::jobs, person_reads::job, + person_reads::providers, person_reads::triggers, person_reads::ticks, person_reads::trigger_changes, diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index e9ec240fc8..8ed3139a1a 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -10,4 +10,5 @@ pub mod list_triggers; pub mod pause_development_triggers; pub(crate) mod publish_definitions; pub mod read_person_job; +pub mod read_person_providers; pub mod snapshot_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/read_person_providers.rs b/crates/sqlbuild-orchestration/src/declarations/main/read_person_providers.rs new file mode 100644 index 0000000000..27182c8166 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/read_person_providers.rs @@ -0,0 +1,61 @@ +//! A person reads the providers of the current snapshot with view on every current node, like jobs. + +use std::collections::BTreeSet; + +use serde::Deserialize; +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::declarations::_helpers::reads::jobs_in; +use crate::declarations::models::{ProviderRecord, ProvidersRead}; +use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn read_person_providers( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location_id, target): (Uuid, &str), + default_role: DefaultRole, +) -> Result { + let (transaction, _) = authorize_listing( + begin(store, TransactionMode::Write).await?, + credentials, + (location_id, target), + default_role, + ) + .await?; + let (transaction, snapshot) = read_current_record(transaction, (location_id, target)).await?; + let Some(snapshot) = snapshot else { + commit(transaction).await?; + return Ok(ProvidersRead { + snapshot_id: None, + providers: Vec::new(), + }); + }; + let (transaction, documents) = + read_snapshot_documents(transaction, snapshot.snapshot_id, Some(&["providers"])).await?; + let (transaction, jobs) = jobs_in(transaction, (location_id, target)).await?; + commit(transaction).await?; + let mut providers = match documents.get("providers") { + Some(document) => Vec::::deserialize(&document["providers"]) + .map_err(|error| ControlError::Invalid(format!("providers document: {error}")))?, + None => Vec::new(), + }; + for provider in &mut providers { + provider.jobs.clear(); + let nodes: BTreeSet<&String> = provider.nodes.iter().collect(); + for job in &jobs { + if job.selection.iter().any(|key| nodes.contains(key)) { + provider.jobs.push(job.job_key.clone()); + } + } + } + Ok(ProvidersRead { + snapshot_id: Some(snapshot.snapshot_id), + providers, + }) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index fecd2146a4..ca2000dfa9 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -256,3 +256,53 @@ pub(crate) struct GroupWaits { pub(crate) edges: BTreeMap>, pub(crate) reasons: BTreeMap<(String, String), String>, } + +/// Providers the current snapshot declares; empty before a publish or when the project has none. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProvidersRead { + pub snapshot_id: Option, + pub providers: Vec, +} + +/// A provider as its location published it, from the class alone: never a setting value. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProviderRecord { + pub name: String, + /// The class as `module.qualname`. + pub class: String, + /// The class docstring. + pub description: String, + /// The file declaring the class, relative to the project. + pub path: String, + pub settings: Vec, + /// Code that injects the provider, in discovery order. + pub used_by: Vec, + /// Snapshot nodes that run that code: Python nodes, models calling a hook or using a materialization, and sources a loader fills. + #[serde(default)] + pub nodes: Vec, + /// Jobs whose selection includes one of those nodes. + #[serde(default)] + pub jobs: Vec, +} + +/// One declared setting field; its value stays in the location's environment. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProviderSetting { + pub name: String, + /// The annotation as written, such as `str` or `SecretStr | None`. + #[serde(rename = "type")] + pub type_name: String, + pub required: bool, + /// Whether the annotation is a pydantic secret type. + pub secret: bool, + /// The environment variable the setting is read from. + pub env: String, + pub description: Option, +} + +/// Code that injects a provider: a node (`loader`, `task`, `asset`, `check`, `hook` or `materialization`, keyed `kind:name`), an `event_exporter` or a `sink`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProviderConsumer { + pub kind: String, + pub name: String, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index a8e01d7cd0..41553f29dd 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -78,6 +78,8 @@ mod test_person_location_work; mod test_person_plans; #[path = "test_person_sessions.rs"] mod test_person_sessions; +#[path = "test_providers.rs"] +mod test_providers; #[path = "test_proxy.rs"] mod test_proxy; #[path = "test_publish_encoding.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_providers.rs b/crates/sqlbuild-orchestration/tests/api/test_providers.rs new file mode 100644 index 0000000000..01e1518569 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_providers.rs @@ -0,0 +1,141 @@ +//! Published providers reach people with the jobs that use them, on both control stores. + +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::test_types::ProvidersTestCase; +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::read_person_providers::read_person_providers; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_published_providers_when_people_read_them_then_jobs_using_their_nodes_are_listed() { + let test_cases = [ + ProvidersTestCase { + description: "SQLite providers", + backend: Backend::Sqlite, + expected_jobs: vec!["order_exports", "stock_alerts"], + }, + ProvidersTestCase { + description: "PostgreSQL providers", + backend: Backend::Postgres, + expected_jobs: vec!["order_exports", "stock_alerts"], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let admin = bootstrap_user(store, &audit, "orders-admin") + .await + .expect("administrator"); + let read = |person: Uuid| { + read_person_providers( + store, + (&audit, person), + (location_id, "development"), + DefaultRole::None, + ) + }; + let before = read(admin).await.expect("read before a publish"); + assert_eq!( + (before.snapshot_id, before.providers.len()), + (None, 0), + "{}: nothing is published yet", + test_case.description + ); + let mut input = snapshot(location_id); + let model = input.nodes["orders"].clone(); + for (key, kind) in [ + ("model:orders", "model"), + ("model:inventory", "model"), + ("check:stock_levels", "check"), + ] { + let mut node = model.clone(); + node.kind = kind.into(); + input.nodes.insert(key.into(), node); + } + input.documents.insert( + "jobs".into(), + json!({ + "order_exports": {"description": "Export orders", "selection": ["model:orders"]}, + "inventory_refresh": {"description": "Refresh stock", "selection": ["model:inventory"]}, + "stock_alerts": {"description": "Alert on stock", "selection": ["check:stock_levels"]}, + }), + ); + input.documents.insert( + "providers".into(), + json!({"providers": [{ + "name": "fulfillment_api", + "class": "providers.fulfillment.FulfillmentApi", + "description": "Client for the fulfillment service.", + "path": "providers/fulfillment.py", + "settings": [ + {"name": "api_key", "type": "SecretStr", "required": true, "secret": true, "env": "API_KEY", "description": null}, + {"name": "endpoint", "type": "str", "required": false, "secret": false, "env": "ENDPOINT", "description": "Base URL"}, + ], + "used_by": [ + {"kind": "hook", "name": "notify_fulfillment"}, + {"kind": "check", "name": "stock_levels"}, + {"kind": "sink", "name": "order_events"}, + ], + "nodes": ["check:stock_levels", "model:orders"], + "jobs": ["unpublished_job"], + }]}), + ); + publish_snapshot(store, &input).await.expect("publish"); + let read_back = read(admin).await.expect("providers"); + assert_eq!( + json!(read_back.providers), + json!([{ + "name": "fulfillment_api", + "class": "providers.fulfillment.FulfillmentApi", + "description": "Client for the fulfillment service.", + "path": "providers/fulfillment.py", + "settings": [ + {"name": "api_key", "type": "SecretStr", "required": true, "secret": true, "env": "API_KEY", "description": null}, + {"name": "endpoint", "type": "str", "required": false, "secret": false, "env": "ENDPOINT", "description": "Base URL"}, + ], + "used_by": [ + {"kind": "hook", "name": "notify_fulfillment"}, + {"kind": "check", "name": "stock_levels"}, + {"kind": "sink", "name": "order_events"}, + ], + "nodes": ["check:stock_levels", "model:orders"], + "jobs": test_case.expected_jobs, + }]), + "{}: jobs selecting a node that runs the provider, including a model through its hook, and none the document names", + test_case.description + ); + let stranger = create_principal(store, &audit, "orders-stranger") + .await + .expect("person"); + assert!( + matches!(read(stranger).await, Err(ControlError::Denied { .. })), + "{}: providers need view on every node", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 94f15bc1ef..abbef0d410 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -254,3 +254,9 @@ pub(crate) struct AcknowledgementTestCase { pub(crate) expected_applied: [bool; 3], pub(crate) expected_acknowledged_by: &'static str, } + +pub(crate) struct ProvidersTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_jobs: Vec<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index c372c93cc4..8306217f22 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -14,3 +14,11 @@ pub(crate) const COMMAND_IMPLEMENTATIONS: [(RunCommand, &str); 5] = [ (RunCommand::Audit, "run_audit_command"), (RunCommand::Freshness, "run_freshness_command"), ]; + +/// Snapshot node kinds a provider consumer can run inside. +pub(crate) const MODEL_NODE_KIND: &str = "model"; +pub(crate) const SOURCE_NODE_KIND: &str = "source"; + +/// `env_prefix_target` values under which pydantic-settings prefixes alias names too. +pub(crate) const ALIAS_PREFIX_TARGET: &str = "alias"; +pub(crate) const ALL_PREFIX_TARGET: &str = "all"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index 5c4301a8d5..f4a9eb49c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -3,4 +3,5 @@ mod constants; mod runtime; mod snapshot; mod snapshot_lineage; +mod snapshot_providers; mod worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index e98184f7c9..455e20cba4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -25,6 +25,7 @@ use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::discovery::project_facts::{named_node_rows, project_facts}; use crate::bindings::_helpers::discovery::project_validation::validation_error; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; +use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; use crate::bindings::_helpers::native_host::orchestration_declarations::{ collect_declarations, orchestration_names, }; @@ -110,6 +111,9 @@ pub(crate) fn orchestration_snapshot<'py>( } input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; drop(owner); + if let Some(providers) = providers_document(&discovered_inputs, &input.nodes)? { + input.documents.insert("providers".into(), providers); + } let declarations = match declarations { Some(declarations) => declarations .into_iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs new file mode 100644 index 0000000000..55d170d62e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs @@ -0,0 +1,169 @@ +//! The project's providers and what uses them, published from the classes alone, so setting values never leave the location. + +use std::collections::{BTreeMap, BTreeSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use serde_json::{Value, json}; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_project::snapshot_boundary::relative_path; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::discovery::imported_modules::shim; +use crate::bindings::_helpers::location_command::constants::{ + ALIAS_PREFIX_TARGET, ALL_PREFIX_TARGET, MODEL_NODE_KIND, SOURCE_NODE_KIND, +}; +use crate::bindings::_helpers::records::fields::field; + +/// How a consumer's code runs inside other snapshot nodes, besides its own node. +#[derive(Clone, Copy)] +enum RunsIn { + Itself, + ModelHooks, + ModelMaterialization, + SourceLoader, +} + +/// Discovered collections whose items inject providers, the kind each is published as, and where it runs. +const CONSUMERS: [(&str, &str, RunsIn); 8] = [ + ("loader", "loader_functions", RunsIn::SourceLoader), + ("task", "task_functions", RunsIn::Itself), + ("asset", "asset_functions", RunsIn::Itself), + ("check", "check_functions", RunsIn::Itself), + ("hook", "hook_functions", RunsIn::ModelHooks), + ( + "materialization", + "materialization_files", + RunsIn::ModelMaterialization, + ), + ("event_exporter", "event_exporters", RunsIn::Itself), + ("sink", "command_output_sinks", RunsIn::Itself), +]; + +/// Each provider's name, class, description, file, setting fields, consumers and the snapshot nodes that run them; `None` when the project declares none. +pub(crate) fn providers_document( + discovered_inputs: &Bound<'_, PyAny>, + nodes: &BTreeMap, +) -> PyResult> { + let py = discovered_inputs.py(); + let providers = field(discovered_inputs, "providers")?; + if providers.len()? == 0 { + return Ok(None); + } + let mut used_by: BTreeMap> = BTreeMap::new(); + let mut running: BTreeMap> = BTreeMap::new(); + for (kind, collection, runs_in) in CONSUMERS { + for consumer in field(discovered_inputs, collection)?.try_iter()? { + let consumer = consumer?; + let name: String = field(&consumer, "name")?.extract()?; + let keys = running_nodes(nodes, (kind, &name, runs_in)); + for usage in field(&consumer, "provider_usages")?.try_iter()? { + let provider: String = field(&usage?, "provider_name")?.extract()?; + used_by + .entry(provider.clone()) + .or_default() + .push(json!({ "kind": kind, "name": name })); + running.entry(provider).or_default().extend(keys.clone()); + } + } + } + let read_class = shim(py, UserCodeHost::ProviderSettings)?; + let mut rows: Vec = Vec::new(); + for provider in providers.try_iter()? { + let provider = provider?; + let name: String = field(&provider, "name")?.extract()?; + let class = plain_value(&call_with( + &read_class, + &[("provider_class", field(&provider, "provider_class")?)], + )?)?; + rows.push(json!({ + "name": name, + "class": class["class"], + "description": class["description"], + "path": relative_path(&provider)?, + "settings": settings(&class), + "nodes": running.remove(&name).unwrap_or_default(), + "used_by": used_by.remove(&name).unwrap_or_default(), + })); + } + Ok(Some(json!({ "providers": rows }))) +} + +/// Snapshot nodes that run one consumer: its own node, models calling a hook or using a materialization, and sources a loader fills. +fn running_nodes( + nodes: &BTreeMap, + (kind, name, runs_in): (&str, &str, RunsIn), +) -> BTreeSet { + let mut keys: BTreeSet = BTreeSet::new(); + let own = format!("{kind}:{name}"); + if nodes.contains_key(&own) { + keys.insert(own); + } + for (key, node) in nodes { + let definition = &node.definition; + let runs = match runs_in { + RunsIn::Itself => false, + RunsIn::ModelHooks => calls_hook(definition, name), + RunsIn::ModelMaterialization => { + node.kind == MODEL_NODE_KIND && definition["materialization"].as_str() == Some(name) + } + RunsIn::SourceLoader => { + node.kind == SOURCE_NODE_KIND + && definition["entry"]["loader"].as_str() == Some(name) + } + }; + if runs { + keys.insert(key.clone()); + } + } + keys +} + +/// Whether a model's pre or post hooks call the hook, written as its name or as `{"name": ...}`. +fn calls_hook(definition: &Value, hook: &str) -> bool { + for hooks in ["pre_hooks", "post_hooks"] { + let Some(calls) = definition["config"][hooks].as_array() else { + continue; + }; + for call in calls { + if call.as_str() == Some(hook) || call["name"].as_str() == Some(hook) { + return true; + } + } + } + false +} + +/// Declared fields with the environment variable each is read from, which pydantic-settings derives from the prefix unless an alias names it. +fn settings(class: &Value) -> Vec { + let prefix = class["env_prefix"].as_str().unwrap_or_default(); + let case_sensitive = class["case_sensitive"].as_bool().unwrap_or_default(); + let target = class["env_prefix_target"].as_str(); + let alias_prefix = match target { + Some(ALIAS_PREFIX_TARGET | ALL_PREFIX_TARGET) => prefix, + _ => "", + }; + let name_prefix = match target { + Some(ALIAS_PREFIX_TARGET) => "", + _ => prefix, + }; + let mut rows: Vec = Vec::new(); + for field in class["fields"].as_array().into_iter().flatten() { + let name = field["name"].as_str().unwrap_or_default(); + let type_name = field["type"].as_str().unwrap_or_default(); + let env = field["alias"].as_str().map_or_else( + || format!("{name_prefix}{name}"), + |alias| format!("{alias_prefix}{alias}"), + ); + rows.push(json!({ + "name": name, + "type": type_name, + "required": field["required"], + "secret": type_name.contains("Secret"), + "env": if case_sensitive { env } else { env.to_uppercase() }, + "description": field["description"], + })); + } + rows +} diff --git a/fensu.toml b/fensu.toml index 415e477619..a3fc3f871f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -234,7 +234,7 @@ reason = "Native code resolves these through the sqlbuild._native_host name tabl [[targets.python.dead_code.roots]] modules = ["sqlbuild.compiler.discovery._helpers.filesystem.project_modules"] -symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_orchestration_module", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] +symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_orchestration_module", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_settings", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] reason = "Native code resolves these through the sqlbuild._native_host name table." [[targets.python.dead_code.roots]] diff --git a/python-boundary.toml b/python-boundary.toml index 0aa3f9f4ee..7f8009a8b5 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -490,7 +490,7 @@ note = "Reads loaded user node functions: identities over live functions and dec [[stays]] path = "compiler/discovery/_helpers/filesystem/project_modules.py" rule = 2 -note = "Rule 2 shim: imports project Python node, hook, provider, sink and materialization modules and reads live members, signatures, provider parameter annotations and source lines; record building, provider naming and validation, settings errors, factory results and duplicate checks are native." +note = "Rule 2 shim: imports project Python node, hook, provider, sink and materialization modules and reads live members, signatures, provider parameter annotations, provider class setting declarations and source lines; record building, provider naming and validation, settings errors, factory results and duplicate checks are native." [[stays]] path = "compiler/discovery/_helpers/validation/discovery.py" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 7ca1ede23e..97118100ed 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -405,7 +405,7 @@ permissions = ["view.evidence"] | Permissions | Any non-administrative action; `user.manage`, `role.assign` and `audit.view` are rejected | | Runs | `job.run`, `run.rerun` and `run.rewrite_history` need the permission on every node the run writes; an [ad-hoc command](#run-ad-hoc-commands) needs `run.adhoc` and its flags' permissions on every node it selects; a priority above 0 also needs `run.prioritise` | | Triggers | `trigger.toggle` and `sensor.reset_cursor` need the permission on every node the trigger's job writes | -| Listings | The catalog, statistics and run list need `view` on every node of the current publication; partial coverage is denied, not filtered | +| Listings | The catalog, statistics, providers and run list need `view` on every node of the current publication; partial coverage is denied, not filtered | | Several roles | Grants add up across a person's roles | Assign a project role with `POST /api/admin/roles`, using the role key as `role`. Each diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index 335a237f59..d1b9d7117e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -17,6 +17,7 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/providers/ - Configuration from environment variables - Discovery rules - Plan output +- Providers in the web UI - Project layout Providers are shared runtime services that SQLBuild discovers, configures, and injects into your Python nodes and hooks. Use them for external connections, API clients, or any stateful service that multiple nodes need access to. @@ -202,6 +203,18 @@ Providers Use `--verbose` to see which specific nodes consume each provider. +## Providers in the web UI + +When a location publishes to [`sqb server`](../../cli/server.md), the **Providers** page lists each +provider with its class and docstring. A provider's page shows: + +| Tab | Shows | +| --- | --- | +| Settings | Each setting's type, environment variable and whether it is required; secret settings are marked | +| Used by | The loaders, tasks, hooks and other code that take the provider, and the jobs that run that code | + +Setting values, defaults included, are never published. + ## Project layout ``` diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index c48bcb38e5..262daa42dc 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -187,6 +187,10 @@ "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", "provider_usages", ), + "provider_settings": ( + "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", + "provider_settings", + ), "requires_arguments": ( "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", "requires_arguments", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index a796297196..ed8076e4d7 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -151,6 +151,9 @@ from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( parameter_annotations as parameter_annotations, ) +from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( + provider_settings as provider_settings, +) from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( provider_usages as provider_usages, ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index 1eef8c4987..9bfc5a5b17 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -9,8 +9,11 @@ from collections.abc import Callable from importlib.machinery import ModuleSpec from pathlib import Path -from types import ModuleType -from typing import get_type_hints +from types import ModuleType, UnionType +from typing import Annotated, Literal, Union, get_args, get_origin, get_type_hints + +from pydantic import AliasChoices, AliasPath +from pydantic.fields import FieldInfo from sqlbuild._native import ( PYTHON_INIT_MODULE_STEM, @@ -415,3 +418,63 @@ def provider_usages( ) ) return tuple(usages) + + +def provider_settings(*, provider_class: type) -> dict[str, object]: + """A provider class's path, docstring and declared setting fields; never a setting value.""" + + config: dict[str, object] = dict(getattr(provider_class, "model_config", {})) + fields: dict[str, FieldInfo] = getattr(provider_class, "model_fields", {}) + rows: list[dict[str, object]] = [] + for name, info in fields.items(): + rows.append( + { + "name": name, + "type": _annotation_text(info.annotation), + "required": info.is_required(), + "alias": _environment_alias(info), + "description": info.description, + } + ) + return { + "class": f"{provider_class.__module__}.{provider_class.__qualname__}", + "description": inspect.cleandoc(provider_class.__doc__ or ""), + "env_prefix": config.get("env_prefix") or "", + "case_sensitive": bool(config.get("case_sensitive")), + "env_prefix_target": config.get("env_prefix_target"), + "fields": rows, + } + + +def _environment_alias(info: FieldInfo) -> str | None: + """The first environment variable an alias names: an `AliasChoices` choice or path head.""" + + alias: object = info.validation_alias or info.alias + if isinstance(alias, AliasChoices | AliasPath): + first: object = next(iter(alias.convert_to_aliases()), None) + if isinstance(first, list): + first = next(iter(first), None) + return first if isinstance(first, str) else None + return alias if isinstance(alias, str) else None + + +def _annotation_text(annotation: object) -> str: + """An annotation as a reader writes it: `SecretStr | None`, `list[str]` or `Literal['gzip']`.""" + + if annotation is None or annotation is type(None): + return "None" + origin: object = get_origin(annotation) + arguments: tuple[object, ...] = get_args(annotation) + if origin is None or not arguments: + if isinstance(annotation, type): + return annotation.__name__ + return str(annotation).replace("typing.", "") + if origin is Annotated: + return _annotation_text(arguments[0]) + if origin is Literal: + return f"Literal[{', '.join(repr(argument) for argument in arguments)}]" + parts: list[str] = [_annotation_text(argument) for argument in arguments] + if origin is Union or origin is UnionType: + return " | ".join(parts) + name: str = origin.__name__ if isinstance(origin, type) else str(origin).replace("typing.", "") + return f"{name}[{', '.join(parts)}]" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index a11bcbf230..f3aa56721b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -252,6 +252,14 @@ class SnapshotLineageTestCase: expected_edges: list[list[str]] +@dataclass(frozen=True) +class SnapshotProvidersTestCase: + description: str + files: dict[str, str] + environment: dict[str, str] + expected_providers: dict[str, object] | None + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 5bfb5e9d9a..cdbdd98489 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -41,6 +41,7 @@ SnapshotExpressionRangeTestCase, SnapshotIdentityTestCase, SnapshotLineageTestCase, + SnapshotProvidersTestCase, SnapshotRejectionTestCase, SnapshotScheduleTestCase, ) @@ -61,6 +62,12 @@ ) _DUCKSHOP: Path = Path("benchmarks/duckshop/project") +_PROVIDER_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/orders.sql": "MODEL (description 'Orders.');\n\nSELECT 1 AS order_id\n", +} + + _PRICE_MODELS: dict[str, str] = { "models/stg_prices.sql": ( 'MODEL (description "Staged prices");\n' @@ -411,6 +418,195 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s assert sources | targets <= set(publication["nodes"]) +@pytest.mark.parametrize( + "test_case", + [ + SnapshotProvidersTestCase( + description="a provider is published with its fields and consumers but no values", + files=_PROVIDER_PROJECT + | { + "providers/fulfillment.py": ( + "from typing import Literal\n\n" + "from pydantic import AliasChoices, Field, SecretStr\n" + "from pydantic_settings import SettingsConfigDict\n" + "from sqlbuild.providers import Provider\n\n\n" + "class FulfillmentApi(Provider):\n" + ' """Client for the fulfillment service.\n\n Sends shipped orders.\n """\n\n' + " model_config = SettingsConfigDict(env_prefix='FULFILLMENT_')\n" + " api_key: SecretStr\n" + " backup_key: SecretStr | None = None\n" + " regions: list[str] = ['eu']\n" + " compression: Literal['none', 'gzip'] = 'gzip'\n" + " endpoint: str = Field('https://fulfillment.example.com', description='Base URL')\n" + " timeout_seconds: int | None = Field(\n" + " None, validation_alias=AliasChoices('SHIP_TIMEOUT', 'TIMEOUT')\n" + " )\n" + ), + "python/tasks/publish_orders.py": ( + "from providers.fulfillment import FulfillmentApi\n" + "from sqlbuild.tasks import task\n\n\n" + "@task\n" + "def publish_orders(fulfillment_api: FulfillmentApi):\n" + ' """Send orders to fulfillment."""\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', post_hooks [python(\"notify_fulfillment\")]);\n\n" + "SELECT 1 AS order_id\n" + ), + "hooks/python/notify_fulfillment.py": ( + "from sqlbuild.hooks import hook\n\n\n" + "@hook\n" + "def notify_fulfillment(ctx, fulfillment_api):\n" + ' """Tell fulfillment a model changed."""\n' + ), + }, + environment={"FULFILLMENT_API_KEY": "synthetic-fulfillment-key"}, + expected_providers={ + "providers": [ + { + "name": "fulfillment_api", + "class": "providers.fulfillment.FulfillmentApi", + "description": "Client for the fulfillment service.\n\nSends shipped orders.", + "path": "providers/fulfillment.py", + "settings": [ + { + "name": "api_key", + "type": "SecretStr", + "required": True, + "secret": True, + "env": "FULFILLMENT_API_KEY", + "description": None, + }, + { + "name": "backup_key", + "type": "SecretStr | None", + "required": False, + "secret": True, + "env": "FULFILLMENT_BACKUP_KEY", + "description": None, + }, + { + "name": "regions", + "type": "list[str]", + "required": False, + "secret": False, + "env": "FULFILLMENT_REGIONS", + "description": None, + }, + { + "name": "compression", + "type": "Literal['none', 'gzip']", + "required": False, + "secret": False, + "env": "FULFILLMENT_COMPRESSION", + "description": None, + }, + { + "name": "endpoint", + "type": "str", + "required": False, + "secret": False, + "env": "FULFILLMENT_ENDPOINT", + "description": "Base URL", + }, + { + "name": "timeout_seconds", + "type": "int | None", + "required": False, + "secret": False, + "env": "SHIP_TIMEOUT", + "description": None, + }, + ], + "used_by": [ + {"kind": "task", "name": "publish_orders"}, + {"kind": "hook", "name": "notify_fulfillment"}, + ], + "nodes": ["hook:notify_fulfillment", "model:orders", "task:publish_orders"], + } + ] + }, + ), + SnapshotProvidersTestCase( + description="a prefix that applies only to aliases leaves plain field names unprefixed", + files=_PROVIDER_PROJECT + | { + "providers/inventory.py": ( + "from pydantic import Field\n" + "from pydantic_settings import SettingsConfigDict\n" + "from sqlbuild.providers import Provider\n\n\n" + "class InventoryApi(Provider):\n" + ' """Client for the inventory service."""\n\n' + " model_config = SettingsConfigDict(\n" + " env_prefix='INVENTORY_', env_prefix_target='alias'\n" + " )\n" + " region: str = 'eu'\n" + " token: str = Field('unset', alias='TOKEN')\n" + ), + }, + environment={}, + expected_providers={ + "providers": [ + { + "name": "inventory_api", + "class": "providers.inventory.InventoryApi", + "description": "Client for the inventory service.", + "path": "providers/inventory.py", + "settings": [ + { + "name": "region", + "type": "str", + "required": False, + "secret": False, + "env": "REGION", + "description": None, + }, + { + "name": "token", + "type": "str", + "required": False, + "secret": False, + "env": "INVENTORY_TOKEN", + "description": None, + }, + ], + "used_by": [], + "nodes": [], + } + ] + }, + ), + SnapshotProvidersTestCase( + description="a project without providers publishes no providers document", + files=_PROVIDER_PROJECT, + environment={}, + expected_providers=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_project_providers_when_publishing_then_they_are_a_snapshot_document_without_values( + test_case: SnapshotProvidersTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + for name, value in test_case.environment.items(): + monkeypatch.setenv(name, value) + write_files(project_dir=tmp_path, files=test_case.files) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + published: str = build_orchestration_snapshot( + project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered + ).to_json() + assert json.loads(published)["documents"].get("providers") == test_case.expected_providers + assert "synthetic-fulfillment-key" not in published + assert "fulfillment.example.com" not in published + + @pytest.mark.parametrize( "test_case", [ diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index d0e54d123e..ef45f56d6e 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -10,6 +10,7 @@ import { toSnapshotChanges } from './http-changes'; import { manualConnectionTest, snapshotOf, toLocationDetail, toLocationSummary, toWarehouseConnection, warehouseSyncOf } from './http-locations'; import { toAccount, toRoleAssignments, toServerSettings } from './http-people'; import { toMaintenanceHistory } from './http-maintenance'; +import { toProviders } from './http-providers'; import { overviewOf } from './http-overview'; import { toPlatformEventRecords, toPlatformEvents, toPlatformHealth } from './http-problems'; import { roleDetailOf, toRoleDefinition } from './http-roles'; @@ -18,7 +19,7 @@ import { timelineOf } from './http-timeline'; import { RUN_WINDOW, groupRunNodes, parseRunCursor, planResultOf, queueEntriesOf, toAdhocCheck, runDetailOf, runRowPageOver, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs'; import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals'; import type { AuthMode, Bootstrap, Capability, Location, NavCounts, PlatformEvent, Scope, Snapshot, WarehouseSync } from './types'; -import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, ServerSettings, WarehouseConnection, LocationSummary, PlatformEventQuery, PlatformEventRecord, PlatformHealth, MaintenanceHistory, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; +import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, ServerSettings, WarehouseConnection, LocationSummary, PlatformEventQuery, PlatformEventRecord, PlatformHealth, MaintenanceHistory, ProviderConnection, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; import type { Acknowledgement, CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, Overview, SnapshotChange, ColumnLineage } from './types/build'; import type { AdhocRunCheck, AdhocRunRequest, BulkRunResult, PlanResult, QueueEntry, Timeline, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, StatementSql } from './types/operate'; @@ -85,6 +86,7 @@ const SUPPORTED: ReadonlySet = new Set([ 'maintenanceHistory', 'testConnection', 'listWarehouseConnections', + 'listProviders', 'serverSettings', 'listSnapshotNodes', 'checkAdhocRun', @@ -597,6 +599,13 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, } throw new ApiError('The location did not answer the connection test in time; check that it is running', 504); }, + async listProviders(scope: Scope): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/providers', { params: { path: { location_id: locationId, target } } }); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return toProviders(data); + }, async maintenanceHistory(scope: Scope): Promise { const { locationId, target } = parseLocationKey(scope.locationId); const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/maintenance', { params: { path: { location_id: locationId, target } } }); diff --git a/ui/src/lib/api/http-providers.test.ts b/ui/src/lib/api/http-providers.test.ts new file mode 100644 index 0000000000..b2e319b581 --- /dev/null +++ b/ui/src/lib/api/http-providers.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest'; +import type { components } from '$control/schema.d.ts'; +import { toProviders } from './http-providers'; + +type W = components['schemas']; + +describe('toProviders', () => { + it('maps settings, consumers and jobs without values', () => { + const wire: W['ProvidersRead'] = { + snapshot_id: '00000000-0000-0000-0000-000000000001', + providers: [ + { + name: 'fulfillment_api', + class: 'providers.fulfillment.FulfillmentApi', + description: 'Client for the fulfillment service.', + path: 'providers/fulfillment.py', + settings: [{ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null }], + used_by: [ + { kind: 'task', name: 'publish_orders' }, + { kind: 'sink', name: 'order_events' }, + { kind: 'event_exporter', name: 'run_events' } + ], + jobs: ['order_exports'] + } + ] + }; + const [provider] = toProviders(wire); + expect(provider.className).toBe('providers.fulfillment.FulfillmentApi'); + expect(provider.settings[0]).toEqual({ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null }); + expect(provider.usedBy.map((c) => `${c.kind}:${c.name}`)).toEqual(['task:publish_orders', 'sink:order_events', 'event_exporter:run_events']); + expect(provider.jobs).toEqual(['order_exports']); + }); +}); diff --git a/ui/src/lib/api/http-providers.ts b/ui/src/lib/api/http-providers.ts new file mode 100644 index 0000000000..773d59cdca --- /dev/null +++ b/ui/src/lib/api/http-providers.ts @@ -0,0 +1,16 @@ +import type { components } from '$control/schema.d.ts'; +import type { ProviderConnection, ProviderConsumer } from './types'; + +type Wire = components['schemas']; + +export function toProviders(wire: Wire['ProvidersRead']): ProviderConnection[] { + return wire.providers.map((p) => ({ + name: p.name, + className: p.class, + description: p.description, + sourceFile: p.path, + settings: p.settings.map((s) => ({ name: s.name, type: s.type, required: s.required, secret: s.secret, env: s.env, description: s.description ?? null })), + usedBy: p.used_by.map((c) => ({ kind: c.kind as ProviderConsumer['kind'], name: c.name })), + jobs: p.jobs ?? [] + })); +} diff --git a/ui/src/lib/api/mock/platform.ts b/ui/src/lib/api/mock/platform.ts index 944f0e0cbb..b05532e71e 100644 --- a/ui/src/lib/api/mock/platform.ts +++ b/ui/src/lib/api/mock/platform.ts @@ -43,6 +43,7 @@ import type { Scope, ServerSettings, SettingField, + ProviderSetting, SsoMapping, WarehouseConnection } from '../types'; @@ -368,60 +369,72 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { } ] }; + const setting = (name: string, type: string, env: string, description: string, extra: Partial = {}): ProviderSetting => ({ + name, + type, + required: false, + secret: type.includes('Secret'), + env, + description, + ...extra + }); const providers: ProviderConnection[] = [ { name: 'orders_api', - type: 'rest', + className: 'providers.orders_api.OrdersApi', description: 'Partner orders API used by the raw orders loader.', sourceFile: 'providers/orders_api.py', settings: [ - { key: 'base_url', type: 'string', required: true, description: 'API base URL' }, - { key: 'page_size', type: 'int', default: 500, min: 1, max: 5000, description: 'Records per request' }, - { key: 'timeout_seconds', type: 'int', default: 30, min: 1, max: 300, description: 'Request timeout' }, - { key: 'api_key', type: 'secret', required: true, secretSource: 'env: ORDERS_API_KEY', description: 'API key' } + setting('base_url', 'str', 'ORDERS_API_BASE_URL', 'API base URL', { required: true }), + setting('page_size', 'int', 'ORDERS_API_PAGE_SIZE', 'Records per request'), + setting('timeout_seconds', 'int', 'ORDERS_API_TIMEOUT_SECONDS', 'Request timeout'), + setting('api_key', 'SecretStr', 'ORDERS_API_KEY', 'API key', { required: true }) ], - values: { base_url: 'https://orders.example.com/v2' }, - usedBy: { nodes: ['raw__orders'], jobs: ['load_sources', 'nightly_all'], triggers: [] } + usedBy: [{ kind: 'loader', name: 'raw__orders' }], + jobs: ['load_sources', 'nightly_all'] }, { name: 'export_bucket', - type: 's3', + className: 'providers.storage.ExportBucket', description: 'Object storage for CSV exports of reporting models.', sourceFile: 'providers/storage.py', settings: [ - { key: 'bucket', type: 'string', required: true, description: 'Bucket name' }, - { key: 'prefix', type: 'string', default: 'exports/', description: 'Key prefix' }, - { key: 'region', type: 'string', required: true, description: 'Region' }, - { key: 'compression', type: 'enum', enum: ['none', 'gzip', 'zstd'], default: 'gzip', description: 'File compression' }, - { key: 'secret_access_key', type: 'secret', required: true, secretSource: 'secret store: exports/bucket', description: 'Access key secret' } + setting('bucket', 'str', 'EXPORT_BUCKET_BUCKET', 'Bucket name', { required: true }), + setting('prefix', 'str', 'EXPORT_BUCKET_PREFIX', 'Key prefix'), + setting('region', 'str', 'EXPORT_BUCKET_REGION', 'Region', { required: true }), + setting('compression', "Literal['none', 'gzip', 'zstd']", 'EXPORT_BUCKET_COMPRESSION', 'File compression'), + setting('secret_access_key', 'SecretStr', 'EXPORT_BUCKET_SECRET_ACCESS_KEY', 'Access key secret', { required: true }) + ], + usedBy: [ + { kind: 'task', name: 'export_revenue_csv' }, + { kind: 'sink', name: 'run_log_archive' } ], - values: { bucket: 'example-exports', region: 'eu-west-1' }, - usedBy: { nodes: ['export_revenue_csv'], jobs: ['revenue_refresh', 'reporting_exports'], triggers: [] } + jobs: ['revenue_refresh', 'reporting_exports'] }, { name: 'support_desk', - type: 'rest', + className: 'providers.support_desk.SupportDesk', description: 'Support ticket system read by the tickets loader.', sourceFile: 'providers/support_desk.py', settings: [ - { key: 'base_url', type: 'string', required: true, description: 'API base URL' }, - { key: 'include_closed', type: 'bool', default: false, description: 'Also load closed tickets' }, - { key: 'token', type: 'secret', required: true, secretSource: 'env: SUPPORT_DESK_TOKEN', description: 'Access token' } + setting('base_url', 'str', 'SUPPORT_DESK_BASE_URL', 'API base URL', { required: true }), + setting('include_closed', 'bool', 'SUPPORT_DESK_INCLUDE_CLOSED', 'Also load closed tickets'), + setting('token', 'SecretStr', 'SUPPORT_DESK_TOKEN', 'Access token', { required: true }) ], - values: { base_url: 'https://support.example.com/api', include_closed: true }, - usedBy: { nodes: ['raw__support_tickets'], jobs: ['load_sources'], triggers: [] } + usedBy: [{ kind: 'loader', name: 'raw__support_tickets' }], + jobs: ['load_sources'] }, { name: 'chat_webhook', - type: 'webhook', - description: 'Incoming webhook of the team chat room; sensors post messages to it.', + className: 'providers.chat.ChatWebhook', + description: 'Incoming webhook of the team chat room.', sourceFile: 'providers/chat.py', settings: [ - { key: 'url', type: 'secret', required: true, secretSource: 'env: CHAT_WEBHOOK_URL', description: 'Webhook URL' }, - { key: 'timeout_seconds', type: 'int', default: 10, min: 1, max: 60, description: 'Request timeout' } + setting('url', 'SecretStr', 'CHAT_WEBHOOK_URL', 'Webhook URL', { required: true }), + setting('timeout_seconds', 'int', 'CHAT_WEBHOOK_TIMEOUT_SECONDS', 'Request timeout') ], - values: {}, - usedBy: { nodes: [], jobs: [], triggers: [{ id: triggerId('platform_health_webhook'), name: 'platform_health_webhook' }] } + usedBy: [], + jobs: [] } ]; diff --git a/ui/src/lib/api/types/platform.ts b/ui/src/lib/api/types/platform.ts index b374e8b8b0..4b71eed302 100644 --- a/ui/src/lib/api/types/platform.ts +++ b/ui/src/lib/api/types/platform.ts @@ -239,16 +239,35 @@ export type WarehouseTableSync = { lastSyncedAt: Iso | null; }; -/** A connection to something other than the warehouse (an API, object storage, a chat webhook), declared in project code. */ -export type ProviderConnection = { +/** One setting a provider declares. Its value stays in the location's environment and never reaches the server. */ +export type ProviderSetting = { name: string; + /** The annotation as written, e.g. `str` or `SecretStr | None`. */ type: string; + required: boolean; + secret: boolean; + /** The environment variable the setting is read from. */ + env: string; + description: string | null; +}; + +/** Python code that injects a provider. None of it is a catalog node, so it has no page to open. */ +export type ProviderConsumer = { + kind: 'loader' | 'task' | 'asset' | 'check' | 'hook' | 'materialization' | 'event_exporter' | 'sink'; + name: string; +}; + +/** A provider declared in project code (an API client, object storage, a chat webhook), as the location published it. */ +export type ProviderConnection = { + name: string; + /** The class as `module.Class`. */ + className: string; description: string; sourceFile: string; - settings: SettingField[]; - values: Record; - /** `triggers` are sensors whose code uses the provider, by trigger ID and name. */ - usedBy: { nodes: string[]; jobs: string[]; triggers: { id: string; name: string }[] }; + settings: ProviderSetting[]; + usedBy: ProviderConsumer[]; + /** Jobs whose selection includes a node that uses the provider. */ + jobs: string[]; }; // ---- maintenance (synced warehouse state tables) ---- diff --git a/ui/src/routes/providers/+page.svelte b/ui/src/routes/providers/+page.svelte index c91ca64899..cba7f76342 100644 --- a/ui/src/routes/providers/+page.svelte +++ b/ui/src/routes/providers/+page.svelte @@ -13,7 +13,7 @@ let query = $state(''); const all = $derived(providers.data ?? []); const q = $derived(query.trim().toLowerCase()); - const shown = $derived(all.filter((p) => !q || `${p.name} ${p.type} ${p.description}`.toLowerCase().includes(q))); + const shown = $derived(all.filter((p) => !q || `${p.name} ${p.className} ${p.description}`.toLowerCase().includes(q))); @@ -28,7 +28,7 @@ Provider - Type + Class Used by Defined in @@ -45,7 +45,7 @@ - {p.type} + {p.className} {usedBySummary(p)} {p.sourceFile} diff --git a/ui/src/routes/providers/[name]/+page.svelte b/ui/src/routes/providers/[name]/+page.svelte index daf5582a61..9bc9cac8b5 100644 --- a/ui/src/routes/providers/[name]/+page.svelte +++ b/ui/src/routes/providers/[name]/+page.svelte @@ -5,12 +5,11 @@ import LockIcon from '@lucide/svelte/icons/lock'; import AppTopbar from '$lib/components/app-topbar.svelte'; import Button from '$lib/components/ui/button/button.svelte'; - import ConfigForm from '$lib/config/config-form.svelte'; import KeyValues from '$lib/components/platform/key-values.svelte'; import PageTabs from '$lib/components/platform/page-tabs.svelte'; import { app } from '$lib/state/app.svelte'; import { createResource } from '$lib/state/resource.svelte'; - import { usedCount } from '../utils'; + import { consumerKindLabel, usedCount } from '../utils'; const name = $derived(page.params.name ?? ''); const providers = createResource((client, scope) => client.listProviders(scope)); @@ -28,7 +27,7 @@
← Providers {name} - {#if provider}{provider.type}{/if} + {#if provider}{provider.className}{/if}
{#if providers.error} @@ -46,44 +45,55 @@ ]} /> {#if tab === 'settings'} -
-
+
+
- Read-only. Declared in {provider.sourceFile}; secrets show only where they come from. + Values come from each location's environment and never reach the server. Declared in {provider.sourceFile}.
- + {#if provider.settings.length} + + + + + + {#each provider.settings as f (f.name)} + + + + + + + {/each} + +
SettingTypeEnvironment variableRequired
+
{f.name}{#if f.secret} secret{/if}
+ {#if f.description}
{f.description}
{/if} +
{f.type}{f.env}{f.required ? 'Required' : 'Optional'}
+ {:else} +

This provider declares no settings.

+ {/if}
{:else}
{#if !usedCount(provider)}

Nothing uses this provider yet.

{/if} - {#if provider.usedBy.nodes.length} -
-
Nodes · {provider.usedBy.nodes.length}
-
- {#each provider.usedBy.nodes as m (m)} - {m} - {/each} -
-
- {/if} - {#if provider.usedBy.jobs.length} + {#if provider.usedBy.length}
-
Jobs · {provider.usedBy.jobs.length}
+
Code · {provider.usedBy.length}
- {#each provider.usedBy.jobs as j (j)} - {j} + {#each provider.usedBy as c (c.kind + c.name)} + {c.name}{consumerKindLabel[c.kind]} {/each}
{/if} - {#if provider.usedBy.triggers.length} + {#if provider.jobs.length}
-
Sensors · {provider.usedBy.triggers.length}
+
Jobs that run this code · {provider.jobs.length}
- {#each provider.usedBy.triggers as t (t.id)} - {t.name} + {#each provider.jobs as j (j)} + {j} {/each}
@@ -96,14 +106,26 @@
Provider
-

{provider.description}

+

{provider.description}

{:else}
{/if}
+ + diff --git a/ui/src/routes/providers/utils.ts b/ui/src/routes/providers/utils.ts index 0d2167e55c..661c59b9df 100644 --- a/ui/src/routes/providers/utils.ts +++ b/ui/src/routes/providers/utils.ts @@ -1,17 +1,26 @@ -import type { ProviderConnection } from '$lib/api'; +import type { ProviderConnection, ProviderConsumer } from '$lib/api'; const plural = (count: number, noun: string) => `${count} ${noun}${count === 1 ? '' : 's'}`; -/** e.g. `1 node · 2 jobs · 1 sensor`, or `Not used yet`. */ +export const consumerKindLabel: Record = { + loader: 'loader', + task: 'task', + asset: 'asset', + check: 'check', + hook: 'hook', + materialization: 'materialization', + event_exporter: 'event exporter', + sink: 'sink' +}; + +/** e.g. `1 loader · 1 sink · 2 jobs`, or `Not used yet`. */ export function usedBySummary(p: ProviderConnection): string { - const parts = [ - p.usedBy.nodes.length ? plural(p.usedBy.nodes.length, 'node') : null, - p.usedBy.jobs.length ? plural(p.usedBy.jobs.length, 'job') : null, - p.usedBy.triggers.length ? plural(p.usedBy.triggers.length, 'sensor') : null - ].filter(Boolean); + const kinds = new Map(); + for (const c of p.usedBy) kinds.set(c.kind, (kinds.get(c.kind) ?? 0) + 1); + const parts = [...[...kinds].map(([kind, count]) => plural(count, consumerKindLabel[kind])), p.jobs.length ? plural(p.jobs.length, 'job') : null].filter(Boolean); return parts.join(' · ') || 'Not used yet'; } export function usedCount(p: ProviderConnection): number { - return p.usedBy.nodes.length + p.usedBy.jobs.length + p.usedBy.triggers.length; + return p.usedBy.length + p.jobs.length; } diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 9ebd2dad66..f3008d5598 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -391,7 +391,7 @@ permissions = ["view.evidence"] | Permissions | Any non-administrative action; `user.manage`, `role.assign` and `audit.view` are rejected | | Runs | `job.run`, `run.rerun` and `run.rewrite_history` need the permission on every node the run writes; an [ad-hoc command](#run-ad-hoc-commands) needs `run.adhoc` and its flags' permissions on every node it selects; a priority above 0 also needs `run.prioritise` | | Triggers | `trigger.toggle` and `sensor.reset_cursor` need the permission on every node the trigger's job writes | -| Listings | The catalog, statistics and run list need `view` on every node of the current publication; partial coverage is denied, not filtered | +| Listings | The catalog, statistics, providers and run list need `view` on every node of the current publication; partial coverage is denied, not filtered | | Several roles | Grants add up across a person's roles | Assign a project role with `POST /api/admin/roles`, using the role key as `role`. Each diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index 2eb9b35d0c..770ff28adc 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -186,6 +186,18 @@ Providers Use `--verbose` to see which specific nodes consume each provider. +## Providers in the web UI + +When a location publishes to [`sqb server`](/docs/cli/server/), the **Providers** page lists each +provider with its class and docstring. A provider's page shows: + +| Tab | Shows | +| --- | --- | +| Settings | Each setting's type, environment variable and whether it is required; secret settings are marked | +| Used by | The loaders, tasks, hooks and other code that take the provider, and the jobs that run that code | + +Setting values, defaults included, are never published. + ## Project layout ``` From c2793d293869395e8fe83106e8b5794fb6e746f0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:12:30 +0000 Subject: [PATCH 2/2] fix(providers): load pydantic only when reading provider settings The CLI imports project_modules at startup, so a module-level pydantic import broke the startup import footprint. --- .../discovery/_helpers/filesystem/project_modules.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index 9bfc5a5b17..7813ad8e63 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -10,10 +10,7 @@ from importlib.machinery import ModuleSpec from pathlib import Path from types import ModuleType, UnionType -from typing import Annotated, Literal, Union, get_args, get_origin, get_type_hints - -from pydantic import AliasChoices, AliasPath -from pydantic.fields import FieldInfo +from typing import TYPE_CHECKING, Annotated, Literal, Union, get_args, get_origin, get_type_hints from sqlbuild._native import ( PYTHON_INIT_MODULE_STEM, @@ -30,6 +27,9 @@ from sqlbuild.runtime.user_code.main.register_user_code_root import register_user_code_root from sqlbuild.runtime.user_code.main.user_code_spec import user_code_spec +if TYPE_CHECKING: + from pydantic.fields import FieldInfo + _PROJECT_PYTHON_PACKAGE_MARKER: str = "__sqlbuild_project_python__" @@ -449,6 +449,8 @@ def provider_settings(*, provider_class: type) -> dict[str, object]: def _environment_alias(info: FieldInfo) -> str | None: """The first environment variable an alias names: an `AliasChoices` choice or path head.""" + from pydantic import AliasChoices, AliasPath + alias: object = info.validation_alias or info.alias if isinstance(alias, AliasChoices | AliasPath): first: object = next(iter(alias.convert_to_aliases()), None)