Skip to content

Commit f9108d1

Browse files
committed
Merge remote-tracking branch 'origin/wip/job-detail' into feat/ui-node-errors-job-detail-redaction
2 parents 3191f27 + f912c79 commit f9108d1

8 files changed

Lines changed: 94 additions & 19 deletions

File tree

‎clients/typescript/src/schema.d.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1524,7 +1524,7 @@ export interface components {
15241524
/** @description A group's member jobs in run order; empty for a job. */
15251525
members: components["schemas"]["JobMember"][];
15261526
selection: string[];
1527-
/** @description Command settings; `vars` shows as `[REDACTED]` and text values pass through secret redaction. */
1527+
/** @description Command settings; `vars` and a `defer_*` value that is not a target name show as `[REDACTED]`, and text values pass through secret redaction. */
15281528
settings: {
15291529
[key: string]: unknown;
15301530
};

‎crates/sqlbuild-orchestration/src/declarations/_helpers/job_detail.rs‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use std::collections::{BTreeMap, BTreeSet};
55
use serde_json::Value;
66
use sqlbuild_core::redaction::main::redact_text::redact_text;
77

8-
use crate::declarations::constants::{VARS_SETTING, WITHHELD_SETTING};
8+
use crate::declarations::constants::{DEFER_SETTING_PREFIX, VARS_SETTING, WITHHELD_SETTING};
99
use crate::declarations::models::{Definitions, GroupDefinition, JobMember};
1010
use crate::snapshots::models::SnapshotNode;
1111

@@ -45,13 +45,13 @@ fn all_placed(waits: &BTreeSet<String>, placed: &BTreeSet<&str>) -> bool {
4545
waits.iter().all(|wait| placed.contains(wait.as_str()))
4646
}
4747

48-
/// `vars` is free-form and can carry credentials, so only its presence shows.
48+
/// `vars` can carry credentials and a deferral can name a storage path, so only their presence shows.
4949
pub(crate) fn shown_settings(settings: &BTreeMap<String, Value>) -> BTreeMap<String, Value> {
5050
settings
5151
.iter()
5252
.map(|(name, value)| {
5353
let shown = match value {
54-
_ if name == VARS_SETTING => Value::String(WITHHELD_SETTING.into()),
54+
_ if withheld(name, value) => Value::String(WITHHELD_SETTING.into()),
5555
Value::String(text) => Value::String(redact_text(text)),
5656
other => other.clone(),
5757
};
@@ -60,6 +60,19 @@ pub(crate) fn shown_settings(settings: &BTreeMap<String, Value>) -> BTreeMap<Str
6060
.collect()
6161
}
6262

63+
fn withheld(name: &str, value: &Value) -> bool {
64+
name == VARS_SETTING
65+
|| (name.starts_with(DEFER_SETTING_PREFIX) && !value.as_str().is_some_and(target_name))
66+
}
67+
68+
/// Target names are TOML bare keys in the project file: letters, digits, `_` and `-`.
69+
fn target_name(text: &str) -> bool {
70+
!text.is_empty()
71+
&& text
72+
.bytes()
73+
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-')
74+
}
75+
6376
/// The groups of the snapshot that run `job_key`.
6477
pub(crate) fn groups_running(definitions: &Definitions, job_key: &str) -> BTreeSet<String> {
6578
definitions

‎crates/sqlbuild-orchestration/src/declarations/constants.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,5 @@
44
pub(crate) const VARS_SETTING: &str = "vars";
55
/// What a withheld setting shows instead of its value.
66
pub(crate) const WITHHELD_SETTING: &str = "[REDACTED]";
7+
/// Settings that name another target; a value that isn't a target name, such as a state path, is withheld.
8+
pub(crate) const DEFER_SETTING_PREFIX: &str = "defer_";

‎crates/sqlbuild-orchestration/src/declarations/models.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ pub struct JobDetailRecord {
117117
pub snapshot_id: uuid::Uuid,
118118
pub selection: BTreeSet<String>,
119119
pub concurrency_groups: BTreeSet<String>,
120-
/// Command settings; `vars` shows as `[REDACTED]` and text values pass through secret redaction.
120+
/// Command settings; `vars` and a `defer_*` value that is not a target name show as `[REDACTED]`, and text values pass through secret redaction.
121121
pub settings: BTreeMap<String, serde_json::Value>,
122122
/// A group's member jobs in run order; empty for a job.
123123
pub members: Vec<JobMember>,

‎crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
//! Initial run and node rows written when a run is queued.
1+
//! Initial run and node rows written when a run is queued; `rewrite_history` records what the request's own frozen inputs rewrite.
22
33
use serde_json::json;
44
use sqlbuild_recall::records::main::sha256_hex::sha256_hex;
55

6+
use crate::runs::_helpers::requests::permissions::rewrites_history;
67
use crate::runs::models::RunRequest;
78
use crate::store::errors::ControlError;
89
use crate::store::main::query::query;
@@ -22,20 +23,21 @@ pub(crate) async fn project_queued_run(
2223
);
2324
(transaction, _) = query(transaction,
2425
"INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, image_id, kind, origin, job_key, parent_run_id, root_run_id, trigger_id, tick_id, rerun_of_run_id, \
25-
requested_by, priority, selection_json, write_set_digest, status, queued_at, node_counts_json, last_event_storage_id, request_json) \
26-
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,'queued',$18,$19,$20,$21) \
26+
requested_by, priority, selection_json, write_set_digest, status, queued_at, node_counts_json, last_event_storage_id, request_json, rewrite_history) \
27+
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,'queued',$18,$19,$20,$21,$22) \
2728
ON CONFLICT (run_id) DO UPDATE SET location_id=excluded.location_id,target=excluded.target,snapshot_id=excluded.snapshot_id, \
2829
image_id=excluded.image_id,kind=excluded.kind,origin=excluded.origin,job_key=excluded.job_key,parent_run_id=excluded.parent_run_id, \
2930
root_run_id=excluded.root_run_id,trigger_id=excluded.trigger_id,tick_id=excluded.tick_id,rerun_of_run_id=excluded.rerun_of_run_id, \
3031
requested_by=excluded.requested_by,priority=excluded.priority,selection_json=excluded.selection_json,write_set_digest=excluded.write_set_digest, \
3132
status=excluded.status,queued_at=excluded.queued_at,node_counts_json=excluded.node_counts_json,last_event_storage_id=excluded.last_event_storage_id, \
32-
request_json=excluded.request_json,rewrite_history=FALSE,status_reason=NULL,launched_at=NULL,started_at=NULL,ended_at=NULL,cancel_requested_at=NULL",
33+
request_json=excluded.request_json,rewrite_history=excluded.rewrite_history,status_reason=NULL,launched_at=NULL,started_at=NULL,ended_at=NULL,cancel_requested_at=NULL",
3334
vec![Parameter::Id(request.run_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Id(request.snapshot_id),
3435
Parameter::OptionalText(request.image_id.clone()), Parameter::Text(request.kind.as_str().into()), Parameter::Text(request.origin.as_str().into()),
3536
Parameter::OptionalText(request.job_key.clone()), Parameter::OptionalId(request.parent_run_id), Parameter::Id(request.root_run_id),
3637
Parameter::OptionalId(request.trigger_id), Parameter::OptionalId(request.tick_id), Parameter::OptionalId(request.rerun_of_run_id),
3738
Parameter::Text(request.requested_by.clone()), Parameter::Integer(request.priority), Parameter::Json(selection), Parameter::Text(digest),
38-
Parameter::Time(now), Parameter::Json(json!({"pending": request.selection.len()})), Parameter::Integer(cursor), Parameter::Json(json!(request))]).await?;
39+
Parameter::Time(now), Parameter::Json(json!({"pending": request.selection.len()})), Parameter::Integer(cursor), Parameter::Json(json!(request)),
40+
Parameter::Boolean(rewrites_history(request, std::slice::from_ref(request)))]).await?;
3941
for node in &request.selection {
4042
(transaction, _) = query(transaction,
4143
"INSERT INTO sqb_run_nodes (run_id, node_key, status, writes) VALUES ($1,$2,'pending',$3)",

‎crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,8 @@ pub(crate) fn execution_scopes(
9191
.collect()
9292
}
9393

94-
fn rewrites_history(request: &RunRequest, executions: &[RunRequest]) -> bool {
94+
/// A window, a full refresh, a cursor setting or an ad-hoc destructive flag on any of `executions`.
95+
pub(in crate::runs) fn rewrites_history(request: &RunRequest, executions: &[RunRequest]) -> bool {
9596
if request.window.is_some() {
9697
return true;
9798
}

‎crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs‎

Lines changed: 55 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
//! One job's declaration reads like the job list, and a full-refresh launch needs run.rewrite_history down to group children.
1+
//! One job's declaration reads like the job list; a full-refresh launch needs run.rewrite_history down to group children and records it.
2+
3+
use std::collections::BTreeMap;
24

35
use serde_json::json;
46
use uuid::Uuid;
@@ -10,10 +12,13 @@ use crate::credentials::tests::role_helpers::{
1012
};
1113
use crate::declarations::main::read_person_job::read_person_job;
1214
use crate::runs::_helpers::requests::job::frozen_group_children;
13-
use crate::runs::main::request_person_run::request_person_run;
14-
use crate::runs::models::{JobRunRequest, PersonRunRequest, RunRequest, RunWindow};
15+
use crate::runs::main::{
16+
advance_groups::advance_groups, list_runs::list_runs, request_person_run::request_person_run,
17+
};
18+
use crate::runs::models::{JobRunRequest, PersonRunRequest, RunRecord, RunRequest, RunWindow};
1519
use crate::runs::tests::helpers::{json_column, rows, snapshot};
1620
use crate::runs::tests::test_types::JobDetailTestCase;
21+
use crate::scheduler::tests::contracts::helpers::acquire_lease;
1722
use crate::snapshots::main::publish_snapshot::publish_snapshot;
1823
use crate::store::errors::ControlError;
1924
use crate::store::main::migrate_store::migrate_store;
@@ -55,7 +60,9 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r
5560
input.documents.insert("jobs".into(), json!({
5661
"daily_orders": {"description": "Build orders", "selection": ["orders"], "settings": {
5762
"vars": "{\"api_key\": \"orders-key-123\"}",
58-
"defer_to": "postgres://orders:orders-db-pass@warehouse.example.test/orders",
63+
"defer_to": "/srv/orders/state",
64+
"defer_sources_to": "production",
65+
"warehouse": "postgres://orders:orders-db-pass@warehouse.example.test/orders",
5966
"concurrency": 4}},
6067
"customers": {"description": "Build customers", "selection": ["customers"]},
6168
}));
@@ -94,13 +101,14 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r
94101
job.members.len()
95102
),
96103
(
97-
json!({"concurrency": 4, "vars": "[REDACTED]",
98-
"defer_to": "postgres://orders:[REDACTED]@warehouse.example.test/orders"}),
104+
json!({"concurrency": 4, "vars": "[REDACTED]", "defer_to": "[REDACTED]",
105+
"defer_sources_to": "production",
106+
"warehouse": "postgres://orders:[REDACTED]@warehouse.example.test/orders"}),
99107
json!(["orders"]),
100108
json!(["orders_group"]),
101109
0
102110
),
103-
"{}: free-form vars and URI passwords are withheld",
111+
"{}: vars and a deferral path are withheld, a target name shows, URI passwords are redacted",
104112
test_case.description
105113
);
106114
let group = read(administrator, "orders_group")
@@ -189,7 +197,8 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r
189197
test_case.description
190198
);
191199
let group_launch = launch("orders_group");
192-
for request in [launch("daily_orders"), group_launch.clone()] {
200+
let full_job = launch("daily_orders");
201+
for request in [full_job.clone(), group_launch.clone()] {
193202
assert!(
194203
request_person_run(store, administrator, PersonRunRequest::Job(&request))
195204
.await
@@ -210,6 +219,44 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r
210219
test_case.description
211220
);
212221
}
222+
let plain = JobRunRequest {
223+
full_refresh: false,
224+
..launch("customers")
225+
};
226+
request_person_run(store, administrator, PersonRunRequest::Job(&plain))
227+
.await
228+
.expect("ordinary launch");
229+
let lease = acquire_lease(store, Uuid::now_v7())
230+
.await
231+
.expect("elect")
232+
.expect("leader");
233+
let scope = (location, "development");
234+
advance_groups(store, &lease, (&BTreeMap::new(), scope))
235+
.await
236+
.expect("start the group");
237+
let runs = list_runs(store, scope, 50).await.expect("runs");
238+
let rewrites = |matches: &dyn Fn(&RunRecord) -> bool| {
239+
runs.iter()
240+
.filter(|run| matches(run))
241+
.map(|run| run.rewrite_history)
242+
.collect::<Vec<_>>()
243+
};
244+
assert_eq!(
245+
(
246+
rewrites(&|run| run.run_id == full_job.run_id),
247+
rewrites(&|run| run.run_id == group_launch.run_id),
248+
rewrites(&|run| run.run_id == plain.run_id),
249+
),
250+
(vec![true], vec![true], vec![false]),
251+
"{}: full refreshes record that they rewrite history, an ordinary run does not",
252+
test_case.description
253+
);
254+
let children = rewrites(&|run| run.parent_run_id == Some(group_launch.run_id));
255+
assert!(
256+
!children.is_empty() && children.iter().all(|flag| *flag),
257+
"{}: started group children record the full refresh: {children:?}",
258+
test_case.description
259+
);
213260
let stored = rows(
214261
store,
215262
"SELECT request_json FROM sqb_runs WHERE run_id=$1",

‎crates/sqlbuild-orchestration/tests/runs/test_reruns.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,16 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_
269269
execution.description
270270
);
271271
let runs = list_runs(store, scope, 20).await.expect("list reruns");
272+
assert!(
273+
runs.iter()
274+
.filter(|run| {
275+
run.run_id == request.run_id
276+
|| run.parent_run_id == Some(request.run_id)
277+
})
278+
.all(|run| run.rewrite_history),
279+
"{}: a backfill's rerun and its children record that they rewrite history",
280+
execution.description
281+
);
272282
assert_eq!(
273283
runs.iter()
274284
.find(|run| run.run_id == request.run_id)

0 commit comments

Comments
 (0)