Skip to content

Commit 97dd18a

Browse files
committed
feat: record where each Recall run came from and expose its integration job and run
Invocations record origin (integration, ci or local) and the mode of sqb debug, and the invocations view exposes the integration, job, run and retry from external_context, NULL for older non-JSON rows.
1 parent 8c020ab commit 97dd18a

26 files changed

Lines changed: 639 additions & 49 deletions

File tree

‎crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,3 +114,7 @@ pub(crate) const RECORD_PARAMETER: &str = "record";
114114
/// How Recall timestamps mark UTC, and the offset `datetime.fromisoformat` reads.
115115
pub(crate) const UTC_SUFFIX: &str = "Z";
116116
pub(crate) const UTC_OFFSET: &str = "+00:00";
117+
/// The command whose flags name its Recall `command_mode`.
118+
pub(crate) const DEBUG_COMMAND: &str = "debug";
119+
/// The namespace attribute of `sqb debug --no-connection`.
120+
pub(crate) const NO_CONNECTION_ATTRIBUTE: &str = "no_connection";

‎crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,9 @@ pub(crate) fn render_result(
161161
fn relation_note(name: &str) -> &'static str {
162162
match name {
163163
"invocations" => {
164-
"one row per invocation; status is completed, failed, running, abandoned or unknown"
164+
"one row per invocation; status is completed, failed, running, abandoned or unknown; \
165+
origin is integration, ci or local; integration, integration_job, \
166+
integration_run_id and integration_retry come from external_context"
165167
}
166168
"statements" => "one row per statement; outcome no_terminal means no terminal fact arrived",
167169
"audit_results" => "one row per evaluated audit",

‎crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ use sqlbuild_cli::dispatch::main::closing_status::closing_status;
1111
use sqlbuild_cli::dispatch::models::CommandFacts;
1212
use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV;
1313
use sqlbuild_host::entries::models::ExecutorHost;
14+
use sqlbuild_recall::records::main::command_mode::command_mode;
1415
use sqlbuild_recall::records::main::format_timestamp::format_timestamp;
16+
use sqlbuild_recall::records::main::invocation_origin::invocation_origin;
1517
use sqlbuild_recall::records::main::sha256_hex::sha256_hex;
1618
use sqlbuild_recall::records::models::{InvocationContext, OutputStream, RowEnvelope};
1719
use sqlbuild_recall::schema::main::create_database_statement::create_database_statement;
@@ -39,9 +41,10 @@ use crate::bindings::_helpers::recall::clickhouse_store::{
3941
};
4042
use crate::bindings::_helpers::recall::console::write_stderr;
4143
use crate::bindings::_helpers::recall::constants::{
42-
CLOSE_POLL_MS, DDL_NOTICE_PREFIX, GIT_DIRTY_ENVIRONMENT, GIT_REVISION_ENVIRONMENT,
43-
HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS,
44-
QUARANTINE_START_WAIT_MS, RECALL_HISTORY, STDOUT_STREAM_INDEX,
44+
CLOSE_POLL_MS, DDL_NOTICE_PREFIX, DEBUG_COMMAND, GIT_DIRTY_ENVIRONMENT,
45+
GIT_REVISION_ENVIRONMENT, HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, MINIMUM_HEARTBEAT_SECONDS,
46+
NO_CONNECTION_ATTRIBUTE, NOTICE_INTERVAL_SECONDS, QUARANTINE_START_WAIT_MS, RECALL_HISTORY,
47+
STDOUT_STREAM_INDEX,
4548
};
4649
use crate::bindings::_helpers::recall::events::escaped_text;
4750
use crate::bindings::_helpers::recall::replacements::{RecallReplacements, in_replacement_sink};
@@ -71,8 +74,12 @@ fn text_value(value: &Bound<'_, PyAny>) -> PyResult<Option<String>> {
7174
escaped_text(&value.str()?).map(Some)
7275
}
7376

77+
fn attribute<'py>(args: &Bound<'py, PyAny>, name: &str) -> PyResult<Bound<'py, PyAny>> {
78+
args.getattr(name)
79+
}
80+
7481
fn list_attribute(args: &Bound<'_, PyAny>, name: &str) -> PyResult<Vec<String>> {
75-
let value: Bound<'_, PyAny> = args.getattr(name)?;
82+
let value: Bound<'_, PyAny> = attribute(args, name)?;
7683
let mut items: Vec<String> = Vec::new();
7784
if value.is_none() {
7885
return Ok(items);
@@ -231,7 +238,12 @@ fn invocation_facts(
231238
settings.destination_database.as_deref().unwrap_or_default(),
232239
settings.destination_schema.as_deref().unwrap_or_default()
233240
);
241+
let external_context: Option<String> = external_context(py)?;
242+
let no_connection: bool = facts.command.as_deref() == Some(DEBUG_COMMAND)
243+
&& attribute(args, NO_CONNECTION_ATTRIBUTE)?.is_truthy()?;
234244
Ok(InvocationContext {
245+
origin: Some(invocation_origin(external_context.is_some(), environment).to_owned()),
246+
command_mode: command_mode(facts.command.as_deref(), no_connection).map(str::to_owned),
235247
command: facts.command.clone(),
236248
subcommand: facts.subcommand.clone(),
237249
selection: list_attribute(args, "select")?,
@@ -248,7 +260,7 @@ fn invocation_facts(
248260
git_dirty: recorded_git_dirty(),
249261
host_token: host_token(),
250262
pid: Some(std::process::id()),
251-
external_context: external_context(py)?,
263+
external_context,
252264
recall_filter_active: false,
253265
argv_digest: None,
254266
})

‎crates/sqlbuild-recall/src/records/constants.rs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,33 @@ pub(crate) const MARCH_BASED_JANUARY_INDEX: i64 = 10;
4545
pub const DROPPED_BEFORE_COLUMN: &str = "dropped_before";
4646
/// Initial capacity of the buffer a row's content hash is computed over.
4747
pub(crate) const CONTENT_HASH_BUFFER_BYTES: usize = 1_024;
48+
/// The origin of an invocation an integration such as Dagster started.
49+
pub const INTEGRATION_ORIGIN: &str = "integration";
50+
/// The origin of an invocation a CI system ran.
51+
pub const CI_ORIGIN: &str = "ci";
52+
/// The origin of every other invocation.
53+
pub const LOCAL_ORIGIN: &str = "local";
54+
/// Environment variables a CI system sets; any non-empty value other than `false` or `0` marks a CI run.
55+
pub const CI_ENVIRONMENT_VARIABLES: [&str; 10] = [
56+
"CI",
57+
"GITHUB_ACTIONS",
58+
"GITLAB_CI",
59+
"BUILDKITE",
60+
"JENKINS_URL",
61+
"CIRCLECI",
62+
"TF_BUILD",
63+
"TEAMCITY_VERSION",
64+
"BITBUCKET_BUILD_NUMBER",
65+
"CODEBUILD_BUILD_ID",
66+
];
67+
/// Values of a CI variable that do not mark a CI run, besides an empty one.
68+
pub const CI_DISABLED_VALUES: [&str; 2] = ["false", "0"];
69+
/// The `command_mode` of `sqb debug --no-connection`: configuration checks only.
70+
pub const CONFIG_ONLY_MODE: &str = "config_only";
71+
/// The `command_mode` of `sqb debug`: configuration checks and a connection test.
72+
pub const CONFIG_AND_CONNECTION_MODE: &str = "config_and_connection";
73+
/// The command whose mode Recall records.
74+
pub(crate) const DEBUG_COMMAND: &str = "debug";
4875
/// The output column holding the bytes a truncation gap replaces.
4976
pub const OMITTED_BYTES_COLUMN: &str = "omitted_bytes";
5077
/// The output column holding the sequences a truncation gap accounts for.
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
//! `command_mode`: what a diagnostic command checked, named from its flags.
2+
3+
use crate::records::constants::{CONFIG_AND_CONNECTION_MODE, CONFIG_ONLY_MODE, DEBUG_COMMAND};
4+
5+
/// The mode of `sqb debug`, from whether `--no-connection` was given; `None` for other commands.
6+
#[must_use]
7+
pub fn command_mode(command: Option<&str>, no_connection: bool) -> Option<&'static str> {
8+
(command == Some(DEBUG_COMMAND)).then_some(if no_connection {
9+
CONFIG_ONLY_MODE
10+
} else {
11+
CONFIG_AND_CONNECTION_MODE
12+
})
13+
}
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
//! `invocation_origin`: whether an integration, a CI system or a person ran an invocation.
2+
3+
use crate::records::constants::{
4+
CI_DISABLED_VALUES, CI_ENVIRONMENT_VARIABLES, CI_ORIGIN, INTEGRATION_ORIGIN, LOCAL_ORIGIN,
5+
};
6+
7+
/// `integration` when an integration supplied external context, `ci` when `environment` names a CI system, else `local`.
8+
#[must_use]
9+
pub fn invocation_origin(
10+
has_external_context: bool,
11+
environment: impl Fn(&str) -> Option<String>,
12+
) -> &'static str {
13+
if has_external_context {
14+
return INTEGRATION_ORIGIN;
15+
}
16+
let set = |name: &str| {
17+
environment(name).is_some_and(|value| {
18+
let value: String = value.trim().to_ascii_lowercase();
19+
!value.is_empty() && !CI_DISABLED_VALUES.contains(&value.as_str())
20+
})
21+
};
22+
if CI_ENVIRONMENT_VARIABLES.iter().any(|name| set(name)) {
23+
CI_ORIGIN
24+
} else {
25+
LOCAL_ORIGIN
26+
}
27+
}

‎crates/sqlbuild-recall/src/records/main/invocation_row.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,11 @@ pub fn invocation_row(envelope: &RowEnvelope, context: &InvocationContext) -> Re
6969
boolean(context.recall_filter_active),
7070
),
7171
("argv_digest", optional_text(context.argv_digest.as_deref())),
72+
("origin", optional_text(context.origin.as_deref())),
73+
(
74+
"command_mode",
75+
optional_text(context.command_mode.as_deref()),
76+
),
7277
],
7378
)
7479
}

‎crates/sqlbuild-recall/src/records/main/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
pub mod artifact_rows;
2+
pub mod command_mode;
23
pub mod conflict_row;
34
pub mod delivery_row;
45
pub mod diagnostic_input_from_json;
@@ -8,6 +9,7 @@ pub mod event_object_row;
89
pub mod event_severity;
910
pub mod format_timestamp;
1011
pub mod heartbeat_row;
12+
pub mod invocation_origin;
1113
pub mod invocation_row;
1214
pub mod meta_row;
1315
pub mod output_row;

‎crates/sqlbuild-recall/src/records/models.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,10 @@ pub struct InvocationContext {
121121
pub external_context: Option<String>,
122122
pub recall_filter_active: bool,
123123
pub argv_digest: Option<String>,
124+
/// `integration`, `ci` or `local`: where the invocation came from.
125+
pub origin: Option<String>,
126+
/// What a diagnostic command checked, from its flags; never its raw arguments.
127+
pub command_mode: Option<String>,
124128
}
125129

126130
/// The correlation identifiers of a fact.

‎crates/sqlbuild-recall/src/records/tests/test_records.rs‎

Lines changed: 88 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,18 @@ use std::time::{Duration, UNIX_EPOCH};
33
use crate::records::_helpers::ids::content_hash;
44
use crate::records::_helpers::ids::diagnostic_record_id;
55
use crate::records::_helpers::ids::output_record_id;
6+
use crate::records::main::command_mode::command_mode;
67
use crate::records::main::diagnostic_input_from_json::diagnostic_input_from_json;
78
use crate::records::main::format_timestamp::format_timestamp;
9+
use crate::records::main::invocation_origin::invocation_origin;
810
use crate::records::main::output_row::output_row;
911
use crate::records::main::parse_timestamp::parse_timestamp;
1012
use crate::records::models::{OutputChunk, OutputStream, RecallValue};
1113
use crate::records::tests::helpers::{envelope, event_row, present, row_columns, text};
1214
use crate::records::tests::test_types::{
13-
ContentHashTestCase, DiagnosticTestCase, ParseTimestampTestCase, RecordIdTestCase,
14-
RejectedInputTestCase, RowColumnsTestCase, TimestampTestCase,
15+
CommandModeTestCase, ContentHashTestCase, DiagnosticTestCase, OriginTestCase,
16+
ParseTimestampTestCase, RecordIdTestCase, RejectedInputTestCase, RowColumnsTestCase,
17+
TimestampTestCase,
1518
};
1619

1720
#[test]
@@ -274,3 +277,86 @@ fn given_incomplete_diagnostic_json_when_parsed_then_it_is_rejected() {
274277
);
275278
}
276279
}
280+
281+
#[test]
282+
fn given_context_and_environment_when_classifying_then_the_origin_is_named() {
283+
let test_cases = [
284+
OriginTestCase {
285+
description: "external context wins over CI",
286+
has_external_context: true,
287+
environment: vec![("CI", "true")],
288+
expected_origin: "integration",
289+
},
290+
OriginTestCase {
291+
description: "a CI system's variable marks CI",
292+
environment: vec![("GITHUB_ACTIONS", "true")],
293+
has_external_context: false,
294+
expected_origin: "ci",
295+
},
296+
OriginTestCase {
297+
description: "a URL value marks CI",
298+
environment: vec![("JENKINS_URL", "https://ci.example.com/")],
299+
has_external_context: false,
300+
expected_origin: "ci",
301+
},
302+
OriginTestCase {
303+
description: "false, 0 and empty values are not CI",
304+
environment: vec![("CI", "false"), ("GITLAB_CI", "0"), ("BUILDKITE", " ")],
305+
has_external_context: false,
306+
expected_origin: "local",
307+
},
308+
OriginTestCase {
309+
description: "no variables is local",
310+
environment: Vec::new(),
311+
has_external_context: false,
312+
expected_origin: "local",
313+
},
314+
];
315+
for test_case in test_cases {
316+
let environment = |name: &str| {
317+
test_case
318+
.environment
319+
.iter()
320+
.find(|(key, _)| *key == name)
321+
.map(|(_, value)| (*value).to_owned())
322+
};
323+
assert_eq!(
324+
invocation_origin(test_case.has_external_context, environment),
325+
test_case.expected_origin,
326+
"{}",
327+
test_case.description
328+
);
329+
}
330+
}
331+
332+
#[test]
333+
fn given_a_command_and_its_flags_when_naming_its_mode_then_only_debug_has_one() {
334+
let test_cases = [
335+
CommandModeTestCase {
336+
description: "debug without a connection checks configuration only",
337+
command: Some("debug"),
338+
no_connection: true,
339+
expected_mode: Some("config_only"),
340+
},
341+
CommandModeTestCase {
342+
description: "debug also tests the connection",
343+
command: Some("debug"),
344+
no_connection: false,
345+
expected_mode: Some("config_and_connection"),
346+
},
347+
CommandModeTestCase {
348+
description: "other commands record no mode",
349+
command: Some("build"),
350+
no_connection: false,
351+
expected_mode: None,
352+
},
353+
];
354+
for test_case in test_cases {
355+
assert_eq!(
356+
command_mode(test_case.command, test_case.no_connection),
357+
test_case.expected_mode,
358+
"{}",
359+
test_case.description
360+
);
361+
}
362+
}

0 commit comments

Comments
 (0)