Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"description": "Four ad-hoc runs of one location target, each with a worker log at its claim and a nested session that shares one SQL text with the others and keeps one of its own. Runs ended 20 days ago, 2 hours ago and 1 hour ago; the fourth is still running. Keeping 14 days and 1 run drops the first two logs whole, keeps the shared SQL, and leaves every run, its nodes and the scheduler's events.",
"keep_days": 14,
"keep_runs": 1,
"ended_hours_ago": [480, 2, 1, null],
"expected_pruned": 2,
"expected_logged_runs": [3, 4]
}
20 changes: 20 additions & 0 deletions clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1726,6 +1726,11 @@ export interface components {
/** @description Digest of the schema; a server starts only on the exact schema it creates. */
schema_checksum: string;
storage: components["schemas"]["StorageKind"];
/**
* Format: int64
* @description The database's size on disk now: SQLite's main file, or the whole PostgreSQL database.
*/
store_bytes: number;
};
/** @description One job or group as declared, for a person who may list the target; free-form setting values are withheld. */
JobDetailRecord: {
Expand Down Expand Up @@ -2330,6 +2335,11 @@ export interface components {
kind: "heartbeat";
/** Format: uuid */
replica_id: string;
/**
* Format: int64
* @description The control store's database size on disk when the replica checked in.
*/
store_bytes: number;
} | {
/** @enum {string} */
kind: "scheduler_no_leader";
Expand Down Expand Up @@ -3041,6 +3051,16 @@ export interface components {
location_health_days: number;
/** Format: int64 */
platform_alert_days: number;
/**
* Format: int64
* @description Days a finished run keeps its log of events, statements and output.
*/
run_log_days: number;
/**
* Format: int64
* @description Finished runs per location target that keep their logs.
*/
run_logs_kept: number;
/** Format: int64 */
sensor_evaluation_days: number;
/**
Expand Down
22 changes: 22 additions & 0 deletions crates/sqlbuild-cli/src/argv/command_table.json
Original file line number Diff line number Diff line change
Expand Up @@ -1124,6 +1124,28 @@
"help": "published snapshots to keep per location and target; older snapshots are deleted unless a run, trigger, job or other retained record still references them (default: 10)",
"group": 1
},
{
"kind": "store",
"options": [
"--run-log-retention-days"
],
"dest": "control_run_log_retention_days",
"default": 14,
"type": "int",
"help": "days a finished run keeps its log of events, statements, SQL and output; the run, its node results and history stay (default: 14)",
"group": 1
},
{
"kind": "store",
"options": [
"--run-log-retention-count"
],
"dest": "control_run_log_retention_count",
"default": 200,
"type": "int",
"help": "finished runs per location and target that keep their logs; older runs lose theirs, whichever limit comes first (default: 200)",
"group": 1
},
{
"kind": "store",
"options": ["--trusted-proxy-config"],
Expand Down
7 changes: 7 additions & 0 deletions crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use sqlbuild_orchestration::api::models::{ApiAuthentication, HumanLogin, ServerF
use sqlbuild_orchestration::credentials::models::{AuditRetention, CredentialScope};
use sqlbuild_orchestration::declarations::models::TriggerChange;
use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest};
use sqlbuild_orchestration::runs::models::RunLogRetention;
use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest, RunWindow};
use sqlbuild_orchestration::snapshots::models::SnapshotRetention;
use sqlbuild_orchestration::store::errors::ControlError;
Expand Down Expand Up @@ -280,6 +281,11 @@ pub(crate) fn control_api_settings(
Some(count) => SnapshotRetention::count(count)?,
None => SnapshotRetention::default(),
};
let defaults = RunLogRetention::default();
let run_logs = RunLogRetention::new(
retention_setting(namespace, "run-log-retention-days")?.unwrap_or(defaults.days()),
retention_setting(namespace, "run-log-retention-count")?.unwrap_or(defaults.count()),
)?;
Ok((
control_authentication(namespace)?,
ServerFacts {
Expand All @@ -288,6 +294,7 @@ pub(crate) fn control_api_settings(
audit,
statistics,
snapshots,
run_logs,
},
))
}
Expand Down
11 changes: 4 additions & 7 deletions crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,29 +21,26 @@ use sqlbuild_config::project::models::ControlConfig;
use sqlbuild_orchestration::api::models::{
ApiAuthentication, DevelopmentScope, HumanLogin, ServerFacts,
};
use sqlbuild_orchestration::credentials::models::AuditRetention;
use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE;
use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health;
use sqlbuild_orchestration::platform::main::run_maintenance::run_maintenance;
use sqlbuild_orchestration::platform::models::{
LocationHealth, LocationHealthReport, LocationWorkClient, LocationWorkPoll,
LocationHealth, LocationHealthReport, LocationWorkClient, LocationWorkPoll, RetentionPolicies,
WarehouseSyncFailure,
};
use sqlbuild_orchestration::runs::types::RunHandoff;
use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler;
use sqlbuild_orchestration::scheduler::models::SchedulerState;
use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation;
use sqlbuild_orchestration::snapshots::models::SnapshotRetention;
use sqlbuild_orchestration::store::errors::ControlError;
use sqlbuild_orchestration::store::main::migrate_store::migrate_store;
use sqlbuild_orchestration::store::main::open_store::open_store;
use sqlbuild_orchestration::store::main::set_release_version::set_release_version;
use sqlbuild_orchestration::store::models::ControlStore;
use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS;
use sqlbuild_orchestration::warehouse_metadata::models::{
MetadataBatch, MetadataScan, StatisticsDueRead, StatisticsRead, StatisticsRetention,
WarehouseHistoryKind, WarehouseHistoryPage, WarehouseStateLocation, WarehouseSyncDueRead,
WarehouseWatermarkRead,
MetadataBatch, MetadataScan, StatisticsDueRead, StatisticsRead, WarehouseHistoryKind,
WarehouseHistoryPage, WarehouseStateLocation, WarehouseSyncDueRead, WarehouseWatermarkRead,
};
use sqlbuild_orchestration::webhooks::constants::SECURITY_AUDIT_WEBHOOK_VARIABLE;
use sqlbuild_orchestration::webhooks::main::{
Expand Down Expand Up @@ -891,7 +888,7 @@ pub(crate) fn location_child(options: &LocationServe, script: &Path) -> Command
pub(crate) fn start_platform_roles(
tasks: &mut JoinSet<Result<(), ControlError>>,
(store, shutdown): (&ControlStore, &watch::Receiver<bool>),
retention: (AuditRetention, StatisticsRetention, SnapshotRetention),
retention: RetentionPolicies,
) -> Result<(), ControlError> {
let mut sinks: Vec<(Webhook, &str)> = Vec::new();
for (variable, source, label) in [
Expand Down
8 changes: 7 additions & 1 deletion crates/sqlbuild-cli/src/native_commands/main/_run_control.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
//! Supervise native API/scheduler roles together; a failed role stops its siblings.

use sqlbuild_orchestration::api::main::serve_api::serve_api;
use sqlbuild_orchestration::platform::models::RetentionPolicies;
use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler;
use sqlbuild_orchestration::scheduler::models::SchedulerState;
use sqlbuild_orchestration::store::errors::ControlError;
Expand Down Expand Up @@ -56,7 +57,12 @@ pub(crate) async fn run_control(
start_platform_roles(
&mut tasks,
(&store, &shutdown),
(facts.audit, facts.statistics, facts.snapshots),
RetentionPolicies {
audit: facts.audit,
statistics: facts.statistics,
snapshots: facts.snapshots,
run_logs: facts.run_logs,
},
)?;
}
if facts.scheduler {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,8 @@ pub(crate) async fn server(
audit_days: i64::from(facts.audit.kept_days()),
statistics_days: i64::from(facts.statistics.kept_days()),
snapshots_kept: i64::from(facts.snapshots.kept_count()),
run_log_days: i64::from(facts.run_logs.days()),
run_logs_kept: i64::from(facts.run_logs.count()),
platform_alert_days: PLATFORM_ALERT_RETENTION_DAYS,
heartbeat_days: HEARTBEAT_RETENTION_DAYS,
location_health_days: LOCATION_RETENTION_DAYS,
Expand Down
5 changes: 5 additions & 0 deletions crates/sqlbuild-orchestration/src/api/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,7 @@ pub struct ServerFacts {
pub audit: crate::credentials::models::AuditRetention,
pub statistics: crate::warehouse_metadata::models::StatisticsRetention,
pub snapshots: crate::snapshots::models::SnapshotRetention,
pub run_logs: crate::runs::models::RunLogRetention,
}

/// The server's version, storage, sign-in and retention, for administrators.
Expand All @@ -541,6 +542,10 @@ pub struct ServerRetention {
pub statistics_days: i64,
/// Publications kept per location target.
pub snapshots_kept: i64,
/// Days a finished run keeps its log of events, statements and output.
pub run_log_days: i64,
/// Finished runs per location target that keep their logs.
pub run_logs_kept: i64,
pub platform_alert_days: i64,
pub heartbeat_days: i64,
pub location_health_days: i64,
Expand Down
133 changes: 127 additions & 6 deletions crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs
Original file line number Diff line number Diff line change
@@ -1,21 +1,25 @@
//! Every retention policy runs in one bounded pass; each part is safe to repeat.

use std::collections::BTreeSet;

use chrono::{DateTime, TimeDelta, Utc};
use sqlbuild_recall::schema::constants::INVOCATION_TABLES;
use sqlbuild_recall::schema::main::table_spec::table_spec;

use crate::credentials::main::audit::prune_audit::prune_audit;
use crate::credentials::models::AuditRetention;
use crate::platform::constants::{
HEARTBEAT_PAGE, HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS,
PLATFORM_ALERT_RETENTION_DAYS, SENSOR_EVALUATION_RETENTION_DAYS,
};
use crate::platform::models::RetentionPolicies;
use crate::runs::constants::{RUN_LOG_DIGEST_CHUNK, RUN_LOG_RETENTION_BATCH};
use crate::runs::models::RunLogRetention;
use crate::snapshots::main::prune_snapshots::prune_snapshots;
use crate::snapshots::models::SnapshotRetention;
use crate::store::errors::ControlError;
use crate::store::main::{query::query, write::write};
use crate::store::models::{ControlStore, Parameter};
use crate::store::main::{begin::begin, commit::commit, query::query, write::write};
use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode};
use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS;
use crate::warehouse_metadata::main::ingest::prune_statistics::prune_statistics;
use crate::warehouse_metadata::models::StatisticsRetention;

/// Old evaluation phases go once a later attempt of the same sensor exists.
const SENSOR_EVALUATIONS: &str = "DELETE FROM sqb_sensor_evaluation_events WHERE event_id IN (SELECT e.event_id \
Expand All @@ -37,12 +41,35 @@ const LOCATION_INSTANCES: &str = "DELETE FROM sqb_location_instances WHERE insta
FROM sqb_location_work w WHERE w.session_id=i.instance_id) \
ORDER BY i.last_seen_at LIMIT $2) RETURNING instance_id";

/// Claims of finished runs past either limit whose worker log is still stored, oldest first. Worker invocations are `worker/<claim>` and `worker/<claim>/<session>`; the stored logs, not every run ever finished, bound the join. A finished run no longer names its worker, so the scheduler's `run_claimed` event, whose ID is the claim, links the two.
const EXPIRED_CLAIMS: &str = "WITH logged AS (\
SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_invocations \
WHERE invocation_id > 'worker/' AND invocation_id < 'worker0' \
AND (length(invocation_id) = 43 OR substr(invocation_id, 44, 1) = '/') \
UNION SELECT DISTINCT substr(invocation_id, 8, 36) AS claim FROM recall_output \
WHERE invocation_id > 'worker/' AND invocation_id < 'worker0' \
AND (length(invocation_id) = 43 OR substr(invocation_id, 44, 1) = '/')), \
claimed AS (SELECT DISTINCT event_id AS claim, run_id FROM recall_events \
WHERE event_type = 'run_claimed' AND producer = 'scheduler'), \
ranked AS (SELECT CAST(run_id AS TEXT) AS run_id, ended_at, ROW_NUMBER() OVER (PARTITION BY location_id, target \
ORDER BY ended_at DESC, run_id DESC) AS newest FROM sqb_runs \
WHERE ended_at IS NOT NULL AND status IN ('success', 'warn', 'partial', 'failed', 'cancelled')) \
SELECT logged.claim FROM ranked JOIN claimed ON claimed.run_id = ranked.run_id \
JOIN logged ON logged.claim = claimed.claim \
WHERE ranked.newest > $1 OR ranked.ended_at < $2 ORDER BY ranked.ended_at LIMIT $3";

/// Returns how many audit rows outlive their horizon because a sink has not acknowledged them.
pub(crate) async fn retain(
store: &ControlStore,
(audit, statistics, snapshots): (AuditRetention, StatisticsRetention, SnapshotRetention),
RetentionPolicies {
audit,
statistics,
snapshots,
run_logs,
}: RetentionPolicies,
sinks: &[String],
) -> Result<i64, ControlError> {
prune_run_logs(store, run_logs).await?;
prune_snapshots(store, snapshots).await?;
while prune_statistics(store, statistics).await? == MAX_HISTORY_RECORDS {
tokio::task::yield_now().await;
Expand Down Expand Up @@ -91,3 +118,97 @@ async fn drain(
tokio::task::yield_now().await;
}
}

/// A finished run's worker log goes whole once it is too old or too many newer runs finished; the run, its nodes and the scheduler's node history stay. Batches commit independently; a repeated or interrupted pass finds only logs still stored.
pub(crate) async fn prune_run_logs(
store: &ControlStore,
retention: RunLogRetention,
) -> Result<usize, ControlError> {
let mut deleted = 0;
loop {
let transaction = begin(store, TransactionMode::Write).await?;
let now = transaction.now;
let (transaction, rows) = query(
transaction,
EXPIRED_CLAIMS,
vec![
Parameter::Integer(i64::from(retention.count())),
Parameter::Time(retention.cutoff(now)),
Parameter::Integer(RUN_LOG_RETENTION_BATCH as i64),
],
)
.await?;
let claims: Vec<String> = rows
.iter()
.filter_map(|row| row["claim"].as_str().map(str::to_owned))
.collect();
let transaction = delete_logs(transaction, &claims).await?;
commit(transaction).await?;
deleted += claims.len();
if claims.len() < RUN_LOG_RETENTION_BATCH {
return Ok(deleted);
}
}
}

/// Every Recall row of the claims' invocations, then stored artifacts no remaining reference names. `worker/<claim>/<session>` sorts between `worker/<claim>/` and `worker/<claim>0` in byte order, which SQLite uses and PostgreSQL control stores declare for `invocation_id`, so each range reads the index.
async fn delete_logs(
mut transaction: ControlTransaction,
claims: &[String],
) -> Result<ControlTransaction, ControlError> {
let parameters: Vec<Parameter> = claims
.iter()
.flat_map(|claim| {
let namespace = format!("worker/{claim}");
[format!("{namespace}/"), format!("{namespace}0"), namespace]
})
.map(Parameter::Text)
.collect();
let ranges: Vec<String> = (0..claims.len())
.map(|index| {
let first = 3 * index + 1;
format!(
"invocation_id=${} OR (invocation_id>${first} AND invocation_id<${})",
first + 2,
first + 1
)
})
.collect();
if ranges.is_empty() {
return Ok(transaction);
}
let owned = format!("({})", ranges.join(" OR "));
let rows;
(transaction, rows) = query(
transaction,
&format!("SELECT DISTINCT digest FROM recall_artifact_refs WHERE {owned}"),
parameters.clone(),
)
.await?;
let digests: BTreeSet<String> = rows
.iter()
.filter_map(|row| row["digest"].as_str().map(str::to_owned))
.collect();
for table in INVOCATION_TABLES {
(transaction, _) = query(
transaction,
&format!("DELETE FROM {} WHERE {owned}", table_spec(table).name),
parameters.clone(),
)
.await?;
}
let digests: Vec<String> = digests.into_iter().collect();
for chunk in digests.chunks(RUN_LOG_DIGEST_CHUNK) {
(transaction, _) = query(
transaction,
&format!(
"DELETE FROM recall_artifacts WHERE digest IN ({}) AND NOT EXISTS \
(SELECT 1 FROM recall_artifact_refs AS ref WHERE ref.digest = recall_artifacts.digest)",
Parameter::placeholders(1..=chunk.len())
),
chunk.iter().cloned().map(Parameter::Text).collect(),
)
.await?;
}
Ok(transaction)
}
Loading
Loading