diff --git a/cli/Cargo.lock b/cli/Cargo.lock index c364cdce0..ed2f23e9f 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -3823,6 +3823,7 @@ dependencies = [ "inquire", "jsonschema", "keyring-core", + "libc", "murmur3", "opentelemetry", "opentelemetry-http", @@ -3832,6 +3833,7 @@ dependencies = [ "quint-connect", "rand 0.8.6", "reqwest", + "security-framework", "serde", "serde_json", "sha2 0.11.0", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index a842db904..6e5ce785d 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -58,11 +58,15 @@ opentelemetry = { version = "=0.33.1", default-features = false, features = ["tr async-trait = "0.1" opentelemetry_sdk = { version = "=0.33.1", default-features = false, features = ["trace", "logs"] } +[target.'cfg(unix)'.dependencies] +libc = "0.2.186" + [target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies] zbus-secret-service-keyring-store = { version = "1.0.0", features = ["rt-tokio-crypto-rust"] } [target.'cfg(target_os = "macos")'.dependencies] apple-native-keyring-store = { version = "1", features = ["keychain"] } +security-framework = "3" [target.'cfg(target_os = "windows")'.dependencies] windows-native-keyring-store = "1" diff --git a/cli/src/app.rs b/cli/src/app.rs index e7ec4d71c..2a863fc0d 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -6,6 +6,7 @@ use std::time::Duration; use crate::services; use services::app_support::{self, RunOutcome}; use services::error::CliError; +use services::managed_telemetry::trace_export::ManagedSpanKind; use services::managed_telemetry::RuntimeTelemetry; use services::observability::traits::{ Logger as LoggerTrait, NoopTelemetry, Telemetry as TelemetryTrait, @@ -15,6 +16,9 @@ const MIN_WORKER_THREADS: usize = 2; const RUNTIME_SHUTDOWN_GRACE: Duration = Duration::from_millis(500); const REPEATED_COMMAND_DISPATCH_ERROR: &str = "Command lifecycle telemetry attempted to execute command dispatch more than once"; +const CLAUDE_ONLY_HOOK_SUBCOMMANDS: [&str; 2] = ["claude-mutation-scope", "claude-model-state"]; +const SHARED_CLAUDE_HOOK_SUBCOMMANDS: [&str; 2] = ["diff-trace", "conversation-trace"]; +const CLAUDE_HOOK_CALLER_MARKERS: [&str; 2] = ["CLAUDECODE", "CLAUDE_PROJECT_DIR"]; struct StartupContext { observability_config: services::config::ResolvedObservabilityRuntimeConfig, @@ -275,7 +279,7 @@ where StderrW: Write, { let args = args.into_iter().collect::>(); - let managed_allowed = !is_hook_invocation(&args); + let span_kind = managed_span_kind(&args, is_claude_hook_caller()); let startup = perform_dependency_check(dependency_check) .and_then(|()| build_startup_context()) .and_then(initialize_runtime); @@ -293,24 +297,36 @@ where ); } }; - runtime.telemetry = RuntimeTelemetry::resolve(managed_allowed).await; + runtime.telemetry = RuntimeTelemetry::resolve(span_kind, &runtime.logger).await; let rendered = run_command_lifecycle(args, &runtime, stdout, stderr).await; runtime.telemetry.shutdown().await; app_support::exit_code_for(&rendered) } -fn is_hook_invocation(args: &[String]) -> bool { +fn managed_span_kind(args: &[String], claude_hook_caller: bool) -> Option { let mut words = args .iter() .skip(1) .map(String::as_str) .filter(|arg| !arg.starts_with('-')); - match (words.next(), words.next()) { - (Some(first), _) if first == services::hooks::NAME => true, - (Some("policy"), Some("bash")) => true, - (Some(first), Some("telemetry-headers")) if first == services::auth_command::NAME => true, - _ => false, - } + let claude_hook = match (words.next(), words.next()) { + (Some(first), Some(subcommand)) if first == services::hooks::NAME => { + CLAUDE_ONLY_HOOK_SUBCOMMANDS.contains(&subcommand) + || (claude_hook_caller && SHARED_CLAUDE_HOOK_SUBCOMMANDS.contains(&subcommand)) + } + (Some(first), _) if first == services::hooks::NAME => false, + (Some(first), _) if first == services::setup::NAME => false, + (Some(first), Some("bash")) if first == services::bash_policy::NAME => claude_hook_caller, + (Some(first), Some("telemetry-headers")) if first == services::auth_command::NAME => false, + _ => return Some(ManagedSpanKind::Command), + }; + claude_hook.then_some(ManagedSpanKind::ClaudeHook) +} + +fn is_claude_hook_caller() -> bool { + CLAUDE_HOOK_CALLER_MARKERS + .iter() + .all(|marker| std::env::var_os(marker).is_some()) } fn perform_dependency_check anyhow::Result<()>>( @@ -924,10 +940,24 @@ pub(crate) mod tests { } let runtime = build_runtime().unwrap(); runtime.block_on(async { - let keychain = RuntimeTelemetry::resolve_with(stalled_keychain_lookup()).await; + let (keychain, fallback) = + RuntimeTelemetry::resolve_with(stalled_keychain_lookup(), ManagedSpanKind::Command) + .await; assert!(matches!(keychain, RuntimeTelemetry::Noop(_))); - let database = RuntimeTelemetry::resolve_with(stalled_blocking_database_lookup()).await; + assert_eq!( + fallback.map(|fallback| fallback.reason), + Some(services::managed_telemetry::TelemetryFallbackReason::LookupTimeout) + ); + let (database, fallback) = RuntimeTelemetry::resolve_with( + stalled_blocking_database_lookup(), + ManagedSpanKind::ClaudeHook, + ) + .await; assert!(matches!(database, RuntimeTelemetry::Noop(_))); + assert_eq!( + fallback.map(|fallback| fallback.reason), + Some(services::managed_telemetry::TelemetryFallbackReason::LookupTimeout) + ); }); finish_runtime(runtime); } @@ -951,6 +981,7 @@ pub(crate) mod tests { let result = runtime.block_on(services::auth_command::telemetry_headers_with( Some(repo.path().to_path_buf()), false, + true, |_: &std::path::Path| Some("repository-a".to_string()), |_: String| stalled_keychain_lookup(), )); @@ -1073,24 +1104,45 @@ pub(crate) mod tests { } #[test] - fn hook_invocations_never_resolve_managed_telemetry() { - let hook = ["sce", "hooks", "pre-commit"].map(String::from); - let flagged_hook = ["sce", "--verbose", "hooks", "pre-commit"].map(String::from); - let other = ["sce", "config", "show"].map(String::from); - let policy = ["sce", "policy", "bash"].map(String::from); - let headers = ["sce", "auth", "telemetry-headers"].map(String::from); - let auth_login = ["sce", "auth", "login"].map(String::from); - assert!(is_hook_invocation(&policy)); - assert!(is_hook_invocation(&headers)); - assert!(!is_hook_invocation(&auth_login)); - assert!(is_hook_invocation(&hook)); - assert!(is_hook_invocation(&flagged_hook)); - assert!(!is_hook_invocation(&other)); + fn only_claude_hook_invocations_are_admitted_to_managed_hook_telemetry() { + const HOOK: Option = Some(ManagedSpanKind::ClaudeHook); + const COMMAND: Option = Some(ManagedSpanKind::Command); + let cases: [(&[&str], bool, Option); 20] = [ + (&["sce", "hooks", "claude-mutation-scope"], false, HOOK), + (&["sce", "hooks", "claude-model-state"], false, HOOK), + (&["sce", "hooks", "diff-trace"], true, HOOK), + (&["sce", "hooks", "diff-trace"], false, None), + (&["sce", "hooks", "conversation-trace"], true, HOOK), + (&["sce", "hooks", "conversation-trace"], false, None), + (&["sce", "policy", "bash"], true, HOOK), + (&["sce", "policy", "bash"], false, None), + (&["sce", "hooks", "pre-commit"], true, None), + (&["sce", "--verbose", "hooks", "codex"], true, None), + (&["sce", "hooks", "codex-mutation-scope"], true, None), + (&["sce", "hooks", "opencode-mutation-scope"], true, None), + (&["sce", "hooks", "pi-mutation-scope"], true, None), + (&["sce", "hooks"], true, None), + (&["sce", "auth", "telemetry-headers"], true, None), + (&["sce", "setup"], true, None), + (&["sce", "setup", "--claude"], true, None), + (&["sce", "auth", "login"], true, COMMAND), + (&["sce", "config", "show"], false, COMMAND), + (&["sce", "doctor"], true, COMMAND), + ]; + for (args, claude_caller, expected) in cases { + let args = args.iter().map(ToString::to_string).collect::>(); + assert_eq!( + managed_span_kind(&args, claude_caller), + expected, + "{args:?} claude_caller={claude_caller}" + ); + } } #[tokio::test] async fn disallowed_managed_telemetry_resolves_to_noop_without_credential_lookup() { - let telemetry = RuntimeTelemetry::resolve(false).await; + let telemetry = + RuntimeTelemetry::resolve(None, &services::observability::traits::NoopLogger).await; assert!(matches!(telemetry, RuntimeTelemetry::Noop(_))); } } diff --git a/cli/src/app/doctor_reconciliation_cli_tests.rs b/cli/src/app/doctor_reconciliation_cli_tests.rs index ed1dfff00..bc798c2a0 100644 --- a/cli/src/app/doctor_reconciliation_cli_tests.rs +++ b/cli/src/app/doctor_reconciliation_cli_tests.rs @@ -5,6 +5,7 @@ use serde_json::Value; use super::run_with_dependency_check_and_streams; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::db::encryption_key::AUTH_DB_ENCRYPTION_KEY_ENV; use crate::services::default_paths::agent_trace_db_path_for_repository_at; use crate::services::repository_identity::resolve::resolve_repository_identity; @@ -15,6 +16,10 @@ const STATE_FILE: &str = "ref-maintenance.json"; const LOCK_FILE: &str = "mutation-cursor.lock"; const RECONCILIATION_DETAIL_PREFIX: &str = "Snapshot ref reconciliation"; +const AUTH_DB_TEST_SECRET: &str = "doctor-cli-test-secret"; +const UNREADABLE_AUTH_DB: &[u8] = b"not an encrypted auth database"; +const FAR_FUTURE_READINESS: &[u8] = br#"{"expires_at_unix_seconds":4102444800}"#; + const EXIT_SUCCESS: &str = "success"; const EXIT_FAILURE: &str = "failure"; @@ -138,6 +143,10 @@ impl Sandbox { } fn run(&self, args: &[&str]) -> CliRun { + self.run_with_env(args, &[]) + } + + fn run_with_env(&self, args: &[&str], envs: &[(&str, &str)]) -> CliRun { let out = self.dir.path().join(format!("out-{}", args.join("-"))); std::fs::create_dir_all(&out).expect("out dir"); let output = Command::new(std::env::current_exe().expect("current exe")) @@ -151,6 +160,8 @@ impl Sandbox { .env("XDG_CACHE_HOME", self.dir.path().join("cache")) .env("NO_COLOR", "1") .env_remove("SCE_CONFIG_FILE") + .env_remove(AUTH_DB_ENCRYPTION_KEY_ENV) + .envs(envs.iter().copied()) .output() .expect("child runs"); assert!( @@ -594,3 +605,88 @@ async fn doctor_fix_recovers_from_every_initial_maintenance_state_condition() { ); } } + +fn global_state_warnings(report: &Value) -> Vec { + report["problems"] + .as_array() + .expect("problems array") + .iter() + .filter(|problem| problem["category"] == "global_state" && problem["severity"] == "warning") + .cloned() + .collect() +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_on_fresh_state_reports_unconfigured_telemetry_without_a_warning() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + let auth_dir = sandbox.state_root().join("sce"); + + let report = sandbox + .run_with_env( + &["doctor", "--format", "json"], + &[(AUTH_DB_ENCRYPTION_KEY_ENV, "")], + ) + .json(); + + assert_eq!(report["managed_telemetry"]["state"], "not_configured"); + let warnings = global_state_warnings(&report); + assert!(warnings.is_empty(), "{warnings:#?}"); + assert!(!auth_dir.join("auth.db").exists()); + assert!(!auth_dir.join("telemetry-ready.json").exists()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_reports_unreadable_telemetry_credentials_without_auth_side_effects() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + let auth_dir = sandbox.state_root().join("sce"); + std::fs::create_dir_all(&auth_dir).expect("auth dir"); + let auth_db = auth_dir.join("auth.db"); + let readiness = auth_dir.join("telemetry-ready.json"); + std::fs::write(&auth_db, UNREADABLE_AUTH_DB).expect("auth DB"); + std::fs::write(&readiness, FAR_FUTURE_READINESS).expect("readiness"); + let env = [(AUTH_DB_ENCRYPTION_KEY_ENV, AUTH_DB_TEST_SECRET)]; + + let fix = sandbox.run_with_env(&["doctor", "--fix", "--format", "json"], &env); + let report = fix.json(); + + assert_eq!( + report["managed_telemetry"], + serde_json::json!({ + "state": "disabled", + "reason": "database_error", + "kind": "managed_telemetry_database_error", + "severity": "warning", + "next_action": "manual_steps", + }), + "{}", + fix.stdout + ); + let warnings = global_state_warnings(&report); + assert_eq!(warnings.len(), 1, "{}", fix.stdout); + assert_eq!(warnings[0]["fixability"], "manual_only"); + assert_eq!(warnings[0]["remediation"]["next_action"], "manual_steps"); + let summary = warnings[0]["summary"].as_str().expect("summary"); + let fixes = report["fix_results"] + .as_array() + .expect("fix_results array") + .iter() + .filter(|entry| { + entry["category"] == "global_state" + && entry["detail"] + .as_str() + .is_some_and(|detail| detail.starts_with(summary)) + }) + .collect::>(); + assert_eq!(fixes.len(), 1, "{}", fix.stdout); + assert_eq!(fixes[0]["outcome"], "manual"); + assert_eq!( + std::fs::read(&auth_db).expect("auth DB kept"), + UNREADABLE_AUTH_DB + ); + assert_eq!( + std::fs::read(&readiness).expect("readiness kept"), + FAR_FUTURE_READINESS + ); + assert!(!fix.stdout.contains(AUTH_DB_TEST_SECRET)); + assert!(!fix.stderr.contains(AUTH_DB_TEST_SECRET)); +} diff --git a/cli/src/services/agent_trace_sync/control_plane.rs b/cli/src/services/agent_trace_sync/control_plane.rs index 693e9ad93..852368fa8 100644 --- a/cli/src/services/agent_trace_sync/control_plane.rs +++ b/cli/src/services/agent_trace_sync/control_plane.rs @@ -11,6 +11,8 @@ use crate::services::agent_trace_export::{ self, AgentTraceAgentTraceExportRow, AgentTraceMessageExportRow, AgentTracePartExportRow, }; use crate::services::auth::{self, AuthError, TokenResponse}; +use crate::services::db::encryption_key::CredentialAccess; +use crate::services::error::AuthStorageFailure; use crate::services::resilience::{run_with_retry, RetryPolicy}; use crate::services::telemetry_credential_storage::{self, TelemetryCredential, TelemetryIdentity}; use crate::services::token_storage::{self, StoredTokens, TokenStorageError}; @@ -107,7 +109,7 @@ pub enum ControlPlaneError { Conflict(String), ServerError(String), InvalidResponse(String), - Storage(String), + Storage(AuthStorageFailure, String), Protocol { status: StatusCode, message: String }, } @@ -136,7 +138,7 @@ impl fmt::Display for ControlPlaneError { Self::InvalidResponse(reason) => { write!(f, "Control-plane returned an unexpected response: {reason}") } - Self::Storage(reason) => write!(f, "Local credential storage error: {reason}"), + Self::Storage(_, reason) => write!(f, "Local credential storage error: {reason}"), Self::Protocol { status, message } => write!( f, "Control-plane rejected the request ({status}): {message}" @@ -155,14 +157,17 @@ impl ControlPlaneError { ) } - pub fn is_storage_failure(&self) -> bool { - matches!(self, Self::Storage(_)) + pub fn storage_failure(&self) -> Option { + match self { + Self::Storage(failure, _) => Some(*failure), + _ => None, + } } } impl From for ControlPlaneError { fn from(value: TokenStorageError) -> Self { - Self::Storage(value.to_string()) + Self::Storage(value.failure(), value.to_string()) } } @@ -171,7 +176,7 @@ impl From for ControlPlaneError { match value { AuthError::Unauthorized(reason) => Self::AuthenticationFailed(reason), AuthError::RequestFailed(error) => Self::Transport(error.to_string()), - AuthError::Storage(error) => Self::Storage(error.to_string()), + AuthError::Storage(error) => Self::Storage(error.failure(), error.to_string()), other => Self::AuthenticationFailed(other.to_string()), } } @@ -197,15 +202,17 @@ pub trait CredentialStore: Send + Sync + 'static { ) -> impl Future> + Send; } -pub struct SystemCredentialStore; +pub struct SystemCredentialStore { + access: CredentialAccess, +} impl CredentialStore for SystemCredentialStore { async fn load(&self) -> Result, ControlPlaneError> { - Ok(token_storage::load_tokens().await?) + Ok(token_storage::load_tokens(self.access).await?) } async fn save(&self, token: &TokenResponse) -> Result { - Ok(token_storage::save_tokens(token).await?) + Ok(token_storage::save_tokens(token, self.access).await?) } } @@ -300,13 +307,14 @@ impl AuthenticatedControlPlaneClient { base_url: impl Into, workos_api_base_url: impl Into, workos_client_id: impl Into, + access: CredentialAccess, ) -> Self { Self::with_credential_store( http, base_url, workos_api_base_url, workos_client_id, - SystemCredentialStore, + SystemCredentialStore { access }, ) } } @@ -445,9 +453,9 @@ impl AuthenticatedControlPlaneClient { Fut: Future>, { let credential = self.issue_telemetry_credential().await?; - save(credential.clone()) - .await - .map_err(|_| ControlPlaneError::Storage(TELEMETRY_STORAGE_MESSAGE.to_string()))?; + save(credential.clone()).await.map_err(|error| { + ControlPlaneError::Storage(error.failure(), TELEMETRY_STORAGE_MESSAGE.to_string()) + })?; Ok(credential) } @@ -1259,7 +1267,7 @@ mod tests { &unauthorized, ControlPlaneError::AuthenticationFailed(_) | ControlPlaneError::Transport(_) - | ControlPlaneError::Storage(_) + | ControlPlaneError::Storage(..) ), "{unauthorized:?}" ); diff --git a/cli/src/services/agent_trace_sync/mod.rs b/cli/src/services/agent_trace_sync/mod.rs index cc9f2ed01..c10f607fd 100644 --- a/cli/src/services/agent_trace_sync/mod.rs +++ b/cli/src/services/agent_trace_sync/mod.rs @@ -5,6 +5,7 @@ use std::fmt; use crate::services::agent_trace_export::{ AgentTraceAgentTraceExportRow, AgentTraceMessageExportRow, AgentTracePartExportRow, }; +use crate::services::error::AuthStorageFailure; use control_plane::ControlPlaneError; pub const RECONCILIATION_MAX_ATTEMPTS: u32 = 5; @@ -93,10 +94,10 @@ impl StreamSyncError { /// or `Terminal` failure) means local credential storage is unavailable. /// `Read`, `InvalidResponse`, and `DidNotConverge` never carry a /// `ControlPlaneError` and are never storage failures. - pub fn is_storage_failure(&self) -> bool { + pub fn storage_failure(&self) -> Option { match self { - Self::Refresh(error) | Self::Terminal(error) => error.is_storage_failure(), - Self::Read(_) | Self::InvalidResponse(_) | Self::DidNotConverge => false, + Self::Refresh(error) | Self::Terminal(error) => error.storage_failure(), + Self::Read(_) | Self::InvalidResponse(_) | Self::DidNotConverge => None, } } } diff --git a/cli/src/services/auth_command/mod.rs b/cli/src/services/auth_command/mod.rs index bd36da304..384fc9643 100644 --- a/cli/src/services/auth_command/mod.rs +++ b/cli/src/services/auth_command/mod.rs @@ -12,7 +12,8 @@ use crate::services::agent_trace_sync::control_plane::{ }; use crate::services::auth::{self, AuthError, DeviceAuthFlowResult}; use crate::services::config; -use crate::services::error::{CliError, UserError}; +use crate::services::db::encryption_key::CredentialAccess; +use crate::services::error::{AuthStorageFailure, CliError, UserError}; use crate::services::managed_telemetry::claude_helper::{ repository_id_for, visible_destination_conflict, }; @@ -20,6 +21,7 @@ use crate::services::managed_telemetry::{run_with_deadline, CREDENTIAL_LOOKUP_DE use crate::services::output_format::OutputFormat; use crate::services::style::{label, prompt_label, prompt_value, value}; use crate::services::telemetry_credential_storage; +use crate::services::telemetry_readiness; use crate::services::token_storage::{self, StoredTokens, TokenStorageError}; pub const NAME: &str = "auth"; @@ -106,6 +108,7 @@ async fn run_telemetry_headers() -> Result { telemetry_headers_with( std::env::current_dir().ok(), visible_destination_conflict(|key| std::env::var(key).ok()), + telemetry_readiness::is_ready(current_unix_seconds()), repository_id_for, telemetry_credential_storage::load_repository_telemetry_credential, ) @@ -115,6 +118,7 @@ async fn run_telemetry_headers() -> Result { pub(crate) async fn telemetry_headers_with( working_dir: Option, destination_conflict: bool, + ready: bool, resolve_repository: R, lookup: L, ) -> Result @@ -129,7 +133,7 @@ where > + Send + 'static, { - let Some(working_dir) = working_dir.filter(|_| !destination_conflict) else { + let Some(working_dir) = working_dir.filter(|_| ready && !destination_conflict) else { return Err(telemetry_headers_unavailable()); }; let credential = run_with_deadline(CREDENTIAL_LOOKUP_DEADLINE, async move { @@ -171,10 +175,11 @@ pub async fn run_login(format: AuthFormat) -> Result { auth_config.control_plane_base_url.value.unwrap_or_default(), auth::WORKOS_DEFAULT_BASE_URL, client_id.clone(), + CredentialAccess::Interactive, ); - let stored_tokens = token_storage::load_tokens() + let stored_tokens = token_storage::load_tokens(CredentialAccess::Interactive) .await - .map_err(auth_storage_error)?; + .map_err(login_storage_error)?; let client = &client; let control_plane = &control_plane; @@ -199,21 +204,25 @@ pub async fn run_login(format: AuthFormat) -> Result { pub async fn run_logout(format: AuthFormat) -> Result { run_logout_with( format, + telemetry_readiness::clear, token_storage::delete_tokens(), telemetry_credential_storage::delete_telemetry_credential(), ) .await } -async fn run_logout_with( +async fn run_logout_with( format: AuthFormat, + clear_readiness: C, delete_workos: W, delete_telemetry: T, ) -> Result where + C: FnOnce() -> Result<()>, W: Future>, T: Future>, { + clear_readiness().map_err(unexpected_auth_command_error)?; let workos_result = delete_workos.await; let telemetry_result = delete_telemetry.await; let workos_deleted = workos_result.map_err(auth_storage_error)?; @@ -223,7 +232,7 @@ where } pub async fn run_whoami(format: AuthFormat) -> Result { - if token_storage::load_tokens() + if token_storage::load_tokens(CredentialAccess::Interactive) .await .map_err(auth_storage_error)? .is_none() @@ -241,6 +250,7 @@ pub async fn run_whoami(format: AuthFormat) -> Result { auth_config.control_plane_base_url.value.unwrap_or_default(), auth::WORKOS_DEFAULT_BASE_URL, auth_config.workos_client_id.value.unwrap_or_default(), + CredentialAccess::Interactive, ); let profile = client .me() @@ -263,9 +273,9 @@ async fn maybe_renew_stored_credentials( ) .await { - Ok(token) => token_storage::save_tokens(&token) + Ok(token) => token_storage::save_tokens(&token, CredentialAccess::Interactive) .await - .map_err(auth_storage_error) + .map_err(login_storage_error) .map(Some), Err(_) => Ok(None), } @@ -299,35 +309,66 @@ where async fn save_new_session_tokens(token: &auth::TokenResponse) -> Result { replace_session_with( + telemetry_readiness::clear, telemetry_credential_storage::delete_telemetry_credential(), - || token_storage::save_tokens(token), + || token_storage::save_tokens(token, CredentialAccess::Interactive), ) .await } -async fn replace_session_with( +async fn replace_session_with( + clear_readiness: C, invalidate_telemetry: D, save_session: S, ) -> Result where + C: FnOnce() -> Result<()>, D: Future>, S: FnOnce() -> SF, SF: Future>, { - invalidate_telemetry.await.map_err(auth_storage_error)?; - save_session().await.map_err(auth_storage_error) + clear_readiness().map_err(unexpected_auth_command_error)?; + invalidate_telemetry.await.map_err(login_storage_error)?; + save_session().await.map_err(login_storage_error) } async fn provision_telemetry_credential( client: &AuthenticatedControlPlaneClient, ) -> TelemetryProvisioningResult { - match client.issue_and_save_telemetry_credential().await { - Ok(_) => { - enroll_current_repository_if_unbound().await; - TelemetryProvisioningResult::Issued - } - Err(_) => TelemetryProvisioningResult::Unavailable, + let provisioned = client.issue_and_save_telemetry_credential().await; + let result = record_telemetry_readiness( + provisioned.as_ref().ok(), + telemetry_readiness::mark_ready, + telemetry_readiness::clear, + ); + if result == TelemetryProvisioningResult::Issued { + enroll_current_repository_if_unbound().await; + } + result +} + +fn record_telemetry_readiness( + provisioned: Option<&telemetry_credential_storage::TelemetryCredential>, + mark_ready: M, + clear: C, +) -> TelemetryProvisioningResult +where + M: FnOnce(u64) -> Result<()>, + C: FnOnce() -> Result<()>, +{ + let recorded = provisioned + .is_some_and(|credential| mark_ready(credential.expires_at_unix_seconds).is_ok()); + if recorded { + return TelemetryProvisioningResult::Issued; } + let _ = clear(); + TelemetryProvisioningResult::Unavailable +} + +fn current_unix_seconds() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) } async fn enroll_current_repository_if_unbound() { @@ -443,7 +484,8 @@ fn write_login_prompt(authorization: &auth::DeviceAuthorizationResponse) -> Resu fn map_login_error(error: AuthError) -> CliError { let user_error = match &error { - AuthError::Io(_) | AuthError::Storage(_) => UserError::AuthStorageUnavailable, + AuthError::Io(_) => UserError::LoginAuthStorageUnavailable(AuthStorageFailure::Database), + AuthError::Storage(storage) => UserError::LoginAuthStorageUnavailable(storage.failure()), _ => UserError::UnexpectedFailure, }; CliError::user_with_source(user_error, error) @@ -646,8 +688,8 @@ fn render_whoami_result(profile: &MeResponse, format: AuthFormat) -> Result CliError { let user_error = if error.is_authentication_failure() { UserError::NotAuthenticated - } else if error.is_storage_failure() { - UserError::AuthStorageUnavailable + } else if let Some(failure) = error.storage_failure() { + UserError::AuthStorageUnavailable(failure) } else { UserError::UnexpectedFailure }; @@ -659,7 +701,14 @@ fn map_whoami_control_plane_error(error: &ControlPlaneError) -> CliError { } fn auth_storage_error(error: TokenStorageError) -> CliError { - CliError::user_with_source(UserError::AuthStorageUnavailable, error) + CliError::user_with_source(UserError::AuthStorageUnavailable(error.failure()), error) +} + +fn login_storage_error(error: TokenStorageError) -> CliError { + CliError::user_with_source( + UserError::LoginAuthStorageUnavailable(error.failure()), + error, + ) } fn unexpected_auth_command_error(error: anyhow::Error) -> CliError { @@ -668,8 +717,6 @@ fn unexpected_auth_command_error(error: anyhow::Error) -> CliError { #[cfg(test)] mod tests { - use std::sync::atomic::{AtomicBool, Ordering}; - use super::*; fn storage_failure() -> TokenStorageError { @@ -704,6 +751,10 @@ mod tests { async fn new_session_invalidates_previous_telemetry_credential_before_saving() { let order = std::sync::Mutex::new(Vec::new()); let saved = replace_session_with( + || { + order.lock().unwrap().push("clear"); + Ok(()) + }, async { order.lock().unwrap().push("invalidate"); Ok(true) @@ -716,16 +767,112 @@ mod tests { .await .expect("replacement should succeed"); assert_eq!(saved, stored_tokens()); - assert_eq!(*order.lock().unwrap(), ["invalidate", "save"]); + assert_eq!(*order.lock().unwrap(), ["clear", "invalidate", "save"]); let save_called = std::sync::atomic::AtomicBool::new(false); - let failed = replace_session_with(async { Err(storage_failure()) }, || async { - save_called.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(stored_tokens()) - }) + let failed = replace_session_with( + || Ok(()), + async { Err(storage_failure()) }, + || async { + save_called.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(stored_tokens()) + }, + ) .await; - assert!(failed.is_err()); assert!(!save_called.load(std::sync::atomic::Ordering::SeqCst)); + assert_login_storage_guidance( + &failed.expect_err("invalidation failure fails login"), + AuthStorageFailure::Database, + ); + + let failed = replace_session_with( + || Ok(()), + async { Ok(true) }, + || async { Err(TokenStorageError::CredentialAccessUnavailable) }, + ) + .await; + assert_login_storage_guidance( + &failed.expect_err("save failure fails login"), + AuthStorageFailure::CredentialAccess, + ); + } + + fn assert_login_storage_guidance(error: &CliError, failure: AuthStorageFailure) { + let CliError::User { + error: user_error, .. + } = error + else { + panic!("expected a user-facing storage error"); + }; + assert_eq!(*user_error, UserError::LoginAuthStorageUnavailable(failure)); + assert_eq!(user_error.key(), "auth.storage_unavailable"); + assert_eq!(error.class().exit_code(), 4); + let message = error + .to_string() + .split_whitespace() + .collect::>() + .join(" "); + assert!(!message.contains("Retry the original command")); + let keychain_guidance = message.contains("encryption key"); + assert_eq!( + keychain_guidance, + failure == AuthStorageFailure::CredentialAccess + ); + if keychain_guidance { + assert!(message.contains("Running login again does not help until")); + assert!(!message.contains("Run `sce auth login` in an interactive terminal.")); + } + } + + #[test] + fn login_device_flow_storage_failure_uses_noncircular_guidance() { + for (storage, failure) in [ + ( + TokenStorageError::CredentialAccessUnavailable, + AuthStorageFailure::CredentialAccess, + ), + ( + TokenStorageError::DatabaseLocked("busy".to_string()), + AuthStorageFailure::DatabaseLocked, + ), + ( + TokenStorageError::Database("corrupt".to_string()), + AuthStorageFailure::Database, + ), + ] { + assert_login_storage_guidance(&map_login_error(AuthError::Storage(storage)), failure); + } + assert_login_storage_guidance( + &map_login_error(AuthError::Io(std::io::Error::other("io"))), + AuthStorageFailure::Database, + ); + let CliError::User { error, .. } = map_login_error(AuthError::MissingClientId) else { + panic!("expected a user-facing error"); + }; + assert_eq!(error, UserError::UnexpectedFailure); + } + + #[tokio::test] + async fn logout_storage_failure_keeps_command_storage_guidance() { + let error = run_logout_with( + AuthFormat::Json, + || Ok(()), + async { Err(TokenStorageError::CredentialAccessUnavailable) }, + async { Ok(false) }, + ) + .await + .expect_err("storage failure fails logout"); + let CliError::User { + error: user_error, .. + } = &error + else { + panic!("expected a user-facing storage error"); + }; + assert_eq!( + *user_error, + UserError::AuthStorageUnavailable(AuthStorageFailure::CredentialAccess) + ); + assert!(error.to_string().contains("sce auth login")); } #[tokio::test] @@ -739,6 +886,17 @@ mod tests { }; let working_dir = Some(PathBuf::from("/workspace/repo")); + let error = telemetry_headers_with( + working_dir.clone(), + false, + false, + |_: &Path| Some("repo-a".to_string()), + lookup(std::sync::Arc::clone(&lookup_called)), + ) + .await + .expect_err("helper must fail closed without telemetry readiness"); + assert_eq!(error.to_string(), TELEMETRY_HEADERS_UNAVAILABLE); + for (dir, conflict, repository) in [ (working_dir.clone(), true, Some("repo-a")), (None, false, Some("repo-a")), @@ -747,6 +905,7 @@ mod tests { let error = telemetry_headers_with( dir, conflict, + true, move |_: &Path| repository.map(str::to_string), lookup(std::sync::Arc::clone(&lookup_called)), ) @@ -759,6 +918,7 @@ mod tests { let headers = telemetry_headers_with( working_dir, false, + true, |dir: &Path| (dir == Path::new("/workspace/repo")).then(|| "repo-a".to_string()), |repository_id: String| async move { assert_eq!(repository_id, "repo-a"); @@ -809,55 +969,39 @@ mod tests { } #[tokio::test] - async fn valid_stored_credentials_are_provisioned_after_workos_persistence() { - let provision_calls = AtomicBool::new(false); - let tokens = stored_tokens(); - let output = run_login_with_stored_credentials( - AuthFormat::Json, - Some(tokens.clone()), - |stored| async move { - assert_eq!(stored, tokens); - Ok(Some(stored)) - }, - || async { - provision_calls.store(true, Ordering::SeqCst); - TelemetryProvisioningResult::Issued - }, - |_format| async { panic!("valid stored credentials must not use device login") }, - ) - .await - .expect("valid stored login should succeed"); - - assert!(provision_calls.load(Ordering::SeqCst)); - assert!(output.contains("\"renewed\": true")); - assert!(output.contains("\"status\": \"issued\"")); - } - - #[tokio::test] - async fn renewed_stored_credentials_are_provisioned_after_workos_persistence() { - let provision_calls = AtomicBool::new(false); + async fn stored_credentials_are_provisioned_only_after_renewal_completes() { let original = stored_tokens(); let renewed = StoredTokens { access_token: "renewed-access".to_string(), refresh_token: "renewed-refresh".to_string(), ..original.clone() }; - let output = run_login_with_stored_credentials( - AuthFormat::Json, - Some(original), - |_stored| async move { Ok(Some(renewed)) }, - || async { - provision_calls.store(true, Ordering::SeqCst); - TelemetryProvisioningResult::Issued - }, - |_format| async { panic!("renewed stored credentials must not use device login") }, - ) - .await - .expect("renewed stored login should succeed"); + for returned in [original.clone(), renewed] { + let events = std::sync::Mutex::new(Vec::new()); + let events = &events; + let expected_stored = original.clone(); + let output = run_login_with_stored_credentials( + AuthFormat::Json, + Some(original.clone()), + |stored| async move { + assert_eq!(stored, expected_stored); + tokio::task::yield_now().await; + events.lock().unwrap().push("renewal-complete"); + Ok(Some(returned)) + }, + || async move { + events.lock().unwrap().push("provision"); + TelemetryProvisioningResult::Issued + }, + |_format| async { panic!("stored credentials must not use device login") }, + ) + .await + .expect("stored login should succeed"); - assert!(provision_calls.load(Ordering::SeqCst)); - assert!(output.contains("\"renewed\": true")); - assert!(output.contains("\"status\": \"issued\"")); + assert_eq!(*events.lock().unwrap(), ["renewal-complete", "provision"]); + assert!(output.contains("\"renewed\": true")); + assert!(output.contains("\"status\": \"issued\"")); + } } #[tokio::test] @@ -923,38 +1067,108 @@ mod tests { } #[tokio::test] - async fn telemetry_credential_logout_reports_removal_when_only_telemetry_deleted() { - let output = run_logout_with(AuthFormat::Json, async { Ok(false) }, async { Ok(true) }) - .await - .expect("logout should succeed"); - assert!(output.contains("\"credentials_removed\": true")); + async fn logout_clears_readiness_and_attempts_both_deletions_before_reporting() { + use AuthStorageFailure::{CredentialAccess, DatabaseLocked}; + + fn deletion(result: Result) -> Result { + result.map_err(|failure| match failure { + CredentialAccess => TokenStorageError::CredentialAccessUnavailable, + _ => TokenStorageError::DatabaseLocked("busy".to_string()), + }) + } + + for (workos, telemetry, expected) in [ + (Ok(false), Ok(true), Ok(())), + (Ok(true), Err(DatabaseLocked), Err(DatabaseLocked)), + (Err(CredentialAccess), Ok(true), Err(CredentialAccess)), + ( + Err(CredentialAccess), + Err(DatabaseLocked), + Err(CredentialAccess), + ), + ] { + let events = std::sync::Mutex::new(Vec::new()); + let result = run_logout_with( + AuthFormat::Json, + || { + events.lock().unwrap().push("clear-readiness"); + Ok(()) + }, + async { + events.lock().unwrap().push("delete-workos"); + deletion(workos) + }, + async { + events.lock().unwrap().push("delete-telemetry"); + deletion(telemetry) + }, + ) + .await; + + assert_eq!( + *events.lock().unwrap(), + ["clear-readiness", "delete-workos", "delete-telemetry"], + "{workos:?} {telemetry:?}" + ); + match (result, expected) { + (Ok(output), Ok(())) => assert!(output.contains("\"credentials_removed\": true")), + (Err(CliError::User { error, .. }), Err(failure)) => { + assert_eq!(error, UserError::AuthStorageUnavailable(failure)); + } + (result, expected) => { + panic!("{workos:?} {telemetry:?}: {result:?} != {expected:?}") + } + } + } } - #[tokio::test] - async fn telemetry_credential_logout_attempts_both_deletions_on_telemetry_failure() { - let workos_called = AtomicBool::new(false); - let result = run_logout_with( - AuthFormat::Json, - async { - workos_called.store(true, Ordering::SeqCst); - Ok(true) + #[test] + fn telemetry_readiness_is_recorded_only_after_credential_persistence() { + let credential = telemetry_credential("tok-ready"); + let marked = std::cell::Cell::new(None); + let cleared = std::cell::Cell::new(false); + + let issued = record_telemetry_readiness( + Some(&credential), + |expires_at| { + marked.set(Some(expires_at)); + Ok(()) }, - async { Err(storage_failure()) }, - ) - .await; - assert!(result.is_err()); - assert!(workos_called.load(Ordering::SeqCst)); - } + || { + cleared.set(true); + Ok(()) + }, + ); + assert_eq!(issued, TelemetryProvisioningResult::Issued); + assert_eq!(marked.get(), Some(credential.expires_at_unix_seconds)); + assert!(!cleared.get()); - #[tokio::test] - async fn telemetry_credential_logout_deletes_telemetry_on_workos_failure() { - let telemetry_called = AtomicBool::new(false); - let result = run_logout_with(AuthFormat::Json, async { Err(storage_failure()) }, async { - telemetry_called.store(true, Ordering::SeqCst); - Ok(true) - }) - .await; - assert!(result.is_err()); - assert!(telemetry_called.load(Ordering::SeqCst)); + marked.set(None); + let failed = record_telemetry_readiness( + None, + |expires_at| { + marked.set(Some(expires_at)); + Ok(()) + }, + || { + cleared.set(true); + Ok(()) + }, + ); + assert_eq!(failed, TelemetryProvisioningResult::Unavailable); + assert_eq!(marked.get(), None); + assert!(cleared.get()); + + cleared.set(false); + let unrecorded = record_telemetry_readiness( + Some(&credential), + |_| Err(anyhow!("read-only state directory")), + || { + cleared.set(true); + Ok(()) + }, + ); + assert_eq!(unrecorded, TelemetryProvisioningResult::Unavailable); + assert!(cleared.get()); } } diff --git a/cli/src/services/auth_db/lifecycle.rs b/cli/src/services/auth_db/lifecycle.rs index 79a5e3160..61826dd25 100644 --- a/cli/src/services/auth_db/lifecycle.rs +++ b/cli/src/services/auth_db/lifecycle.rs @@ -8,7 +8,7 @@ use crate::services::lifecycle::{ HealthProblemKind, HealthSeverity, ServiceLifecycle, SetupOutcome, }; -use super::{AuthDb, AuthDbSpec}; +use super::AuthDbSpec; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct AuthDbLifecycle; @@ -49,13 +49,8 @@ impl ServiceLifecycle for AuthDbLifecycle { } async fn setup(&self, _ctx: &C) -> Result { - if auth_db_path()?.exists() { - return Ok(SetupOutcome::default()); - } - - AuthDb::new() - .await - .context("Auth DB lifecycle setup failed while initializing auth DB")?; + bootstrap_auth_db_parent() + .context("Auth DB lifecycle setup failed while preparing the auth DB directory")?; Ok(SetupOutcome::default()) } } diff --git a/cli/src/services/auth_db/mod.rs b/cli/src/services/auth_db/mod.rs index 5e2934bb0..5e464738d 100644 --- a/cli/src/services/auth_db/mod.rs +++ b/cli/src/services/auth_db/mod.rs @@ -35,3 +35,6 @@ impl DbSpec for AuthDbSpec { pub type AuthDb = EncryptedTursoDb; pub mod lifecycle; + +#[cfg(test)] +mod multiprocess_tests; diff --git a/cli/src/services/auth_db/multiprocess_tests.rs b/cli/src/services/auth_db/multiprocess_tests.rs new file mode 100644 index 000000000..e35be5e2a --- /dev/null +++ b/cli/src/services/auth_db/multiprocess_tests.rs @@ -0,0 +1,1028 @@ +use std::collections::BTreeSet; +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, ChildStdin, Command, Stdio}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use anyhow::Result; + +use super::{AuthDb, AuthDbSpec}; +use crate::generated_migrations; +use crate::services::bash_policy::{ + run_bash_policy_request, BashPolicyRequest, PolicyInputMode, PolicyOutputMode, +}; +use crate::services::db::encryption_key::AUTH_DB_ENCRYPTION_KEY_ENV; +use crate::services::db::{DatabaseKind, DbSpec, EncryptedTursoDb}; +use crate::services::default_paths::auth_db_path; +use crate::services::error::CliError; +use crate::services::hooks::{run_hooks_subcommand, HookSubcommand}; +use crate::services::managed_telemetry::claude_helper::repository_id_for; +use crate::services::managed_telemetry::claude_helper::tests::git_repository_with_remote; +use crate::services::managed_telemetry::trace_export::tests::{ + capture_requests_until, loopback_hook_telemetry, trace_spans, +}; +use crate::services::managed_telemetry::trace_export::ClaudeHookCommand; +use crate::services::managed_telemetry::{inspect_health, ManagedTelemetryHealth}; +use crate::services::observability::traits::{NoopLogger, Telemetry}; +use crate::services::telemetry_credential_storage::{ + bind_repository_telemetry, load_repository_telemetry_credential, load_telemetry_credential, + save_telemetry_credential, test_access_token, RepositoryBindingMode, TelemetryCredential, + TelemetryIdentity, +}; + +const CHILD_TEST: &str = "services::auth_db::multiprocess_tests::auth_db_child_process"; +const ROLE_ENV: &str = "SCE_AUTH_DB_TEST_CHILD_ROLE"; +const COLLECTOR_ENV: &str = "SCE_AUTH_DB_TEST_COLLECTOR"; +const HOOK_INDEX_ENV: &str = "SCE_AUTH_DB_TEST_HOOK_INDEX"; +const TEST_SECRET: &str = "auth-db-multiprocess-test-secret"; +const WRONG_SECRET: &str = "auth-db-multiprocess-wrong-secret"; +const TOKEN: &str = "tok-multiprocess"; +const USER_ID: &str = "user_mp"; +const WORKSPACE_ID: &str = "workspace_mp"; +const READY: &str = "SCE_AUTH_DB_CHILD_READY"; +const TOKEN_PREFIX: &str = "SCE_AUTH_DB_CHILD_TOKEN="; +const HEALTH_PREFIX: &str = "SCE_AUTH_DB_CHILD_TELEMETRY_HEALTH="; +const CONCURRENT_PROCESSES: usize = 32; +const CONCURRENT_MIGRATORS: usize = 4; +const CHILD_KILL_AFTER: Duration = Duration::from_mins(1); +const CHILD_POLL_INTERVAL: Duration = Duration::from_millis(10); +const FAR_FUTURE: u64 = 4_102_444_800; +const HOOK_COMMANDS: [ClaudeHookCommand; 5] = [ + ClaudeHookCommand::DiffTrace, + ClaudeHookCommand::ConversationTrace, + ClaudeHookCommand::ClaudeMutationScope, + ClaudeHookCommand::ClaudeModelState, + ClaudeHookCommand::PolicyBash, +]; +const PRE_BINDING_MIGRATIONS: usize = 3; +const BINDING_MIGRATION_ID: &str = "004_bind_telemetry_credentials"; +const WRITER_ENV: &str = "SCE_AUTH_DB_TEST_WRITER"; +const COMMITTED_PREFIX: &str = "SCE_AUTH_DB_CHILD_COMMITTED="; +const ROW_PREFIX: &str = "SCE_AUTH_DB_CHILD_ROW="; +const WRITER_ROWS: usize = 8; +const WRITER_PAUSE: Duration = Duration::from_millis(20); +const CONCURRENT_WRITERS: usize = 4; +const INSERT_BINDING_SQL: &str = "INSERT INTO telemetry_repository_bindings \ + (repository_id, user_id, workspace_id, bound_at_unix_seconds) VALUES (?1, ?2, ?3, 1)"; +const AUTH_DB_FILES: [&str; 3] = ["auth.db", "auth.db-wal", "auth.db-tshm"]; +const HEADERS_REPOSITORY: &str = "bound-repo"; +const UNBOUND_REPOSITORY: &str = "unbound-repo"; +const HEADERS_AUTHORIZATION: &str = r#"{"Authorization":"Bearer tok-multiprocess"}"#; +const HEADERS_UNAVAILABLE: &str = "Telemetry credential is unavailable."; +const STDIN_SECRET: &str = "SECRET_STDIN_VALUE"; +const FAIL_OPEN_HOOK_COMMANDS: [ClaudeHookCommand; 3] = [ + ClaudeHookCommand::DiffTrace, + ClaudeHookCommand::ConversationTrace, + ClaudeHookCommand::ClaudeModelState, +]; + +struct PreBindingAuthDbSpec; + +impl DbSpec for PreBindingAuthDbSpec { + fn db_name() -> &'static str { + AuthDbSpec::db_name() + } + + fn db_path() -> Result { + auth_db_path() + } + + fn migrations() -> &'static [(&'static str, &'static str)] { + &generated_migrations::AUTH_MIGRATIONS[..PRE_BINDING_MIGRATIONS] + } + + const KIND: DatabaseKind = DatabaseKind::Auth; +} + +static FAN_OUT_LOCK: Mutex<()> = Mutex::new(()); + +fn serialize_fan_out() -> MutexGuard<'static, ()> { + FAN_OUT_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +struct SchemalessAuthDbSpec; + +impl DbSpec for SchemalessAuthDbSpec { + fn db_name() -> &'static str { + AuthDbSpec::db_name() + } + + fn db_path() -> Result { + auth_db_path() + } + + fn migrations() -> &'static [(&'static str, &'static str)] { + &[] + } + + const KIND: DatabaseKind = DatabaseKind::Auth; +} + +struct Sandbox { + dir: tempfile::TempDir, +} + +impl Sandbox { + fn new() -> Self { + Self { + dir: tempfile::tempdir().expect("sandbox"), + } + } + + fn command(&self, role: &str) -> Command { + let root = self.dir.path(); + let mut command = Command::new(std::env::current_exe().expect("test binary")); + command + .args(["--exact", CHILD_TEST, "--nocapture", "--test-threads=1"]) + .env(ROLE_ENV, role) + .env("HOME", root) + .env("XDG_STATE_HOME", root.join("state")) + .env("XDG_CONFIG_HOME", root.join("config")) + .env("XDG_CACHE_HOME", root.join("cache")) + .env("XDG_DATA_HOME", root.join("data")) + .env(AUTH_DB_ENCRYPTION_KEY_ENV, TEST_SECRET) + .env_remove("TRACEPARENT") + .env_remove("TRACESTATE") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("OTEL_") { + command.env_remove(key); + } + } + command + } + + fn rows(&self) -> BTreeSet { + prefixed_lines(&self.run("verify-rows"), ROW_PREFIX) + } + + fn sce_state_dir(&self) -> PathBuf { + let root = self.dir.path(); + let state_root = if cfg!(target_os = "macos") { + root.join("Library").join("Application Support") + } else { + root.join("state") + }; + state_root.join("sce") + } + + fn auth_db_files(&self) -> Vec { + let state = self.sce_state_dir(); + AUTH_DB_FILES + .iter() + .filter(|name| state.join(name).exists()) + .map(ToString::to_string) + .collect() + } + + fn run(&self, role: &str) -> String { + wait_success(role, self.command(role).spawn().expect("spawn child")) + } + + fn hold(&self, role: &str) -> HeldChild { + Self::hold_command(role, self.command(role)) + } + + fn hold_command(role: &str, mut command: Command) -> HeldChild { + let mut child = command.stdin(Stdio::piped()).spawn().expect("spawn holder"); + let stdin = child.stdin.take().expect("holder stdin"); + let stdout = child.stdout.take().expect("holder stdout"); + let (sender, lines) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines().map_while(Result::ok) { + if sender.send(line).is_err() { + break; + } + } + }); + let mut held = HeldChild { + role: role.to_string(), + child, + stdin, + lines, + }; + held.wait_ready(); + held + } + + fn seed(&self) { + self.run("seed"); + } +} + +struct HeldChild { + role: String, + child: Child, + stdin: ChildStdin, + lines: Receiver, +} + +impl HeldChild { + fn signal(&mut self) { + writeln!(self.stdin).expect("release holder"); + } + + fn wait_ready(&mut self) -> String { + let deadline = Instant::now() + CHILD_KILL_AFTER; + let mut captured = String::new(); + loop { + match self + .lines + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + { + Ok(line) if line.trim_end().ends_with(READY) => return captured, + Ok(line) => { + captured.push_str(&line); + captured.push('\n'); + } + Err(RecvTimeoutError::Timeout) => self.abort( + &format!("was not ready within {CHILD_KILL_AFTER:?}"), + &captured, + ), + Err(RecvTimeoutError::Disconnected) => self.abort("exited before ready", &captured), + } + } + } + + fn abort(&mut self, reason: &str, stdout: &str) -> ! { + let _ = self.child.kill(); + let _ = self.child.wait(); + let mut stderr = String::new(); + if let Some(mut pipe) = self.child.stderr.take() { + let _ = pipe.read_to_string(&mut stderr); + } + panic!("{} holder {reason}:\n{stdout}\n{stderr}", self.role); + } + + fn release(mut self) { + self.signal(); + wait_bounded(&self.role, &mut self.child); + let rest = self.lines.iter().collect::>().join("\n"); + let output = self.child.wait_with_output().expect("holder output"); + assert!( + output.status.success() && rest.contains("1 passed"), + "{} holder failed:\n{rest}\n{}", + self.role, + String::from_utf8_lossy(&output.stderr) + ); + } +} + +fn wait_bounded(role: &str, child: &mut Child) { + let started = Instant::now(); + while child.try_wait().expect("child status").is_none() { + if started.elapsed() > CHILD_KILL_AFTER { + let _ = child.kill(); + let _ = child.wait(); + panic!("{role} child did not exit within {CHILD_KILL_AFTER:?}"); + } + std::thread::sleep(CHILD_POLL_INTERVAL); + } +} + +fn wait_success(role: &str, mut child: Child) -> String { + wait_bounded(role, &mut child); + let output = child.wait_with_output().expect("child output"); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + assert!( + output.status.success() && stdout.contains("1 passed"), + "{role} child failed:\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + stdout +} + +fn prefixed_lines(stdout: &str, prefix: &str) -> BTreeSet { + stdout + .lines() + .filter_map(|line| line.split_once(prefix).map(|(_, value)| value.to_string())) + .collect() +} + +fn coordinated_writer_rows(sandbox: &Sandbox) -> BTreeSet { + // Open every connection before writes start, and keep every connection + // alive until all writes finish so WAL startup/teardown cannot race them. + let mut writers = (0..CONCURRENT_WRITERS) + .map(|index| { + let mut command = sandbox.command("coordinated-writer"); + command.env(WRITER_ENV, format!("writer-{index}")); + Sandbox::hold_command("coordinated-writer", command) + }) + .collect::>(); + for writer in &mut writers { + writer.signal(); + } + let mut committed = BTreeSet::new(); + for writer in &mut writers { + committed.extend(prefixed_lines(&writer.wait_ready(), COMMITTED_PREFIX)); + } + for writer in writers { + writer.release(); + } + committed +} + +fn expected_writer_rows() -> BTreeSet { + (0..CONCURRENT_WRITERS) + .flat_map(|index| (0..WRITER_ROWS).map(move |row| format!("writer-{index}-{row:02}"))) + .collect() +} + +fn read_token(stdout: &str) -> Option<&str> { + stdout + .lines() + .find_map(|line| line.split_once(TOKEN_PREFIX).map(|(_, token)| token)) +} + +fn spawn_readers(sandbox: &Sandbox) -> Vec { + (0..CONCURRENT_PROCESSES) + .map(|_| sandbox.command("read").spawn().expect("spawn reader")) + .collect() +} + +fn wait_readers(readers: Vec) -> usize { + readers + .into_iter() + .map(|child| wait_success("read", child)) + .filter(|stdout| read_token(stdout) == Some(TOKEN)) + .count() +} + +#[test] +fn reader_succeeds_while_another_process_holds_auth_db_open() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let holder = sandbox.hold("hold-open"); + let reader = sandbox.run("read"); + assert_eq!(read_token(&reader), Some(TOKEN)); + holder.release(); +} + +#[test] +fn thirty_two_reader_processes_all_resolve_the_credential() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + assert_eq!(wait_readers(spawn_readers(&sandbox)), CONCURRENT_PROCESSES); +} + +#[test] +fn current_schema_reader_performs_no_migration_write_under_held_write_lock() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let writer = sandbox.hold("hold-write"); + let reader = sandbox.run("read"); + assert_eq!(read_token(&reader), Some(TOKEN)); + writer.release(); + sandbox.run("verify-schema"); +} + +struct NoRepositoryContext; + +impl crate::app::HasRepoRoot for NoRepositoryContext { + fn repo_root(&self) -> Option<&Path> { + None + } +} + +fn run_setup_lifecycle_child(sandbox: &Sandbox, role: &str) { + wait_success( + role, + sandbox + .command(role) + .env(AUTH_DB_ENCRYPTION_KEY_ENV, "") + .spawn() + .expect("spawn setup child"), + ); +} + +#[test] +fn setup_lifecycle_on_fresh_state_creates_no_auth_db_and_resolves_no_key() { + let _fan_out = serialize_fan_out(); + run_setup_lifecycle_child(&Sandbox::new(), "setup-fresh"); +} + +#[test] +fn setup_lifecycle_leaves_existing_auth_db_unchanged_and_resolves_no_key() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + run_setup_lifecycle_child(&sandbox, "setup-existing"); +} + +#[test] +fn telemetry_lookup_never_creates_a_missing_auth_db() { + let _fan_out = serialize_fan_out(); + Sandbox::new().run("lookup-missing"); +} + +fn telemetry_health(sandbox: &Sandbox, secret: &str) -> String { + let stdout = wait_success( + "telemetry-health", + sandbox + .command("telemetry-health") + .env(AUTH_DB_ENCRYPTION_KEY_ENV, secret) + .spawn() + .expect("spawn health child"), + ); + let health = prefixed_lines(&stdout, HEALTH_PREFIX); + assert_eq!(health.len(), 1, "{stdout}"); + health.into_iter().next().expect("one health line") +} + +#[test] +fn doctor_telemetry_health_on_fresh_state_resolves_no_key_and_creates_no_auth_db() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + assert_eq!(telemetry_health(&sandbox, ""), "not_configured"); + assert!(sandbox.auth_db_files().is_empty()); +} + +#[test] +fn doctor_telemetry_health_classifies_real_auth_db_states_without_mutating_them() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + let state = sandbox.sce_state_dir(); + let auth_db = std::fs::read(state.join("auth.db")).expect("seeded auth DB"); + let readiness = state.join("telemetry-ready.json"); + + assert_eq!(telemetry_health(&sandbox, TEST_SECRET), "healthy"); + assert_eq!(telemetry_health(&sandbox, WRONG_SECRET), "database_error"); + assert_eq!(telemetry_health(&sandbox, ""), "database_error"); + assert_eq!( + std::fs::read(state.join("auth.db")).expect("auth DB"), + auth_db + ); + + std::fs::write(&readiness, br#"{"expires_at_unix_seconds":1}"#).expect("expire readiness"); + assert_eq!(telemetry_health(&sandbox, ""), "credential_expired"); + std::fs::remove_file(&readiness).expect("remove readiness"); + assert_eq!(telemetry_health(&sandbox, ""), "not_configured"); + assert_eq!(read_token(&sandbox.run("read")), Some(TOKEN)); +} + +#[test] +fn concurrent_migration_of_schemaless_db_applies_each_migration_once() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.run("create-schemaless"); + + let initializers = (0..CONCURRENT_MIGRATORS) + .map(|_| sandbox.command("init").spawn().expect("spawn initializer")) + .collect::>(); + for child in initializers { + wait_success("init", child); + } + sandbox.run("verify-schema"); +} + +#[test] +fn outdated_schema_is_migrated_and_partial_schema_fails_clearly() { + let _fan_out = serialize_fan_out(); + Sandbox::new().run("upgrade"); + Sandbox::new().run("partial"); +} + +#[test] +fn concurrent_current_version_writers_persist_every_row() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + assert_eq!(coordinated_writer_rows(&sandbox), expected_writer_rows()); + assert_eq!(sandbox.rows(), expected_writer_rows()); +} + +#[cfg(target_os = "macos")] +#[test] +fn macos_current_version_processes_share_the_lock_concurrently() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let holders = [sandbox.hold("hold-open"), sandbox.hold("hold-open")]; + assert_eq!(coordinated_writer_rows(&sandbox), expected_writer_rows()); + assert_eq!(read_token(&sandbox.run("read")), Some(TOKEN)); + for holder in holders { + holder.release(); + } + assert_eq!(sandbox.rows(), expected_writer_rows()); +} + +#[test] +fn repository_telemetry_lookup_never_creates_a_missing_auth_db() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + let mut lookup = sandbox.command("repository-lookup-missing"); + lookup.env(AUTH_DB_ENCRYPTION_KEY_ENV, ""); + wait_success( + "repository-lookup-missing", + lookup.spawn().expect("spawn lookup"), + ); + assert!(sandbox.auth_db_files().is_empty()); +} + +#[test] +fn concurrent_telemetry_headers_on_missing_auth_db_fail_closed_without_creating_it() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + let repository = git_repository_with_remote( + sandbox.dir.path(), + HEADERS_REPOSITORY, + "https://github.com/acme/bound-repo", + ); + + let helpers = (0..CONCURRENT_PROCESSES) + .map(|_| spawn_headers(&sandbox, &repository)) + .collect::>(); + for helper in helpers { + assert_headers_unavailable(helper); + } + assert!(sandbox.auth_db_files().is_empty()); +} + +#[test] +fn telemetry_headers_serve_only_the_bound_repository() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + let bound = git_repository_with_remote( + sandbox.dir.path(), + HEADERS_REPOSITORY, + "https://github.com/acme/bound-repo", + ); + let unbound = git_repository_with_remote( + sandbox.dir.path(), + UNBOUND_REPOSITORY, + "https://github.com/acme/unbound-repo", + ); + sandbox.seed(); + let mut bind = sandbox.command("bind"); + bind.current_dir(&bound); + wait_success("bind", bind.spawn().expect("spawn bind")); + + let output = spawn_headers(&sandbox, &bound) + .wait_with_output() + .expect("bound headers output"); + assert!(output.status.success(), "bound headers failed"); + assert!(String::from_utf8_lossy(&output.stdout).contains(HEADERS_AUTHORIZATION)); + assert_headers_unavailable(spawn_headers(&sandbox, &unbound)); +} + +fn spawn_headers(sandbox: &Sandbox, repository: &Path) -> Child { + let mut helper = sandbox.command("headers"); + helper.current_dir(repository); + helper.spawn().expect("spawn headers") +} + +fn assert_headers_unavailable(helper: Child) { + let output = helper.wait_with_output().expect("headers output"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "headers child failed: {stderr}"); + assert!(stdout.contains("1 passed"), "{stdout}"); + assert!(!stdout.contains("Authorization"), "{stdout}"); + assert!(stderr.contains(HEADERS_UNAVAILABLE), "{stderr}"); +} + +#[test] +fn every_claude_hook_entry_point_records_its_command_identity() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + std::fs::create_dir_all(sandbox.dir.path().join("work")).expect("work dir"); + tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .expect("test runtime") + .block_on(export_hook_entry_spans(&sandbox)); +} + +fn hook_stdin_cases(command: ClaudeHookCommand) -> [(Vec, String); 4] { + let tool_use_id = format!("toolu_entry_{}", command.as_str()); + let valid = serde_json::json!({ + "hook_event_name": "PreToolUse", + "tool_name": "Bash", + "session_id": "session-entry", + "tool_use_id": tool_use_id, + "tool_input": { "command": "true" }, + }) + .to_string(); + [ + (valid.into_bytes(), tool_use_id), + (Vec::new(), String::new()), + ( + format!("{{\"tool_use_id\":\"{STDIN_SECRET}").into_bytes(), + String::new(), + ), + ( + [b"\xff\xfe".as_slice(), STDIN_SECRET.as_bytes()].concat(), + String::new(), + ), + ] +} + +async fn export_hook_entry_spans(sandbox: &Sandbox) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let address = listener.local_addr().expect("addr"); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let receiver = tokio::spawn(capture_requests_until(listener, stopped)); + + let mut expected = Vec::new(); + let mut hooks = Vec::new(); + for (index, command) in HOOK_COMMANDS.iter().enumerate() { + for (stdin, tool_use_id) in hook_stdin_cases(*command) { + let mut child = sandbox + .command("hook-entry") + .current_dir(sandbox.dir.path().join("work")) + .env(COLLECTOR_ENV, address.to_string()) + .env(HOOK_INDEX_ENV, index.to_string()) + .stdin(Stdio::piped()) + .spawn() + .expect("spawn hook entry"); + let mut child_stdin = child.stdin.take().expect("hook entry stdin"); + child_stdin.write_all(&stdin).expect("write hook stdin"); + drop(child_stdin); + hooks.push(child); + expected.push((command.as_str().to_string(), tool_use_id)); + } + } + tokio::task::spawn_blocking(move || { + for child in hooks { + wait_success("hook-entry", child); + } + }) + .await + .expect("hook entry children join"); + stop.send(()).expect("stop collector"); + + let requests = receiver.await.expect("collector join"); + for request in &requests { + let body = String::from_utf8_lossy(&request.body); + assert!(!body.contains(STDIN_SECRET), "hook payload leaked"); + assert!(!body.contains("bao-token-value"), "token leaked"); + } + let mut observed = Vec::new(); + for span in trace_spans(&requests) + .iter() + .filter(|span| span.name == "sce.hook") + { + let attribute = |key: &str| { + span.attributes + .iter() + .find(|(name, _)| name == key) + .map(|(_, value)| value.clone()) + .unwrap_or_default() + }; + let command = attribute("sce.hook.command"); + let tool_use_id = attribute("sce.claude.tool_use_id"); + let outcome = attribute("sce.outcome"); + if FAIL_OPEN_HOOK_COMMANDS + .iter() + .any(|hook| hook.as_str() == command) + { + assert_eq!(outcome, "success", "{command} must fail open"); + } + if command == ClaudeHookCommand::PolicyBash.as_str() && tool_use_id.is_empty() { + assert_eq!(outcome, "failure", "malformed policy-bash input must fail"); + } + observed.push((command, tool_use_id)); + } + observed.sort(); + expected.sort(); + assert_eq!(observed, expected); +} + +async fn run_hook_entry_child() { + let index = std::env::var(HOOK_INDEX_ENV) + .expect("hook index") + .parse::() + .expect("numeric hook index"); + let address = std::env::var(COLLECTOR_ENV) + .expect("collector address") + .parse() + .expect("socket address"); + let telemetry = loopback_hook_telemetry(address, "bao-token-value"); + let command = HOOK_COMMANDS[index]; + let _ = telemetry + .with_default_subscriber(&mut || async move { + let subcommand = match command { + ClaudeHookCommand::DiffTrace => HookSubcommand::DiffTrace, + ClaudeHookCommand::ConversationTrace => HookSubcommand::ConversationTrace, + ClaudeHookCommand::ClaudeMutationScope => HookSubcommand::ClaudeMutationScope, + ClaudeHookCommand::ClaudeModelState => HookSubcommand::ClaudeModelState, + ClaudeHookCommand::PolicyBash => { + return run_bash_policy_request(&BashPolicyRequest { + input: PolicyInputMode::ClaudePreToolUse, + output: PolicyOutputMode::ClaudeHook, + }); + } + }; + run_hooks_subcommand(&subcommand, None::<&NoopLogger>) + .await + .map_err(CliError::runtime) + }) + .await; + telemetry.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn auth_db_child_process() { + let Ok(role) = std::env::var(ROLE_ENV) else { + return; + }; + match role.as_str() { + "seed" => seed().await, + "hold-open" => { + let _db = AuthDb::new().await.expect("holder opens auth DB"); + wait_for_release(); + } + "hold-write" => { + let db = AuthDb::new().await.expect("writer opens auth DB"); + db.execute("BEGIN IMMEDIATE", ()) + .await + .expect("writer takes write lock"); + wait_for_release(); + db.execute("ROLLBACK", ()).await.expect("writer rollback"); + } + "read" => { + let credential = load_telemetry_credential() + .await + .expect("credential lookup succeeds") + .expect("credential present"); + println!("{TOKEN_PREFIX}{}", credential.token); + } + "telemetry-health" => { + let label = match inspect_health().await { + ManagedTelemetryHealth::Healthy => "healthy", + ManagedTelemetryHealth::Disabled(reason) => reason.as_str(), + }; + println!("{HEALTH_PREFIX}{label}"); + } + "lookup-missing" => { + assert_eq!( + load_telemetry_credential() + .await + .expect("missing auth DB is not an error"), + None + ); + assert!(!auth_db_path().expect("auth DB path").exists()); + } + "setup-fresh" | "setup-existing" => { + use crate::services::lifecycle::ServiceLifecycle; + let path = auth_db_path().expect("auth DB path"); + let before = std::fs::read(&path).ok(); + assert_eq!(before.is_some(), role == "setup-existing"); + super::lifecycle::AuthDbLifecycle + .setup(&NoRepositoryContext) + .await + .expect("setup needs no auth DB key"); + assert!(path.parent().expect("auth DB parent").is_dir()); + assert_eq!(std::fs::read(&path).ok(), before); + } + "create-schemaless" => { + let db = EncryptedTursoDb::::new() + .await + .expect("schemaless auth DB opens"); + assert!(applied_migrations(&db).await.is_empty()); + } + "init" => { + let db = AuthDb::new().await.expect("initializer opens auth DB"); + assert_eq!(applied_migrations(&db).await.len(), expected_migrations()); + } + "verify-schema" => { + let db = AuthDb::new().await.expect("verifier opens auth DB"); + assert_eq!(applied_migrations(&db).await.len(), expected_migrations()); + let check = db + .query_map("PRAGMA integrity_check", (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("integrity check"); + assert_eq!(check, vec!["ok".to_string()]); + } + "upgrade" => upgrade_outdated_schema().await, + "partial" => reject_partial_schema().await, + "verify-rows" => print_verified_rows().await, + "coordinated-writer" => run_coordinated_writer_child().await, + "hook-entry" => run_hook_entry_child().await, + other => run_binding_child(other).await, + } +} + +async fn run_binding_child(role: &str) { + match role { + "repository-lookup-missing" => { + assert_eq!( + load_repository_telemetry_credential("repo-missing".to_string()) + .await + .expect("missing auth DB is not an error"), + None + ); + assert_eq!( + load_telemetry_credential() + .await + .expect("missing auth DB is not an error"), + None + ); + } + "bind" => { + let repository_id = repository_id_for(&std::env::current_dir().expect("cwd")) + .expect("repository identity"); + assert!( + bind_repository_telemetry(repository_id, RepositoryBindingMode::Enroll) + .await + .expect("binding succeeds") + ); + } + "headers" => { + let code = Box::pin(crate::app::run( + ["sce", "auth", "telemetry-headers"] + .into_iter() + .map(String::from), + )) + .await; + println!(); + if code != std::process::ExitCode::SUCCESS { + assert_eq!(code, std::process::ExitCode::from(4)); + } + } + other => panic!("unknown child role {other}"), + } +} + +async fn run_coordinated_writer_child() { + let db = AuthDb::new() + .await + .expect("coordinated writer opens auth DB"); + wait_for_release(); + for row in writer_rows() { + db.execute(INSERT_BINDING_SQL, (row.as_str(), USER_ID, WORKSPACE_ID)) + .await + .unwrap_or_else(|error| panic!("writer failed at {row}: {error:#}")); + println!("{COMMITTED_PREFIX}{row}"); + tokio::time::sleep(WRITER_PAUSE).await; + } + wait_for_release(); +} + +async fn seed() { + let db = AuthDb::new().await.expect("seed opens auth DB"); + db.execute( + "INSERT OR REPLACE INTO auth_credentials \ + (id, access_token, token_type, expires_in, refresh_token, scope, stored_at_unix_seconds) \ + VALUES (1, ?1, 'Bearer', 3600, 'refresh', NULL, 1)", + (test_access_token(USER_ID, Some(WORKSPACE_ID)),), + ) + .await + .expect("session seeds"); + drop(db); + save_telemetry_credential(&TelemetryCredential { + token: TOKEN.to_string(), + accessor: "accessor-multiprocess".to_string(), + expires_at_unix_seconds: FAR_FUTURE, + identity: TelemetryIdentity { + user_id: USER_ID.to_string(), + workspace_id: WORKSPACE_ID.to_string(), + }, + }) + .await + .expect("telemetry credential seeds"); + crate::services::telemetry_readiness::mark_ready(FAR_FUTURE).expect("readiness seeds"); +} + +fn writer_rows() -> Vec { + let writer = std::env::var(WRITER_ENV).expect("writer id"); + (0..WRITER_ROWS) + .map(|row| format!("{writer}-{row:02}")) + .collect() +} + +async fn print_verified_rows() { + let db = AuthDb::new().await.expect("verifier opens auth DB"); + assert_credential_rows_intact(&db).await; + for row in db + .query_map( + "SELECT repository_id FROM telemetry_repository_bindings ORDER BY repository_id", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("binding rows") + { + println!("{ROW_PREFIX}{row}"); + } +} + +async fn assert_credential_rows_intact(db: &AuthDb) { + let tokens = db + .query_map( + "SELECT token, user_id, workspace_id FROM telemetry_credentials", + (), + |row| { + Ok(( + row.get::(0)?, + row.get::(1)?, + row.get::(2)?, + )) + }, + ) + .await + .expect("telemetry credential rows"); + assert_eq!( + tokens, + vec![( + TOKEN.to_string(), + USER_ID.to_string(), + WORKSPACE_ID.to_string() + )] + ); + let sessions = db + .query_map("SELECT access_token FROM auth_credentials", (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("session rows"); + assert_eq!( + sessions, + vec![test_access_token(USER_ID, Some(WORKSPACE_ID))] + ); + let check = db + .query_map("PRAGMA integrity_check", (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("integrity check"); + assert_eq!(check, vec!["ok".to_string()]); +} + +fn wait_for_release() { + println!("{READY}"); + let mut line = String::new(); + let _ = std::io::stdin().read_line(&mut line); +} + +fn expected_migrations() -> usize { + generated_migrations::AUTH_MIGRATIONS.len() +} + +async fn applied_migrations(db: &EncryptedTursoDb) -> Vec { + db.query_map("SELECT id FROM __sce_migrations ORDER BY id", (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("migration ids") +} + +async fn upgrade_outdated_schema() { + let old = EncryptedTursoDb::::new() + .await + .expect("pre-binding schema opens"); + assert_eq!(applied_migrations(&old).await.len(), PRE_BINDING_MIGRATIONS); + drop(old); + let db = AuthDb::new().await.expect("outdated schema upgrades"); + assert_eq!(applied_migrations(&db).await.len(), expected_migrations()); + db.query_map( + "SELECT user_id, workspace_id FROM telemetry_credentials", + (), + |_| Ok(()), + ) + .await + .expect("binding columns exist"); +} + +async fn reject_partial_schema() { + let old = EncryptedTursoDb::::new() + .await + .expect("pre-binding schema opens"); + old.execute( + "ALTER TABLE telemetry_credentials ADD COLUMN user_id TEXT NOT NULL DEFAULT ''", + (), + ) + .await + .expect("partial migration applies"); + drop(old); + let error = AuthDb::new() + .await + .err() + .expect("partially migrated schema must not open"); + let chain = format!("{error:#}"); + assert!(chain.contains(BINDING_MIGRATION_ID), "{chain}"); + let old = EncryptedTursoDb::::new() + .await + .expect("pre-binding schema reopens"); + assert_eq!(applied_migrations(&old).await.len(), PRE_BINDING_MIGRATIONS); +} diff --git a/cli/src/services/bash_policy.rs b/cli/src/services/bash_policy.rs index c0712bd1a..c4a53b88d 100644 --- a/cli/src/services/bash_policy.rs +++ b/cli/src/services/bash_policy.rs @@ -456,7 +456,11 @@ struct JsonPolicyResult<'a> { } pub fn run_bash_policy_request(request: &BashPolicyRequest) -> Result { + crate::services::managed_telemetry::trace_export::record_claude_hook_command( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::PolicyBash, + ); let stdin_payload = read_stdin_payload()?; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); run_bash_policy_request_from_payload(request, &stdin_payload) } diff --git a/cli/src/services/db/encryption_key.rs b/cli/src/services/db/encryption_key.rs index ef9cba9e1..f79f0243c 100644 --- a/cli/src/services/db/encryption_key.rs +++ b/cli/src/services/db/encryption_key.rs @@ -1,19 +1,8 @@ -//! Encryption key management backed by an env-secret override or the OS credential store. -//! -//! Provides a single entry point to get-or-create a 64-character hex -//! encryption key sourced from `SCE_AUTH_DB_ENCRYPTION_KEY` when set, or -//! stored in the platform-native credential store (macOS Keychain, -//! Linux Secret Service via zbus, Windows Credential Store) otherwise. -//! -//! On first use for a given database name (when the database file does -//! not yet exist), a random 32-byte key is generated, hex-encoded, -//! persisted in the credential store, and returned. Subsequent calls -//! read the key from the credential store. - +use std::fmt; use std::path::Path; use std::sync::{ atomic::{AtomicBool, Ordering}, - OnceLock, + Mutex, OnceLock, PoisonError, }; use std::thread; @@ -21,9 +10,70 @@ use anyhow::{Context, Result}; use keyring_core::Entry; use sha2::{Digest, Sha256}; -/// Environment variable that overrides keyring-backed auth DB encryption. pub const AUTH_DB_ENCRYPTION_KEY_ENV: &str = "SCE_AUTH_DB_ENCRYPTION_KEY"; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CredentialAccess { + Interactive, + NonInteractive, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CredentialAccessUnavailable { + db_name: String, +} + +impl fmt::Display for CredentialAccessUnavailable { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "encryption key for '{}' is not accessible without user interaction. \ + Try: run 'sce auth login' to authorize credential store access.", + self.db_name + ) + } +} + +impl std::error::Error for CredentialAccessUnavailable {} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CredentialAccessDenied { + db_name: String, +} + +impl fmt::Display for CredentialAccessDenied { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "credential store denied access to the encryption key for '{}'", + self.db_name + ) + } +} + +impl std::error::Error for CredentialAccessDenied {} + +#[cfg(target_os = "macos")] +const ERR_SEC_USER_CANCELED: i32 = -128; +#[cfg(target_os = "macos")] +const ERR_SEC_NO_ACCESS_FOR_ITEM: i32 = -25243; +#[cfg(target_os = "macos")] +const ERR_SEC_AUTH_FAILED: i32 = -25293; +#[cfg(target_os = "macos")] +const ERR_SEC_INTERACTION_NOT_ALLOWED: i32 = -25308; +#[cfg(target_os = "macos")] +const ERR_SEC_INTERACTION_REQUIRED: i32 = -25315; +#[cfg(target_os = "macos")] +const MACOS_AUTHORIZATION_FAILURE_CODES: [i32; 5] = [ + ERR_SEC_USER_CANCELED, + ERR_SEC_NO_ACCESS_FOR_ITEM, + ERR_SEC_AUTH_FAILED, + ERR_SEC_INTERACTION_NOT_ALLOWED, + ERR_SEC_INTERACTION_REQUIRED, +]; + +static CREDENTIAL_STORE_ACCESS: Mutex<()> = Mutex::new(()); + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum CredentialStorePlatform { #[cfg(target_os = "linux")] @@ -110,34 +160,42 @@ fn register_default_store() -> Result<()> { Ok(()) } -/// Retrieve or create the encryption key for the named database. -/// -/// The key is stored in the platform-native credential store under the -/// service name `"sce"` with the database name as the user/account -/// identifier. -/// -/// # Arguments -/// * `db_path` — Canonical path to the database file. Used to decide -/// whether this is a first-use (generate new key) or a subsequent -/// access (read existing key). -/// * `db_name` — Logical database name used as the credential store -/// username (e.g. `"auth_db"`, `"agent_trace_db"`). -/// -/// # Returns -/// A 64-character lowercase hex string that is the encryption key. -/// -/// # Errors -/// - Returns an error if the credential store cannot be initialised on -/// the current platform. -/// - Returns an error if the database file exists but the keyring entry -/// is missing (e.g. keyring was cleared on Linux). -/// - Returns an error if key generation or credential store I/O fails. -pub fn get_or_create_encryption_key(db_path: &Path, db_name: &str) -> Result { - get_or_create_encryption_key_with_keyring_resolver( - db_path, - db_name, - get_or_create_keyring_encryption_key, - ) +pub fn get_or_create_encryption_key( + db_path: &Path, + db_name: &str, + access: CredentialAccess, +) -> Result { + match access { + CredentialAccess::Interactive => get_or_create_encryption_key_with_keyring_resolver( + db_path, + db_name, + get_or_create_keyring_encryption_key, + ), + CredentialAccess::NonInteractive => { + get_or_create_encryption_key_with_keyring_resolver(db_path, db_name, |_, db_name| { + read_keyring_encryption_key_without_interaction(db_name) + }) + } + } +} + +fn read_keyring_encryption_key_without_interaction(db_name: &str) -> Result { + let unavailable = || { + anyhow::Error::new(CredentialAccessUnavailable { + db_name: db_name.to_owned(), + }) + }; + ensure_default_store().map_err(|_| unavailable())?; + let _serialized = CREDENTIAL_STORE_ACCESS + .lock() + .unwrap_or_else(PoisonError::into_inner); + #[cfg(target_os = "macos")] + let _interaction_disabled = + security_framework::os::macos::keychain::SecKeychain::disable_user_interaction() + .map_err(|_| unavailable())?; + Entry::new("sce", db_name) + .and_then(|entry| entry.get_password()) + .map_err(|_| unavailable()) } fn get_or_create_encryption_key_with_keyring_resolver( @@ -166,29 +224,39 @@ fn get_or_create_keyring_encryption_key(db_path: &Path, db_name: &str) -> Result ) })?; - // Try to retrieve an existing password from the credential store. - if let Ok(password) = entry.get_password() { - return Ok(password); + let _serialized = CREDENTIAL_STORE_ACCESS + .lock() + .unwrap_or_else(PoisonError::into_inner); + + match entry.get_password() { + Ok(password) => return Ok(password), + Err(keyring_core::Error::NoEntry) => {} + Err(error) => { + return Err(keyring_error(error, db_name)).with_context(|| { + format!( + "failed to read encryption key for '{db_name}' from credential store. {}", + credential_store_unavailable_message(current_credential_store_platform()) + ) + }); + } } - // No existing key was found. If the database file does not exist, - // this is a first-use scenario: generate and store a new key. if !db_path.exists() { let hex_key = generate_key(); - entry.set_password(&hex_key).with_context(|| { - format!( - "failed to store encryption key for '{db_name}' in credential store. {}", - credential_store_unavailable_message(current_credential_store_platform()) - ) - })?; + entry + .set_password(&hex_key) + .map_err(|error| keyring_error(error, db_name)) + .with_context(|| { + format!( + "failed to store encryption key for '{db_name}' in credential store. {}", + credential_store_unavailable_message(current_credential_store_platform()) + ) + })?; return Ok(hex_key); } - // The database file exists but the keyring entry is missing. - // This can happen when credentials were removed from the OS keyring. - // Provide clear remediation. anyhow::bail!( "encryption key for '{db_name}' not found in credential store.\n\ The database file exists at '{}' but no matching credential was found \ @@ -203,6 +271,29 @@ fn get_or_create_keyring_encryption_key(db_path: &Path, db_name: &str) -> Result ); } +fn keyring_error(error: keyring_core::Error, db_name: &str) -> anyhow::Error { + let denied = is_authorization_failure(&error); + let error = anyhow::Error::new(error); + if denied { + error.context(CredentialAccessDenied { + db_name: db_name.to_owned(), + }) + } else { + error + } +} + +fn is_authorization_failure(error: &keyring_core::Error) -> bool { + match error { + keyring_core::Error::NoStorageAccess(_) => true, + #[cfg(target_os = "macos")] + keyring_core::Error::PlatformFailure(platform) => platform + .downcast_ref::() + .is_some_and(|error| MACOS_AUTHORIZATION_FAILURE_CODES.contains(&error.code())), + _ => false, + } +} + fn current_credential_store_platform() -> CredentialStorePlatform { #[cfg(target_os = "linux")] { @@ -291,3 +382,48 @@ fn hex_encode(bytes: &[u8]) -> String { } hex } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn storage_access_failures_are_tagged_as_credential_access_denied() { + let error = keyring_error( + keyring_core::Error::NoStorageAccess(Box::new(std::io::Error::other("locked"))), + "auth_db", + ) + .context("failed to read encryption key"); + assert!(error.is::()); + assert_eq!(error.to_string(), "failed to read encryption key"); + + for untagged in [ + keyring_core::Error::NoEntry, + keyring_core::Error::PlatformFailure(Box::new(std::io::Error::other("other"))), + ] { + assert!(!keyring_error(untagged, "auth_db").is::()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_authorization_status_codes_are_tagged_by_code() { + let platform_failure = |code| { + keyring_core::Error::PlatformFailure(Box::new( + security_framework::base::Error::from_code(code), + )) + }; + for code in MACOS_AUTHORIZATION_FAILURE_CODES { + assert!( + keyring_error(platform_failure(code), "auth_db").is::(), + "{code}" + ); + } + for code in [-50, -25299, -34018] { + assert!( + !keyring_error(platform_failure(code), "auth_db").is::(), + "{code}" + ); + } + } +} diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index 05f9d76d1..67941f18c 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -8,6 +8,7 @@ use std::{ fs, marker::PhantomData, path::{Path, PathBuf}, + sync::atomic::{AtomicBool, Ordering}, }; use anyhow::{Context, Result}; @@ -29,13 +30,19 @@ const ENCRYPTION_CIPHER_AEGIS256: &str = "aegis256"; const CONNECTION_OPEN_RETRY_POLICY: RetryPolicy = RetryPolicy::builtin(3, 1_000, 25, 200); const CONNECTION_OPEN_RETRY_HINT: &str = "retry after the database lock clears; if the issue persists, stop other SCE processes using this database and rerun the command"; const QUERY_RETRY_POLICY: RetryPolicy = RetryPolicy::builtin(5, 200, 25, 100); +const MIGRATION_RETRY_POLICY: RetryPolicy = RetryPolicy::builtin(10, 1_000, 25, 200); const QUERY_RETRY_HINT: &str = "retry after the database lock clears; if the issue persists, stop other SCE processes using this database and rerun the command"; const AGENT_TRACE_DB_BUSY_TIMEOUT_MS: u64 = 1_000; +const AUTH_DB_BUSY_TIMEOUT_MS: u64 = 1_000; +const MIGRATIONS_TABLE_EXISTS_SQL: &str = + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = '__sce_migrations' LIMIT 1"; +const SELECT_APPLIED_MIGRATIONS_SQL: &str = "SELECT id FROM __sce_migrations"; const AGENT_TRACE_DB_CONTENTION_DEADLINE_MS: u64 = 2_250; const AGENT_TRACE_DB_WRITE_CONTENTION_MAX_ATTEMPTS: u32 = 2; const AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS: u64 = 100; pub mod encryption_key; +mod open_lock; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum DatabaseKind { @@ -316,6 +323,42 @@ fn is_retryable_turso_error(error: &turso::Error) -> bool { matches!(error, turso::Error::Busy(_) | turso::Error::BusySnapshot(_)) } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct DatabaseLocked { + db_name: &'static str, +} + +impl std::fmt::Display for DatabaseLocked { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} database is locked by another process", self.db_name) + } +} + +impl std::error::Error for DatabaseLocked {} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct TursoBusy; + +impl std::fmt::Display for TursoBusy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "database is busy") + } +} + +impl std::error::Error for TursoBusy {} + +fn turso_open_error(error: &turso::Error, message: String) -> anyhow::Error { + if is_retryable_turso_error(error) { + anyhow::Error::new(TursoBusy).context(message) + } else { + anyhow::Error::msg(message) + } +} + +fn is_database_lock_conflict(error: &anyhow::Error) -> bool { + error.is::() || error.is::() +} + enum WriteAttemptFailure { Retryable(anyhow::Error), Deterministic(anyhow::Error), @@ -532,6 +575,99 @@ impl TursoConnectionCore { async fn run_migrations(&self) -> Result<()> { run_embedded_migrations(&self.conn, M::db_name(), M::migrations()).await } + + async fn has_pending_migrations(&self) -> Result { + let db_name = M::db_name(); + let mut tables = self + .conn + .query(MIGRATIONS_TABLE_EXISTS_SQL, ()) + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration readiness query failed: {e}"))?; + let table_exists = tables + .next() + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration readiness row fetch failed: {e}"))? + .is_some(); + drop(tables); + if !table_exists { + return Ok(true); + } + + let mut rows = self + .conn + .query(SELECT_APPLIED_MIGRATIONS_SQL, ()) + .await + .map_err(|e| anyhow::anyhow!("{db_name} applied migration query failed: {e}"))?; + let mut applied = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| anyhow::anyhow!("{db_name} applied migration row fetch failed: {e}"))? + { + applied.push( + row.get::(0) + .map_err(|e| anyhow::anyhow!("{db_name} applied migration id invalid: {e}"))?, + ); + } + + Ok(M::migrations() + .iter() + .any(|(id, _)| !applied.iter().any(|applied_id| applied_id == id))) + } + + async fn run_migrations_serialized(&mut self) -> std::result::Result<(), WriteAttemptFailure> { + let db_name = M::db_name(); + let tx = turso::transaction::Transaction::new( + &mut self.conn, + turso::transaction::TransactionBehavior::Immediate, + ) + .await + .map_err(|e| classify_turso_error(db_name, "failed to begin migration transaction", &e))?; + + match run_embedded_migrations(&tx, db_name, M::migrations()).await { + Ok(()) => tx.commit().await.map_err(|e| { + classify_turso_error(db_name, "failed to commit migration transaction", &e) + }), + Err(error) => { + let _ = tx.rollback().await; + Err(WriteAttemptFailure::Deterministic(error)) + } + } + } + + async fn ensure_migrations_current(&mut self) -> Result<()> { + if !self.has_pending_migrations().await? { + return Ok(()); + } + + let db_name = M::db_name(); + let operation_name = format!("migrate {db_name} database"); + run_with_retry_elapsed( + MIGRATION_RETRY_POLICY, + &operation_name, + QUERY_RETRY_HINT, + MigrationAttempt(self), + ) + .await??; + + if self.has_pending_migrations().await? { + anyhow::bail!("{db_name} schema is incomplete after running migrations"); + } + + Ok(()) + } +} + +struct MigrationAttempt<'a, M: DbSpec>(&'a mut TursoConnectionCore); + +impl RetryOperation> for MigrationAttempt<'_, M> { + async fn run(&mut self, _attempt: u32) -> Result> { + match self.0.run_migrations_serialized().await { + Ok(()) => Ok(Ok(())), + Err(WriteAttemptFailure::Retryable(error)) => Err(error), + Err(WriteAttemptFailure::Deterministic(error)) => Ok(Err(error)), + } + } } fn resolve_connection_open_retry_policy() -> RetryPolicy { @@ -566,20 +702,6 @@ fn resolve_query_retry_policy() -> RetryPolicy { QUERY_RETRY_POLICY } -/// Resolve the Turso busy timeout applied to every connection opened for `M`. -/// -/// Multiprocess WAL provides cross-process correctness and locking; it does -/// not wait for a contended lock. The busy timeout is Turso's own wait policy -/// for a `Busy` result: while another process holds the write lock, Turso -/// sleeps in short phases until the lock clears or the timeout elapses, and -/// only then returns `Busy`. Because `Transaction::new(.., Immediate)` -/// runs `BEGIN IMMEDIATE` through `Connection::execute` on the same connection, -/// the wait also covers writer-lock acquisition. The timeout is connection-wide. -/// -/// Only the Agent Trace DB has a busy timeout, taken from -/// `policies.database_retry.agent_trace_db.busy_timeout_ms` when configured; -/// every other database resolves to zero, which leaves Turso's busy handler -/// unset. fn resolve_busy_timeout() -> std::time::Duration { busy_timeout_from_config::(crate::services::config::get_database_retry_config()) } @@ -587,6 +709,9 @@ fn resolve_busy_timeout() -> std::time::Duration { fn busy_timeout_from_config( config: Option<&DatabaseRetryConfig>, ) -> std::time::Duration { + if M::KIND == DatabaseKind::Auth { + return std::time::Duration::from_millis(AUTH_DB_BUSY_TIMEOUT_MS); + } agent_trace_db_millis::( config, |db| db.busy_timeout_ms, @@ -788,16 +913,27 @@ fn apply_busy_timeout( .map_err(|e| anyhow::anyhow!("failed to set {db_name} database busy timeout: {e}")) } +async fn resolve_encryption_key( + db_path: &Path, + db_name: &str, + access: encryption_key::CredentialAccess, +) -> Result { + let key_path = db_path.to_path_buf(); + let key_name = db_name.to_owned(); + tokio::task::spawn_blocking(move || { + encryption_key::get_or_create_encryption_key(&key_path, &key_name, access) + }) + .await + .context("encryption-key resolution worker failed")? +} + pub struct TursoDb { core: TursoConnectionCore, } -/// Generic encrypted Turso database adapter. -/// -/// Mirrors the structural seams of [`TursoDb`] while reserving encrypted local -/// database initialization for services that require at-rest encryption. pub struct EncryptedTursoDb { core: TursoConnectionCore, + _open_lock: fs::File, } impl TursoDb { @@ -1266,70 +1402,124 @@ impl TursoDb { } impl EncryptedTursoDb { - /// Open or create the encrypted database at the spec-provided canonical - /// path. - /// - /// This constructor is the encrypted counterpart to [`TursoDb::new`] and - /// uses a strict encrypted local-builder path. pub async fn new() -> Result { let db_name = M::db_name(); let db_path = M::db_path().with_context(|| format!("failed to resolve {db_name} path"))?; - let key_path = db_path.clone(); - let key_name = db_name.to_owned(); - let encryption_key = tokio::task::spawn_blocking(move || { - encryption_key::get_or_create_encryption_key(&key_path, &key_name) - }) - .await - .context("encryption-key resolution worker failed")??; - + let encryption_key = resolve_encryption_key( + &db_path, + db_name, + encryption_key::CredentialAccess::Interactive, + ) + .await?; ensure_db_parent_dir(db_name, &db_path)?; + let open_lock = open_lock::open_lock_file(&db_path, true)? + .ok_or_else(|| anyhow::anyhow!("failed to create {db_name} database file"))?; + Box::pin(Self::open_locked(db_path, encryption_key, open_lock)).await + } + + pub async fn open_existing_at( + db_path: impl AsRef, + access: encryption_key::CredentialAccess, + ) -> Result> { + let db_path = db_path.as_ref().to_path_buf(); + let Some(open_lock) = open_lock::open_lock_file(&db_path, false)? else { + return Ok(None); + }; + let encryption_key = resolve_encryption_key(&db_path, M::db_name(), access).await?; + Box::pin(Self::open_locked(db_path, encryption_key, open_lock)) + .await + .map(Some) + } + async fn open_locked( + db_path: PathBuf, + encryption_key: String, + open_lock: fs::File, + ) -> Result { + let db_name = M::db_name(); let retry_policy = resolve_connection_open_retry_policy::(); + let busy_timeout = resolve_busy_timeout::(); let operation_name = format!("open encrypted {db_name} database connection"); + let last_attempt_locked = AtomicBool::new(false); + let conn = run_with_retry_elapsed( retry_policy, &operation_name, CONNECTION_OPEN_RETRY_HINT, |_| async { - let path_str = db_path.to_str().ok_or_else(|| { - anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) - })?; - - let encryption_opts = turso::EncryptionOpts { - hexkey: encryption_key.clone(), - cipher: ENCRYPTION_CIPHER_AEGIS256.to_string(), - }; - - let db = turso::Builder::new_local(path_str) - .experimental_encryption(true) - .with_encryption(encryption_opts) - .build() - .await - .map_err(|e| { - anyhow::anyhow!( - "failed to open encrypted {db_name} database at {} with cipher {ENCRYPTION_CIPHER_AEGIS256}. Try: verify the credential store encryption key is valid and that local Turso encryption support is available: {e}", - db_path.display() - ) - })?; - - db.connect().map_err(|e| { - anyhow::anyhow!("failed to connect to encrypted {db_name} database: {e}") - }) + let attempt = + Self::connect_locked(&db_path, &encryption_key, &open_lock, busy_timeout).await; + last_attempt_locked.store( + attempt.as_ref().is_err_and(is_database_lock_conflict), + Ordering::Relaxed, + ); + attempt }, - ).await?; + ) + .await + .map_err(|error| { + if last_attempt_locked.load(Ordering::Relaxed) { + anyhow::Error::new(DatabaseLocked { db_name }).context(format!("{error:#}")) + } else { + error + } + })?; - let db = Self { + let mut db = Self { core: TursoConnectionCore::new(conn), + _open_lock: open_lock, }; - db.run_migrations() + db.core + .ensure_migrations_current() .await .with_context(|| format!("failed to run {db_name} migrations"))?; Ok(db) } + async fn connect_locked( + db_path: &Path, + encryption_key: &str, + open_lock: &fs::File, + busy_timeout: std::time::Duration, + ) -> Result { + let db_name = M::db_name(); + open_lock::lock_shared(open_lock, db_name, db_path)?; + let _cold_open = open_lock::lock_cold_open(db_path, db_name).await?; + let path_str = db_path.to_str().ok_or_else(|| { + anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) + })?; + + let encryption_opts = turso::EncryptionOpts { + hexkey: encryption_key.to_owned(), + cipher: ENCRYPTION_CIPHER_AEGIS256.to_string(), + }; + + let db = turso::Builder::new_local(path_str) + .experimental_multiprocess_wal(true) + .experimental_encryption(true) + .with_encryption(encryption_opts) + .build() + .await + .map_err(|e| { + turso_open_error(&e, format!( + "failed to open encrypted {db_name} database at {} with cipher {ENCRYPTION_CIPHER_AEGIS256}. Try: verify the credential store encryption key is valid and that local Turso encryption support is available: {e}", + db_path.display() + )) + })?; + + let conn = db.connect().map_err(|e| { + turso_open_error( + &e, + format!("failed to connect to encrypted {db_name} database: {e}"), + ) + })?; + apply_busy_timeout(&conn, db_name, busy_timeout)?; + Ok(conn) + } + /// Execute a SQL statement that does not return rows. /// /// # Arguments @@ -1417,16 +1607,6 @@ impl EncryptedTursoDb { Ok(results) } - - /// Run all embedded migrations in order. - /// - /// Applied migration IDs are recorded in `__sce_migrations` so later - /// initializations apply only migrations that were not already recorded. - /// Existing databases without migration metadata are brought forward by - /// re-applying the current idempotent migration set and recording each ID. - pub async fn run_migrations(&self) -> Result<()> { - self.core.run_migrations().await - } } #[cfg(test)] @@ -1458,7 +1638,10 @@ mod database_kind_tests { busy_timeout_from_config::(None), Duration::ZERO ); - assert_eq!(busy_timeout_from_config::(None), Duration::ZERO); + assert_eq!( + busy_timeout_from_config::(None), + Duration::from_secs(1) + ); assert_eq!( busy_timeout_from_config::(None), Duration::from_secs(1) diff --git a/cli/src/services/db/open_lock.rs b/cli/src/services/db/open_lock.rs new file mode 100644 index 000000000..f87c27cb8 --- /dev/null +++ b/cli/src/services/db/open_lock.rs @@ -0,0 +1,140 @@ +use std::fmt; +use std::fs::{File, OpenOptions}; +use std::io; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Context, Result}; + +const COLD_OPEN_LOCK_SUFFIX: &str = "-open.lock"; + +#[cfg(target_os = "linux")] +const SHARED_LOCK_COMMAND: libc::c_int = libc::F_OFD_SETLK; +#[cfg(all(unix, not(any(target_os = "linux", target_os = "macos"))))] +const SHARED_LOCK_COMMAND: libc::c_int = libc::F_SETLK; + +pub(super) fn open_lock_file(db_path: &Path, create: bool) -> Result> { + match OpenOptions::new() + .read(true) + .write(create) + .create(create) + .truncate(false) + .open(db_path) + { + Ok(file) => Ok(Some(file)), + Err(error) if !create && error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(anyhow!( + "failed to open database file {} for its compatibility lock: {error}", + db_path.display() + )), + } +} + +#[derive(Debug)] +pub(super) struct SharedLockConflict { + db_name: String, + db_path: PathBuf, +} + +impl fmt::Display for SharedLockConflict { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{} database at {} is open in an older SCE process that does not support shared access. Try: wait for older SCE processes to exit, upgrade every installed 'sce' binary, and rerun the command", + self.db_name, + self.db_path.display() + ) + } +} + +impl std::error::Error for SharedLockConflict {} + +pub(super) fn lock_shared(file: &File, db_name: &str, db_path: &Path) -> Result<()> { + match try_lock_shared(file) { + Ok(()) => Ok(()), + Err(error) if is_lock_conflict(&error) => Err(SharedLockConflict { + db_name: db_name.to_owned(), + db_path: db_path.to_path_buf(), + } + .into()), + Err(error) => Err(anyhow!( + "failed to lock {db_name} database at {}: {error}", + db_path.display() + )), + } +} + +fn cold_open_lock_path(db_path: &Path) -> PathBuf { + let mut path = db_path.as_os_str().to_owned(); + path.push(COLD_OPEN_LOCK_SUFFIX); + PathBuf::from(path) +} + +pub(super) async fn lock_cold_open(db_path: &Path, db_name: &str) -> Result { + let lock_path = cold_open_lock_path(db_path); + let db_name = db_name.to_owned(); + tokio::task::spawn_blocking(move || { + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .and_then(|file| file.lock().map(|()| file)) + .map_err(|error| { + anyhow!( + "failed to serialize opening the {db_name} database with {}: {error}", + lock_path.display() + ) + }) + }) + .await + .context("database cold-open lock worker failed")? +} + +#[cfg(target_os = "macos")] +fn try_lock_shared(file: &File) -> io::Result<()> { + use std::os::fd::AsRawFd; + + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH | libc::LOCK_NB) } == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +#[cfg(all(unix, not(target_os = "macos")))] +fn try_lock_shared(file: &File) -> io::Result<()> { + use std::os::fd::AsRawFd; + + let mut lock: libc::flock = unsafe { std::mem::zeroed() }; + lock.l_type = lock_field(libc::F_RDLCK)?; + lock.l_whence = lock_field(libc::SEEK_SET)?; + if unsafe { libc::fcntl(file.as_raw_fd(), SHARED_LOCK_COMMAND, &raw const lock) } == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +#[cfg(all(unix, not(target_os = "macos")))] +fn lock_field(value: impl TryInto) -> io::Result { + value + .try_into() + .map_err(|_| io::Error::from(io::ErrorKind::InvalidInput)) +} + +#[cfg(not(unix))] +fn try_lock_shared(_file: &File) -> io::Result<()> { + Ok(()) +} + +#[cfg(unix)] +fn is_lock_conflict(error: &io::Error) -> bool { + matches!( + error.raw_os_error(), + Some(code) if code == libc::EWOULDBLOCK || code == libc::EAGAIN || code == libc::EACCES + ) +} + +#[cfg(not(unix))] +fn is_lock_conflict(_error: &io::Error) -> bool { + false +} diff --git a/cli/src/services/default_paths.rs b/cli/src/services/default_paths.rs index 86ec44208..5f9a3eae3 100644 --- a/cli/src/services/default_paths.rs +++ b/cli/src/services/default_paths.rs @@ -250,21 +250,14 @@ pub fn local_db_path() -> anyhow::Result { Ok(sce_state_dir()?.join("local.db")) } -/// Returns the canonical path to the encrypted auth Turso database file. -/// -/// The path is `/sce/auth.db`, where `state_root` comes -/// from the shared default-path catalog (`XDG_STATE_HOME` or platform -/// equivalent). pub fn auth_db_path() -> anyhow::Result { Ok(sce_state_dir()?.join("auth.db")) } -/// Returns the canonical repository-scoped Agent Trace Turso database file path. -/// -/// The path is `/sce/repos//agent-trace.db`, where -/// `state_root` comes from the shared default-path catalog (`XDG_STATE_HOME` or -/// platform equivalent) and `repository_id` is the stable repository identity -/// hash from `services::repository_identity`. +pub fn telemetry_readiness_path() -> anyhow::Result { + Ok(sce_state_dir()?.join("telemetry-ready.json")) +} + pub fn agent_trace_db_path_for_repository(repository_id: &str) -> anyhow::Result { let state_root = resolve_sce_default_locations()? .roots() diff --git a/cli/src/services/doctor/managed_telemetry.rs b/cli/src/services/doctor/managed_telemetry.rs new file mode 100644 index 000000000..825e67ee9 --- /dev/null +++ b/cli/src/services/doctor/managed_telemetry.rs @@ -0,0 +1,336 @@ +use serde_json::{json, Value}; + +use crate::services::error::{CredentialStorePlatform, CURRENT_CREDENTIAL_STORE_PLATFORM}; +use crate::services::managed_telemetry::{ + ManagedTelemetryHealth, TelemetryFallbackReason, CREDENTIAL_LOOKUP_DEADLINE, +}; + +use super::types::{ + DoctorProblem, ProblemCategory, ProblemFixability, ProblemKind, ProblemSeverity, +}; + +const STATE_HEALTHY: &str = "healthy"; +const STATE_NOT_CONFIGURED: &str = "not_configured"; +const STATE_DISABLED: &str = "disabled"; +const SEVERITY_WARNING: &str = "warning"; +const NEXT_ACTION_AUTH_LOGIN: &str = "auth_login"; +const NEXT_ACTION_MANUAL_STEPS: &str = "manual_steps"; +pub(super) const HOOK_IMPACT: &str = + "Claude Code hooks continue working, but telemetry export is disabled."; +const MACOS_INACCESSIBLE_SUMMARY: &str = + "SCE cannot access its encryption key in macOS Keychain without user interaction."; +const MACOS_INACCESSIBLE_REMEDIATION: &str = "To fix: + 1. Run `sce auth login` in an interactive terminal. + 2. When macOS requests access to the SCE Keychain item, + select \"Always Allow\". + 3. Run `sce doctor` again to verify access."; +const RELOGIN_REMEDIATION: &str = "To fix: + 1. Run `sce auth login` in an interactive terminal to provision + a new telemetry credential. + 2. Run `sce doctor` again to verify access."; +const DATABASE_ERROR_REMEDIATION: &str = "To fix: + 1. Make sure no older `sce` executable is running; mixed versions + cannot share the auth DB. + 2. Run `sce doctor` again. + 3. If the problem persists, check the SCE log files for + `sce.telemetry.fallback` events with reason `database_error`."; +const PATH_RESOLUTION_REMEDIATION: &str = "To fix: + 1. Make sure HOME (and XDG_STATE_HOME, if set) point to a writable directory. + 2. Run `sce doctor` again."; +const EXPORTER_INIT_REMEDIATION: &str = "To fix: + 1. Run `sce doctor` again. + 2. If the problem persists, check the SCE log files for + `sce.telemetry.fallback` events with reason `exporter_init_failed`."; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct ManagedTelemetryDiagnostic { + pub(super) reason: &'static str, + pub(super) kind: &'static str, + pub(super) summary: String, + pub(super) remediation: String, + pub(super) next_action: &'static str, +} + +pub(super) fn diagnostic(health: ManagedTelemetryHealth) -> Option { + diagnostic_for_platform(health, CURRENT_CREDENTIAL_STORE_PLATFORM) +} + +fn diagnostic_for_platform( + health: ManagedTelemetryHealth, + platform: CredentialStorePlatform, +) -> Option { + let ManagedTelemetryHealth::Disabled(reason) = health else { + return None; + }; + let (kind, summary, remediation, next_action) = match reason { + TelemetryFallbackReason::NotConfigured => return None, + TelemetryFallbackReason::CredentialInaccessible => { + let (summary, remediation) = inaccessible_guidance(platform); + ( + "managed_telemetry_credential_inaccessible", + summary, + remediation, + NEXT_ACTION_AUTH_LOGIN, + ) + } + TelemetryFallbackReason::CredentialUnavailable => ( + "managed_telemetry_credential_unavailable", + "Managed telemetry was configured, but its credential is no longer available in the auth DB." + .to_string(), + RELOGIN_REMEDIATION.to_string(), + NEXT_ACTION_AUTH_LOGIN, + ), + TelemetryFallbackReason::CredentialExpired => ( + "managed_telemetry_credential_expired", + "The managed telemetry credential has expired.".to_string(), + RELOGIN_REMEDIATION.to_string(), + NEXT_ACTION_AUTH_LOGIN, + ), + TelemetryFallbackReason::LookupTimeout => ( + "managed_telemetry_lookup_timeout", + format!( + "The managed telemetry credential lookup did not finish within {} seconds.", + CREDENTIAL_LOOKUP_DEADLINE.as_secs() + ), + format!( + "To fix:\n 1. Run `sce doctor` again; another SCE process may have held the auth DB.\n 2. If the timeout persists, make sure {} is responsive and unlocked.", + platform.store_name() + ), + NEXT_ACTION_MANUAL_STEPS, + ), + TelemetryFallbackReason::DatabaseError => ( + "managed_telemetry_database_error", + "The auth DB could not be read to load the managed telemetry credential.".to_string(), + DATABASE_ERROR_REMEDIATION.to_string(), + NEXT_ACTION_MANUAL_STEPS, + ), + TelemetryFallbackReason::PathResolutionError => ( + "managed_telemetry_path_resolution_error", + "The SCE state directory could not be resolved, so the managed telemetry credential could not be located." + .to_string(), + PATH_RESOLUTION_REMEDIATION.to_string(), + NEXT_ACTION_MANUAL_STEPS, + ), + TelemetryFallbackReason::ExporterInitFailed => ( + "managed_telemetry_exporter_init_failed", + "The managed telemetry exporter could not be initialized.".to_string(), + EXPORTER_INIT_REMEDIATION.to_string(), + NEXT_ACTION_MANUAL_STEPS, + ), + }; + Some(ManagedTelemetryDiagnostic { + reason: reason.as_str(), + kind, + summary, + remediation, + next_action, + }) +} + +fn inaccessible_guidance(platform: CredentialStorePlatform) -> (String, String) { + if platform == CredentialStorePlatform::Macos { + return ( + MACOS_INACCESSIBLE_SUMMARY.to_string(), + MACOS_INACCESSIBLE_REMEDIATION.to_string(), + ); + } + let store = platform.store_name(); + ( + format!("SCE cannot access its encryption key in {store} without user interaction."), + format!( + "To fix:\n 1. Make sure {store} is available and unlocked.\n 2. Run `sce auth login` in an interactive terminal.\n 3. Run `sce doctor` again to verify access." + ), + ) +} + +pub(super) fn problem(health: ManagedTelemetryHealth) -> Option { + diagnostic(health).map(|diagnostic| DoctorProblem { + kind: ProblemKind::ManagedTelemetryUnavailable, + category: ProblemCategory::GlobalState, + severity: ProblemSeverity::Warning, + fixability: ProblemFixability::ManualOnly, + summary: format!("{} {HOOK_IMPACT}", diagnostic.summary), + remediation: diagnostic.remediation, + next_action: diagnostic.next_action, + scope: None, + mutation_scope_target: None, + }) +} + +pub(super) fn json(health: ManagedTelemetryHealth) -> Value { + match (health, diagnostic(health)) { + (_, Some(diagnostic)) => json!({ + "state": STATE_DISABLED, + "reason": diagnostic.reason, + "kind": diagnostic.kind, + "severity": SEVERITY_WARNING, + "next_action": diagnostic.next_action, + }), + (ManagedTelemetryHealth::Healthy, None) => json!({ + "state": STATE_HEALTHY, + "reason": null, + "kind": null, + "severity": null, + "next_action": null, + }), + (ManagedTelemetryHealth::Disabled(reason), None) => json!({ + "state": STATE_NOT_CONFIGURED, + "reason": reason.as_str(), + "kind": null, + "severity": null, + "next_action": null, + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn disabled(reason: TelemetryFallbackReason) -> ManagedTelemetryHealth { + ManagedTelemetryHealth::Disabled(reason) + } + + #[test] + fn not_configured_and_healthy_telemetry_report_no_problem() { + assert_eq!(problem(ManagedTelemetryHealth::Healthy), None); + assert_eq!( + problem(disabled(TelemetryFallbackReason::NotConfigured)), + None + ); + assert_eq!( + json(ManagedTelemetryHealth::Healthy), + json!({ + "state": "healthy", + "reason": null, + "kind": null, + "severity": null, + "next_action": null, + }) + ); + assert_eq!( + json(disabled(TelemetryFallbackReason::NotConfigured)), + json!({ + "state": "not_configured", + "reason": "not_configured", + "kind": null, + "severity": null, + "next_action": null, + }) + ); + } + + #[test] + fn macos_inaccessible_credential_is_an_actionable_always_allow_warning() { + let diagnostic = diagnostic_for_platform( + disabled(TelemetryFallbackReason::CredentialInaccessible), + CredentialStorePlatform::Macos, + ) + .expect("inaccessible credentials are diagnosed"); + assert_eq!(diagnostic.reason, "credential_inaccessible"); + assert_eq!(diagnostic.kind, "managed_telemetry_credential_inaccessible"); + assert_eq!(diagnostic.next_action, "auth_login"); + assert_eq!( + diagnostic.summary, + "SCE cannot access its encryption key in macOS Keychain without user interaction." + ); + assert_eq!(diagnostic.remediation, MACOS_INACCESSIBLE_REMEDIATION); + assert!(diagnostic.remediation.contains("select \"Always Allow\".")); + } + + #[test] + fn always_allow_is_only_recommended_for_macos_keychain_authorization() { + use CredentialStorePlatform::{Linux, Macos, Other, Windows}; + use TelemetryFallbackReason::{CredentialInaccessible, LookupTimeout}; + + for (platform, reason, expects_always_allow) in [ + (Macos, CredentialInaccessible, true), + (Macos, LookupTimeout, false), + (Linux, CredentialInaccessible, false), + (Windows, CredentialInaccessible, false), + (Other, CredentialInaccessible, false), + (Linux, LookupTimeout, false), + ] { + let diagnostic = diagnostic_for_platform(disabled(reason), platform) + .expect("disabled telemetry is diagnosed"); + let text = format!("{} {}", diagnostic.summary, diagnostic.remediation); + assert_eq!( + text.contains("Always Allow"), + expects_always_allow, + "{platform:?}/{reason:?}: {text}" + ); + assert_eq!( + text.contains("macOS"), + platform == Macos, + "{platform:?}/{reason:?}: {text}" + ); + } + } + + #[test] + fn every_disabled_reason_has_a_stable_kind_and_safe_remediation() { + let expected = [ + ( + TelemetryFallbackReason::CredentialInaccessible, + "managed_telemetry_credential_inaccessible", + "auth_login", + ), + ( + TelemetryFallbackReason::CredentialUnavailable, + "managed_telemetry_credential_unavailable", + "auth_login", + ), + ( + TelemetryFallbackReason::CredentialExpired, + "managed_telemetry_credential_expired", + "auth_login", + ), + ( + TelemetryFallbackReason::LookupTimeout, + "managed_telemetry_lookup_timeout", + "manual_steps", + ), + ( + TelemetryFallbackReason::DatabaseError, + "managed_telemetry_database_error", + "manual_steps", + ), + ( + TelemetryFallbackReason::PathResolutionError, + "managed_telemetry_path_resolution_error", + "manual_steps", + ), + ( + TelemetryFallbackReason::ExporterInitFailed, + "managed_telemetry_exporter_init_failed", + "manual_steps", + ), + ]; + for (reason, kind, next_action) in expected { + let health = disabled(reason); + let problem = problem(health).expect("disabled telemetry is a problem"); + assert_eq!(problem.kind, ProblemKind::ManagedTelemetryUnavailable); + assert_eq!(problem.severity, ProblemSeverity::Warning); + assert_eq!(problem.fixability, ProblemFixability::ManualOnly); + assert_eq!(problem.next_action, next_action); + assert!(problem.summary.ends_with(HOOK_IMPACT)); + let lowered = format!("{} {}", problem.summary, problem.remediation).to_lowercase(); + for forbidden in ["delete", "regenerat", "reset", "bearer", "token:"] { + assert!( + !lowered.contains(forbidden), + "{reason:?} mentions {forbidden}" + ); + } + assert_eq!( + json(health), + json!({ + "state": "disabled", + "reason": reason.as_str(), + "kind": kind, + "severity": "warning", + "next_action": next_action, + }) + ); + } + } +} diff --git a/cli/src/services/doctor/mod.rs b/cli/src/services/doctor/mod.rs index 8cc796382..74c5dbb28 100644 --- a/cli/src/services/doctor/mod.rs +++ b/cli/src/services/doctor/mod.rs @@ -11,6 +11,7 @@ use crate::services::lifecycle::{ lifecycle_providers, FixOutcome, HealthCategory, HealthFixability, HealthProblem, HealthProblemKind, HealthSeverity, LifecycleProvider, LifecycleProviderId, }; +use crate::services::managed_telemetry::ManagedTelemetryHealth; use crate::services::mutation_trace::runtime::{ inspect_reconciliation_recommendation, run_explicit_reconciliation, ReconciliationCounts, ReconciliationFix, ReconciliationRecommendation, @@ -20,6 +21,7 @@ use crate::services::setup; mod fixes; mod inspect; +mod managed_telemetry; mod render; pub(crate) mod types; @@ -60,10 +62,12 @@ struct DoctorDependencies<'a, G, S, C, P> { resolve_state_root: &'a S, resolve_global_config_path: &'a C, probe_codex_hook_policy: &'a P, + managed_telemetry: ManagedTelemetryHealth, } struct DoctorExecution { report: HookDoctorReport, + managed_telemetry: ManagedTelemetryHealth, fix_results: Vec, ref_reconciliation: Option, ref_reconciliation_fix: Option, @@ -122,6 +126,7 @@ async fn execute_doctor_with_context( Ok(resolve_sce_default_locations()?.global_config_file()) }, probe_codex_hook_policy: &codex_hook_policy::probe_default, + managed_telemetry: crate::services::managed_telemetry::inspect_health().await, }, mutation_scope_seam, reconcile_seam, @@ -150,12 +155,14 @@ async fn execute_doctor_with_lifecycle_providers( recommendation_seam: &impl Fn(&Path) -> Option, ) -> DoctorExecution { let policy_readiness = (dependencies.probe_codex_hook_policy)(); + let managed_telemetry = dependencies.managed_telemetry; let providers = lifecycle_providers(true); let initial_problems = diagnose_lifecycle_providers(context, &providers).await; let initial_doctor_problems = initial_problems .iter() .map(|problem| problem.problem.clone()) + .chain(managed_telemetry::problem(managed_telemetry)) .collect::>(); let initial_report = build_report_with_lifecycle_problems( request.mode, @@ -169,6 +176,7 @@ async fn execute_doctor_with_lifecycle_providers( if request.mode != DoctorMode::Fix { return DoctorExecution { report: initial_report, + managed_telemetry, fix_results: Vec::new(), ref_reconciliation: recommendation_seam(repository_root), ref_reconciliation_fix: None, @@ -186,6 +194,7 @@ async fn execute_doctor_with_lifecycle_providers( let final_doctor_problems = final_problems .into_iter() .map(|problem| problem.problem) + .chain(managed_telemetry::problem(managed_telemetry)) .collect::>(); let final_report = build_report_with_lifecycle_problems( request.mode, @@ -209,6 +218,7 @@ async fn execute_doctor_with_lifecycle_providers( DoctorExecution { report: final_report, + managed_telemetry, fix_results, ref_reconciliation: None, ref_reconciliation_fix: Some(reconciliation_fix), @@ -481,6 +491,7 @@ fn doctor_problem_kind(kind: HealthProblemKind) -> ProblemKind { } HealthProblemKind::MutationScopeHealthBlocked => ProblemKind::MutationScopeHealthBlocked, HealthProblemKind::MutationScopeHealthInvalid => ProblemKind::MutationScopeHealthInvalid, + HealthProblemKind::ManagedTelemetryUnavailable => ProblemKind::ManagedTelemetryUnavailable, } } @@ -560,6 +571,7 @@ fn health_problem_kind(kind: ProblemKind) -> HealthProblemKind { } ProblemKind::MutationScopeHealthBlocked => HealthProblemKind::MutationScopeHealthBlocked, ProblemKind::MutationScopeHealthInvalid => HealthProblemKind::MutationScopeHealthInvalid, + ProblemKind::ManagedTelemetryUnavailable => HealthProblemKind::ManagedTelemetryUnavailable, } } diff --git a/cli/src/services/doctor/render.rs b/cli/src/services/doctor/render.rs index e9555e168..25655df8d 100644 --- a/cli/src/services/doctor/render.rs +++ b/cli/src/services/doctor/render.rs @@ -12,7 +12,8 @@ use super::types::{ IntegrationGroupHealth, IntegrationGroupKey, IntegrationTarget, MutationScopeHealthRow, PostCommitAutoSyncState, ProblemKind, ProblemSeverity, Readiness, }; -use super::{DoctorExecution, DoctorFormat, DoctorMode, DoctorRequest, NAME}; +use super::{managed_telemetry, DoctorExecution, DoctorFormat, DoctorMode, DoctorRequest, NAME}; +use crate::services::managed_telemetry::ManagedTelemetryHealth; use crate::services::mutation_trace::runtime::{ ReconciliationCounts, ReconciliationFix, ReconciliationRecommendation, StateWarning, }; @@ -30,7 +31,7 @@ pub(super) fn render_report(request: DoctorRequest, execution: &DoctorExecution) fn format_execution(execution: &DoctorExecution) -> String { let report = &execution.report; - let base_report = format_report(report); + let base_report = format_report(report, execution.managed_telemetry); let mut lines = base_report .lines() .map(ToOwned::to_owned) @@ -103,11 +104,15 @@ fn format_age(age_ms: i64) -> String { } } -fn format_report(report: &HookDoctorReport) -> String { - format_report_with_color_policy(report, supports_color()) +fn format_report(report: &HookDoctorReport, managed_telemetry: ManagedTelemetryHealth) -> String { + format_report_with_color_policy(report, managed_telemetry, supports_color()) } -fn format_report_with_color_policy(report: &HookDoctorReport, color_enabled: bool) -> String { +fn format_report_with_color_policy( + report: &HookDoctorReport, + managed_telemetry: ManagedTelemetryHealth, + color_enabled: bool, +) -> String { let blocking_problem_count = report .problems .iter() @@ -125,7 +130,7 @@ fn format_report_with_color_policy(report: &HookDoctorReport, color_enabled: boo }); lines.push(format!("\n{}", heading("Environment"))); - for node in environment_nodes(report) { + for node in environment_nodes(report, managed_telemetry) { render_display_node(&mut lines, &node, color_enabled, 2, true); } @@ -248,7 +253,10 @@ fn format_human_text_row( ) } -fn environment_nodes(report: &HookDoctorReport) -> Vec { +fn environment_nodes( + report: &HookDoctorReport, + managed_telemetry: ManagedTelemetryHealth, +) -> Vec { vec![ top_level_node( "State", @@ -283,9 +291,32 @@ fn environment_nodes(report: &HookDoctorReport) -> Vec { ) }), ), + managed_telemetry_node(managed_telemetry), ] } +fn managed_telemetry_node(health: ManagedTelemetryHealth) -> DoctorDisplayNode { + match (health, managed_telemetry::diagnostic(health)) { + (_, Some(diagnostic)) => top_level_node( + "Managed telemetry", + DoctorDisplayStatus::Warn, + vec![DoctorDisplayDetail::ManagedTelemetry { + reason: diagnostic.reason, + summary: diagnostic.summary, + remediation: diagnostic.remediation, + }], + ), + (ManagedTelemetryHealth::Healthy, None) => { + top_level_node("Managed telemetry", DoctorDisplayStatus::Pass, Vec::new()) + } + (ManagedTelemetryHealth::Disabled(_), None) => top_level_node( + "Managed telemetry (not configured)", + DoctorDisplayStatus::Pass, + Vec::new(), + ), + } +} + fn top_level_node( label: &str, status: DoctorDisplayStatus, @@ -696,6 +727,16 @@ fn render_display_detail(lines: &mut Vec, detail: &DoctorDisplayDetail, lines.push(format!("{prefix}Remediation: {remediation}")); } } + DoctorDisplayDetail::ManagedTelemetry { + reason, + summary, + remediation, + } => { + lines.push(format!("{prefix}Reason: {reason}")); + lines.push(format!("{prefix}{summary}")); + lines.push(format!("{prefix}{}", managed_telemetry::HOOK_IMPACT)); + lines.extend(remediation.lines().map(|line| format!("{prefix}{line}"))); + } } } @@ -868,8 +909,6 @@ fn render_report_json(execution: &DoctorExecution) -> Result { }) .collect::>(); - let mutation_scope_health = mutation_scope_health_json(&report.mutation_scope_health); - let payload = json!({ "status": "ok", "command": NAME, @@ -917,7 +956,8 @@ fn render_report_json(execution: &DoctorExecution) -> Result { }, "config_paths": config_paths, "hooks": hooks, - "mutation_scope_health": mutation_scope_health, + "mutation_scope_health": mutation_scope_health_json(&report.mutation_scope_health), + "managed_telemetry": managed_telemetry::json(execution.managed_telemetry), "problems": report.problems.iter().map(|problem| json!({ "category": problem_category(problem.category), "severity": problem_severity(problem.severity), @@ -1228,3 +1268,55 @@ mod reconciliation_render_tests { } } } + +#[cfg(test)] +mod managed_telemetry_render_tests { + use super::{managed_telemetry_node, render_display_node}; + use crate::services::managed_telemetry::{ManagedTelemetryHealth, TelemetryFallbackReason}; + + fn rendered(health: ManagedTelemetryHealth) -> Vec { + let mut lines = Vec::new(); + render_display_node(&mut lines, &managed_telemetry_node(health), false, 2, true); + lines + } + + #[test] + fn healthy_and_unconfigured_telemetry_render_as_single_pass_rows() { + assert_eq!( + rendered(ManagedTelemetryHealth::Healthy), + [" [PASS] Managed telemetry"] + ); + assert_eq!( + rendered(ManagedTelemetryHealth::Disabled( + TelemetryFallbackReason::NotConfigured + )), + [" [PASS] Managed telemetry (not configured)"] + ); + } + + #[test] + fn inaccessible_credentials_render_reason_impact_and_login_steps() { + let lines = rendered(ManagedTelemetryHealth::Disabled( + TelemetryFallbackReason::CredentialInaccessible, + )); + assert_eq!(lines[0], " [WARN] Managed telemetry"); + assert_eq!(lines[1], " Reason: credential_inaccessible"); + assert!(lines[2].contains("without user interaction.")); + assert_eq!( + lines[3], + " Claude Code hooks continue working, but telemetry export is disabled." + ); + assert_eq!(lines[4], " To fix:"); + assert!(lines + .iter() + .any(|line| line.ends_with(". Run `sce auth login` in an interactive terminal."))); + assert_eq!( + lines.last().map(String::as_str), + Some(" 3. Run `sce doctor` again to verify access.") + ); + assert_eq!( + lines.iter().any(|line| line.contains("Always Allow")), + cfg!(target_os = "macos") + ); + } +} diff --git a/cli/src/services/doctor/types.rs b/cli/src/services/doctor/types.rs index f03489545..57ea0fa51 100644 --- a/cli/src/services/doctor/types.rs +++ b/cli/src/services/doctor/types.rs @@ -278,6 +278,11 @@ pub(super) enum DoctorDisplayDetail { detail: Option, remediation: Option, }, + ManagedTelemetry { + reason: &'static str, + summary: String, + remediation: String, + }, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -464,6 +469,7 @@ pub(crate) enum ProblemKind { MutationScopeHealthRecovering, MutationScopeHealthBlocked, MutationScopeHealthInvalid, + ManagedTelemetryUnavailable, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/cli/src/services/error.rs b/cli/src/services/error.rs index a99303543..786cf9a09 100644 --- a/cli/src/services/error.rs +++ b/cli/src/services/error.rs @@ -56,17 +56,28 @@ pub enum UserError { NotAuthenticated, NotGitRepository, NotGitRemote, - AuthStorageUnavailable, + AuthStorageUnavailable(AuthStorageFailure), + LoginAuthStorageUnavailable(AuthStorageFailure), UnexpectedFailure, } +/// Typed cause of an authentication storage failure, selected from typed +/// error sources so user guidance never depends on diagnostic text. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AuthStorageFailure { + CredentialAccess, + DatabaseLocked, + Database, +} + impl UserError { pub fn class(&self) -> FailureClass { match self { Self::NotAuthenticated | Self::NotGitRepository | Self::NotGitRemote - | Self::AuthStorageUnavailable + | Self::AuthStorageUnavailable(_) + | Self::LoginAuthStorageUnavailable(_) | Self::UnexpectedFailure => FailureClass::Runtime, } } @@ -76,7 +87,9 @@ impl UserError { Self::NotAuthenticated => "auth.not_authenticated", Self::NotGitRepository => "setup.not_git_repository", Self::NotGitRemote { .. } => "setup.not_git_remote", - Self::AuthStorageUnavailable => "auth.storage_unavailable", + Self::AuthStorageUnavailable(_) | Self::LoginAuthStorageUnavailable(_) => { + "auth.storage_unavailable" + } Self::UnexpectedFailure => "general.unexpected_failure", } } @@ -92,9 +105,16 @@ impl UserError { Self::NotGitRemote => { "The Git repository has no configured remote URL. Please run `git remote add `, then retry." } - Self::AuthStorageUnavailable => { - "Authentication storage is unavailable. Verify local credential storage is available, then retry the command." - } + Self::AuthStorageUnavailable(failure) => auth_storage_unavailable_message( + CURRENT_CREDENTIAL_STORE_PLATFORM, + AuthStorageOperation::Command, + *failure, + ), + Self::LoginAuthStorageUnavailable(failure) => auth_storage_unavailable_message( + CURRENT_CREDENTIAL_STORE_PLATFORM, + AuthStorageOperation::Login, + *failure, + ), Self::UnexpectedFailure => { "An unexpected error occurred. Check the log files for more details." } @@ -102,6 +122,196 @@ impl UserError { } } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CredentialStorePlatform { + Macos, + Linux, + Windows, + Other, +} + +impl CredentialStorePlatform { + pub(crate) const fn store_name(self) -> &'static str { + match self { + Self::Macos => "macOS Keychain", + Self::Linux => "the system keyring (Secret Service)", + Self::Windows => "Windows Credential Manager", + Self::Other => "the OS credential store", + } + } +} + +pub(crate) const CURRENT_CREDENTIAL_STORE_PLATFORM: CredentialStorePlatform = + if cfg!(target_os = "macos") { + CredentialStorePlatform::Macos + } else if cfg!(target_os = "linux") { + CredentialStorePlatform::Linux + } else if cfg!(target_os = "windows") { + CredentialStorePlatform::Windows + } else { + CredentialStorePlatform::Other + }; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AuthStorageOperation { + Command, + Login, +} + +const MACOS_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in macOS Keychain. + +To fix: + 1. Run `sce auth login` in an interactive terminal. + 2. When macOS asks for permission to access the SCE Keychain item, + choose \"Always Allow\". + 3. Retry the original command, or run `sce doctor` to verify + telemetry access. + +If no prompt appears, verify that your login Keychain is unlocked +and retry using the same installed SCE executable."; + +const MACOS_LOGIN_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in macOS Keychain, so `sce auth login` +could not use its stored session. Running login again does not help until +Keychain access is allowed. + +To fix: + 1. Unlock your login Keychain (for example, in Keychain Access). + 2. Run `sce auth login` again in an interactive terminal. + 3. When macOS asks for permission to access the SCE Keychain item, + choose \"Always Allow\" to authorize this SCE executable. + +If no prompt appears, verify that your login Keychain is unlocked +and retry using the same installed SCE executable."; + +const LINUX_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in the system keyring (Secret Service). + +To fix: + 1. Make sure a Secret Service provider (such as GNOME Keyring or KWallet) + is running and its default collection is unlocked. + 2. Run `sce auth login` in an interactive terminal. + 3. Retry the original command."; + +const LINUX_LOGIN_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in the system keyring (Secret Service), +so `sce auth login` could not use its stored session. Running login again does +not help until the keyring is available. + +To fix: + 1. Make sure a Secret Service provider (such as GNOME Keyring or KWallet) + is running and its default collection is unlocked. + 2. Retry `sce auth login` in an interactive terminal."; + +const WINDOWS_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in Windows Credential Manager. + +To fix: + 1. Make sure you are signed in to your Windows user account and + Windows Credential Manager is available. + 2. Run `sce auth login` in an interactive terminal. + 3. Retry the original command."; + +const WINDOWS_LOGIN_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in Windows Credential Manager, +so `sce auth login` could not use its stored session. Running login again +does not help until Credential Manager is available. + +To fix: + 1. Make sure you are signed in to your Windows user account and + Windows Credential Manager is available. + 2. Retry `sce auth login` in an interactive terminal."; + +const OTHER_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in the OS credential store. + +To fix: + 1. Make sure a supported OS credential store is available and unlocked. + 2. Run `sce auth login` in an interactive terminal. + 3. Retry the original command."; + +const OTHER_LOGIN_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in the OS credential store, +so `sce auth login` could not use its stored session. Running login again +does not help until the credential store is available. + +To fix: + 1. Make sure a supported OS credential store is available and unlocked. + 2. Retry `sce auth login` in an interactive terminal."; + +const AUTH_DATABASE_LOCKED: &str = "Authentication storage is unavailable. + +The SCE authentication database is locked by another SCE process. + +To fix: + 1. Wait for other running `sce` processes, including Claude Code hooks, + to finish. + 2. Make sure every installed `sce` executable is the same version; + older versions cannot share the authentication database. + 3. Retry the command."; + +const AUTH_DATABASE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not open or read its local authentication database. + +To fix: + 1. Run `sce doctor` to check the SCE state directory. + 2. Retry the command. + 3. If the problem persists, check the SCE log files for details."; + +const fn auth_storage_unavailable_message( + platform: CredentialStorePlatform, + operation: AuthStorageOperation, + failure: AuthStorageFailure, +) -> &'static str { + match failure { + AuthStorageFailure::CredentialAccess => credential_access_message(platform, operation), + AuthStorageFailure::DatabaseLocked => AUTH_DATABASE_LOCKED, + AuthStorageFailure::Database => AUTH_DATABASE_UNAVAILABLE, + } +} + +const fn credential_access_message( + platform: CredentialStorePlatform, + operation: AuthStorageOperation, +) -> &'static str { + match (platform, operation) { + (CredentialStorePlatform::Macos, AuthStorageOperation::Command) => { + MACOS_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Macos, AuthStorageOperation::Login) => { + MACOS_LOGIN_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Linux, AuthStorageOperation::Command) => { + LINUX_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Linux, AuthStorageOperation::Login) => { + LINUX_LOGIN_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Windows, AuthStorageOperation::Command) => { + WINDOWS_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Windows, AuthStorageOperation::Login) => { + WINDOWS_LOGIN_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Other, AuthStorageOperation::Command) => { + OTHER_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Other, AuthStorageOperation::Login) => { + OTHER_LOGIN_AUTH_STORAGE_UNAVAILABLE + } + } +} + impl std::fmt::Display for UserError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.message()) @@ -305,4 +515,177 @@ mod tests { assert_eq!(validation_error.class(), FailureClass::Validation); assert_eq!(validation_error.to_string(), "bad value"); } + + const ALL_PLATFORMS: [CredentialStorePlatform; 4] = [ + CredentialStorePlatform::Macos, + CredentialStorePlatform::Linux, + CredentialStorePlatform::Windows, + CredentialStorePlatform::Other, + ]; + const ALL_OPERATIONS: [AuthStorageOperation; 2] = + [AuthStorageOperation::Command, AuthStorageOperation::Login]; + const ALL_FAILURES: [AuthStorageFailure; 3] = [ + AuthStorageFailure::CredentialAccess, + AuthStorageFailure::DatabaseLocked, + AuthStorageFailure::Database, + ]; + + #[test] + fn auth_storage_unavailable_keeps_stable_key_class_and_exit_code() { + for failure in ALL_FAILURES { + for user_error in [ + UserError::AuthStorageUnavailable(failure), + UserError::LoginAuthStorageUnavailable(failure), + ] { + let error = CliError::user_with_source( + user_error.clone(), + anyhow::anyhow!("keychain item secret-key-material denied"), + ); + assert_eq!(user_error.key(), "auth.storage_unavailable"); + assert_eq!(error.class(), FailureClass::Runtime); + assert_eq!(error.code(), "SCE-ERR-RUNTIME"); + assert_eq!(error.class().exit_code(), 4); + assert_eq!(error.to_string(), user_error.message()); + assert!(!error.to_string().contains("secret-key-material")); + } + } + } + + #[test] + fn macos_auth_storage_guidance_explains_login_and_always_allow() { + let message = auth_storage_unavailable_message( + CredentialStorePlatform::Macos, + AuthStorageOperation::Command, + AuthStorageFailure::CredentialAccess, + ); + assert!(message.starts_with("Authentication storage is unavailable.\n")); + assert!(message.contains("macOS Keychain")); + assert!(message.contains("Run `sce auth login` in an interactive terminal.")); + assert!(message.contains("choose \"Always Allow\".")); + assert!(message.contains("Retry the original command, or run `sce doctor`")); + assert!(message.contains("login Keychain is unlocked")); + } + + #[test] + fn macos_login_storage_guidance_is_not_circular() { + let message = auth_storage_unavailable_message( + CredentialStorePlatform::Macos, + AuthStorageOperation::Login, + AuthStorageFailure::CredentialAccess, + ); + assert!(message.contains("Running login again does not help until")); + assert!(message.contains("Unlock your login Keychain")); + assert!(message.contains("\"Always Allow\" to authorize this SCE executable")); + assert!(!message.contains("Retry the original command")); + } + + #[test] + fn database_failures_never_blame_the_credential_store() { + for platform in ALL_PLATFORMS { + for operation in ALL_OPERATIONS { + let locked = auth_storage_unavailable_message( + platform, + operation, + AuthStorageFailure::DatabaseLocked, + ); + assert!(locked.contains("locked by another SCE process")); + let generic = auth_storage_unavailable_message( + platform, + operation, + AuthStorageFailure::Database, + ); + assert!( + generic.contains("could not open or read its local authentication database") + ); + for message in [locked, generic] { + assert!(message.starts_with("Authentication storage is unavailable.\n")); + assert!( + !message.contains("encryption key"), + "{platform:?}/{operation:?}" + ); + assert!(!message.contains("Keychain"), "{platform:?}/{operation:?}"); + assert!( + !message.contains("Always Allow"), + "{platform:?}/{operation:?}" + ); + assert!( + !message.contains("sce auth login"), + "{platform:?}/{operation:?}" + ); + } + } + } + } + + #[test] + fn non_macos_auth_storage_guidance_never_mentions_macos() { + for platform in [ + CredentialStorePlatform::Linux, + CredentialStorePlatform::Windows, + CredentialStorePlatform::Other, + ] { + for operation in ALL_OPERATIONS { + for failure in ALL_FAILURES { + let message = auth_storage_unavailable_message(platform, operation, failure); + assert!(message.starts_with("Authentication storage is unavailable.\n")); + assert!(!message.contains("macOS"), "{platform:?}"); + assert!(!message.contains("Keychain"), "{platform:?}"); + assert!(!message.contains("Always Allow"), "{platform:?}"); + } + assert!( + auth_storage_unavailable_message( + platform, + operation, + AuthStorageFailure::CredentialAccess + ) + .contains("sce auth login"), + "{platform:?}" + ); + } + } + } + + #[test] + fn auth_storage_guidance_never_recommends_destructive_repair() { + for platform in ALL_PLATFORMS { + for operation in ALL_OPERATIONS { + for failure in ALL_FAILURES { + let message = auth_storage_unavailable_message(platform, operation, failure) + .to_lowercase(); + for forbidden in ["auth.db", "delete", "regenerat", "reset", "permanent"] { + assert!( + !message.contains(forbidden), + "{platform:?}/{operation:?}/{failure:?} mentions {forbidden:?}" + ); + } + } + } + } + } + + #[test] + fn current_platform_message_is_selected_for_display() { + for failure in ALL_FAILURES { + assert_eq!( + UserError::AuthStorageUnavailable(failure).message(), + auth_storage_unavailable_message( + CURRENT_CREDENTIAL_STORE_PLATFORM, + AuthStorageOperation::Command, + failure + ) + ); + assert_eq!( + UserError::LoginAuthStorageUnavailable(failure).message(), + auth_storage_unavailable_message( + CURRENT_CREDENTIAL_STORE_PLATFORM, + AuthStorageOperation::Login, + failure + ) + ); + } + assert_eq!( + CURRENT_CREDENTIAL_STORE_PLATFORM == CredentialStorePlatform::Macos, + cfg!(target_os = "macos") + ); + } } diff --git a/cli/src/services/hooks/claude_model_state.rs b/cli/src/services/hooks/claude_model_state.rs index 17a3e4f99..42f2730df 100644 --- a/cli/src/services/hooks/claude_model_state.rs +++ b/cli/src/services/hooks/claude_model_state.rs @@ -29,6 +29,9 @@ pub(super) async fn run_claude_model_state_subcommand< repository_root: &Path, logger: Option<&L>, ) -> String { + crate::services::managed_telemetry::trace_export::record_claude_hook_command( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::ClaudeModelState, + ); let stdin_payload = match read_hook_stdin() { Ok(payload) => payload, Err(error) => { @@ -36,6 +39,7 @@ pub(super) async fn run_claude_model_state_subcommand< return String::new(); } }; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); let session_id = fail_open_session_id(&stdin_payload); let observed_at_ms = match current_unix_time_ms() { diff --git a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs index 85f01c29b..97f0a9bbb 100644 --- a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs +++ b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::Result; use crate::services::hooks; +use crate::services::managed_telemetry::trace_export::{in_child_span, ManagedChildSpan}; use crate::services::mutation_trace::runtime::resolve_git_dir; use super::events::BASH_TOOL_NAME; @@ -46,7 +47,11 @@ pub(crate) async fn run_claude_mutation_scope_subcommand< >( logger: Option<&L>, ) -> Result { + crate::services::managed_telemetry::trace_export::record_claude_hook_command( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::ClaudeMutationScope, + ); let stdin_payload = hooks::read_hook_stdin()?; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); run_claude_mutation_scope_from_payload(&stdin_payload, logger).await } @@ -93,7 +98,21 @@ pub(super) async fn run_claude_mutation_scope_from_payload_with_resolver< seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { let event = parse_claude_hook_event(stdin_payload)?; - dispatch_claude_hook_event(event, logger, resolve_git_dir, model_state_resolver, seam).await + let traced_seam = async |repository_root: &Path, payload: &str, logger: Option<&L>| { + in_child_span( + ManagedChildSpan::MutationScope, + seam(repository_root, payload, logger), + ) + .await + }; + dispatch_claude_hook_event( + event, + logger, + resolve_git_dir, + model_state_resolver, + &traced_seam, + ) + .await } pub(super) async fn dispatch_claude_hook_event< @@ -504,3 +523,259 @@ pub(crate) async fn repair_blocked>, + seam_payloads: Vec, + state: state::AdapterState, + } + + async fn run_hook_sequence(telemetry: &T) -> HookRun { + let git_dir = tempfile::tempdir().expect("temp dir"); + let seam_payloads = Mutex::new(Vec::new()); + let resolve = async |_: &str| Ok::<_, anyhow::Error>(git_dir.path().to_path_buf()); + let model_state = async |_: &Path, _: &str, _: &str| Ok::<_, anyhow::Error>(None); + let seam = async |_: &Path, payload: &str, _: Option<&NoopLogger>| { + let mut payloads = seam_payloads.lock().expect("seam payloads"); + payloads.push(payload.to_string()); + if payloads.len() == FAILING_SEAM_CALL { + anyhow::bail!("seam failure"); + } + Ok(String::new()) + }; + let mut outputs = Vec::new(); + for payload in HOOK_SEQUENCE { + let output = telemetry + .with_default_subscriber(&mut || async { + record_claude_hook_command(ClaudeHookCommand::ClaudeMutationScope); + record_claude_hook_payload(payload); + run_claude_mutation_scope_from_payload_with_resolver::( + payload, + None, + &resolve, + &model_state, + &seam, + ) + .await + .map_err(CliError::runtime) + }) + .await; + outputs.push(output.ok()); + } + HookRun { + outputs, + seam_payloads: seam_payloads.into_inner().expect("seam payloads"), + state: state::read_state(git_dir.path()).await.expect("state"), + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn failing_managed_telemetry_cannot_change_claude_hook_behavior() { + let baseline = run_hook_sequence(&NoopTelemetry).await; + assert!(baseline + .outputs + .iter() + .flatten() + .any(|output| output.contains("\"permissionDecision\":\"deny\""))); + + let refused = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let refused_address = refused.local_addr().expect("addr"); + drop(refused); + let stalled = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let stalled_address = stalled.local_addr().expect("addr"); + let holder = tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((stream, _)) = stalled.accept().await { + held.push(stream); + } + }); + + for address in [refused_address, stalled_address] { + let telemetry = loopback_hook_telemetry(address, "bao-token-value"); + let observed = run_hook_sequence(&telemetry).await; + let started = Instant::now(); + telemetry.shutdown().await; + assert!(started.elapsed() < SHUTDOWN_TIMEOUT + SHUTDOWN_MARGIN); + assert_eq!(observed, baseline); + } + holder.abort(); + } + + fn git(root: &Path, args: &[&str]) { + let status = Command::new("git") + .args(args) + .current_dir(root) + .status() + .expect("git"); + assert!(status.success(), "git {args:?} failed"); + } + + fn init_repo(root: &Path) { + std::fs::create_dir_all(root).expect("repo dir"); + git(root, &["init", "-q"]); + git(root, &["config", "user.email", "t@example.invalid"]); + git(root, &["config", "user.name", "T"]); + git(root, &["config", "commit.gpgsign", "false"]); + std::fs::write(root.join("tracked.txt"), "tracked\n").expect("write file"); + git(root, &["add", "tracked.txt"]); + git(root, &["commit", "-q", "-m", "initial"]); + } + + async fn export_claude_hook_spans(payload: &str) -> (Option, Vec) { + let dir = tempfile::tempdir().expect("temp dir"); + let root = dir.path().join("repo"); + init_repo(&root); + let git_dir = root.join(".git"); + let db_path = dir.path().join("agent-trace.db"); + RepositoryAgentTraceDb::new_at(&db_path) + .await + .expect("create db") + .verify_or_initialize_repository_metadata(TEST_REPOSITORY_ID) + .await + .expect("initialize metadata"); + let resolve = async |_: &str| Ok::<_, anyhow::Error>(git_dir.clone()); + let model_state = async |_: &Path, _: &str, _: &str| Ok::<_, anyhow::Error>(None); + let seam = async |_: &Path, payload: &str, _: Option<&NoopLogger>| { + drive_mutation_scope( + &root, + parse_mutation_scope_payload(payload)?, + None::<&NoopLogger>, + &mut Vec::new(), + async |root: &Path, boundary| { + coordinate(root, boundary, async || { + RepositoryAgentTraceDb::open_without_migrations_at(&db_path).await + }) + .await + }, + async |root: &Path, scope| { + abandon_scope(root, scope, async || { + RepositoryAgentTraceDb::open_without_migrations_at(&db_path).await + }) + .await + }, + advise_after_completed_boundary, + ) + .await + }; + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr"); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let receiver = tokio::spawn(capture_requests_until(listener, stopped)); + let telemetry = loopback_hook_telemetry(address, "bao-token-value"); + let output = telemetry + .with_default_subscriber(&mut || async { + record_claude_hook_command(ClaudeHookCommand::ClaudeMutationScope); + record_claude_hook_payload(payload); + run_claude_mutation_scope_from_payload_with_resolver::( + payload, + None, + &resolve, + &model_state, + &seam, + ) + .await + .map_err(CliError::runtime) + }) + .await; + telemetry.shutdown().await; + stop.send(()).expect("stop receiver"); + let spans = trace_spans(&receiver.await.expect("receiver join")); + (output.ok(), spans) + } + + fn span_named<'a>(spans: &'a [DecodedSpan], name: &str) -> &'a DecodedSpan { + spans + .iter() + .find(|span| span.name == name) + .unwrap_or_else(|| panic!("missing {name}: {spans:?}")) + } + + fn sorted_names(spans: &[DecodedSpan]) -> Vec<&str> { + let mut names = spans + .iter() + .map(|span| span.name.as_str()) + .collect::>(); + names.sort_unstable(); + names + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn tracked_claude_hook_exports_mutation_snapshot_and_persistence_children() { + let (output, spans) = export_claude_hook_spans(WRITE_PRE_TOOL_USE).await; + + assert_eq!(output.as_deref(), Some("")); + assert_eq!( + sorted_names(&spans), + [ + "sce.git_snapshot", + "sce.hook", + "sce.mutation_scope", + "sce.persistence" + ] + ); + let hook = span_named(&spans, "sce.hook"); + let mutation_scope = span_named(&spans, "sce.mutation_scope"); + let git_snapshot = span_named(&spans, "sce.git_snapshot"); + let persistence = span_named(&spans, "sce.persistence"); + assert!(hook.parent_span_id.is_empty()); + assert_eq!(mutation_scope.parent_span_id, hook.span_id); + assert_eq!(git_snapshot.parent_span_id, mutation_scope.span_id); + assert_eq!(persistence.parent_span_id, mutation_scope.span_id); + let success = vec![("sce.outcome".to_string(), "success".to_string())]; + for child in [mutation_scope, git_snapshot, persistence] { + assert_eq!(child.trace_id, hook.trace_id, "{}", child.name); + assert_eq!(child.sce_attributes(), success, "{}", child.name); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn read_only_claude_hook_exports_no_operation_children() { + let payload = + WRITE_PRE_TOOL_USE.replace("\"tool_name\":\"Write\"", "\"tool_name\":\"Read\""); + assert_ne!(payload, WRITE_PRE_TOOL_USE); + + let (output, spans) = export_claude_hook_spans(&payload).await; + + assert_eq!(output.as_deref(), Some("")); + assert_eq!(sorted_names(&spans), ["sce.hook"]); + } +} diff --git a/cli/src/services/hooks/conversation_trace.rs b/cli/src/services/hooks/conversation_trace.rs index 739bbd826..8237bee58 100644 --- a/cli/src/services/hooks/conversation_trace.rs +++ b/cli/src/services/hooks/conversation_trace.rs @@ -70,10 +70,14 @@ pub(crate) async fn run_conversation_trace_subcommand< repository_root: &Path, logger: Option<&L>, ) -> String { + crate::services::managed_telemetry::trace_export::record_claude_hook_command( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::ConversationTrace, + ); let stdin_payload = match read_hook_stdin() { Ok(payload) => payload, Err(error) => return log_conversation_trace_fail_open(&error, logger, None), }; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); let session_id = conversation_trace_fail_open_session_id(&stdin_payload); match run_conversation_trace_subcommand_from_payload( diff --git a/cli/src/services/hooks/diff_trace.rs b/cli/src/services/hooks/diff_trace.rs index 65ee8e892..05d918909 100644 --- a/cli/src/services/hooks/diff_trace.rs +++ b/cli/src/services/hooks/diff_trace.rs @@ -67,10 +67,14 @@ pub(crate) async fn run_diff_trace_subcommand, ) -> String { + crate::services::managed_telemetry::trace_export::record_claude_hook_command( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::DiffTrace, + ); let stdin_payload = match read_hook_stdin() { Ok(payload) => payload, Err(error) => return log_diff_trace_fail_open(&error, logger, None), }; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); let session_id = diff_trace_fail_open_session_id(&stdin_payload); match run_diff_trace_subcommand_from_payload(repository_root, &stdin_payload, logger).await { diff --git a/cli/src/services/lifecycle.rs b/cli/src/services/lifecycle.rs index a343f1b2c..20490aca9 100644 --- a/cli/src/services/lifecycle.rs +++ b/cli/src/services/lifecycle.rs @@ -76,6 +76,7 @@ pub enum HealthProblemKind { MutationScopeHealthRecovering, MutationScopeHealthBlocked, MutationScopeHealthInvalid, + ManagedTelemetryUnavailable, } #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/cli/src/services/managed_telemetry.rs b/cli/src/services/managed_telemetry.rs index 74831a1ce..6b9a5ac16 100644 --- a/cli/src/services/managed_telemetry.rs +++ b/cli/src/services/managed_telemetry.rs @@ -7,12 +7,13 @@ pub mod log_export; pub mod trace_export; use crate::services::error::CliError; -use crate::services::observability::traits::{NoopTelemetry, Telemetry}; -use trace_export::ManagedTraceTelemetry; +use crate::services::observability::traits::{Logger, NoopTelemetry, Telemetry}; +use trace_export::{ManagedSpanKind, ManagedTraceTelemetry}; use crate::services::telemetry_credential_storage::{ load_telemetry_credential, TelemetryCredential, }; +use crate::services::telemetry_readiness::{self, ReadinessState}; use crate::services::token_storage::TokenStorageError; pub const MANAGED_OTLP_ENDPOINT: &str = "https://otel.crocoderlab.dev"; @@ -26,6 +27,11 @@ const TRACES_PATH: &str = "/v1/traces"; #[cfg(test)] const LOGS_PATH: &str = "/v1/logs"; const REDACTED: &str = ""; +const TELEMETRY_FALLBACK_EVENT_ID: &str = "sce.telemetry.fallback"; +const TELEMETRY_FALLBACK_MESSAGE: &str = + "Managed telemetry disabled for this invocation; continuing without export"; +const FALLBACK_REASON_FIELD: &str = "reason"; +const FALLBACK_ERROR_FIELD: &str = "error"; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ManagedOtlpEndpoints { @@ -108,14 +114,153 @@ where joined.ok()?.ok() } -pub(crate) async fn load_credential_within_deadline(lookup: Fut) -> Option +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum TelemetryFallbackReason { + NotConfigured, + CredentialInaccessible, + CredentialUnavailable, + CredentialExpired, + LookupTimeout, + DatabaseError, + PathResolutionError, + ExporterInitFailed, +} + +impl TelemetryFallbackReason { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::NotConfigured => "not_configured", + Self::CredentialInaccessible => "credential_inaccessible", + Self::CredentialUnavailable => "credential_unavailable", + Self::CredentialExpired => "credential_expired", + Self::LookupTimeout => "lookup_timeout", + Self::DatabaseError => "database_error", + Self::PathResolutionError => "path_resolution_error", + Self::ExporterInitFailed => "exporter_init_failed", + } + } + + const fn is_failure(self) -> bool { + !matches!( + self, + Self::NotConfigured + | Self::CredentialInaccessible + | Self::CredentialUnavailable + | Self::CredentialExpired + ) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TelemetryFallback { + pub(crate) reason: TelemetryFallbackReason, + detail: Option, +} + +impl TelemetryFallback { + const fn new(reason: TelemetryFallbackReason) -> Self { + Self { + reason, + detail: None, + } + } + + fn from_storage_error(error: &TokenStorageError) -> Self { + let reason = match error { + TokenStorageError::CredentialAccessUnavailable => { + return Self::new(TelemetryFallbackReason::CredentialInaccessible); + } + TokenStorageError::Database(_) | TokenStorageError::DatabaseLocked(_) => { + TelemetryFallbackReason::DatabaseError + } + TokenStorageError::PathResolution(_) => TelemetryFallbackReason::PathResolutionError, + }; + Self { + reason, + detail: Some(error.to_string()), + } + } + + fn log(&self, logger: &impl Logger) { + let mut fields = vec![(FALLBACK_REASON_FIELD, self.reason.as_str())]; + if let Some(detail) = &self.detail { + fields.push((FALLBACK_ERROR_FIELD, detail.as_str())); + } + if self.reason.is_failure() { + logger.warn( + TELEMETRY_FALLBACK_EVENT_ID, + TELEMETRY_FALLBACK_MESSAGE, + &fields, + None, + ); + } else { + logger.debug( + TELEMETRY_FALLBACK_EVENT_ID, + TELEMETRY_FALLBACK_MESSAGE, + &fields, + None, + ); + } + } +} + +pub(crate) async fn load_credential_within_deadline( + lookup: Fut, +) -> Result where Fut: Future, TokenStorageError>> + Send + 'static, { - run_with_deadline(CREDENTIAL_LOOKUP_DEADLINE, lookup) - .await - .and_then(Result::ok) - .flatten() + match run_with_deadline(CREDENTIAL_LOOKUP_DEADLINE, lookup).await { + None => Err(TelemetryFallback::new( + TelemetryFallbackReason::LookupTimeout, + )), + Some(Err(error)) => Err(TelemetryFallback::from_storage_error(&error)), + Some(Ok(None)) => Err(TelemetryFallback::new( + TelemetryFallbackReason::CredentialUnavailable, + )), + Some(Ok(Some(credential))) => Ok(credential), + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ManagedTelemetryHealth { + Healthy, + Disabled(TelemetryFallbackReason), +} + +pub(crate) async fn inspect_health() -> ManagedTelemetryHealth { + let now = now_unix_seconds(); + let Ok(readiness) = telemetry_readiness::state(now) else { + return ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::PathResolutionError); + }; + inspect_health_with(readiness, load_telemetry_credential(), now).await +} + +pub(crate) async fn inspect_health_with( + readiness: ReadinessState, + lookup: Fut, + now_unix_seconds: u64, +) -> ManagedTelemetryHealth +where + Fut: Future, TokenStorageError>> + Send + 'static, +{ + match readiness { + ReadinessState::NotConfigured => { + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::NotConfigured) + } + ReadinessState::Expired => { + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::CredentialExpired) + } + ReadinessState::Ready => match load_credential_within_deadline(lookup).await { + Err(fallback) => ManagedTelemetryHealth::Disabled(fallback.reason), + Ok(credential) => match select_mode(Some(credential), now_unix_seconds) { + ManagedTelemetryMode::Managed { .. } => ManagedTelemetryHealth::Healthy, + ManagedTelemetryMode::Noop => { + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::CredentialExpired) + } + }, + }, + } } fn now_unix_seconds() -> u64 { @@ -130,29 +275,77 @@ pub enum RuntimeTelemetry { } impl RuntimeTelemetry { - pub async fn resolve(managed_allowed: bool) -> Self { - if !managed_allowed { + pub async fn resolve(span_kind: Option, logger: &impl Logger) -> Self { + let Some(span_kind) = span_kind else { return Self::Noop(NoopTelemetry); + }; + let (telemetry, fallback) = Self::resolve_when_ready( + telemetry_readiness::is_ready(now_unix_seconds()), + load_telemetry_credential(), + span_kind, + ) + .await; + if let Some(fallback) = fallback { + fallback.log(logger); } - Self::resolve_with(load_telemetry_credential()).await + telemetry } - pub(crate) async fn resolve_with(lookup: Fut) -> Self + pub(crate) async fn resolve_when_ready( + ready: bool, + lookup: Fut, + span_kind: ManagedSpanKind, + ) -> (Self, Option) where Fut: Future, TokenStorageError>> + Send + 'static, { - let credential = load_credential_within_deadline(lookup).await; + if !ready { + return ( + Self::Noop(NoopTelemetry), + Some(TelemetryFallback::new( + TelemetryFallbackReason::NotConfigured, + )), + ); + } + Self::resolve_with(lookup, span_kind).await + } + + pub(crate) async fn resolve_with( + lookup: Fut, + span_kind: ManagedSpanKind, + ) -> (Self, Option) + where + Fut: Future, TokenStorageError>> + + Send + + 'static, + { + let credential = match load_credential_within_deadline(lookup).await { + Ok(credential) => credential, + Err(fallback) => return (Self::Noop(NoopTelemetry), Some(fallback)), + }; let Ok(runtime) = tokio::runtime::Handle::try_current() else { - return Self::Noop(NoopTelemetry); + return (Self::Noop(NoopTelemetry), None); }; - match select_mode(credential, now_unix_seconds()) { + match select_mode(Some(credential), now_unix_seconds()) { ManagedTelemetryMode::Managed { bearer, endpoints } => { - ManagedTraceTelemetry::new(&bearer, endpoints, runtime) - .map_or(Self::Noop(NoopTelemetry), Self::Managed) + match ManagedTraceTelemetry::new(&bearer, endpoints, runtime, span_kind) { + Ok(telemetry) => (Self::Managed(telemetry), None), + Err(_) => ( + Self::Noop(NoopTelemetry), + Some(TelemetryFallback::new( + TelemetryFallbackReason::ExporterInitFailed, + )), + ), + } } - ManagedTelemetryMode::Noop => Self::Noop(NoopTelemetry), + ManagedTelemetryMode::Noop => ( + Self::Noop(NoopTelemetry), + Some(TelemetryFallback::new( + TelemetryFallbackReason::CredentialExpired, + )), + ), } } @@ -240,6 +433,185 @@ mod tests { } } + #[tokio::test] + async fn claude_hook_without_readiness_never_looks_up_credentials() { + let looked_up = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = looked_up.clone(); + + let (telemetry, fallback) = RuntimeTelemetry::resolve_when_ready( + false, + async move { + flag.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(None) + }, + ManagedSpanKind::ClaudeHook, + ) + .await; + + assert!(matches!(telemetry, RuntimeTelemetry::Noop(_))); + let fallback = fallback.expect("disabled telemetry is diagnosed"); + assert_eq!(fallback.reason, TelemetryFallbackReason::NotConfigured); + assert!(!fallback.reason.is_failure()); + assert!(!looked_up.load(std::sync::atomic::Ordering::SeqCst)); + } + + #[tokio::test] + async fn claude_hook_with_inaccessible_credential_store_skips_quietly() { + let (telemetry, fallback) = RuntimeTelemetry::resolve_when_ready( + true, + async { Err(TokenStorageError::CredentialAccessUnavailable) }, + ManagedSpanKind::ClaudeHook, + ) + .await; + + assert!(matches!(telemetry, RuntimeTelemetry::Noop(_))); + let fallback = fallback.expect("inaccessible credentials are diagnosed"); + assert_eq!( + fallback.reason, + TelemetryFallbackReason::CredentialInaccessible + ); + assert!(!fallback.reason.is_failure()); + let output = telemetry + .with_default_subscriber(&mut || async { Ok("hook-output".to_string()) }) + .await; + assert_eq!(output.expect("hook result unaffected"), "hook-output"); + } + + #[tokio::test] + async fn claude_hook_credential_failure_or_expiry_resolves_noop_with_reason() { + let (failed, fallback) = RuntimeTelemetry::resolve_with( + async { Err(TokenStorageError::Database("locked".to_string())) }, + ManagedSpanKind::ClaudeHook, + ) + .await; + assert!(matches!(failed, RuntimeTelemetry::Noop(_))); + let fallback = fallback.expect("database failure is diagnosed"); + assert_eq!(fallback.reason, TelemetryFallbackReason::DatabaseError); + assert!(fallback.reason.is_failure()); + assert!(fallback + .detail + .as_deref() + .is_some_and(|detail| detail.contains("locked"))); + + let (path, fallback) = RuntimeTelemetry::resolve_with( + async { Err(TokenStorageError::PathResolution("no home".to_string())) }, + ManagedSpanKind::ClaudeHook, + ) + .await; + assert!(matches!(path, RuntimeTelemetry::Noop(_))); + assert_eq!( + fallback.map(|fallback| fallback.reason), + Some(TelemetryFallbackReason::PathResolutionError) + ); + + let (expired, fallback) = RuntimeTelemetry::resolve_with( + async { Ok(Some(credential("bao-token", 1))) }, + ManagedSpanKind::ClaudeHook, + ) + .await; + assert!(matches!(expired, RuntimeTelemetry::Noop(_))); + let fallback = fallback.expect("expiry is diagnosed"); + assert_eq!(fallback.reason, TelemetryFallbackReason::CredentialExpired); + assert!(!fallback.reason.is_failure()); + assert_eq!(fallback.detail, None); + + let (missing, fallback) = + RuntimeTelemetry::resolve_with(async { Ok(None) }, ManagedSpanKind::ClaudeHook).await; + assert!(matches!(missing, RuntimeTelemetry::Noop(_))); + assert_eq!( + fallback.map(|fallback| fallback.reason), + Some(TelemetryFallbackReason::CredentialUnavailable) + ); + } + + fn tracked_lookup( + result: Result, TokenStorageError>, + ) -> ( + std::sync::Arc, + impl Future, TokenStorageError>> + Send + 'static, + ) { + let looked_up = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = looked_up.clone(); + (looked_up, async move { + flag.store(true, std::sync::atomic::Ordering::SeqCst); + result + }) + } + + #[tokio::test] + async fn health_without_readiness_or_with_expired_readiness_never_looks_up_credentials() { + for (readiness, reason) in [ + ( + ReadinessState::NotConfigured, + TelemetryFallbackReason::NotConfigured, + ), + ( + ReadinessState::Expired, + TelemetryFallbackReason::CredentialExpired, + ), + ] { + let (looked_up, lookup) = tracked_lookup(Ok(Some(credential("bao-token", NOW + 60)))); + let health = inspect_health_with(readiness, lookup, NOW).await; + assert_eq!(health, ManagedTelemetryHealth::Disabled(reason)); + assert!(!looked_up.load(std::sync::atomic::Ordering::SeqCst)); + } + } + + #[tokio::test] + async fn health_with_readiness_classifies_lookup_outcomes() { + let cases: Vec<( + Result, TokenStorageError>, + ManagedTelemetryHealth, + )> = vec![ + ( + Ok(Some(credential("bao-token", NOW + 60))), + ManagedTelemetryHealth::Healthy, + ), + ( + Err(TokenStorageError::CredentialAccessUnavailable), + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::CredentialInaccessible), + ), + ( + Ok(None), + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::CredentialUnavailable), + ), + ( + Ok(Some(credential("bao-token", NOW))), + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::CredentialExpired), + ), + ( + Err(TokenStorageError::Database("locked".to_string())), + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::DatabaseError), + ), + ( + Err(TokenStorageError::PathResolution("no home".to_string())), + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::PathResolutionError), + ), + ]; + for (result, expected) in cases { + let (looked_up, lookup) = tracked_lookup(result); + let health = inspect_health_with(ReadinessState::Ready, lookup, NOW).await; + assert_eq!(health, expected); + assert!(looked_up.load(std::sync::atomic::Ordering::SeqCst)); + } + } + + #[tokio::test] + async fn health_lookup_is_bounded_by_the_credential_lookup_deadline() { + let started = std::time::Instant::now(); + let health = inspect_health_with( + ReadinessState::Ready, + std::future::pending::, TokenStorageError>>(), + NOW, + ) + .await; + assert!(started.elapsed() < CREDENTIAL_LOOKUP_DEADLINE * 2); + assert_eq!( + health, + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::LookupTimeout) + ); + } + #[test] fn absent_expired_or_blank_credential_selects_noop() { assert_eq!(select_mode(None, NOW), ManagedTelemetryMode::Noop); diff --git a/cli/src/services/managed_telemetry/trace_export.rs b/cli/src/services/managed_telemetry/trace_export.rs index f629d90c4..00e104219 100644 --- a/cli/src/services/managed_telemetry/trace_export.rs +++ b/cli/src/services/managed_telemetry/trace_export.rs @@ -1,19 +1,23 @@ +use std::collections::HashMap; use std::future::Future; use std::time::Duration; use opentelemetry::logs::LoggerProvider as _; -use opentelemetry::trace::TracerProvider as _; -use opentelemetry::KeyValue; +use opentelemetry::propagation::TextMapPropagator as _; +use opentelemetry::trace::{TraceContextExt as _, TracerProvider as _}; +use opentelemetry::{Context, KeyValue}; use opentelemetry_http::{Bytes, HttpClient, HttpError, Request, Response}; use opentelemetry_otlp::{LogExporter, SpanExporter, WithExportConfig, WithHttpConfig}; use opentelemetry_sdk::logs::{ BatchConfigBuilder as LogBatchConfigBuilder, BatchLogProcessor, SdkLoggerProvider, }; +use opentelemetry_sdk::propagation::TraceContextPropagator; use opentelemetry_sdk::trace::{BatchConfigBuilder, BatchSpanProcessor, SdkTracerProvider}; use opentelemetry_sdk::Resource; use reqwest::header::{HeaderValue, AUTHORIZATION, CONTENT_TYPE}; use tracing::instrument::WithSubscriber; use tracing::{field, Dispatch, Instrument, Span}; +use tracing_opentelemetry::OpenTelemetrySpanExt as _; use tracing_subscriber::filter::filter_fn; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::{Layer, Registry}; @@ -27,7 +31,33 @@ const SERVICE_NAME: &str = "sce"; const TRACER_NAME: &str = "sce.cli"; pub const COMMAND_SPAN_TARGET: &str = "sce.managed.trace"; const COMMAND_SPAN_NAME: &str = "sce.command"; +const HOOK_SPAN_NAME: &str = "sce.hook"; +const MUTATION_SCOPE_SPAN_NAME: &str = "sce.mutation_scope"; +const GIT_SNAPSHOT_SPAN_NAME: &str = "sce.git_snapshot"; +const PERSISTENCE_SPAN_NAME: &str = "sce.persistence"; +const HOOK_EXPORTED_SPAN_NAMES: [&str; 4] = [ + HOOK_SPAN_NAME, + MUTATION_SCOPE_SPAN_NAME, + GIT_SNAPSHOT_SPAN_NAME, + PERSISTENCE_SPAN_NAME, +]; pub const COMMAND_NAME_FIELD: &str = "sce.command.name"; +const AGENT_CLAUDE: &str = "claude"; +const HOOK_EVENT_FIELD: &str = "sce.hook.event"; +const HOOK_TOOL_FIELD: &str = "sce.hook.tool"; +const HOOK_COMMAND_FIELD: &str = "sce.hook.command"; +const CLAUDE_SESSION_ID_FIELD: &str = "sce.claude.session_id"; +const CLAUDE_TOOL_USE_ID_FIELD: &str = "sce.claude.tool_use_id"; +const CLAUDE_PAYLOAD_FIELDS: [(&str, &str); 4] = [ + ("hook_event_name", HOOK_EVENT_FIELD), + ("tool_name", HOOK_TOOL_FIELD), + ("session_id", CLAUDE_SESSION_ID_FIELD), + ("tool_use_id", CLAUDE_TOOL_USE_ID_FIELD), +]; +const TRACEPARENT_ENV: &str = "TRACEPARENT"; +const TRACESTATE_ENV: &str = "TRACESTATE"; +const TRACEPARENT_HEADER: &str = "traceparent"; +const TRACESTATE_HEADER: &str = "tracestate"; const OUTCOME_FIELD: &str = "sce.outcome"; const ERROR_CLASS_FIELD: &str = "sce.error.class"; const OUTCOME_SUCCESS: &str = "success"; @@ -70,7 +100,49 @@ impl HttpClient for PinnedBearerClient { } fn is_command_span(metadata: &tracing::Metadata<'_>) -> bool { - metadata.is_span() && metadata.target() == COMMAND_SPAN_TARGET + metadata.is_span() + && metadata.target() == COMMAND_SPAN_TARGET + && metadata.name() == COMMAND_SPAN_NAME +} + +fn is_hook_span(metadata: &tracing::Metadata<'_>) -> bool { + metadata.is_span() + && metadata.target() == COMMAND_SPAN_TARGET + && HOOK_EXPORTED_SPAN_NAMES.contains(&metadata.name()) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ManagedSpanKind { + Command, + ClaudeHook, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ClaudeHookCommand { + DiffTrace, + ConversationTrace, + ClaudeMutationScope, + ClaudeModelState, + PolicyBash, +} + +impl ClaudeHookCommand { + pub const fn as_str(self) -> &'static str { + match self { + Self::DiffTrace => "diff-trace", + Self::ConversationTrace => "conversation-trace", + Self::ClaudeMutationScope => "claude-mutation-scope", + Self::ClaudeModelState => "claude-model-state", + Self::PolicyBash => "policy-bash", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ManagedChildSpan { + MutationScope, + GitSnapshot, + Persistence, } #[derive(Debug)] @@ -78,6 +150,8 @@ pub struct ManagedTraceTelemetry { provider: SdkTracerProvider, log_provider: SdkLoggerProvider, dispatch: Dispatch, + span_kind: ManagedSpanKind, + parent_context: Option, } impl ManagedTraceTelemetry { @@ -85,12 +159,20 @@ impl ManagedTraceTelemetry { bearer: &ManagedBearerToken, endpoints: ManagedOtlpEndpoints, runtime: tokio::runtime::Handle, + span_kind: ManagedSpanKind, ) -> anyhow::Result { let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .timeout(REQUEST_TIMEOUT) .build()?; - Self::with_client(bearer, endpoints, runtime, client) + let mut telemetry = Self::with_client(bearer, endpoints, runtime, client, span_kind)?; + if span_kind == ManagedSpanKind::ClaudeHook { + telemetry.parent_context = extract_parent_context( + std::env::var(TRACEPARENT_ENV).ok(), + std::env::var(TRACESTATE_ENV).ok(), + ); + } + Ok(telemetry) } fn with_client( @@ -98,6 +180,7 @@ impl ManagedTraceTelemetry { endpoints: ManagedOtlpEndpoints, runtime: tokio::runtime::Handle, client: reqwest::Client, + span_kind: ManagedSpanKind, ) -> anyhow::Result { let mut authorization = HeaderValue::from_str(&bearer.authorization_header_value())?; authorization.set_sensitive(true); @@ -157,11 +240,13 @@ impl ManagedTraceTelemetry { .build(), ) .build(); + let span_filter: fn(&tracing::Metadata<'_>) -> bool = match span_kind { + ManagedSpanKind::Command => is_command_span, + ManagedSpanKind::ClaudeHook => is_hook_span, + }; let span_layer = tracing_opentelemetry::layer() .with_tracer(provider.tracer(TRACER_NAME)) - .with_filter(filter_fn( - is_command_span as fn(&tracing::Metadata<'_>) -> bool, - )); + .with_filter(filter_fn(span_filter)); let log_layer = LogBridgeLayer::new(log_provider.logger(TRACER_NAME)).with_filter( filter_fn(is_exportable_event as fn(&tracing::Metadata<'_>) -> bool), ); @@ -170,6 +255,8 @@ impl ManagedTraceTelemetry { provider, log_provider, dispatch, + span_kind, + parent_context: None, }) } @@ -179,6 +266,7 @@ impl ManagedTraceTelemetry { provider, log_provider, dispatch, + .. } = self; drop(dispatch); let traces = @@ -193,15 +281,35 @@ impl ManagedTraceTelemetry { .await; } - fn command_span(&self) -> Span { - tracing::dispatcher::with_default(&self.dispatch, || { - tracing::info_span!( + fn root_span(&self) -> Span { + let span = self.unparented_root_span(); + if let Some(parent_context) = &self.parent_context { + let _ = span.set_parent(parent_context.clone()); + } + span + } + + fn unparented_root_span(&self) -> Span { + tracing::dispatcher::with_default(&self.dispatch, || match self.span_kind { + ManagedSpanKind::Command => tracing::info_span!( target: COMMAND_SPAN_TARGET, COMMAND_SPAN_NAME, sce.command.name = field::Empty, sce.outcome = field::Empty, sce.error.class = field::Empty, - ) + ), + ManagedSpanKind::ClaudeHook => tracing::info_span!( + target: COMMAND_SPAN_TARGET, + HOOK_SPAN_NAME, + sce.agent = AGENT_CLAUDE, + sce.hook.command = field::Empty, + sce.hook.event = field::Empty, + sce.hook.tool = field::Empty, + sce.claude.session_id = field::Empty, + sce.claude.tool_use_id = field::Empty, + sce.outcome = field::Empty, + sce.error.class = field::Empty, + ), }) } } @@ -212,7 +320,7 @@ impl Telemetry for ManagedTraceTelemetry { F: FnMut() -> Fut, Fut: Future>, { - let span = self.command_span(); + let span = self.root_span(); let result = action() .instrument(span.clone()) .with_subscriber(self.dispatch.clone()) @@ -231,10 +339,81 @@ impl Telemetry for ManagedTraceTelemetry { } } +fn extract_parent_context( + traceparent: Option, + tracestate: Option, +) -> Option { + let traceparent = traceparent?; + let mut carrier = HashMap::from([(TRACEPARENT_HEADER.to_string(), traceparent)]); + if let Some(tracestate) = tracestate { + carrier.insert(TRACESTATE_HEADER.to_string(), tracestate); + } + let context = TraceContextPropagator::new().extract(&carrier); + context.span().span_context().is_valid().then_some(context) +} + +fn child_span(kind: ManagedChildSpan) -> Span { + match kind { + ManagedChildSpan::MutationScope => tracing::info_span!( + target: COMMAND_SPAN_TARGET, + MUTATION_SCOPE_SPAN_NAME, + sce.outcome = field::Empty, + ), + ManagedChildSpan::GitSnapshot => tracing::info_span!( + target: COMMAND_SPAN_TARGET, + GIT_SNAPSHOT_SPAN_NAME, + sce.outcome = field::Empty, + ), + ManagedChildSpan::Persistence => tracing::info_span!( + target: COMMAND_SPAN_TARGET, + PERSISTENCE_SPAN_NAME, + sce.outcome = field::Empty, + ), + } +} + +pub async fn in_child_span( + kind: ManagedChildSpan, + operation: impl Future>, +) -> Result { + let span = child_span(kind); + let result = Box::pin(operation.instrument(span.clone())).await; + let outcome = if result.is_ok() { + OUTCOME_SUCCESS + } else { + OUTCOME_FAILURE + }; + span.record(OUTCOME_FIELD, outcome); + result +} + pub fn record_command_name(name: &str) { Span::current().record(COMMAND_NAME_FIELD, name); } +pub fn record_claude_hook_command(command: ClaudeHookCommand) { + Span::current().record(HOOK_COMMAND_FIELD, command.as_str()); +} + +pub fn record_claude_hook_payload(stdin_payload: &str) { + let span = Span::current(); + if span.is_disabled() { + return; + } + let Ok(serde_json::Value::Object(payload)) = serde_json::from_str(stdin_payload) else { + return; + }; + for (payload_key, span_field) in CLAUDE_PAYLOAD_FIELDS { + if let Some(value) = payload + .get(payload_key) + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + { + span.record(span_field, value); + } + } +} + #[cfg(test)] pub(crate) mod tests { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -267,6 +446,27 @@ pub(crate) mod tests { captured } + pub(crate) async fn capture_requests_until( + listener: TcpListener, + stop: tokio::sync::oneshot::Receiver<()>, + ) -> Vec { + capture_requests_with_status_until(listener, "HTTP/1.1 200 OK", stop).await + } + + async fn capture_requests_with_status_until( + listener: TcpListener, + status_line: &'static str, + mut stop: tokio::sync::oneshot::Receiver<()>, + ) -> Vec { + let mut captured = Vec::new(); + loop { + tokio::select! { + request = capture_one_request(&listener, status_line) => captured.push(request), + _ = &mut stop => return captured, + } + } + } + async fn capture_one_request( listener: &TcpListener, status_line: &'static str, @@ -308,9 +508,185 @@ pub(crate) mod tests { } } + #[derive(Debug, Default)] + pub(crate) struct DecodedSpan { + pub(crate) name: String, + pub(crate) trace_id: Vec, + pub(crate) span_id: Vec, + pub(crate) parent_span_id: Vec, + pub(crate) attributes: Vec<(String, String)>, + pub(crate) resource_attributes: Vec<(String, String)>, + } + + impl DecodedSpan { + pub(crate) fn sce_attributes(&self) -> Vec<(String, String)> { + let mut attributes = self + .attributes + .iter() + .filter(|(key, _)| key.starts_with("sce.")) + .cloned() + .collect::>(); + attributes.sort(); + attributes + } + } + + fn read_varint(bytes: &[u8], position: &mut usize) -> u64 { + let mut value = 0_u64; + let mut shift = 0; + loop { + let byte = bytes[*position]; + *position += 1; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return value; + } + shift += 7; + } + } + + fn length_delimited_fields(bytes: &[u8]) -> Vec<(u64, &[u8])> { + let mut fields = Vec::new(); + let mut position = 0; + while position < bytes.len() { + let key = read_varint(bytes, &mut position); + match key & 0x7 { + 0 => { + read_varint(bytes, &mut position); + } + 1 => position += 8, + 2 => { + let length = usize::try_from(read_varint(bytes, &mut position)).unwrap(); + fields.push((key >> 3, &bytes[position..position + length])); + position += length; + } + 5 => position += 4, + wire => panic!("unsupported protobuf wire type {wire}"), + } + } + fields + } + + fn fields_numbered(bytes: &[u8], number: u64) -> impl Iterator { + length_delimited_fields(bytes) + .into_iter() + .filter(move |(field, _)| *field == number) + .map(|(_, value)| value) + } + + fn decode_string_attribute(bytes: &[u8]) -> Option<(String, String)> { + let key = fields_numbered(bytes, 1).next()?; + let value = fields_numbered(bytes, 2).next()?; + let string = fields_numbered(value, 1).next()?; + Some(( + String::from_utf8_lossy(key).to_string(), + String::from_utf8_lossy(string).to_string(), + )) + } + + pub(crate) fn decode_spans(body: &[u8]) -> Vec { + let mut spans = Vec::new(); + for resource_spans in fields_numbered(body, 1) { + let resource_attributes = fields_numbered(resource_spans, 1) + .flat_map(|resource| fields_numbered(resource, 1)) + .filter_map(decode_string_attribute) + .collect::>(); + for scope_spans in fields_numbered(resource_spans, 2) { + for span in fields_numbered(scope_spans, 2) { + let mut decoded = DecodedSpan { + resource_attributes: resource_attributes.clone(), + ..DecodedSpan::default() + }; + for (field, value) in length_delimited_fields(span) { + match field { + 1 => decoded.trace_id = value.to_vec(), + 2 => decoded.span_id = value.to_vec(), + 4 => decoded.parent_span_id = value.to_vec(), + 5 => decoded.name = String::from_utf8_lossy(value).to_string(), + 9 => decoded.attributes.extend(decode_string_attribute(value)), + _ => {} + } + } + spans.push(decoded); + } + } + } + spans + } + + pub(crate) fn trace_spans(requests: &[CapturedRequest]) -> Vec { + requests + .iter() + .filter(|request| { + request + .head + .to_ascii_lowercase() + .starts_with("post /v1/traces ") + }) + .flat_map(|request| decode_spans(&request.body)) + .collect() + } + pub(crate) fn loopback_telemetry( address: std::net::SocketAddr, token: &str, + ) -> ManagedTraceTelemetry { + loopback_telemetry_of_kind(address, token, ManagedSpanKind::Command) + } + + pub(crate) fn loopback_hook_telemetry( + address: std::net::SocketAddr, + token: &str, + ) -> ManagedTraceTelemetry { + loopback_telemetry_of_kind(address, token, ManagedSpanKind::ClaudeHook) + } + + fn loopback_hook_telemetry_with_traceparent( + address: std::net::SocketAddr, + traceparent: Option<&str>, + ) -> ManagedTraceTelemetry { + let mut telemetry = loopback_hook_telemetry(address, "bao-token-value"); + telemetry.parent_context = extract_parent_context( + traceparent.map(str::to_string), + Some("vendor=opaque".to_string()), + ); + telemetry + } + + const UPSTREAM_TRACE_ID: [u8; 16] = [ + 0x4b, 0xf9, 0x2f, 0x35, 0x77, 0xb3, 0x4d, 0xa6, 0xa3, 0xce, 0x92, 0x9d, 0x0e, 0x0e, 0x47, + 0x36, + ]; + const UPSTREAM_SPAN_ID: [u8; 8] = [0x00, 0xf0, 0x67, 0xaa, 0x0b, 0xa9, 0x02, 0xb7]; + const UPSTREAM_TRACEPARENT: &str = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + + async fn export_hook_spans( + traceparent: Option<&str>, + action: impl Future + Send + 'static, + ) -> Vec { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr"); + let receiver = tokio::spawn(capture_requests(listener, "HTTP/1.1 200 OK", 1)); + let telemetry = loopback_hook_telemetry_with_traceparent(address, traceparent); + let mut action = Some(action); + let result = telemetry + .with_default_subscriber(&mut || { + let action = action.take().expect("single invocation"); + async move { + action.await; + Ok(String::new()) + } + }) + .await; + assert_eq!(result.expect("hook result"), ""); + telemetry.shutdown().await; + trace_spans(&receiver.await.expect("receiver join")) + } + + fn loopback_telemetry_of_kind( + address: std::net::SocketAddr, + token: &str, + span_kind: ManagedSpanKind, ) -> ManagedTraceTelemetry { let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) @@ -318,13 +694,14 @@ pub(crate) mod tests { .tls_certs_only(Vec::new()) .build() .expect("test http client"); - loopback_telemetry_with_client(address, token, client) + loopback_telemetry_with_client(address, token, client, span_kind) } fn loopback_telemetry_with_client( address: std::net::SocketAddr, token: &str, client: reqwest::Client, + span_kind: ManagedSpanKind, ) -> ManagedTraceTelemetry { let traces_endpoint: &'static str = Box::leak(format!("http://{address}/v1/traces").into_boxed_str()); @@ -357,6 +734,7 @@ pub(crate) mod tests { endpoints, tokio::runtime::Handle::current(), client, + span_kind, ) .expect("telemetry") } @@ -432,110 +810,333 @@ pub(crate) mod tests { ] { assert!(!log_body.contains(leaked), "leaked {leaked}"); } + let spans = decode_spans(&trace.body); + assert_eq!(spans.len(), 1, "{spans:?}"); + assert_eq!(spans[0].name, "sce.command"); + assert_eq!( + spans[0].sce_attributes(), + [("sce.command.name", "config"), ("sce.outcome", "success")] + .map(|(key, value)| (key.to_string(), value.to_string())) + ); + assert!(spans[0] + .resource_attributes + .contains(&("service.name".to_string(), "sce".to_string()))); let body = String::from_utf8_lossy(&trace.body).to_string(); - for expected in [ - "sce.command", - "sce.command.name", - "config", - "service.name", - "success", - ] { - assert!(body.contains(expected), "missing {expected}"); + for leaked in ["local-accessor-secret", "bao-token-value"] { + assert!(!body.contains(leaked), "leaked {leaked}"); } - assert!(!body.contains("local-accessor-secret")); - assert!(!body.contains("bao-token-value")); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn managed_trace_export_does_not_follow_redirects_or_alter_results() { + async fn claude_hook_export_posts_one_hook_span_with_correlation_ids_only() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); let address = listener.local_addr().expect("addr"); - let receiver = tokio::spawn(capture_requests( - listener, - "HTTP/1.1 307 Temporary Redirect\r\nlocation: http://127.0.0.1:1/steal", - 1, - )); - let telemetry = loopback_telemetry(address, "bao-token-value"); + let receiver = tokio::spawn(capture_requests(listener, "HTTP/1.1 200 OK", 1)); + let telemetry = loopback_hook_telemetry(address, "bao-token-value"); + let payload = r#"{"session_id":"session-7d07","transcript_path":"/home/u/private-repo/transcript.jsonl","cwd":"/home/u/private-repo","prompt":"PROMPT_TEXT_VALUE","hook_event_name":"PostToolUse","tool_name":"Write","tool_input":{"file_path":"/home/u/private-repo/src/secret.rs","content":"FILE_CONTENT_VALUE Bearer leak-value"},"tool_response":{"success":true,"output":"TOOL_RESPONSE_VALUE"},"tool_use_id":"toolu_01KN1s"}"#; let result = telemetry .with_default_subscriber(&mut || async { - Err(CliError::runtime(anyhow::anyhow!("boom"))) + record_command_name("hooks"); + record_claude_hook_command(ClaudeHookCommand::DiffTrace); + record_claude_hook_payload(payload); + tokio::task::yield_now().await; + Ok(String::new()) }) .await; - assert!(result.is_err()); + assert_eq!(result.expect("hook result"), ""); telemetry.shutdown().await; let requests = receiver.await.expect("receiver join"); - assert!(!requests.is_empty()); + let spans = trace_spans(&requests); + assert_eq!(spans.len(), 1, "{spans:?}"); + let span = &spans[0]; + assert_eq!(span.name, "sce.hook"); + assert!(span.parent_span_id.is_empty()); + assert!(span + .resource_attributes + .contains(&("service.name".to_string(), "sce".to_string()))); + let expected = [ + ("sce.agent", "claude"), + ("sce.claude.session_id", "session-7d07"), + ("sce.claude.tool_use_id", "toolu_01KN1s"), + ("sce.hook.command", "diff-trace"), + ("sce.hook.event", "PostToolUse"), + ("sce.hook.tool", "Write"), + ("sce.outcome", "success"), + ] + .map(|(key, value)| (key.to_string(), value.to_string())); + assert_eq!(span.sce_attributes(), expected); for request in &requests { - let head = request.head.to_ascii_lowercase(); - assert!(head.starts_with("post /v1/traces ") || head.starts_with("post /v1/logs ")); + let body = String::from_utf8_lossy(&request.body).to_string(); + for leaked in [ + "private-repo", + "PROMPT_TEXT_VALUE", + "FILE_CONTENT_VALUE", + "TOOL_RESPONSE_VALUE", + "leak-value", + "bao-token-value", + "local-accessor-secret", + "sce.command", + ] { + assert!(!body.contains(leaked), "leaked {leaked}"); + } } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn unreachable_collector_does_not_block_command_or_shutdown() { + async fn claude_hook_span_joins_valid_upstream_trace_with_fresh_span_id() { + let spans = export_hook_spans(Some(UPSTREAM_TRACEPARENT), async { + tokio::task::yield_now().await; + }) + .await; + assert_eq!(spans.len(), 1, "{spans:?}"); + let span = &spans[0]; + assert_eq!(span.name, "sce.hook"); + assert_eq!(span.trace_id, UPSTREAM_TRACE_ID); + assert_eq!(span.parent_span_id, UPSTREAM_SPAN_ID); + assert_eq!(span.span_id.len(), 8); + assert_ne!(span.span_id, UPSTREAM_SPAN_ID); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn claude_hook_span_starts_independent_root_without_usable_traceparent() { + for traceparent in [ + None, + Some(""), + Some("not-a-traceparent"), + Some("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7"), + Some("00-4BF92F3577B34DA6A3CE929D0E0E4736-00f067aa0ba902b7-01"), + Some("ff-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"), + Some("00-00000000000000000000000000000000-00f067aa0ba902b7-01"), + Some("00-4bf92f3577b34da6a3ce929d0e0e4736-0000000000000000-01"), + ] { + let spans = export_hook_spans(traceparent, async {}).await; + assert_eq!(spans.len(), 1, "{traceparent:?}: {spans:?}"); + let span = &spans[0]; + assert_eq!(span.name, "sce.hook"); + assert!(span.parent_span_id.is_empty(), "{traceparent:?}"); + assert_eq!(span.trace_id.len(), 16, "{traceparent:?}"); + assert_ne!(span.trace_id, UPSTREAM_TRACE_ID, "{traceparent:?}"); + assert!( + span.trace_id.iter().any(|byte| *byte != 0), + "{traceparent:?}" + ); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn claude_hook_child_spans_inherit_upstream_trace_across_polls_and_workers() { + let spans = export_hook_spans(Some(UPSTREAM_TRACEPARENT), async { + for _ in 0..8 { + tokio::task::yield_now().await; + } + in_child_span(ManagedChildSpan::MutationScope, async { + tokio::task::yield_now().await; + tokio::spawn( + async { + tokio::task::yield_now().await; + in_child_span(ManagedChildSpan::GitSnapshot, async { + for _ in 0..4 { + tokio::task::yield_now().await; + } + Ok::<(), ()>(()) + }) + .await + } + .in_current_span() + .with_current_subscriber(), + ) + .await + .expect("spawned child join") + }) + .await + .expect("child span result"); + }) + .await; + assert_eq!(spans.len(), 3, "{spans:?}"); + let by_name = |name: &str| { + spans + .iter() + .find(|span| span.name == name) + .unwrap_or_else(|| panic!("missing {name}")) + }; + let hook = by_name("sce.hook"); + let child = by_name("sce.mutation_scope"); + let grandchild = by_name("sce.git_snapshot"); + for span in [hook, child, grandchild] { + assert_eq!(span.trace_id, UPSTREAM_TRACE_ID, "{}", span.name); + } + assert_eq!(hook.parent_span_id, UPSTREAM_SPAN_ID); + assert_eq!(child.parent_span_id, hook.span_id); + assert_eq!(grandchild.parent_span_id, child.span_id); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn command_export_keeps_single_command_span_when_child_operations_run() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); let address = listener.local_addr().expect("addr"); - drop(listener); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let receiver = tokio::spawn(capture_requests_until(listener, stopped)); let telemetry = loopback_telemetry(address, "bao-token-value"); let result = telemetry - .with_default_subscriber(&mut || async { Ok("ok".to_string()) }) + .with_default_subscriber(&mut || async { + record_command_name("doctor"); + for kind in [ + ManagedChildSpan::MutationScope, + ManagedChildSpan::GitSnapshot, + ManagedChildSpan::Persistence, + ] { + in_child_span(kind, async { Ok::<(), ()>(()) }) + .await + .expect("child operation"); + } + Ok(String::new()) + }) .await; - assert_eq!(result.expect("result"), "ok"); - let started = std::time::Instant::now(); + assert_eq!(result.expect("command result"), ""); telemetry.shutdown().await; - assert!(started.elapsed() < Duration::from_secs(6)); + stop.send(()).expect("stop receiver"); + + let spans = trace_spans(&receiver.await.expect("receiver join")); + let names = spans + .iter() + .map(|span| span.name.as_str()) + .collect::>(); + assert_eq!(names, ["sce.command"]); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn ambient_otlp_environment_cannot_redirect_or_inject_headers() { - let alternate = TcpListener::bind("127.0.0.1:0").await.expect("bind alt"); - let alternate_address = alternate.local_addr().expect("alt addr"); + async fn managed_trace_export_does_not_follow_redirects_or_alter_results() { + let destination = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind destination"); + let destination_address = destination.local_addr().expect("destination addr"); let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); let address = listener.local_addr().expect("addr"); - std::env::set_var( - "OTEL_EXPORTER_OTLP_ENDPOINT", - format!("http://{alternate_address}"), - ); - std::env::set_var( - "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", - format!("http://{alternate_address}/v1/traces"), + let redirect: &'static str = Box::leak( + format!( + "HTTP/1.1 307 Temporary Redirect\r\nlocation: http://{destination_address}/v1/traces" + ) + .into_boxed_str(), ); - std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", "x-ambient-leak=1"); - std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc"); - let receiver = tokio::spawn(capture_requests(listener, "HTTP/1.1 200 OK", 2)); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let receiver = tokio::spawn(capture_requests_with_status_until( + listener, redirect, stopped, + )); let telemetry = loopback_telemetry(address, "bao-token-value"); + let expected = CliError::runtime(anyhow::anyhow!("boom")); - let result = telemetry + let error = telemetry .with_default_subscriber(&mut || async { - record_command_name("config"); - tracing::info!(target: "sce", event_id = "sce.test.event", event_message = "Starting command"); - Ok("payload".to_string()) + Err(CliError::runtime(anyhow::anyhow!("boom"))) }) - .await; - assert_eq!(result.expect("command result"), "payload"); + .await + .expect_err("command error is preserved"); + assert_eq!(error.code(), expected.code()); + assert_eq!(error.to_string(), expected.to_string()); telemetry.shutdown().await; - for name in [ - "OTEL_EXPORTER_OTLP_ENDPOINT", - "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", - "OTEL_EXPORTER_OTLP_HEADERS", - "OTEL_EXPORTER_OTLP_PROTOCOL", - ] { - std::env::remove_var(name); - } + stop.send(()).expect("stop receiver"); let requests = receiver.await.expect("receiver join"); - assert_eq!(requests.len(), 2); + assert!(!trace_spans(&requests).is_empty()); for request in &requests { let head = request.head.to_ascii_lowercase(); assert!(head.contains("authorization: bearer bao-token-value")); - assert!(!head.contains("x-ambient-leak")); - assert!(head.contains("application/x-protobuf")); } - let unexpected = tokio::time::timeout(Duration::from_millis(300), alternate.accept()).await; - assert!(unexpected.is_err(), "alternate receiver was contacted"); + let redirected = + tokio::time::timeout(Duration::from_millis(300), destination.accept()).await; + assert!(redirected.is_err(), "redirect destination was contacted"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn unreachable_collector_does_not_block_command_or_shutdown() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr"); + drop(listener); + let telemetry = loopback_telemetry(address, "bao-token-value"); + + let result = telemetry + .with_default_subscriber(&mut || async { Ok("ok".to_string()) }) + .await; + assert_eq!(result.expect("result"), "ok"); + let started = std::time::Instant::now(); + telemetry.shutdown().await; + assert!(started.elapsed() < Duration::from_secs(6)); + } + + #[test] + fn ambient_otlp_environment_cannot_redirect_or_inject_headers() { + crate::app::tests::run_fault_child( + "services::managed_telemetry::trace_export::tests::isolated_ambient_otlp_environment", + ); + } + + #[test] + fn isolated_ambient_otlp_environment() { + if !crate::app::tests::is_fault_child() { + return; + } + let bind = || { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + listener.set_nonblocking(true).expect("nonblocking"); + listener + }; + let (alternate, collector) = (bind(), bind()); + let alternate_address = alternate.local_addr().expect("alt addr"); + let address = collector.local_addr().expect("addr"); + for (name, value) in [ + ( + "OTEL_EXPORTER_OTLP_ENDPOINT", + format!("http://{alternate_address}"), + ), + ( + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + format!("http://{alternate_address}/v1/traces"), + ), + ( + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + format!("http://{alternate_address}/v1/logs"), + ), + ( + "OTEL_EXPORTER_OTLP_HEADERS", + "x-ambient-leak=1,authorization=Bearer ambient-leak".to_string(), + ), + ("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc".to_string()), + ] { + std::env::set_var(name, value); + } + let runtime = crate::app::build_runtime().expect("runtime"); + runtime.block_on(async { + let alternate = TcpListener::from_std(alternate).expect("alt listener"); + let collector = TcpListener::from_std(collector).expect("listener"); + let receiver = tokio::spawn(capture_requests(collector, "HTTP/1.1 200 OK", 2)); + let telemetry = loopback_telemetry(address, "bao-token-value"); + + let result = telemetry + .with_default_subscriber(&mut || async { + record_command_name("config"); + tracing::info!(target: "sce", event_id = "sce.test.event", event_message = "Starting command"); + Ok("payload".to_string()) + }) + .await; + assert_eq!(result.expect("command result"), "payload"); + telemetry.shutdown().await; + + let requests = receiver.await.expect("receiver join"); + assert_eq!(requests.len(), 2); + for request in &requests { + let head = request.head.to_ascii_lowercase(); + assert!(head.contains("authorization: bearer bao-token-value")); + assert!(!head.contains("ambient-leak")); + assert!(head.contains("application/x-protobuf")); + } + let unexpected = + tokio::time::timeout(Duration::from_millis(300), alternate.accept()).await; + assert!(unexpected.is_err(), "alternate receiver was contacted"); + }); + crate::app::finish_runtime(runtime); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -566,7 +1167,8 @@ pub(crate) mod tests { async fn concurrent_commands_keep_results_and_export_isolated() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); let address = listener.local_addr().expect("addr"); - let receiver = tokio::spawn(capture_requests(listener, "HTTP/1.1 200 OK", 1)); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let receiver = tokio::spawn(capture_requests_until(listener, stopped)); let telemetry = std::sync::Arc::new(loopback_telemetry(address, "bao-token-value")); let first = { @@ -600,21 +1202,30 @@ pub(crate) mod tests { .shutdown() .await; - let requests = receiver.await.expect("receiver join"); - let trace = requests + stop.send(()).expect("stop receiver"); + + let mut commands = trace_spans(&receiver.await.expect("receiver join")) .iter() - .find(|request| { - request - .head - .to_ascii_lowercase() - .starts_with("post /v1/traces ") - }) - .expect("trace request"); - let body = String::from_utf8_lossy(&trace.body).to_string(); - assert!(body.contains("first")); - assert!(body.contains("second")); - assert!(body.contains("success")); - assert!(body.contains("failure")); + .filter(|span| span.name == "sce.command") + .map(DecodedSpan::sce_attributes) + .collect::>(); + commands.sort(); + let failure_class = CliError::runtime(anyhow::anyhow!("boom")).code(); + let expected = [ + vec![("sce.command.name", "first"), ("sce.outcome", "success")], + vec![ + ("sce.command.name", "second"), + ("sce.error.class", failure_class), + ("sce.outcome", "failure"), + ], + ] + .map(|attributes| { + attributes + .into_iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect::>() + }); + assert_eq!(commands, expected); } #[test] @@ -647,7 +1258,12 @@ pub(crate) mod tests { .build() .expect("test http client"); let telemetry = - loopback_telemetry_with_client(address, "bao-token-value", unbounded_client); + loopback_telemetry_with_client( + address, + "bao-token-value", + unbounded_client, + ManagedSpanKind::Command, + ); let result = telemetry .with_default_subscriber(&mut || async { record_command_name("config"); diff --git a/cli/src/services/mod.rs b/cli/src/services/mod.rs index 2612c646b..2f484e8e0 100644 --- a/cli/src/services/mod.rs +++ b/cli/src/services/mod.rs @@ -37,5 +37,6 @@ pub mod structured_patch; pub mod style; pub mod sync; pub mod telemetry_credential_storage; +pub mod telemetry_readiness; pub mod token_storage; pub mod version; diff --git a/cli/src/services/mutation_trace/runtime/coordinator.rs b/cli/src/services/mutation_trace/runtime/coordinator.rs index 94eee0341..ccb3c5aec 100644 --- a/cli/src/services/mutation_trace/runtime/coordinator.rs +++ b/cli/src/services/mutation_trace/runtime/coordinator.rs @@ -5,6 +5,7 @@ use anyhow::Result; use uuid::Uuid; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::managed_telemetry::trace_export::{in_child_span, ManagedChildSpan}; use crate::services::mutation_trace::protocol; use crate::services::mutation_trace::store::{ CasResult, DurableTransition, MutationTraceStore, ScopeProvenance, @@ -353,8 +354,8 @@ async fn coordinate_boundary_inner( worktree_id: &WorktreeId, boundary: &RuntimeBoundary, inherited_external_taint: bool, - mut after_load: AfterLoad, - mut after_recovery: AfterRecovery, + after_load: AfterLoad, + after_recovery: AfterRecovery, ) -> Result where C: SnapshotCapture, @@ -363,11 +364,44 @@ where { let mut store = MutationTraceStore::new(db); - let observed_tree = match capture_and_pin(capture, pin_lease, worktree_id).await { + let observed_tree = match in_child_span( + ManagedChildSpan::GitSnapshot, + capture_and_pin(capture, pin_lease, worktree_id), + ) + .await + { Ok(tree) => tree, Err(source) => return Err(handle_snapshot_failure(&mut store, worktree_id, source).await), }; + in_child_span( + ManagedChildSpan::Persistence, + persist_boundary( + &mut store, + worktree_id, + boundary, + observed_tree, + inherited_external_taint, + after_load, + after_recovery, + ), + ) + .await +} + +async fn persist_boundary( + store: &mut MutationTraceStore<'_>, + worktree_id: &WorktreeId, + boundary: &RuntimeBoundary, + observed_tree: TreeId, + inherited_external_taint: bool, + mut after_load: AfterLoad, + mut after_recovery: AfterRecovery, +) -> Result +where + AfterLoad: FnMut(u32), + AfterRecovery: FnMut(u32) -> Result<()>, +{ store .initialize_worktree(worktree_id, &observed_tree) .await @@ -383,7 +417,7 @@ where None => None, }; - register_start_provenance(&store, boundary, registered_scope.as_ref()).await?; + register_start_provenance(store, boundary, registered_scope.as_ref()).await?; let type_boundary = into_protocol_boundary(boundary, worktree_id); let scope_ref = types::boundary_scope(&type_boundary); diff --git a/cli/src/services/mutation_trace/runtime/scope_runtime.rs b/cli/src/services/mutation_trace/runtime/scope_runtime.rs index fb3397452..11bf84547 100644 --- a/cli/src/services/mutation_trace/runtime/scope_runtime.rs +++ b/cli/src/services/mutation_trace/runtime/scope_runtime.rs @@ -1,6 +1,7 @@ use std::path::Path; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::managed_telemetry::trace_export::{in_child_span, ManagedChildSpan}; use crate::services::mutation_trace::protocol; use crate::services::mutation_trace::store::{CasResult, DurableTransition, MutationTraceStore}; use crate::services::mutation_trace::types::{ProtocolState, ScopeId, ScopeStatus, WorktreeId}; @@ -147,7 +148,11 @@ where }); } - let outcome = abandon_protected(protected.worktree_id(), scope, open_db, after_load).await?; + let outcome = in_child_span( + ManagedChildSpan::Persistence, + abandon_protected(protected.worktree_id(), scope, open_db, after_load), + ) + .await?; if matches!(outcome, AbandonScopeOutcome::RecoveryRequired { .. }) { return Ok(outcome); diff --git a/cli/src/services/setup/command.rs b/cli/src/services/setup/command.rs index ca2c6ebd7..f05f5a9fd 100644 --- a/cli/src/services/setup/command.rs +++ b/cli/src/services/setup/command.rs @@ -1,5 +1,3 @@ -use std::future::Future; - use anyhow::Context; use crate::app::ContextWithRepoRoot; @@ -7,12 +5,6 @@ use crate::services::error::{CliError, UserError}; use crate::services::lifecycle::{ lifecycle_providers, RequiredHookInstallStatus, RequiredHooksInstallOutcome, }; -use crate::services::managed_telemetry::claude_helper::repository_id_for; -use crate::services::managed_telemetry::{run_with_deadline, CREDENTIAL_LOOKUP_DEADLINE}; -use crate::services::telemetry_credential_storage::{ - bind_repository_telemetry, RepositoryBindingMode, -}; -use crate::services::token_storage::TokenStorageError; use crate::services::{config, setup}; pub struct SetupCommand { @@ -108,7 +100,7 @@ impl SetupCommand { sections.push(setup_message); if claude_telemetry_selected { - sections.push(claude_telemetry_report(&repository_root).await); + sections.push(claude_telemetry_report()); } } @@ -124,31 +116,17 @@ fn dispatch_targets_claude(dispatch: Option<&setup::SetupDispatch>) -> bool { ) } -async fn claude_telemetry_report(repository_root: &std::path::Path) -> String { - let repository_id = repository_id_for(repository_root); +fn claude_telemetry_report() -> String { render_claude_telemetry_section( setup::claude_telemetry::resolve_claude_settings_path().as_deref(), &setup::claude_telemetry::managed_claude_settings_path(), - async move { - match repository_id { - Some(repository_id) => { - bind_repository_telemetry(repository_id, RepositoryBindingMode::Enroll).await - } - None => Ok(false), - } - }, ) - .await } -async fn render_claude_telemetry_section( +fn render_claude_telemetry_section( settings_path: Option<&std::path::Path>, managed_settings_path: &std::path::Path, - repository_enrollment: Fut, -) -> String -where - Fut: Future> + Send + 'static, -{ +) -> String { use setup::claude_telemetry::{ format_claude_telemetry_report, install_claude_telemetry, ClaudeTelemetryInstallOutcome, }; @@ -158,11 +136,7 @@ where .unwrap_or(ClaudeTelemetryInstallOutcome::SkippedInvalid), None => ClaudeTelemetryInstallOutcome::SkippedInvalid, }; - let authentication_pending = installed == ClaudeTelemetryInstallOutcome::Configured - && !run_with_deadline(CREDENTIAL_LOOKUP_DEADLINE, repository_enrollment) - .await - .and_then(Result::ok) - .unwrap_or(false); + let authentication_pending = installed == ClaudeTelemetryInstallOutcome::Configured; format_claude_telemetry_report(&installed, authentication_pending) } @@ -275,80 +249,36 @@ mod tests { fn run_claude_telemetry_stage(dispatch: Option<&SetupDispatch>, root: &Path) -> Option { dispatch_targets_claude(dispatch).then(|| { - block_on_current_thread(render_claude_telemetry_section( + render_claude_telemetry_section( Some(&root.join("settings.json")), &root.join("managed.json"), - std::future::ready(Ok(false)), - )) + ) }) } - fn block_on_current_thread(future: F) -> F::Output { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap() - .block_on(future) - } - - #[tokio::test] - async fn setup_claude_telemetry_skips_repository_enrollment_when_install_is_skipped() { + #[test] + fn setup_claude_telemetry_reports_skip_without_repository_enrollment() { let dir = tempfile::tempdir().unwrap(); let settings = dir.path().join("settings.json"); fs::write(&settings, r#"{"otelHeadersHelper":"my-helper"}"#).unwrap(); - let looked_up = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let flag = looked_up.clone(); - let report = render_claude_telemetry_section( - Some(&settings), - &dir.path().join("managed.json"), - async move { - flag.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(false) - }, - ) - .await; + let report = + render_claude_telemetry_section(Some(&settings), &dir.path().join("managed.json")); assert!(report.contains("skipped")); - assert!(!looked_up.load(std::sync::atomic::Ordering::SeqCst)); + assert!(!report.contains("authentication pending")); } - #[tokio::test] - async fn setup_claude_telemetry_reports_pending_when_repository_enrollment_stalls() { + #[test] + fn setup_claude_telemetry_defers_repository_enrollment_to_auth_login() { let dir = tempfile::tempdir().unwrap(); - let started = std::time::Instant::now(); let report = render_claude_telemetry_section( Some(&dir.path().join("settings.json")), &dir.path().join("managed.json"), - std::future::pending(), - ) - .await; + ); assert!(report.contains("authentication pending")); - assert!(started.elapsed() < std::time::Duration::from_secs(5)); - } - - #[tokio::test] - async fn setup_claude_telemetry_reports_ready_only_when_repository_is_enrolled() { - for (enrolled, pending) in [(Ok(true), false), (Ok(false), true), (Err(storage()), true)] { - let dir = tempfile::tempdir().unwrap(); - let report = render_claude_telemetry_section( - Some(&dir.path().join("settings.json")), - &dir.path().join("managed.json"), - std::future::ready(enrolled), - ) - .await; - assert_eq!( - report.contains("authentication pending"), - pending, - "{report}" - ); - } - } - - fn storage() -> TokenStorageError { - TokenStorageError::Database("unavailable".to_string()) } fn assert_installed(root: &Path) { diff --git a/cli/src/services/sync/command.rs b/cli/src/services/sync/command.rs index 77adb11fb..c83165518 100644 --- a/cli/src/services/sync/command.rs +++ b/cli/src/services/sync/command.rs @@ -33,8 +33,8 @@ where #[allow(clippy::needless_pass_by_value)] fn classify_sync_error(err: TraceSyncError) -> CliError { - if err.is_storage_failure() { - CliError::user_with_source(UserError::AuthStorageUnavailable, err) + if let Some(failure) = err.storage_failure() { + CliError::user_with_source(UserError::AuthStorageUnavailable(failure), err) } else if err.is_authentication_failure() { CliError::user_with_source(UserError::NotAuthenticated, err) } else { diff --git a/cli/src/services/sync/sync.rs b/cli/src/services/sync/sync.rs index dd41bee3b..a07ff6a39 100644 --- a/cli/src/services/sync/sync.rs +++ b/cli/src/services/sync/sync.rs @@ -25,6 +25,8 @@ use crate::services::agent_trace_sync::{ }; use crate::services::auth; use crate::services::config; +use crate::services::db::encryption_key::CredentialAccess; +use crate::services::error::AuthStorageFailure; use crate::services::sync::progress::ProgressReporter; #[derive(Clone, Debug, PartialEq, Eq)] @@ -136,11 +138,11 @@ impl TraceSyncError { /// True when the failure came from local credential storage, whether it /// surfaced during the initial state request or a stream batch/refresh /// path. - pub fn is_storage_failure(&self) -> bool { + pub fn storage_failure(&self) -> Option { match self { - Self::ControlPlane(error) => error.is_storage_failure(), - Self::Stream { source, .. } => source.is_storage_failure(), - Self::Runtime(_) => false, + Self::ControlPlane(error) => error.storage_failure(), + Self::Stream { source, .. } => source.storage_failure(), + Self::Runtime(_) => None, } } } @@ -189,6 +191,7 @@ where auth_config.control_plane_base_url.value.unwrap_or_default(), auth::WORKOS_DEFAULT_BASE_URL, auth_config.workos_client_id.value.unwrap_or_default(), + CredentialAccess::NonInteractive, ); let result = run_sync_against_without_progress( @@ -510,7 +513,7 @@ fn is_stream_terminal(error: &ControlPlaneError) -> bool { | ControlPlaneError::AuthenticationFailed(_) | ControlPlaneError::BadRequest(_) | ControlPlaneError::Forbidden(_) - | ControlPlaneError::Storage(_) + | ControlPlaneError::Storage(..) | ControlPlaneError::Protocol { .. } ) } @@ -547,6 +550,7 @@ mod tests { }; use crate::services::agent_trace_sync::StreamSyncError; use crate::services::auth::TokenResponse; + use crate::services::db::encryption_key::CredentialAccess; use crate::services::sync::progress::NoopProgressReporter; use crate::services::token_storage::StoredTokens; @@ -627,6 +631,7 @@ mod tests { "http://127.0.0.1:1", "http://127.0.0.1:1", "client", + CredentialAccess::NonInteractive, ); let mut reporter = NoopProgressReporter; let progress = Rc::new(RefCell::new(&mut reporter)); @@ -704,6 +709,7 @@ mod tests { "http://127.0.0.1:1", "http://127.0.0.1:1", "client", + CredentialAccess::NonInteractive, ) } diff --git a/cli/src/services/telemetry_credential_storage.rs b/cli/src/services/telemetry_credential_storage.rs index 3110e029c..557cdd04d 100644 --- a/cli/src/services/telemetry_credential_storage.rs +++ b/cli/src/services/telemetry_credential_storage.rs @@ -5,8 +5,11 @@ use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine; use serde::Deserialize; +use crate::services::db::encryption_key::CredentialAccess; use crate::services::db::{DbSpec, EncryptedTursoDb}; -use crate::services::token_storage::{get_auth_db, TokenStorageError}; +use crate::services::token_storage::{ + open_existing_auth_db, open_or_create_auth_db, TokenStorageError, +}; const TELEMETRY_CREDENTIAL_ROW_ID: i64 = 1; const WORKOS_SESSION_ROW_ID: i64 = 1; @@ -75,6 +78,7 @@ impl TelemetryCredential { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RepositoryBindingMode { + #[cfg_attr(not(test), expect(dead_code))] Enroll, EnrollIfUnbound, } @@ -82,39 +86,40 @@ pub enum RepositoryBindingMode { pub async fn save_telemetry_credential( credential: &TelemetryCredential, ) -> Result<(), TokenStorageError> { - save_with(get_auth_db().await?, credential).await + save_with(open_or_create_auth_db().await?, credential).await } pub async fn load_telemetry_credential() -> Result, TokenStorageError> { - load_with(get_auth_db().await?, current_unix_timestamp_seconds()?).await + let Some(db) = open_existing_auth_db(CredentialAccess::NonInteractive).await? else { + return Ok(None); + }; + load_with(db, current_unix_timestamp_seconds()?).await } pub async fn load_repository_telemetry_credential( repository_id: String, ) -> Result, TokenStorageError> { - load_for_repository_with( - get_auth_db().await?, - current_unix_timestamp_seconds()?, - &repository_id, - ) - .await + let Some(db) = open_existing_auth_db(CredentialAccess::NonInteractive).await? else { + return Ok(None); + }; + load_for_repository_with(db, current_unix_timestamp_seconds()?, &repository_id).await } pub async fn bind_repository_telemetry( repository_id: String, mode: RepositoryBindingMode, ) -> Result { - bind_repository_with( - get_auth_db().await?, - current_unix_timestamp_seconds()?, - &repository_id, - mode, - ) - .await + let Some(db) = open_existing_auth_db(CredentialAccess::Interactive).await? else { + return Ok(false); + }; + bind_repository_with(db, current_unix_timestamp_seconds()?, &repository_id, mode).await } pub async fn delete_telemetry_credential() -> Result { - delete_with(get_auth_db().await?).await + let Some(db) = open_existing_auth_db(CredentialAccess::Interactive).await? else { + return Ok(false); + }; + delete_with(db).await } fn database_error(message: &str) -> TokenStorageError { @@ -912,6 +917,7 @@ mod tests { telemetry_headers_with( Some(working_dir), visible_destination_conflict(claude_helper_env), + true, repository_id_for, |repository_id: String| async move { let db = TestDb::new() diff --git a/cli/src/services/telemetry_readiness.rs b/cli/src/services/telemetry_readiness.rs new file mode 100644 index 000000000..00e9b2444 --- /dev/null +++ b/cli/src/services/telemetry_readiness.rs @@ -0,0 +1,193 @@ +use std::fs; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; + +use crate::services::default_paths::{auth_db_path, telemetry_readiness_path}; + +const TEMP_SUFFIX: &str = "tmp"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReadinessState { + NotConfigured, + Expired, + Ready, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct TelemetryReadiness { + expires_at_unix_seconds: u64, +} + +pub fn mark_ready(expires_at_unix_seconds: u64) -> Result<()> { + mark_ready_at(&telemetry_readiness_path()?, expires_at_unix_seconds) +} + +pub fn clear() -> Result<()> { + clear_at(&telemetry_readiness_path()?) +} + +pub fn is_ready(now_unix_seconds: u64) -> bool { + state(now_unix_seconds).is_ok_and(|state| state == ReadinessState::Ready) +} + +pub(crate) fn state(now_unix_seconds: u64) -> Result { + Ok(state_at( + &telemetry_readiness_path()?, + &auth_db_path()?, + now_unix_seconds, + )) +} + +pub(crate) fn mark_ready_at(path: &Path, expires_at_unix_seconds: u64) -> Result<()> { + let parent = path.parent().with_context(|| { + format!( + "telemetry readiness path '{}' has no parent", + path.display() + ) + })?; + fs::create_dir_all(parent) + .with_context(|| format!("failed to create '{}'", parent.display()))?; + let bytes = serde_json::to_vec(&TelemetryReadiness { + expires_at_unix_seconds, + }) + .context("failed to serialize telemetry readiness")?; + let temp_path = temp_path_for(path); + fs::write(&temp_path, bytes) + .with_context(|| format!("failed to write '{}'", temp_path.display()))?; + fs::rename(&temp_path, path).with_context(|| { + let _ = fs::remove_file(&temp_path); + format!("failed to replace '{}'", path.display()) + }) +} + +pub(crate) fn clear_at(path: &Path) -> Result<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| format!("failed to remove '{}'", path.display())), + } +} + +#[cfg(test)] +pub(crate) fn is_ready_at(path: &Path, auth_db_path: &Path, now_unix_seconds: u64) -> bool { + state_at(path, auth_db_path, now_unix_seconds) == ReadinessState::Ready +} + +pub(crate) fn state_at(path: &Path, auth_db_path: &Path, now_unix_seconds: u64) -> ReadinessState { + if !auth_db_path.is_file() { + return ReadinessState::NotConfigured; + } + match fs::read(path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + { + None => ReadinessState::NotConfigured, + Some(readiness) if readiness.expires_at_unix_seconds > now_unix_seconds => { + ReadinessState::Ready + } + Some(_) => ReadinessState::Expired, + } +} + +fn temp_path_for(path: &Path) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(format!(".{}.{TEMP_SUFFIX}", std::process::id())); + PathBuf::from(name) +} + +#[cfg(test)] +mod tests { + use super::*; + + const NOW: u64 = 1_000; + + struct Paths { + _dir: tempfile::TempDir, + readiness: PathBuf, + auth_db: PathBuf, + } + + fn paths_with_auth_db() -> Paths { + let dir = tempfile::tempdir().unwrap(); + let readiness = dir.path().join("sce").join("telemetry-ready.json"); + let auth_db = dir.path().join("sce").join("auth.db"); + fs::create_dir_all(auth_db.parent().unwrap()).unwrap(); + fs::write(&auth_db, b"encrypted").unwrap(); + Paths { + _dir: dir, + readiness, + auth_db, + } + } + + #[test] + fn readiness_state_requires_valid_unexpired_marker_and_auth_db() { + for (case, marker, auth_db, expected) in [ + ("missing marker", None, true, ReadinessState::NotConfigured), + ( + "malformed marker", + Some(b"not json".to_vec()), + true, + ReadinessState::NotConfigured, + ), + ( + "valid marker", + Some(format!(r#"{{"expires_at_unix_seconds":{}}}"#, NOW + 1).into_bytes()), + true, + ReadinessState::Ready, + ), + ( + "expired marker", + Some(format!(r#"{{"expires_at_unix_seconds":{NOW}}}"#).into_bytes()), + true, + ReadinessState::Expired, + ), + ( + "missing auth DB", + Some(format!(r#"{{"expires_at_unix_seconds":{}}}"#, NOW + 1).into_bytes()), + false, + ReadinessState::NotConfigured, + ), + ] { + let paths = paths_with_auth_db(); + if let Some(marker) = marker { + fs::write(&paths.readiness, marker).unwrap(); + } + if !auth_db { + fs::remove_file(&paths.auth_db).unwrap(); + } + assert_eq!( + state_at(&paths.readiness, &paths.auth_db, NOW), + expected, + "{case}" + ); + } + } + + #[test] + fn marked_readiness_is_ready_until_expiry_and_holds_no_secret() { + let paths = paths_with_auth_db(); + mark_ready_at(&paths.readiness, NOW + 1).unwrap(); + + assert!(is_ready_at(&paths.readiness, &paths.auth_db, NOW)); + assert!(!is_ready_at(&paths.readiness, &paths.auth_db, NOW + 1)); + assert_eq!( + fs::read_to_string(&paths.readiness).unwrap(), + format!(r#"{{"expires_at_unix_seconds":{}}}"#, NOW + 1) + ); + } + + #[test] + fn clearing_readiness_disables_it_and_is_idempotent() { + let paths = paths_with_auth_db(); + mark_ready_at(&paths.readiness, NOW + 1).unwrap(); + + clear_at(&paths.readiness).unwrap(); + clear_at(&paths.readiness).unwrap(); + + assert!(!is_ready_at(&paths.readiness, &paths.auth_db, NOW)); + } +} diff --git a/cli/src/services/token_storage.rs b/cli/src/services/token_storage.rs index ccdb7977b..afedb8803 100644 --- a/cli/src/services/token_storage.rs +++ b/cli/src/services/token_storage.rs @@ -6,25 +6,51 @@ use serde::{Deserialize, Serialize}; use crate::services::auth::TokenResponse; use crate::services::auth_db::AuthDb; +use crate::services::db::encryption_key::{ + CredentialAccess, CredentialAccessDenied, CredentialAccessUnavailable, +}; +use crate::services::db::DatabaseLocked; +use crate::services::default_paths::auth_db_path; +use crate::services::error::AuthStorageFailure; -/// Constant row ID for the single token row in `auth_credentials`. const DEFAULT_TOKEN_ROW_ID: i64 = 1; -/// Lazy singleton for the encrypted auth database. -/// -/// Stores `Result` so initialization failures are preserved across calls. -static AUTH_DB: OnceCell> = OnceCell::const_new(); - -pub(crate) async fn get_auth_db() -> Result<&'static AuthDb, TokenStorageError> { - let result = AUTH_DB - .get_or_init(async || AuthDb::new().await.map_err(|e| e.to_string())) - .await; - match result { - Ok(db) => Ok(db), - Err(msg) => Err(TokenStorageError::Database(msg.clone())), +static AUTH_DB: OnceCell = OnceCell::const_new(); + +fn open_error(error: &anyhow::Error) -> TokenStorageError { + if error.is::() || error.is::() { + TokenStorageError::CredentialAccessUnavailable + } else if error.is::() { + TokenStorageError::DatabaseLocked(error.to_string()) + } else { + TokenStorageError::Database(error.to_string()) } } +pub(crate) async fn open_or_create_auth_db() -> Result<&'static AuthDb, TokenStorageError> { + AUTH_DB + .get_or_try_init(async || AuthDb::new().await.map_err(|error| open_error(&error))) + .await +} + +pub(crate) async fn open_existing_auth_db( + access: CredentialAccess, +) -> Result, TokenStorageError> { + if let Some(db) = AUTH_DB.get() { + return Ok(Some(db)); + } + let db_path = + auth_db_path().map_err(|error| TokenStorageError::PathResolution(error.to_string()))?; + let Some(db) = AuthDb::open_existing_at(db_path, access) + .await + .map_err(|error| open_error(&error))? + else { + return Ok(None); + }; + let _ = AUTH_DB.set(db); + Ok(AUTH_DB.get()) +} + #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct StoredTokens { pub access_token: String, @@ -53,6 +79,18 @@ impl StoredTokens { pub enum TokenStorageError { PathResolution(String), Database(String), + DatabaseLocked(String), + CredentialAccessUnavailable, +} + +impl TokenStorageError { + pub fn failure(&self) -> AuthStorageFailure { + match self { + Self::CredentialAccessUnavailable => AuthStorageFailure::CredentialAccess, + Self::DatabaseLocked(_) => AuthStorageFailure::DatabaseLocked, + Self::PathResolution(_) | Self::Database(_) => AuthStorageFailure::Database, + } + } } impl fmt::Display for TokenStorageError { @@ -66,14 +104,30 @@ impl fmt::Display for TokenStorageError { f, "Token storage database error: {reason}. Try: ensure the OS credential store is available and the auth database is accessible." ), + Self::DatabaseLocked(reason) => write!( + f, + "Token storage database is locked: {reason}. Try: wait for other SCE processes to exit and retry." + ), + Self::CredentialAccessUnavailable => write!( + f, + "Token storage requires credential store authorization. Try: run 'sce auth login' to authorize access." + ), } } } impl std::error::Error for TokenStorageError {} -pub async fn save_tokens(token: &TokenResponse) -> Result { - let db = get_auth_db().await?; +pub async fn save_tokens( + token: &TokenResponse, + access: CredentialAccess, +) -> Result { + let db = match access { + CredentialAccess::Interactive => open_or_create_auth_db().await?, + CredentialAccess::NonInteractive => open_existing_auth_db(access) + .await? + .ok_or(TokenStorageError::CredentialAccessUnavailable)?, + }; let stored = StoredTokens::from_token_response(token)?; let expires_in = i64::try_from(stored.expires_in).map_err(|error| { @@ -109,8 +163,12 @@ pub async fn save_tokens(token: &TokenResponse) -> Result Result, TokenStorageError> { - let db = get_auth_db().await?; +pub async fn load_tokens( + access: CredentialAccess, +) -> Result, TokenStorageError> { + let Some(db) = open_existing_auth_db(access).await? else { + return Ok(None); + }; let sql = "SELECT access_token, token_type, expires_in, refresh_token, scope, \ stored_at_unix_seconds FROM auth_credentials WHERE id = ?1"; @@ -150,7 +208,9 @@ pub async fn load_tokens() -> Result, TokenStorageError> { } pub async fn delete_tokens() -> Result { - let db = get_auth_db().await?; + let Some(db) = open_existing_auth_db(CredentialAccess::Interactive).await? else { + return Ok(false); + }; let affected = db .execute( diff --git a/context/architecture.md b/context/architecture.md index c13ca8749..6c6d2bbca 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -53,7 +53,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - Command parse-time conversion and run-time handling are separated by an internal static `RuntimeCommand` seam. `cli/src/services/command_registry.rs` defines the `RuntimeCommand` enum with variants for help/help-text, version, completion, auth, config, setup, doctor, hooks, policy, and sync, plus a deterministic `CommandRegistry` name catalog populated by `build_default_registry()`. `parse_command_phase` in `cli/src/app.rs` delegates clap-output conversion to `cli/src/services/parse/command_runtime.rs`, which owns clap error classification, help rendering bridges, longest-valid-parent traversal for unknown command paths, and parsed-request-to-enum conversion while returning concrete enum values. Service-owned `command.rs` modules define command payload structs and generic execution methods with narrow context requirements: context-free commands accept any context, hooks requires logger access, setup/doctor require repo-root scoping, and central dispatch requires the union of logger plus repo-root-scoping capabilities. `services::app_support::execute_command_phase` emits lifecycle logs around `RuntimeCommand::execute_with_stderr(...)`; the enum performs the only central dispatch match and delegates business behavior to the service-owned command structs, with sync receiving the app-owned stderr writer for format-gated progress. - Startup observability bootstrapping in `cli/src/app.rs` still tolerates invalid default-discovered config files by continuing with degraded defaults plus `sce.config.invalid_config` warn-level logs, but the warning/logging work is now isolated behind the startup-context and runtime-initialization phases rather than one inline startup function. - `cli/src/services/observability.rs` provides deterministic runtime observability controls and rendering for app lifecycle logs, including shared config-resolved threshold/format, explicit config-file/default `log_to_file`, and `log_dir` inputs with precedence `env > config file > defaults` for non-flag observability keys, stable event identifiers, severity filtering, the forced-emission warning path used for invalid discovered config startup diagnostics, error-specific stderr suppression when file logging is enabled while non-error records and file-write diagnostics remain on stderr, redaction-safe emission through the shared security helper, and log-directory writes with bounded retention. Config resolution also carries a positive config-file/default-only `log_file_retention_limit` (`10` by default) into startup observability config and `sce config show`; the concrete logger stores that resolved value and threads it through primary and v2 cleanup. When `log_dir` resolves from `SCE_LOG_DIR`, config, or the `/sce/logs` default, each enabled or forced log operation selects `/sce-.log` or `/sce--.log` using the machine-local date and optional logger session context, with deterministic percent-encoding for unsafe session filename bytes; after successfully writing a newly created selected file, retention keeps the configured number of newest direct regular `*.log` files by mtime plus path/name tie-break and fails open on cleanup errors. Its `observability::traits` submodule exposes the current `Logger` API with `Option<&str>` session context plus the generic async, deliberately non-object-safe `Telemetry` capability and `NoopLogger`; the concrete observability `Logger` and `NoopTelemetry` implement those capability traits; `NoopTelemetry` installs no tracing subscriber. `services::app_support::render_run_outcome` consumes the logger through that trait boundary when logging classified errors and stdout-write failures. -- `cli/src/services/managed_telemetry.rs` (+ `managed_telemetry/trace_export.rs`, `managed_telemetry/log_export.rs`) owns hardcoded managed OTLP defaults, credential-backed runtime telemetry selection from the encrypted auth DB, and bounded fail-open OTLP/HTTP protobuf export of `sce.command` spans and allowlisted `Logger` events to the pinned canonical collector (metrics endpoint reserved, not exported); see `context/sce/cli-observability-contract.md`. +- `cli/src/services/managed_telemetry.rs` (+ `managed_telemetry/trace_export.rs`, `managed_telemetry/log_export.rs`) owns hardcoded managed OTLP defaults, credential-backed runtime telemetry selection from the encrypted auth DB, and bounded fail-open OTLP/HTTP protobuf export of `sce.command` spans (or one `sce.hook` span, with `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` operation children opened through `trace_export::in_child_span` at the Claude adapter seam and in `mutation_trace::runtime`, for Claude-invoked hooks admitted by `managed_span_kind` in `cli/src/app.rs`) and allowlisted `Logger` events to the pinned canonical collector (metrics endpoint reserved, not exported); see `context/sce/cli-observability-contract.md`. - `cli/src/services/observability.rs` no longer owns duplicate log enums or parsing helpers; it consumes the canonical primitive seam from `cli/src/services/config/mod.rs` and stays focused on concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary. - `cli/src/cli_schema.rs` is now the canonical owner for top-level command metadata for the real clap-backed command set (`auth`, `config`, `setup`, `doctor`, `hooks`, `policy`, `sync`, `version`, `completion`), including the slim top-level help purpose text and per-command visibility on `sce`, `sce help`, and `sce --help`; `cli/src/command_surface.rs` remains the custom top-level help renderer and known-command classifier, adding the synthetic `help` row plus the ASCII banner while consuming that shared metadata instead of maintaining a parallel command catalog. - `cli/src/services/default_paths.rs` is the canonical production path catalog for the CLI: it resolves config/state/cache roots with platform-aware XDG or `dirs` fallbacks through an internal `roots` seam, exposes named default paths for current persisted artifacts and database/log files (global config, auth tokens, auth DB, local DB, default observability log directory, and the sole Agent Trace DB path helper `agent_trace_db_path_for_repository` under `repos//agent-trace.db`; the former global-sentinel and per-checkout Agent Trace path helpers were removed by the `retire-legacy-agent-trace-db` plan), and owns canonical repo-relative, embedded-asset, install, hook, and context-path accessors so non-test production path definitions have one shared owner. Compile-time generated payload paths are owned by `build.rs` under `OUT_DIR`, not by the default-path catalog. Current production consumers such as config discovery, observability config resolution, doctor reporting, setup/install flows, database adapters, checkout identity, Agent Trace storage resolution, and local hook runtime path resolution consume this shared catalog rather than defining owned path literals in their own modules. diff --git a/context/cli/claude-mutation-scope-integration.md b/context/cli/claude-mutation-scope-integration.md index cd37b8c3e..47daf9ccf 100644 --- a/context/cli/claude-mutation-scope-integration.md +++ b/context/cli/claude-mutation-scope-integration.md @@ -238,7 +238,7 @@ mutation-stack dependency is the reused `RuntimeBoundary` path or spawned `sce` subprocess. The production resolver uses the hooks-layer Agent Trace DB opener only for the exact `claude_model_state` read; the seam retains strict parsing, lazy DB acquisition, -durable-completion classification, and empty-stdout semantics. +durable-completion classification, and empty-stdout semantics. The adapter also wraps each seam call in the managed `sce.mutation_scope` trace span ([cli-observability-contract.md](../sce/cli-observability-contract.md)), which is disabled without a managed hook dispatch. ## Related context diff --git a/context/context-map.md b/context/context-map.md index 40476a1da..4544c0f36 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -56,7 +56,8 @@ Feature/domain context: - `context/sce/cli-version-command-contract.md` (implemented `sce version` contract for deterministic human and machine-readable runtime identification) - `context/sce/cli-shell-completion-contract.md` (implemented `sce completion` contract for deterministic Bash/Zsh/Fish completion script generation) - `context/sce/claude-raw-hook-capture.md` (removed feature: the former `sce hooks claude-capture` raw-capture route and its supporting types, replaced by the active `diff-trace` and `conversation-trace` intakes) -- `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary, logger and deliberately non-object-safe generic async telemetry boundaries with static capability dispatch and future-aware polling scope, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, managed OTLP trace/log export (credential-gated, hardcoded canonical collector, allowlisted fields, bounded fail-open lifecycle, reserved metrics endpoint), and the trimmed `sce config validate` status-only validation surface) +- `context/sce/claude-hook-trace-propagation.md` (probe evidence for Claude Code 2.1.301 command-hook trace context: `TRACEPARENT` names the `claude_code.interaction` span, not the tool span, and only when Claude tracing is active; hook-to-tool correlation by `tool_use_id`; `CLAUDECODE` + `CLAUDE_PROJECT_DIR` presence as the Claude hook-caller signal; T05 live Dash0 verification (doctor and session-boundary hook spans observed, parented tool-hook hierarchy unverified) and the opt-in live smoke procedure; the shipped `sce.hook` contract lives in `cli-observability-contract.md`, admission decision `context/decisions/2026-10-17-claude-hook-managed-trace-admission.md`) +- `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary, logger and deliberately non-object-safe generic async telemetry boundaries with static capability dispatch and future-aware polling scope, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, managed OTLP trace/log export (credential-gated, hardcoded canonical collector, Claude-only hook admission with one `sce.hook` span and correlation-only session/tool-use IDs, its `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` operation child spans with a span-kind-specific export filter, allowlisted fields, bounded fail-open lifecycle, reserved metrics endpoint), and the trimmed `sce config validate` status-only validation surface) - `context/sce/shared-context-code-workflow.md` (canonical `/next-task` task-synchronization lifecycle and validation-only `/validate` lifecycle, package-local phase references with single-skill control flow, and the task-synchronization-scoped `sce-decision` sibling invocation with ADR reuse/blocker propagation) - `context/sce/shared-context-plan-workflow.md` (canonical `/change-to-plan` workflow, package-local context-load/plan-authoring/template references, clarification/readiness gate contract, and one-task/one-atomic-commit task slicing) - [Context workflow rules](sce/context-workflow-rules.md) (canonical bootstrap, ongoing context maintenance, task synchronization, hygiene, discoverability, and feature-existence rules) diff --git a/context/decisions/2026-10-17-claude-hook-managed-trace-admission.md b/context/decisions/2026-10-17-claude-hook-managed-trace-admission.md new file mode 100644 index 000000000..b03919f2b --- /dev/null +++ b/context/decisions/2026-10-17-claude-hook-managed-trace-admission.md @@ -0,0 +1,69 @@ +# Decision: Admit Claude-invoked SCE hooks to managed trace export as one `sce.hook` span + +Date: 2026-10-17 +Status: Accepted +Plan: `context/plans/claude-hook-trace-propagation.md` +Task: `T02` + +## Context + +Managed OTLP export previously forced every `sce hooks ...` invocation, `sce policy bash`, and the hidden `sce auth telemetry-headers` helper to `NoopTelemetry`. That kept hook processes free of credential lookup and exporter shutdown latency, and it kept hook data off the network. Claude Code runs SCE hooks around every tool call. Correlating those hook runs with Claude Code's own exported traces requires SCE hook processes to export something. Probe evidence for Claude Code 2.1.301 showed two things. First, the Claude-only hook subcommands can be recognized from argv. Second, Claude command hooks can be told apart from the Claude Bash tool by the presence of both `CLAUDECODE` and `CLAUDE_PROJECT_DIR`. + +## Decision + +Claude-invoked SCE hook processes are admitted to managed trace export. Each admitted process exports exactly one root-of-process `sce.hook` span in place of `sce.command`. + +- **Admitted from argv:** `sce hooks claude-mutation-scope` and `sce hooks claude-model-state`. +- **Admitted only when `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are both present** (checked for presence only): `sce hooks diff-trace`, `sce hooks conversation-trace`, and `sce policy bash`. +- **Never admitted:** every other hook (commit, Codex, OpenCode, and Pi) and `sce auth telemetry-headers`. These stay on `NoopTelemetry`. +- **Exported attributes:** `sce.agent=claude`, `sce.hook.event`, `sce.hook.tool`, `sce.outcome`, `sce.error.class`, and the Claude `session_id` and `tool_use_id`. The two IDs are correlation-only span attributes. + +## Rationale + +- Admitting only Claude callers delivers the correlation value without changing the latency or privacy profile of other harnesses' hooks. +- Replacing `sce.command` instead of nesting under it keeps one span per hook process. +- Session and tool-use IDs are opaque identifiers, not payload content. They are the only fields that join an SCE hook run to Claude's `claude_code.tool` span. +- Presence-only environment checks avoid reading values. + +## Alternatives considered + +- **Keep all hooks on Noop** — rejected because it gives no way to correlate SCE hook work with Claude Code traces. +- **Admit every hook process** — rejected because it widens latency and data exposure to Codex, OpenCode, Pi, and commit hooks, which have no proven correlation need. +- **Use `CLAUDECODE` alone as the caller signal** — rejected because the Claude Bash tool also inherits it, so commands run by the agent would be misclassified as hooks. +- **Nest `sce.hook` under `sce.command`** — rejected because it adds a redundant wrapper span per hook process. + +## Compatibility and risks + +- This reverses the earlier guarantee that hooks never resolve managed telemetry. +- Admitted Claude hook processes, including the blocking `PreToolUse` path, can wait up to the existing `CREDENTIAL_LOOKUP_DEADLINE` (2s) plus `SHUTDOWN_TIMEOUT` (3s), under the runtime grace bound. Whether hooks need a tighter, hook-specific deadline remains an open question in the plan. +- Claude session and tool-use IDs now leave the machine when a valid telemetry credential is stored. They go only to the pinned managed collector. + +## Guardrails + +- Only the four string fields `hook_event_name`, `tool_name`, `session_id`, and `tool_use_id` are read from the hook payload for telemetry. Nothing is read when the current span is disabled. +- Tool input and response, prompts, `cwd`, transcript paths, file contents, raw payloads, the OpenBao token, and the accessor are never exported. +- Session and tool-use IDs are never metric attributes and are never used to derive a trace ID. +- Telemetry stays fail-open. Credential, exporter, or collector failures cannot change hook stdout, permission decisions, persisted state, or exit codes. +- Endpoint pinning, the bearer-only header, disabled redirects, and the existing bounded shutdown are unchanged. + +## Consequences + +- Claude-invoked SCE hook runs become visible as `sce.hook` spans. They can be joined to Claude Code tool spans by `tool_use_id`. +- Non-Claude hook processes keep their previous Noop behavior and cost. +- The admission predicate in `cli/src/app.rs` is now part of the telemetry privacy contract and is covered by an admission test matrix. + +## Follow-up + +- Parenting `sce.hook` on the Claude-provided `TRACEPARENT` and adding child spans are tracked by later tasks of the plan. + +## References + +- Plan: [`claude-hook-trace-propagation`](../plans/claude-hook-trace-propagation.md) +- Task: `T02` +- Current-state context: [`cli-observability-contract.md`](../sce/cli-observability-contract.md) +- Current-state context: [`claude-hook-trace-propagation.md`](../sce/claude-hook-trace-propagation.md) +- Current-state context: [`telemetry-credential.md`](../sce/telemetry-credential.md) +- Evidence: [`cli/src/app.rs`](../../cli/src/app.rs) (`only_claude_hook_invocations_are_admitted_to_managed_hook_telemetry`) +- Evidence: [`cli/src/services/managed_telemetry/trace_export.rs`](../../cli/src/services/managed_telemetry/trace_export.rs) (`claude_hook_export_posts_one_hook_span_with_correlation_ids_only`) +- Evidence: [`cli/src/services/hooks/claude_mutation_scope/lifecycle.rs`](../../cli/src/services/hooks/claude_mutation_scope/lifecycle.rs) (`failing_managed_telemetry_cannot_change_claude_hook_behavior`) +- Related decision: [`Application-owned async command runtime`](2026-10-07-application-owned-async-command-runtime.md) diff --git a/context/glossary.md b/context/glossary.md index 1bd61e54c..f42dfe56b 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -80,6 +80,7 @@ - `encrypted Turso adapter`: Generic native async adapter seam in `cli/src/services/db/mod.rs` exposed as `EncryptedTursoDb`, structurally parallel to `TursoDb` and owning no Tokio runtime. Its constructor resolves the encryption key via `encryption_key::get_or_create_encryption_key(&db_path, db_name)`, which derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text before falling back to OS credential-store keyring get-or-create behavior; credential-store default registration is guarded by stable `OnceLock` plus an atomic in-progress flag so errors or panics leave initialization retryable without mutex poisoning. The adapter enables Turso local encryption with strict `aegis256` cipher selection through `turso::EncryptionOpts`, wraps encrypted local open/connect in the default DB connection-open retry policy, and runs embedded migrations after retry has produced a connection; execute, query, query_map, and migration operations are async and preserve `__sce_migrations` tracking parity. - `auth DB adapter`: Module in `cli/src/services/auth_db/mod.rs` that defines `AuthDbSpec` and exposes `AuthDb` as an `EncryptedTursoDb` alias. It resolves the canonical `/sce/auth.db` path with `auth_db_path()`, keeps encryption mandatory with `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret precedence before OS keyring fallback and no plaintext mode, and embeds ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. Auth runtime token-storage is now wired through `cli/src/services/token_storage.rs`, which persists tokens via the `auth_credentials` table in the encrypted auth DB instead of a JSON file. - `telemetry credential`: Telemetry-only OpenBao token and accessor issued by the Control Plane `POST /auth/telemetry/credential`, stored as one fixed row in the encrypted auth DB `telemetry_credentials` table by `cli/src/services/telemetry_credential_storage.rs`, returned only while unexpired, removed by `sce auth logout`, not revoked server-side by local deletion, and used as the sole bearer secret for managed OTLP export (never the accessor); see `context/sce/telemetry-credential.md`. +- `sce.hook span`: The single root-of-process managed trace span a Claude-invoked SCE hook exports in place of `sce.command`, carrying `sce.agent=claude`, the hook event, tool name, outcome, and the Claude `session_id`/`tool_use_id` as correlation-only attributes. It is parented on a valid Claude-provided `TRACEPARENT` (the `claude_code.interaction` span, so it sits beside `claude_code.tool`) and is otherwise an independent root. Its only exported children are `sce.mutation_scope` (one per adapter seam call) and, beneath it, `sce.git_snapshot` and `sce.persistence`, each emitted only when that operation actually runs and carrying only `sce.outcome`. Admission is Claude-only (Claude-only hook subcommands by argv; shared hook subcommands and `sce policy bash` only with `CLAUDECODE` and `CLAUDE_PROJECT_DIR` present); see `context/sce/cli-observability-contract.md`. - `AuthDbLifecycle`: Lifecycle provider in `cli/src/services/auth_db/lifecycle.rs` that implements `ServiceLifecycle` for encrypted auth DB setup/doctor integration. `diagnose` collects auth DB path health problems, `fix` bootstraps missing auth DB parent directory, and `setup` calls `AuthDb::new()`. Registered as `LifecycleProviderId::AuthDb` in the shared lifecycle catalog. - `agent trace DB adapter`: Modules under `cli/src/services/agent_trace_db/` that define the sole repository-scoped `RepositoryAgentTraceDb = TursoDb` adapter (the checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter and its 15-file migration chain were removed by the `retire-legacy-agent-trace-db` plan). The repository adapter uses the `agent-trace-repository` migration set (fresh baseline schema plus additive `source_instance_id` and `claude_model_state` migrations) with `repository_metadata`, repository-level `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts` tables, no row-level `checkout_id`, typed parameterized insert helpers, the non-exported `claude_model_state` latest-locally-observed register with exact `(session_id, agent_id)` reads and guarded writes, and chronological recent `diff_traces` query/parse support. `AgentTraceDbLifecycle` initializes/checks repository-scoped storage through `agent_trace_storage`. - `post-commit Agent Trace auto-sync readiness`: The doctor report fact that explains whether the enabled post-commit trigger is ready without invoking it. Doctor compares the installed `post-commit` hook's SCE managed block using the same currency semantics as setup and resolves config-file-only `agent_trace.auto_sync` with source metadata. JSON states are `ready`, `disabled`, `not_ready`, and `not_applicable`; explicit disable is healthy, while existing hook problems continue to own overall readiness and remediation. See [automatic Agent Trace synchronization](cli/agent-trace-auto-sync.md) and [doctor human text](sce/doctor-human-text-contract.md). diff --git a/context/overview.md b/context/overview.md index ddc104775..cef3d0f8b 100644 --- a/context/overview.md +++ b/context/overview.md @@ -65,7 +65,7 @@ Its command loop is implemented with `clap` derive-based argument parsing and `a The current doctor presentation contract supersedes the earlier output-shape scaffolding wording above: human text uses the compact Environment/Repository/Integrations hierarchy with healthy rows collapsed and unhealthy branches expanded, while JSON retains complete path, identity, problem, and fix-result detail. See `context/sce/doctor-human-text-contract.md`. The command loop now enforces a stable exit-code contract in `cli/src/app.rs`: `2` parse failures, `3` invocation validation failures, `4` runtime failures, and `5` dependency startup failures. Auth command orchestration routes authenticated Control Plane authentication failures, token-storage failures, and approved user-facing fallback failures through typed `CliError::User` catalog entries (`NotAuthenticated`, `AuthStorageUnavailable`, `NotGitRepository`, and `UnexpectedFailure`) while retaining technical sources for observability; missing-credential logout/whoami state queries succeed with their documented text/JSON reports, and internal auth failures remain runtime errors. The same runtime also emits stable CLI stderr diagnostics: internal failures use `SCE-ERR-PARSE`, `SCE-ERR-VALIDATION`, `SCE-ERR-RUNTIME`, or `SCE-ERR-DEPENDENCY` in deterministic `Error []: ...` diagnostics with class-default `Try:` remediation appended when missing, while expected failures emit only their redacted, unstyled catalog messages. The command boundary's former flat, string-only `ClassifiedError` has been replaced by typed `CliError` in `cli/src/services/error.rs`: `CliError::User` carries a closed catalog (`NotAuthenticated`, `NotGitRepository`, `NotGitRemote`, authentication-storage `AuthStorageUnavailable`, and general `UnexpectedFailure`) for expected, deliberately-explained failures rendered without the technical source, wrapper, styling, or automatic `Try:` guidance; the general entry renders one fixed static log-files guidance sentence without dynamic path text. `CliError::Internal` carries a live `anyhow::Error` source rendered as the real error chain with the existing styled wrapper and class-default remediation; `app_support` is the sole owner of the distinct terminal paths, and `sce sync` classifies authentication, credential-storage, and all other sync failures as cataloged user errors while preserving their technical sources for observability. Setup preflight errors preserve technical sources, including the configured remote name, while raw remote URLs remain out of user-facing diagnostics. See `context/sce/cli-error-code-taxonomy.md` for the full contract. -The app runtime now also includes a structured observability baseline in `cli/src/services/observability.rs`: deterministic env-controlled log threshold/format (`SCE_LOG_LEVEL` defaults to `error`; `SCE_LOG_FORMAT` defaults to `text`), default-backed log-directory routing (`SCE_LOG_DIR` / config-file `log_dir` / `/sce/logs`) with per-operation machine-local dated file selection, optional session filename partitioning, and creation-triggered retention of direct regular `*.log` files to 10 entries, stable lifecycle event IDs, normal logger records routed only to files when file logging is enabled and only error-level logger records routed to `stderr` when file logging is disabled, direct file-write diagnostics remaining on `stderr`, and `observability::traits` boundaries for logger and telemetry behavior. When the encrypted auth DB holds a valid unexpired telemetry credential, non-hook commands also export `sce.command` spans and allowlisted logger events as OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev` `/v1/traces` and `/v1/logs` paths (bearer token only, redirects disabled, ambient `OTEL_EXPORTER_OTLP_*` and repo config cannot redirect it, fail-open and bounded); without a credential nothing is exported, and `/v1/metrics` is reserved with no metrics pipeline. +The app runtime now also includes a structured observability baseline in `cli/src/services/observability.rs`: deterministic env-controlled log threshold/format (`SCE_LOG_LEVEL` defaults to `error`; `SCE_LOG_FORMAT` defaults to `text`), default-backed log-directory routing (`SCE_LOG_DIR` / config-file `log_dir` / `/sce/logs`) with per-operation machine-local dated file selection, optional session filename partitioning, and creation-triggered retention of direct regular `*.log` files to 10 entries, stable lifecycle event IDs, normal logger records routed only to files when file logging is enabled and only error-level logger records routed to `stderr` when file logging is disabled, direct file-write diagnostics remaining on `stderr`, and `observability::traits` boundaries for logger and telemetry behavior. When the encrypted auth DB holds a valid unexpired telemetry credential, commands also export `sce.command` spans (Claude-invoked SCE hooks export one root `sce.hook` span instead, with `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` children only for operations that ran; other hooks export nothing) and allowlisted logger events as OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev` `/v1/traces` and `/v1/logs` paths (bearer token only, redirects disabled, ambient `OTEL_EXPORTER_OTLP_*` and repo config cannot redirect it, fail-open and bounded); without a credential nothing is exported, and `/v1/metrics` is reserved with no metrics pipeline. One application-level multi-thread Tokio runtime directly awaits static command dispatch through async credentials, lifecycle providers, hooks, sync/export, and Turso persistence. The shared Turso adapters own no runtime or executor bridge, and foreground hooks return after required persistence completes. See [application execution runtime](architecture.md#application-execution-runtime). The app command dispatcher now enforces a centralized stdout/stderr stream contract in `cli/src/app.rs`: command success payloads are emitted on stdout only, while redacted user-facing diagnostics and text-mode sync progress are emitted on stderr; JSON sync remains silent. `cli/src/app.rs` also now runs through explicit startup phases — dependency check, observability config resolution, runtime initialization, command parse/execute, and output rendering — with the app runtime carrying logger/telemetry plus static command-catalog state across those phases while preserving the existing exit-code and degraded-startup contracts. Within that lifecycle, `parse_command_phase` delegates clap-to-runtime conversion to `cli/src/services/parse/command_runtime.rs`, which returns a static `RuntimeCommand` enum, `services::app_support::execute_command_phase` logs around enum-owned `execute_with_stderr(...)` dispatch, and generic `RunOutcome` rendering logs classified errors through the logger trait boundary without coupling render support to the production logger type. Command payload structs for `help`, `version`, `completion`, `auth`, `config`, `setup`, `doctor`, `hooks`, and `sync` live in service-owned `command.rs` files; `cli/src/services/command_registry.rs` owns the deterministic static command-name catalog and enum variants instead of boxed command trait objects. The CLI now also enforces a shared output-format parser contract in `cli/src/services/output_format.rs`, with canonical `--format ` parsing and command-specific actionable invalid-value guidance reused by `config` and `version` services. A compile-safe service lifecycle seam also exists in `cli/src/services/lifecycle.rs`: `ServiceLifecycle` exposes default no-op `diagnose`, `fix`, and `setup` methods against the narrow `HasRepoRoot` accessor, uses lifecycle-owned health/fix/setup result types, and owns the shared static `LifecycleProvider` enum catalog/factory with deterministic config → local*db → auth_db → agent_trace_db → hooks ordering and no boxed provider aggregation. Hooks has a `services/hooks/lifecycle.rs` provider for hook rollout diagnosis/fix/setup, config has a `services/config/lifecycle.rs` provider for global/repo-local config validation plus repo-local config bootstrap, local_db has a `services/local_db/lifecycle.rs` provider for canonical local DB path health, parent-directory readiness/bootstrap, and `LocalDb::new()` setup, auth_db has a `services/auth_db/lifecycle.rs` provider for canonical auth DB path health, parent-directory readiness/bootstrap, and `AuthDb::new()` setup, and agent_trace_db has a `services/agent_trace_db/lifecycle.rs` provider for repository-scoped Agent Trace DB setup and repository DB path health/parent readiness from resolved repository identity, returning an actionable "requires a Git repository" diagnostic outside repository context (no global/checkout fallback path). Doctor runtime aggregates the full shared provider catalog for `diagnose` and `fix` and adapts lifecycle records into doctor-owned output records; setup command aggregates the shared provider catalog for `setup` with hooks included only when requested and adapts lifecycle setup outcomes before rendering setup-owned messages. diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md new file mode 100644 index 000000000..9ba0f4ed4 --- /dev/null +++ b/context/plans/claude-hook-trace-propagation.md @@ -0,0 +1,357 @@ +# Plan: claude-hook-trace-propagation + +## Change summary + +Connect SCE's existing managed OTLP trace export (PRs #306/#325, `cli/src/services/managed_telemetry/trace_export.rs`) to Claude Code hook execution so a Claude-invoked SCE hook emits a dedicated `sce.hook` span, parented to Claude Code's active tool span when Claude Code actually passes valid W3C trace context to command hooks, and otherwise exported as an independent SCE trace carrying safe Claude session/tool correlation attributes. + +This reverses one recorded behavior: today every `sce hooks ...`, `sce policy bash`, and `sce auth telemetry-headers` invocation is forced to `NoopTelemetry` by `is_hook_invocation` in `cli/src/app.rs` (asserted by `hook_invocations_never_resolve_managed_telemetry`, and by AC4 of `claude-native-otel-instrumentation`). This plan admits only Claude-invoked hook processes to managed export; Codex, OpenCode, and Pi hook processes and `sce auth telemetry-headers` stay `Noop`. No trace-context propagator or `trace_context` module exists in the repository today; the W3C extraction is new, but it reuses the pinned `opentelemetry`/`opentelemetry_sdk` 0.33.1 and `tracing-opentelemetry` 0.34.0 crates, the existing exporter, credential lookup, endpoint pinning, and bounded shutdown. Whether parent-child propagation is possible at all is decided by a live probe in T01, not assumed. + +## Acceptance criteria + +How this plan is proven complete. Each criterion is observable and names the check that proves it. `/validate` runs these checks; no task in the stack performs final validation. + +- [x] AC1: Durable context records, from real Claude Code execution of `PreToolUse` and `PostToolUse` around a real tool call, whether command hooks receive `TRACEPARENT`/`TRACESTATE` (or equivalent), whether the value is a valid W3C traceparent, whether its trace ID and parent span ID match a span Claude Code actually exported, the Claude Code version tested, and the documentation consulted; the probe captured only `session_id`, `hook_event_name`, `tool_use_id`, presence/validity flags, trace ID, and parent span ID, and no probe code remains in the repository. + - Validate: Inspect `context/sce/claude-hook-trace-propagation.md` for the recorded evidence table and verdict; `git ls-files` shows no probe script; grep the recorded evidence for tool input, prompt text, `Bearer`, or token-shaped values (none). +- [x] AC2: With a valid stored telemetry credential, each Claude-invoked SCE hook process exports exactly one `sce.hook` root-of-process span (no additional `sce.command` span) to the pinned `/v1/traces` endpoint, carrying `service.name=sce`, `sce.agent=claude`, `sce.hook.event`, `sce.hook.tool` when present, and `sce.outcome`; Codex, OpenCode, and Pi hook invocations and `sce auth telemetry-headers` still resolve `NoopTelemetry`, and ordinary commands keep their unchanged `sce.command` span. + - Validate: Loopback-receiver Rust tests decode the captured OTLP payload for a Claude `PostToolUse` fixture and assert span name, count, and attributes; admission-predicate unit tests cover Claude, non-Claude, and helper invocations; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`. +- [x] AC3: When T01 proved usable parent context: a valid upstream traceparent makes the `sce.hook` span share the upstream trace ID with its parent span ID equal to the upstream span ID, SCE span IDs are freshly generated, child spans inherit the trace across `.await` points and Tokio worker migration, and an invalid, malformed, or missing traceparent yields a new independent SCE root without panic or hook-behavior change. When T01 proved context unavailable: no extraction code ships, and the context file states the parent-child goal is blocked, why, and the smallest future bridge. + - Validate: Loopback-receiver tests for valid, invalid, all-zero, and missing traceparent asserting trace ID / parent span ID; a multi-poll async test asserting child span trace IDs; or, for the unavailable branch, inspection of `context/sce/claude-hook-trace-propagation.md` and source grep showing no propagator wiring. +- [x] AC4: The `sce.hook` span carries Claude `session_id` and `tool_use_id` as span attributes only (never as metric attributes, never reconstructed into a trace ID), and no exported span or log contains tool input, prompts, file contents, repository paths, raw hook payloads, the OpenBao token, or the accessor. + - Validate: Loopback-receiver test injecting secret- and payload-shaped fixture values asserts their absence from the decoded payload and presence of only the two correlation IDs. +- [x] AC5: Child spans `sce.mutation_scope`, `sce.git_snapshot`, and `sce.persistence` appear under `sce.hook` only when the corresponding operation actually executed in that hook process; a hook path that skips an operation exports no span for it. + - Validate: Loopback-receiver tests over a mutation-scope fixture that executes all three and a fixture that executes none (e.g. a non-tracked tool) assert the exact span set and parent links. +- [x] AC6: Telemetry cannot change Claude hook behavior: with credential lookup failure, expired credential, refused or stalled receiver, or exporter timeout, hook stdout, stderr, exit code, permission decisions, Agent Trace DB rows, and mutation-trace state match the telemetry-disabled run, and process lifetime stays within the existing `CREDENTIAL_LOOKUP_DEADLINE` + `SHUTDOWN_TIMEOUT` bounds. + - Validate: Parity tests running the same Claude hook fixture with Noop and with failing managed telemetry, comparing outputs and persisted state, and asserting elapsed-time bounds. +- [x] AC7: Standalone tracing is unchanged: `sce doctor` with a valid credential exports its `sce.command` span through the managed exporter with no Claude context required. + - Validate: Existing `managed_trace_export_posts_allowlisted_span_with_bearer_only` and related trace_export tests still pass; live evidence for `sce doctor` recorded per AC8. +- [x] AC8: Live verification is recorded: `sce doctor` and a Claude Code session triggering SCE hooks in this repository both export through `https://otel.crocoderlab.dev` with their existing managed credentials, and the record states whether each trace was observed in the correct Dash0 workspace dataset (with trace IDs and, where proven, the parent relationship) or exactly what could not be verified; a collector 2xx alone is not recorded as delivery. + - Validate: Inspect the live-verification section of `context/sce/claude-hook-trace-propagation.md` and the T05 completion record. +- [x] AC9: Current-state context describes hook telemetry admission, the `sce.hook` span contract and attributes, the parent-context verdict, correlation-ID privacy rules, and the opt-in live smoke procedure, without claiming distributed tracing unless AC3's parent relationship was verified live. + - Validate: Inspect the files listed under Context sync against the implemented code. + +### Full validation + +- `nix flake check` +- `nix run .#pkl-check-generated` +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry` + +### Context sync + +- `context/sce/claude-hook-trace-propagation.md` (new: probe evidence, verdict, live verification, smoke procedure) +- `context/sce/cli-observability-contract.md` +- `context/sce/telemetry-credential.md` (hook admission now differs from the helper bypass) +- `context/cli/claude-mutation-scope-integration.md` (span boundaries over the adapter) +- `context/architecture.md`, `context/glossary.md`, `context/context-map.md` + +## Task context synchronization lifecycle + +Persist this field in every plan; this is durable plan state, not chat state: + +- **Task context synchronization:** every task carries `pending | synced | blocked`. + A completed task must be `synced` before another task can start or the plan can + finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** + beside the status. Never infer `synced` from conversation history; write every + lifecycle transition to the plan file. + +## Constraints and non-goals + +- **In scope:** `cli/src/app.rs` (hook admission predicate, telemetry resolution), `cli/src/services/managed_telemetry.rs` and `managed_telemetry/trace_export.rs` (hook span, span filter, propagation), the Claude hook entry points in `cli/src/services/hooks/` (`claude_mutation_scope/`, `claude_model_state.rs`, Claude-classified paths of `diff_trace.rs`, `conversation_trace.rs`, and `sce policy bash`), span boundaries at existing call sites in `cli/src/services/mutation_trace/runtime/` (e.g. `git_snapshot.rs`) and Agent Trace DB persistence, focused tests, and the context files listed above. +- **Out of scope:** Codex, OpenCode, and Pi hooks/plugins; Pkl/generated Claude hook config; Claude settings written by `sce setup`; metrics; credential issuance, storage, or lifecycle; Control Plane or Collector changes; a collector, proxy, sidecar, daemon, or correlation engine; new public CLI commands; making `sce auth telemetry-headers` visible. +- **Constraints:** reuse the pinned OTEL crates, `TraceContextPropagator` from `opentelemetry_sdk`, `MANAGED_OTLP_*`, `load_credential_within_deadline`, and existing bounded shutdown; static dispatch, no new `dyn` abstractions or wrappers beyond what the span boundary needs; hooks stay fail-open; never derive a trace ID from Claude session or tool-use IDs; no code comments; Cargo only via `nix develop`; tests stay a minimal focused set using the existing loopback receiver; CI never needs live credentials, Claude Code, or Dash0. +- **Non-goal:** generic distributed-tracing support for every harness or every SCE command; span-instrumenting arbitrary internal functions. + +## Assumptions + +- The repository stays on Rust edition 2021 (`cli/Cargo.toml`, AGENTS.md); the request's "Rust 2024" is treated as "current Rust", since an edition migration is a separate cross-cutting change. +- `sce.hook` replaces `sce.command` for admitted Claude hook processes rather than nesting under it, so each hook process exports one root-of-process span and no redundant wrapper. +- Claude admission is decided before dispatch from argv for Claude-only subcommands (`claude-mutation-scope`, `claude-model-state`) and, for shared subcommands (`diff-trace`, `conversation-trace`, `policy bash`), from a cheap Claude-caller signal T01 confirms (presence-only check of a Claude-set hook environment marker); if no reliable signal exists, shared subcommands stay `Noop`. +- Claude `session_id` and `tool_use_id` are permitted as span attributes: they are opaque identifiers, not hook payload content, and the managed-otel-export privacy contract forbids payloads, paths, and secrets, not correlation IDs. The context sync records this explicitly. +- The "opt-in live smoke test" is a documented manual procedure in `context/sce/claude-hook-trace-propagation.md` plus recorded evidence, not a committed script or ignored Rust test, because driving Claude Code and Dash0 cannot run deterministically from the test suite. +- Probe scripts live only in the session scratchpad and user-level Claude settings changes for the probe are reverted after T01. +- The PR is opened after `/validate` passes, using the plan's recorded evidence for its description; it is not a task in this stack. + +## Task stack + +- [x] T01: `Probe and record Claude Code hook trace-context propagation` (status:done) + - Task ID: T01 + - Scope: In — read Claude Code's current hooks and monitoring/OTEL documentation and any inspectable implementation; run a temporary scratchpad probe registered for `PreToolUse` and `PostToolUse` in a real Claude Code session with native telemetry enabled, triggering a real tool call; capture only `session_id`, `hook_event_name`, `tool_use_id`, `TRACEPARENT`/`TRACESTATE` presence, traceparent validity, trace ID, parent span ID, and presence-only Claude caller env markers; compare captured IDs against Claude Code's exported spans to decide whether the context corresponds to an active tool span; write `context/sce/claude-hook-trace-propagation.md` with evidence, Claude Code version, and verdict. Out — any Rust change; inferring hook behavior from Bash subprocess propagation; committing the probe. + - Dependencies: none + - Done when: The context file records a verdict of `parent_context_available` or `parent_context_unavailable` backed by captured evidence for both events, names the Claude-caller signal (or its absence), the probe is removed, and the user-level Claude settings are restored. + - Verify: Inspect the context file; `git status` shows only the new context file; grep the file for prompt/tool-input/token content (none). + - Completed: 2026-10-17 + - Files changed: `context/sce/claude-hook-trace-propagation.md` (new) + - Result: Verdict `parent_context_available`, qualified. Headless Claude Code 2.1.301 runs with telemetry and the tracing beta on gave every `PreToolUse`/`PostToolUse` hook a valid `TRACEPARENT` with no `TRACESTATE`. Its trace ID matched the exported trace, and its parent span ID was the `claude_code.interaction` root span, never `claude_code.tool` or `claude_code.hook` (detailed beta tracing included). Hooks run outside the tool span; correlate with the tool by `tool_use_id`. Without the tracing beta, or with only `CLAUDE_CODE_PROPAGATE_TRACEPARENT=1`, no `TRACEPARENT` is present. Claude-caller signal for shared subcommands: `CLAUDECODE` and `CLAUDE_PROJECT_DIR` both present (presence only); `CLAUDE_PROJECT_DIR` is absent from the Bash tool environment. Interactive TUI hooks not probed; deferred to T05. The probe ran only in the scratchpad through `claude -p --settings`. It never edited user-level `~/.claude/settings.json` and was deleted afterward. + - Deviation: the plan assumed parenting to Claude's active tool span; the evidence shows only the interaction span is available, so T03 parents `sce.hook` as a sibling of `claude_code.tool` under `claude_code.interaction`. + - Verify outcomes: context file inspected (verdict, evidence for both events, caller signal, version, docs) — passed; `git status --porcelain -uall` shows only `?? context/sce/claude-hook-trace-propagation.md` — passed; grep of the file for `bearer|token=|sk-ant|hvs.|password|tool_input"|user_prompt"` and the probe file content returned no matches — passed; `git ls-files` contains no T01 probe code — passed. + - Context impact: new current-state context file under `context/sce/` (probe evidence and verdict); needs a `context/context-map.md` entry; no code behavior change. + - Context synchronization: synced + +- [x] T02: `Export a dedicated sce.hook span for Claude hook invocations` (status:done) + - Task ID: T02 + - Scope: In — narrow `is_hook_invocation` into a Claude-admission decision in `cli/src/app.rs`; add the `sce.hook` span (replacing `sce.command` for admitted hooks) in `trace_export.rs` with `sce.agent`, `sce.hook.event`, `sce.hook.tool`, `sce.outcome`, `sce.error.class`, and the two correlation IDs recorded from the parsed Claude payload; loopback tests for span shape, admission, secret/payload absence, credential failure, exporter failure/stall, bounded shutdown, and hook output/state parity. Out — parent-context extraction (T03), child spans (T04), non-Claude harnesses. + - Dependencies: T01 + - Done when: AC2, AC4, AC6, and AC7 hold in local tests; `hook_invocations_never_resolve_managed_telemetry` is replaced by an admission test that keeps non-Claude hooks and `telemetry-headers` on `Noop`. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml app::tests`. + - Completed: 2026-10-17 + - Files changed: `cli/src/app.rs`, `cli/src/services/managed_telemetry.rs`, `cli/src/services/managed_telemetry/trace_export.rs`, `cli/src/services/hooks/claude_mutation_scope/lifecycle.rs`, `cli/src/services/hooks/claude_model_state.rs`, `cli/src/services/hooks/diff_trace.rs`, `cli/src/services/hooks/conversation_trace.rs`, `cli/src/services/bash_policy.rs` + - Result: `is_hook_invocation` replaced by `managed_span_kind(args, claude_hook_caller) -> Option`. `hooks claude-mutation-scope` and `hooks claude-model-state` are admitted from argv. `hooks diff-trace`, `hooks conversation-trace`, and `policy bash` are admitted only when `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are both present (presence only). Every other hook subcommand and `auth telemetry-headers` resolve `None` (Noop); every other command resolves `Command`. `RuntimeTelemetry::resolve`/`resolve_with` and `ManagedTraceTelemetry::new` take the span kind. For `ClaudeHook`, the root span is `sce.hook` (same `sce.managed.trace` target and filter, no `sce.command`), with `sce.agent=claude`, `sce.hook.event`, `sce.hook.tool`, `sce.claude.session_id`, `sce.claude.tool_use_id`, `sce.outcome`, and `sce.error.class`. `record_claude_hook_payload` returns early on a disabled span and copies only the four string fields `hook_event_name`, `tool_name`, `session_id`, and `tool_use_id`; it is called once after stdin is read in the five Claude entry points. Tests added: an admission matrix; a loopback test that decodes the OTLP protobuf (new test-only decoder `decode_spans`/`trace_spans`, reusable by T03/T04) and asserts exactly one root `sce.hook` span, its exact `sce.*` attribute set, `service.name=sce`, and the absence of paths, prompt, file content, tool response, `Bearer` value, token, accessor, and `sce.command`; credential failure and expiry resolve Noop for `ClaudeHook`; and a parity test that runs a four-event Claude mutation-scope fixture sequence (including a deny decision) under Noop, refused-receiver, and stalled-receiver managed telemetry, asserting identical outputs, seam (persistence) payloads, and adapter state, with shutdown under `SHUTDOWN_TIMEOUT` + 1s. + - Assumptions: hooks reuse the existing `CREDENTIAL_LOOKUP_DEADLINE` and `SHUTDOWN_TIMEOUT` bounds per AC6, and the open question on a tighter hook deadline stays open; correlation attributes are named `sce.claude.session_id` and `sce.claude.tool_use_id`. AC6 persistence parity is proven at the mutation-scope seam (the Agent Trace DB write boundary), not by opening a real Agent Trace DB. + - Verify outcomes: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`: 22 passed, 0 failed (passed); `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml app::tests`: 12 passed, 0 failed (passed); additionally `nix flake check`: all checks passed (cli-tests, cli-clippy, cli-fmt, actionlint). + - Context impact: important change. Hook telemetry admission reverses the recorded "hooks are always Noop" contract (`context/sce/cli-observability-contract.md`, `context/sce/telemetry-credential.md`, AC4 of `claude-native-otel-instrumentation` context), and the new `sce.hook` span contract and correlation-ID privacy rule affect `context/architecture.md` and `context/glossary.md`. + - Context synchronization: synced + +- [x] T03: `Parent the sce.hook span on Claude-provided W3C trace context` (status:done) + - Task ID: T03 + - Scope: In — only when T01 recorded `parent_context_available`: extract the context Claude provides using `opentelemetry_sdk::propagation::TraceContextPropagator` before the `sce.hook` span is created, attach it with `OpenTelemetrySpanExt::set_parent`, keep it scoped to the instrumented hook future, and add loopback tests for valid, invalid, all-zero, and missing traceparent plus a multi-poll async child-span test. When T01 recorded `parent_context_unavailable`: make no code change, and record in the context file that the parent-child goal is blocked, why, and the smallest future bridge (correlating Claude OTLP export records with `sce.hook` via `session_id`/`tool_use_id`). Out — reconstructing trace IDs from Claude identifiers; global propagator registration beyond the hook boundary; non-Claude callers. + - Dependencies: T02 + - Done when: AC3 holds for the branch T01 selected. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml trace_export`, or inspection of the context file for the unavailable branch. + - Completed: 2026-10-17 + - Files changed: `cli/src/services/managed_telemetry/trace_export.rs` + - Result: `parent_context_available` branch implemented. `ManagedTraceTelemetry` gained a `parent_context: Option` field. Only for `ManagedSpanKind::ClaudeHook`, `ManagedTraceTelemetry::new` fills it from process env `TRACEPARENT`/`TRACESTATE`. The pure `extract_parent_context` helper does this by running `opentelemetry_sdk::propagation::TraceContextPropagator::extract` over a `HashMap` carrier, and it keeps the context only when the remote span context is valid. `root_span` calls `OpenTelemetrySpanExt::set_parent` on the `sce.hook` span and ignores its result (fail-open). The span is entered only through `.instrument(...)` on the hook future, so no global propagator is registered and the parent never leaks outside the hook boundary. `sce.command` and non-Claude paths never read the env. Tests added (loopback receiver plus the existing OTLP decoder): + - A valid traceparent gives `sce.hook` the upstream trace ID, its parent span ID is the upstream span ID, and its own span ID is new. + - A table of missing, empty, malformed, truncated, uppercase, version `ff`, all-zero trace ID, and all-zero span ID values each yields one independent root `sce.hook` with a new non-zero trace ID and no parent. + - On a 4-worker runtime, a child span created after 8 polls and a grandchild inside a `tokio::spawn`ed task (`in_current_span` + `with_current_subscriber`) share the upstream trace ID with a correct parent chain. + - Deviation: per T01, the upstream parent is `claude_code.interaction`, so `sce.hook` is a sibling of `claude_code.tool`, not its child. + - Verify outcomes: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml trace_export`: 13 passed, 0 failed (passed); additionally `nix flake check`: all checks passed (cli-tests, cli-clippy, cli-fmt). + - Context impact: important change. `sce.hook` now joins Claude's trace when Claude Code's tracing beta supplies `TRACEPARENT`, otherwise it stays an independent root. This affects the span contract in `context/sce/cli-observability-contract.md` and the verdict/implementation notes in `context/sce/claude-hook-trace-propagation.md`; the glossary/architecture entries for `sce.hook` gain the parent-context rule. + - Context synchronization: synced + +- [x] T04: `Add mutation-scope, git-snapshot, and persistence child spans under sce.hook` (status:done) + - Task ID: T04 + - Scope: In — spans on the managed-trace target at the existing execution sites: the Claude mutation-scope coordinate call (`sce.mutation_scope`), the mutation-trace Git snapshot/reconciliation call (`sce.git_snapshot`), and Agent Trace DB / mutation store persistence (`sce.persistence`), each with low-cardinality outcome attributes; extend the span filter so only these named spans export; loopback tests asserting the exact span set for an executing and a non-executing hook path. Out — spans inside pure protocol functions or arbitrary helpers; changes to ordering, locking, or failure semantics. + - Dependencies: T03 + - Done when: AC5 holds, existing mutation-trace and Agent Trace tests pass unchanged, and no span is emitted for an operation that did not run. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml claude_mutation_scope`; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`. + - Completed: 2026-10-17 + - Files changed: `cli/src/services/managed_telemetry/trace_export.rs`, `cli/src/services/hooks/claude_mutation_scope/lifecycle.rs`, `cli/src/services/mutation_trace/runtime/coordinator.rs`, `cli/src/services/mutation_trace/runtime/scope_runtime.rs` + - Result: Added `ManagedChildSpan { MutationScope, GitSnapshot, Persistence }` and `in_child_span(kind, future)`. The helper wraps the future in a span named `sce.mutation_scope`, `sce.git_snapshot`, or `sce.persistence` on the `sce.managed.trace` target, boxes the instrumented future, and records `sce.outcome=success|failure` from the `Result`. The span filter is now selected per span kind: a `Command` dispatch exports only `sce.command`, so ordinary commands that reach the coordinator still export exactly one span; a `ClaudeHook` dispatch exports only `sce.hook` and the three child names. Span sites: + - `run_claude_mutation_scope_from_payload_with_resolver` wraps every seam (coordinate/abandon boundary) call in `sce.mutation_scope`. + - `coordinate_boundary_inner` wraps `capture_and_pin` in `sce.git_snapshot`. + - The post-snapshot store phase (initialize, register, provenance, CAS loop) moved unchanged into `persist_boundary`, which is wrapped in `sce.persistence`. + - `abandon_scope_inner` wraps `abandon_protected` in `sce.persistence`. + + Spans are disabled without a managed hook dispatch. Ordering, locking, and errors are unchanged. Tests added: + - A real temporary git repo and Agent Trace DB, with a real `coordinate` seam, run a Claude `Write` PreToolUse. It exports exactly `sce.hook` → `sce.mutation_scope` → {`sce.git_snapshot`, `sce.persistence`} in one trace, all with `success`. + - A read-only `Read` PreToolUse exports only `sce.hook`. + - A `Command` dispatch that runs all three child operations exports only `sce.command`. + + The T03 multi-poll test now uses the real child span names. New test helper `capture_requests_until` collects requests until shutdown, so batch timing cannot change the captured set. + - Assumptions: `sce.persistence` covers the mutation-store phase of coordinate and abandon only. Taint persistence after a snapshot failure, and conversation/diff-trace Agent Trace DB writes, are not wrapped. `sce.git_snapshot` and `sce.persistence` are grandchildren of `sce.hook` through `sce.mutation_scope`. Child spans carry only `sce.outcome`. + - Deviation: The first `nix flake check` failed `cli-clippy` with `large_futures` at 15 call sites, because the instrumented chain exceeded the 16 KiB future limit. Fixed in scope by `Box::pin`-ing the instrumented future inside `in_child_span`. + - Verify outcomes: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml claude_mutation_scope`: 7 passed, 0 failed (passed); `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`: 26 passed, 0 failed (passed); additionally `nix flake check` after the fix: all checks passed (cli-tests incl. existing mutation-trace and Agent Trace suites, cli-clippy, cli-fmt). + - Context impact: important change. The `sce.hook` span contract gains three operation child spans, plus the rule that spans are exported per span kind (`sce.command` alone for commands; `sce.hook` plus children for Claude hooks). This affects `context/sce/cli-observability-contract.md`, `context/sce/claude-hook-trace-propagation.md`, `context/cli/claude-mutation-scope-integration.md` (span boundaries over the adapter seam and coordinator), and the `context/glossary.md` `sce.hook` entry. + - Context synchronization: synced + +- [x] T05: `Record live Dash0 verification and sync hook-telemetry context` (status:done) + - Task ID: T05 + - Scope: In — run `sce doctor` and a Claude Code session in this repository that triggers SCE hooks using a build with T02–T04; confirm both exporters use their existing managed credentials and the canonical collector, and check Dash0 for both traces in the correct workspace dataset, recording trace IDs, observed hierarchy, and anything unverified; write the opt-in live smoke procedure; update the context files listed under Context sync. Out — code changes; Collector/Control Plane changes; claiming delivery from a collector 2xx. + - Dependencies: T04 + - Done when: AC8 and AC9 hold and the context file states the exact observed hierarchy. + - Verify: Inspect the updated context files; `nix run .#pkl-check-generated` unaffected. + - Completed: 2026-10-17 + - Files changed: `context/sce/claude-hook-trace-propagation.md`, `context/sce/cli-observability-contract.md`, `context/sce/telemetry-credential.md`, `context/context-map.md` + - Result: Live run on 2026-10-17 used a debug build of `eae7be8f` and the existing stored credential (presence checked, value never printed). Dash0 evidence was a user-exported 30-span root-span listing; no trace detail view was provided. + - Observed in Dash0: `sce doctor` exported one root `sce.command` (trace `1cc8e0b1eda96df1f757fab2e3798efd`). Headless Claude Code 2.1.301 session `96272753-…` (real `Write` + `Read` in this repo, branch `sce` on `PATH`, tracing beta on) exported root `claude_code.interaction` `dcd919ecc4f5d400df5b642a1a229783`/`3eec7153327c6839`, which is exactly the trace/span every tool hook's `TRACEPARENT` named (scratchpad probe). The session's `SessionStart` and `SessionEnd` `sce.hook` spans are independent roots with `sce.agent=claude`, the session ID, and `success`, because they run outside the interaction span. `sce auth telemetry-headers` produced no span. + - Not verified: the parented tool-hook `sce.hook` spans and their `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` children inside trace `dcd919ec…`. A root-span listing cannot show them (it also omits Claude's own `claude_code.tool` children). No unparented tool-hook `sce.hook` roots appear in the window. Also not verified: the Dash0 dataset name and interactive TUI hooks. + - Follow-up (2026-10-17, user found the trace in Dash0): trace `dcd919ecc4f5d400df5b642a1a229783` is in the SCE CroCoder (`sce-crocoder`) dataset. Root `claude_code.interaction` (service `claude-code`, session `96272753-8c39-4ec3-824d-238f5202e016`, repo `crocoder-dev/shared-context-engineering`), started 12:32:16 UTC, about 7.0s, status UNSET, 0 descendants in error, 0 error or warning logs. It has 11 direct children (21 spans total): 3 `claude_code.llm_request`, 2 `claude_code.tool`, and 6 `sce.hook` spans from service `sce`. Most `sce.hook` spans took 1–26ms; the two with children took 111–154ms. This confirms the parented `sce.hook` spans and the dataset live. The names of the `sce.hook` children and which hook events the six spans belong to were not given in the evidence. + - Recorded the evidence, observed vs. expected hierarchy, and an opt-in live smoke procedure in `claude-hook-trace-propagation.md`. Added session-boundary root behavior and live status to the observability contract, hook admission vs. helper bypass to `telemetry-credential.md`, and a context-map entry update. + - Deviation: AC8's parent relationship was first recorded as unverified; the follow-up above now records it as observed in Dash0. The other context files still describe distributed tracing as locally tested, not live-verified. + - Verify outcomes: context files inspected against T02–T04 code and Dash0 evidence — passed; secret grep of the context file for bearer values, the Dash0 token identifier, prompt/tool-input keys — no matches (passed); `nix run .#pkl-check-generated`: ephemeral generation passed, 142 files (passed); smoke file removed and `git status` shows only the four context files — passed. + - Context impact: localized context-only change; the live-verification record and smoke procedure live in `context/sce/claude-hook-trace-propagation.md` with cross-references from the observability contract, telemetry credential doc, and context map; no code behavior change. + - Context synchronization: synced + +## Open questions + +- Admitting Claude hooks to managed export adds an encrypted `auth.db` credential lookup (up to `CREDENTIAL_LOOKUP_DEADLINE`, 2s) and an exporter shutdown (up to `SHUTDOWN_TIMEOUT`, 3s) to every Claude hook process — several per tool call, including the blocking `PreToolUse` path. `managed-otel-export` and `claude-native-otel-instrumentation` excluded hooks deliberately. Is the existing bound acceptable for hooks, or should hooks get a tighter hook-specific deadline? A smaller version is to admit only `PostToolUse` hooks, leaving the blocking `PreToolUse` path on `Noop`. +- If T01 finds Claude Code does not pass trace context to command hooks, the remaining value is correlation attributes on independent traces. Is that still worth T02–T05, or should the plan stop after T01 and T03's documented blocker? +- The request asked for Rust 2024; the plan keeps edition 2021. Confirm, or plan the edition migration as a separate change. + +## Validation Report + +**Status:** validated +**Date:** 2026-10-17 + +### Commands run + +- `nix flake check` -> exit 0 (all checks passed for x86_64-linux, including cli-tests, cli-clippy, cli-fmt, cli-generated-input, pkl-generated, config-lib and npm Bun/Biome checks, workflow-actionlint) +- `nix run .#pkl-check-generated` -> exit 0 (ephemeral Pkl generation passed: 142 files) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry` -> exit 0 (26 passed, 0 failed) +- `git diff --name-only main...HEAD` plus `git status --porcelain -uall` -> exit 0 (branch changes limited to the planned code and context files; no probe files added; working tree holds only the T05 context edits) +- `grep -inE "bearer [A-Za-z0-9]|tool_input\"|user_prompt|prompt\"|sk-ant|hvs\.|token=|password" context/sce/claude-hook-trace-propagation.md` -> exit 1 (no matches) + +### Success-criteria verification + +- [x] AC1: Probe evidence recorded from real Claude Code hooks -> `context/sce/claude-hook-trace-propagation.md` has the verdict, Claude Code 2.1.301, per-hook traceparent tables for `PreToolUse`/`PostToolUse` matched against exported spans, gating runs, and the docs consulted; the secret/payload grep found nothing; the branch diff adds no probe files (the `probe*` fixtures in `git ls-files` predate this branch and belong to earlier adapter plans). +- [x] AC2: One `sce.hook` span per Claude hook; non-Claude hooks and the helper stay Noop -> `claude_hook_export_posts_one_hook_span_with_correlation_ids_only`, the `app::tests` admission matrix, and `managed_trace_export_posts_allowlisted_span_with_bearer_only` passed in the `managed_telemetry` and flake runs. +- [x] AC3: Parent-context branch -> `claude_hook_span_joins_valid_upstream_trace_with_fresh_span_id`, `claude_hook_span_starts_independent_root_without_usable_traceparent` (missing/malformed/all-zero table), and `claude_hook_child_spans_inherit_upstream_trace_across_polls_and_workers` passed. +- [x] AC4: Only correlation IDs, no payload or secrets -> `claude_hook_export_posts_one_hook_span_with_correlation_ids_only` asserts the exact `sce.*` attribute set and that paths, prompt, file content, tool response, Bearer value, token, and accessor are absent; passed. +- [x] AC5: Child spans only when the operation ran -> the mutation-scope loopback test (`sce.hook` → `sce.mutation_scope` → {`sce.git_snapshot`, `sce.persistence`}) and `read_only_claude_hook_exports_no_operation_children` passed in `nix flake check` cli-tests. +- [x] AC6: Telemetry cannot change hook behavior -> `failing_managed_telemetry_cannot_change_claude_hook_behavior` (Noop vs. refused vs. stalled receiver, output/seam/state parity, bounded shutdown) and `claude_hook_credential_failure_or_expiry_resolves_noop` passed. +- [x] AC7: Standalone tracing unchanged -> `managed_trace_export_posts_allowlisted_span_with_bearer_only` and the related trace_export tests passed; live `sce doctor` `sce.command` span (trace `1cc8e0b1eda96df1f757fab2e3798efd`) recorded. +- [x] AC8: Live verification recorded -> the Live verification section of the context file and the T05 record state the observed Dash0 spans and trace IDs, and say exactly what was not verified: the parented tool-hook hierarchy, the Dash0 dataset name, and interactive TUI hooks. Delivery is not inferred from a collector 2xx. Follow-up 2026-10-17: the user found trace `dcd919ecc4f5d400df5b642a1a229783` in the `sce-crocoder` dataset with 6 parented `sce.hook` spans under `claude_code.interaction` (see the T05 follow-up), so the parent relationship and dataset are now observed live; interactive TUI hooks remain unverified. +- [x] AC9: Current-state context synced -> `cli-observability-contract.md`, `telemetry-credential.md`, `claude-mutation-scope-integration.md`, `architecture.md`, `glossary.md`, and `context-map.md` describe admission, the `sce.hook` contract and children, the parent-context verdict, the correlation-ID privacy rule, and the smoke procedure. They describe distributed tracing as locally tested and not live-verified, which matches the code inspected. + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- Resolved 2026-10-17: the user found trace `dcd919ecc4f5d400df5b642a1a229783` in the `sce-crocoder` Dash0 dataset with 6 `sce.hook` spans as direct children of `claude_code.interaction`, two of them with child spans. The child span names were not listed, so live delivery of `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` by name is still proven only by local loopback tests. +- Interactive TUI sessions, subagent hooks, and other hook events were not probed for `TRACEPARENT`. +- Each Claude hook process can now take up to `CREDENTIAL_LOOKUP_DEADLINE` + `SHUTDOWN_TIMEOUT` longer, including the blocking `PreToolUse` path; the plan's open question on a tighter hook deadline is still unresolved. + +## Follow-up validation: concurrent hook telemetry reliability (PR #330) + +**Date:** 2026-10-19 + +### Change + +- Root cause of missing live `sce.hook` spans (8 of 15 expected in an earlier run): the encrypted `auth.db` opened without Turso multiprocess WAL, so concurrent hook processes failed with `LockingError` and fell back silently to Noop. Fix and evidence: [Concurrent hook span loss](../sce/claude-hook-trace-propagation.md#concurrent-hook-span-loss), [auth-db.md](../sce/auth-db.md#concurrency-and-schema-readiness). +- Added `sce.hook.command`, `sce.telemetry.fallback` diagnostics, migration-free current-schema opens, serialized Busy-retried migrations, a 1000 ms auth DB busy timeout, and a telemetry lookup that never creates a missing `auth.db`. + +### Commands run + +- `nix flake check` -> exit 0 (all checks, including cli-tests, cli-clippy, cli-fmt, pkl-generated) +- `nix build .#checks.x86_64-linux.cli-tests --rebuild` ×2 -> all tests passed both times; the command exits 1 only on Nix's output-determinism comparison +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml auth_db::multiprocess_tests` -> 8 passed +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_credential` -> 31 passed +- Multiprocess suite stress, 6 parallel copies × 8 rounds -> 48 of 48 passed +- Mutation checks: without `experimental_multiprocess_wal(true)`, 5 of 7 multiprocess tests fail; with unconditional migrations, the held-write-lock reader test fails + +### Results + +- 32 reader processes × 3 rounds: 32/32 each round (about 60–190 ms per round, debug build). +- 32 concurrent hook processes: every `(sce.claude.tool_use_id, sce.hook.command)` pair exported exactly once to a loopback collector. A wrong-key child resolves Noop with reason `database_error` and unchanged hook output. +- Mixed versions: unsafe for concurrent writers (silent lost update); sequential use and legacy reads are safe. Superseded by the compatibility lock below; see [Mixed-version compatibility lock](../sce/auth-db.md#mixed-version-compatibility-lock). +- Live: headless session `48a276ea-ce4a-4488-8244-649f20b87a42`, trace `132a62ceed757d0d8431011311ae76af`, 13 SCE hook invocations, zero local telemetry fallbacks. Dash0 delivery is **unverified**. + +### Residual risks + +- Dash0 confirmation of the 13 spans, their `sce.hook.command` values, and `tool_use_id` equality is still needed. +- Mixed old/new `sce` writers can lose updates; rollout guidance is in `auth-db.md`. Resolved on Linux by the follow-up below. +- Turso 0.8.1 has a multiprocess-WAL race when many processes concurrently create or migrate a DB under heavy load. It was not observed on read-only hook paths. + +## Follow-up validation: mixed-version lock, existing-only lookups, hook command identity (PR #330) + +**Date:** 2026-10-19 + +### Change + +- P1: every encrypted auth DB open takes a shared compatibility lock on `auth.db` (`cli/src/services/db/open_lock.rs`): an OFD `F_RDLCK` on Linux, a classic `F_SETLK` `F_RDLCK` on other Unix. It is held for the life of the `EncryptedTursoDb`. It conflicts with the classic whole-file `F_WRLCK` that legacy Turso 0.8.1/0.7.0 takes in pre-multiprocess SCE binaries, so old and new processes can no longer hold the DB at the same time. New processes still share it, and writes still go through multiprocess WAL. Details: [Mixed-version compatibility lock](../sce/auth-db.md#mixed-version-compatibility-lock). +- P2: `load_repository_telemetry_credential` (the `sce auth telemetry-headers` lookup) now uses the same existing-only path as `load_telemetry_credential`: `token_storage::get_existing_auth_db` → `EncryptedTursoDb::open_existing_at`, which opens the file without `O_CREAT` before resolving any key. +- P3: hook entry points call `record_claude_hook_command` before reading stdin. `record_claude_hook_payload` now parses the payload only. + +### Commands run + +- `nix flake check` -> exit 0, all checks passed (x86_64-linux; Darwin systems omitted by the check) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::auth_db::multiprocess_tests` -> 16 passed +- Targeted filters on the test binary: `telemetry_credential` 31, `trace_export` 14, `managed_telemetry` 26, `claude_mutation_scope` 7, `bash_policy` 36, `auth_command` 13, `services::db` 4, all passed +- Multiprocess suite stress, 6 parallel copies × 8 rounds -> 48 of 48 passed (one contended round took about 29 s per copy) +- Mutation checks: with the lock disabled, 3 tests fail, and the mixed-writer test loses 24 of 72 committed rows. With the repository lookup back on `get_auth_db`, the repository missing-DB test and the 32-helper test fail because `auth.db*` files are created. With the command recorded after the stdin read, the hook-entry test fails on the unreadable-stdin `diff-trace` span. +- Real binaries: the pre-fix `sce` (`nix build git+file://…?rev=9d4f5482…#default`) against this branch's debug build in a sandboxed `HOME`/`XDG_*`, with an env-derived key. + +### Results + +- Legacy holder (`/proc/locks`: `POSIX ADVISORY WRITE`): new `sce auth whoami` and `sce auth logout` exit 4 with `Authentication storage is unavailable`, and the log records the shared-access rejection. New holder (`OFDLCK ADVISORY READ`): a second new process succeeds, and legacy `whoami`/`logout` exit 4 (log: `File is locked by another process`). After all new processes exit, legacy `whoami` and `logout` exit 0. +- No silently lost committed write in any tested mixed-version scenario on Linux. +- Missing `auth.db`: both lookups and 32 concurrent `telemetry-headers` runs create no `auth.db`, `-wal`, or `-tshm` file and resolve no key, and the helpers fail closed. +- `sce.hook.command` is present on every `sce.hook` span for valid, empty, malformed, and unreadable stdin across all five commands. Fail-open hooks keep `sce.outcome=success`, and malformed `policy-bash` input records `failure`. No stdin content or token reaches the collector. +- Dash0: **unverified**. This session had no Dash0 access, and no live Claude run was made. + +### Residual risks + +- macOS: resolved by the follow-up below. Other non-Linux Unix still uses a process-scoped `F_SETLK` lock and is untested. Windows has no compatibility lock. +- Mixed old/new processes still cannot run at the same time: one side fails explicitly until the other exits. Every `sce` binary used by the account must be upgraded. +- The upstream Turso multiprocess cold-open race for many concurrent creators or migrators is unchanged. + +## Follow-up validation: macOS compatibility lock lifetime (PR #330) + +**Date:** 2026-10-19 + +### Change + +- `cli/src/services/db/open_lock.rs` on macOS now takes `flock(LOCK_SH | LOCK_NB)` on the SCE-owned `auth.db` descriptor instead of `fcntl(F_SETLK, F_RDLCK)`. The old process-scoped record lock was released whenever the process closed any descriptor for `auth.db`, even though `EncryptedTursoDb` still owned `_open_lock`. A `flock` lock belongs to the open file description, so it lasts until the `EncryptedTursoDb` is dropped or the process exits. Linux (`F_OFD_SETLK` `F_RDLCK`) and other Unix (`F_SETLK` `F_RDLCK`) are unchanged. `EWOULDBLOCK` is now listed with `EAGAIN`/`EACCES` as a lock conflict (they are the same value on Linux and macOS). Retry policy, diagnostics, and the rule that Turso is never opened after a failed lock are unchanged. No separate lock file, helper process, or dependency. +- Four `#[cfg(target_os = "macos")]` tests in `cli/src/services/auth_db/multiprocess_tests.rs`, described in [Mixed-version compatibility lock](../sce/auth-db.md#mixed-version-compatibility-lock). + +### Commands run (macOS 15.4.1, Darwin 24.4.0, arm64) + +- Before the fix: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::auth_db::multiprocess_tests` -> 18 passed. The existing suite did not detect the lifetime bug. +- Raw lock experiment with separate Python processes (`flock` vs `fcntl(F_SETLK)`): shared `flock` and legacy `F_WRLCK` reject each other in both directions with `EAGAIN`; two shared `flock`s coexist; a held `flock` survives an unrelated open/close in its process; an `F_SETLK` `F_RDLCK` holder that opens and closes another descriptor lets a legacy `F_WRLCK` through. +- After the fix: same command -> 22 passed, including `thirty_two_reader_processes_all_resolve_the_credential`, `concurrent_hook_processes_export_correlated_hook_spans`, `concurrent_legacy_and_new_writers_never_lose_committed_rows`, and the 4 new macOS tests. +- Mutation check: with `open_lock.rs` reverted to the old `F_SETLK` version, `macos_new_shared_lock_survives_unrelated_descriptor_close` fails (`legacy lock must be rejected while a new process holds the DB`). The other three macOS tests pass, as expected for the old lock. +- 3 parallel copies of the multiprocess suite -> 3 of 3 passed. +- `nix flake check` -> exit 0 on aarch64-darwin (`cli-clippy`, `cli-tests`, `cli-fmt`). Linux systems were omitted by the check. + +### Residual risks + +- Linux was not re-run in this session. Its lock code is unchanged; only the conflict check now also names `EWOULDBLOCK` (== `EAGAIN` on Linux). Run `nix flake check` on x86_64-linux before merge. +- Tested on macOS 15.4.1 only. Older macOS versions were not tested. +- macOS was not tested against the real pre-fix `sce` binary. The legacy side used the pre-fix Turso builder configuration and raw `F_SETLK` `F_WRLCK` children. + +## Follow-up validation: auth storage guidance and doctor telemetry health (PR #330) + +**Date:** 2026-10-19 + +### Change + +- `UserError::AuthStorageUnavailable` now shows platform-specific recovery steps. On macOS these are: run `sce auth login` interactively, choose "Always Allow" in the Keychain dialog, then retry. A new `LoginAuthStorageUnavailable` variant covers storage failures inside `sce auth login`, so its text does not tell the user to rerun login before Keychain access is allowed. Both keep key `auth.storage_unavailable`, the runtime class, and exit code 4. Details: [User-facing storage guidance](../sce/auth-db.md#user-facing-storage-guidance). +- `sce doctor` has a read-only `Managed telemetry` row and a JSON `managed_telemetry` object. It reuses the readiness hint, the non-interactive existing-DB credential lookup, the lookup deadline, and `TelemetryFallbackReason`. Details: [Doctor health check](../sce/telemetry-credential.md#doctor-health-check). +- `telemetry_readiness::state` separates `NotConfigured`, `Expired`, and `Ready` without opening `auth.db`. `is_ready` behavior is unchanged. +- The multiprocess test sandbox now resolves the macOS state directory (`$HOME/Library/Application Support/sce`). Before, `auth_db_files()` checked `state/sce` on macOS, so its "no `auth.db*` files created" assertions passed without checking anything there. +- No changes to hook telemetry resolution, fallback logging levels, exporter lifecycle, setup, the compatibility lock, migrations, signing, or release workflows. + +### Commands run (macOS 15.4.1, Darwin 24.4.0, arm64) + +- `nix flake check` -> exit 0 on aarch64-darwin (`cli-tests`, `cli-clippy`, `cli-fmt`, and the other checks). The first run failed only in `concurrent_legacy_and_new_writers_never_lose_committed_rows`, on a `turso_core` `wal.rs` panic (`shared WAL frame ids must increase monotonically`). That is the upstream multiprocess-WAL race listed above. The test does not use the changed code, passed 6 of 6 isolated reruns, and passed on the full rerun. +- Targeted filters: `services::error` 15, `services::doctor` 11, `services::managed_telemetry` 29, `services::telemetry_readiness` 6, `services::auth_command` 17, `app::doctor_reconciliation_cli_tests` 9, `services::auth_db::multiprocess_tests` 24. All passed. +- Sandboxed debug binary (`HOME` in the scratchpad, `SCE_AUTH_DB_ENCRYPTION_KEY` set) with an unreadable `auth.db` and a ready hint: text shows `[WARN] Managed telemetry` / `Reason: database_error` and the step list. JSON has `managed_telemetry.kind = managed_telemetry_database_error`. With the hint removed: `[PASS] Managed telemetry (not configured)`. + +### Results + +- Fresh state: doctor reports `not_configured` with no warning and creates no `auth.db`, `telemetry-ready.json`, or key. The child runs with an empty `SCE_AUTH_DB_ENCRYPTION_KEY`, so any key resolution would have failed the check. +- Real seeded auth DB: `healthy` with the right key and `database_error` with a wrong or empty key. An expired hint gives `credential_expired` and a removed hint gives `not_configured`, both without a lookup. `auth.db` bytes stay unchanged and the credential still loads afterwards. +- `sce doctor --fix` with an unreadable `auth.db`: one `manual` fix result, and `auth.db` and the hint are byte-for-byte unchanged. Neither stdout nor stderr contains the secret. +- Unit tests cover the classification table, the lookup deadline (`lookup_timeout`), "Always Allow" only for macOS `credential_inaccessible`, no destructive or secret-bearing text, and noncircular login guidance. + +### Residual risks + +- Not run against the real macOS Keychain. No live `credential_inaccessible` doctor run or real "Always Allow" dialog was exercised in this session. Running it would have read the user's login Keychain. Its text is covered by unit tests, and the non-interactive read path is unchanged. +- "Always Allow" trust is tied to the executable's code signature. Unsigned or ad-hoc-signed upgrades can lose it, and the user then has to authorize again through `sce auth login`. This is out of scope without a Developer ID certificate. +- Interactive Keychain failures reach `AuthStorageUnavailable` as `TokenStorageError::Database`, so a DB failure during login also shows credential-store guidance. Resolved in the follow-up below. +- Doctor repeats the startup credential lookup (cached `AuthDb` on success), adding up to `CREDENTIAL_LOOKUP_DEADLINE` when the store hangs. Hook startup is unaffected. +- Linux was not re-run in this session. Run `nix flake check` on x86_64-linux before merge. + +## Follow-up validation: typed auth storage failure classification (PR #330) + +**Date:** 2026-10-19 + +### Change + +- `UserError::AuthStorageUnavailable` and `LoginAuthStorageUnavailable` now carry a typed `AuthStorageFailure` cause: `CredentialAccess`, `DatabaseLocked`, or `Database`. Turso, lock, path, and I/O failures no longer show Keychain guidance. Key `auth.storage_unavailable`, the runtime class, and exit code 4 are unchanged. Details: [User-facing storage guidance](../sce/auth-db.md#user-facing-storage-guidance). +- Interactive keyring failures get the typed `CredentialAccessDenied` marker only for `NoStorageAccess`, or, on macOS, for the `OSStatus` codes `errSecUserCanceled`, `errSecNoAccessForItem`, `errSecAuthFailed`, `errSecInteractionNotAllowed`, and `errSecInteractionRequired`. +- The compatibility-lock conflict is a typed `SharedLockConflict` error with the same text. When the last encrypted-open retry fails on it, or on a Turso `Busy`/`BusySnapshot`, the error carries `db::DatabaseLocked`, which maps to `TokenStorageError::DatabaseLocked`. Managed telemetry still reports it as `database_error`. +- `ControlPlaneError::Storage` carries the cause, and `is_storage_failure()` became `storage_failure() -> Option` on the control-plane, stream, and sync error types. +- The macOS command guidance now ends with "Retry the original command, or run `sce doctor` to verify telemetry access." +- No changes to the non-interactive read path, `flock(LOCK_SH | LOCK_NB)`, lock lifetime, retry policy, Turso, setup, `doctor --fix`, signing, or release workflows. + +### Commands run (macOS 15.4.1, Darwin 24.4.0, arm64) + +- `nix flake check -L` -> exit 0 on aarch64-darwin (`cli-fmt`, `cli-clippy`, `cli-tests`: 671 passed, 0 failed). This includes all four `macos_*` compatibility-lock tests and the legacy-holder tests. +- Targeted filters: `services::error` 16, `services::auth_command` 17, `services::db::encryption_key` 2, `services::managed_telemetry` 29, `services::doctor` 11, `services::sync` 12, `services::agent_trace_sync` 14, `services::auth_db` 24. All passed. +- New or extended tests: per-cause message selection, with no Keychain text for DB causes on any platform or operation. Login guidance for all three causes plus `AuthError::Io`. Keyring tagging for `NoStorageAccess`, `NoEntry`, and generic platform failures, and for each macOS `OSStatus` code. The legacy-holder child roles now also assert `TokenStorageError::failure() == DatabaseLocked` and `error.is::()`. + +### Real macOS Keychain + +- Verified (read-only, user's login Keychain, existing `sce`/`auth DB` item left unchanged): a freshly built ad-hoc, linker-signed debug binary is a new code identity that is not in the item's ACL. `sce doctor --format json` from that binary exited 0 in about 2 s with empty stderr. It reported `managed_telemetry.reason = credential_inaccessible`, `kind = managed_telemetry_credential_inaccessible`, and the "Always Allow" remediation. It did not report `lookup_timeout`, so the lookup returned `errSecInteractionNotAllowed` without blocking on a dialog. +- Not verified: authorized silent access by the current build, locked-Keychain behavior, interactive authorization during `sce auth login`, persistence after "Always Allow", and real Claude Code hook runs against the Keychain. These need GUI interaction, locking the user's login Keychain, or changing its ACL or search list. The adapter only targets Keychain domains, not a disposable keychain file, so these could not be isolated. Hook fail-open and no-prompt behavior stays covered by the sandboxed multiprocess tests using `SCE_AUTH_DB_ENCRYPTION_KEY`. + +### Residual risks + +- "Always Allow" trust is tied to the executable's code signature. Unsigned or ad-hoc-signed upgrades can lose it. This is out of scope without a Developer ID certificate. +- Non-interactive failures, including a missing Keychain item while `auth.db` exists, still all map to `CredentialAccessUnavailable` (doctor `credential_inaccessible`), as before. +- `DatabaseLocked` is detected only while opening the encrypted DB. Busy errors from later queries after retries still classify as `Database`. +- Linux was not re-run in this session. + diff --git a/context/sce/auth-db.md b/context/sce/auth-db.md index 08e1163e2..e74776f7d 100644 --- a/context/sce/auth-db.md +++ b/context/sce/auth-db.md @@ -48,23 +48,124 @@ Current migration baseline: Storage operations live in `cli/src/services/telemetry_credential_storage.rs`: `save_telemetry_credential` is a single-statement `INSERT OR REPLACE` of the fixed row (last complete writer wins), `load_telemetry_credential` returns the row only when token, accessor, and identity are non-empty, the expiry is non-negative and in the future, and the stored WorkOS session's access-token `(sub, workspace_id)` claims equal the row's identity (it never mints or refreshes), `load_repository_telemetry_credential` additionally requires a matching repository binding, `bind_repository_telemetry` records a binding for the current credential identity, and `delete_telemetry_credential` removes the row. `TelemetryCredential` has a redacted `Debug`, and storage errors use fixed messages without secret values. Saving or deleting never touches `auth_credentials`. Managed OTLP export reads this row at CLI startup through `load_telemetry_credential` and uses only the token as bearer. The shared `AuthDb` singleton comes from `token_storage::get_auth_db`. `sce auth logout` deletes both the `auth_credentials` row and the telemetry row; see [telemetry-credential.md](telemetry-credential.md) for lifecycle and security boundaries. +## Concurrency and schema readiness + +- Many short-lived `sce` processes read `auth.db` at once: every admitted Claude hook resolves the telemetry credential at startup. `EncryptedTursoDb::new()` therefore opens with Turso multiprocess WAL, alongside encryption, so concurrent readers share the file. Multiprocess WAL creates `auth.db-wal` and the `auth.db-tshm` coordination file next to the database. SCE also creates `auth.db-open.lock` once `auth.db` exists, to serialize Turso cold opens (see [Serialized cold opens](#serialized-cold-opens-after-a-legacy-write)). +- A current schema is opened read-only: no `CREATE TABLE IF NOT EXISTS __sce_migrations` or other DDL runs when every embedded migration ID is already recorded. A missing or outdated DB is migrated in one `BEGIN IMMEDIATE` transaction, with a retry only for genuine `Busy`. A partially migrated schema fails with the failing migration ID rather than being treated as current. Details: [shared-turso-db.md](shared-turso-db.md). +- Telemetry credential lookups are existing-DB read paths. Both `load_telemetry_credential` (the managed-telemetry lookup used by every command and admitted hook) and `load_repository_telemetry_credential` (the `sce auth telemetry-headers` lookup that Claude Code's `otelHeadersHelper` runs) go through `token_storage::get_existing_auth_db`. That helper reuses the process `AuthDb` singleton when it is already open. Otherwise it calls `EncryptedTursoDb::open_existing_at`, which opens the database file without `O_CREAT` before resolving the encryption key. An absent file returns no credential, with no DB file, `-wal`/`-tshm` file, encryption key, or migration created. The existence check is the `open()` itself, not a separate `Path::exists()`. Path-resolution errors stay `TokenStorageError::PathResolution`. Only `sce auth`, `sce setup`, and other writers create the DB. This also keeps concurrent hooks and header-helper calls on a never-logged-in machine from racing to create it. +- The auth DB busy timeout is 1000 ms. Hook reads stay bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s), which aborts the lookup regardless of busy waits or retries. +- Regression coverage (`cli/src/services/auth_db/multiprocess_tests.rs`, child processes of the test binary with sandboxed `HOME`/`XDG_*` and an env-provided `SCE_AUTH_DB_ENCRYPTION_KEY`; never the real DB or keyring): a reader succeeds while another process holds the DB open; 32 reader processes × 3 rounds all resolve the credential; a current-schema reader succeeds while another process holds `BEGIN IMMEDIATE`, which proves it performs no migration write; both telemetry lookups on a missing DB return none and create nothing (the repository lookup runs with an empty `SCE_AUTH_DB_ENCRYPTION_KEY`, which fails any key resolution, so it also proves no key is resolved or created); 32 concurrent `sce auth telemetry-headers` runs (through `app::run`, inside a Git repository) on a missing DB all fail closed with `Telemetry credential is unavailable.` and leave no `auth.db*` file; the helper prints the bearer only for the bound repository; 4 processes concurrently migrating an existing schemaless DB record each migration once and pass `PRAGMA integrity_check`; an outdated schema upgrades and a partial one fails clearly and rolls back; 32 concurrent hook processes export correlated `sce.hook` spans. The mixed-version tests are listed under [Mixed-version compatibility lock](#mixed-version-compatibility-lock). + +### Upstream Turso multiprocess race + +Under heavy load, Turso 0.8.1 multiprocess WAL has an internal race on concurrent cold opens that involve writes. Stress runs (6 parallel copies of the suite, with 32 processes creating or migrating the same DB) occasionally failed: `short read on page 3`, a `turso_assert` panic `reader slot released by non-owner` in `shared_wal_coordination.rs`, or, in the Nix sandbox, a writer that stayed `database is locked` through all 10 migration attempts (about 49 s). The read-only paths never failed across about 3,800 cold-opening reader and hook processes, and that is all hooks do on a current schema. None of those runs followed a legacy-mode write; that case did fail, as described under [Serialized cold opens](#serialized-cold-opens-after-a-legacy-write). The migration regression therefore uses 4 concurrent migrators, which was 0 failures in 48 parallel stress runs. Concurrent creation and migration by many writers is left as a documented upstream limitation. That migration limitation affects first-time initialization, and the Agent Trace DB already runs in the same Turso mode. + +The four-writer compatibility regression also reproduced exhausted `database is locked` retries against an already-seeded, current-schema DB when connection startup/teardown overlapped inserts. Its children previously swallowed insert failures, leaving only a short committed-row count in the parent. The regression now opens every writer before releasing concurrent inserts and keeps every connection alive until all writers finish; every insert must succeed, all 32 rows must persist, and a legacy process must reopen afterwards. This passed 80 stress repetitions across four parallel test runs. The macOS shared-lock writer regression uses the same coordination. This isolates the compatibility-lock guarantee from the Turso lifecycle race; production open/write/close behavior and retry limits are unchanged. + +### Serialized cold opens after a legacy write + +A legacy-mode process (any SCE release through `v0.4.0-pre-alpha-v11`) writes `auth.db-wal` without updating the multiprocess coordination authority in `auth.db-tshm`. The next new-mode open finds that its WAL disk scan disagrees with the persisted authority. Turso 0.8.1 then takes the rebuild branch of `WalFile::repair_or_reseed_authority_from_local_disk_scan` (`storage/wal.rs`, reached from `seed_or_sync_authority` during `turso::Builder::build`). That branch discards the durable shared frame index and republishes it from the local scan. It treats the open as exclusive, but its only guard is `!writer_or_checkpoint_lock_active()`. The publish mutex in `SharedWalCoordination::record_frame` is process-local. When several new processes cold-open at once, each runs the rebuild concurrently against the same shared index. This is an upstream Turso defect, not an SCE ordering bug or a test orchestration issue. The compatibility lock keeps legacy and new processes from holding the DB at the same time, but it cannot help here because the legacy process has already exited. + +Observed outcomes, all on read-only `load_telemetry_credential` children (the hook path) against a seeded, current-schema DB: + +- A `turso_assert` panic: `shared WAL frame index length changed while publishing an entry` (`wal.rs:1605`). This is the Linux CI failure in `concurrent_legacy_and_new_writers_never_lose_committed_rows`, where 2 new writers cold-opened together after a legacy writer from the previous round. +- A successful open that read a stale snapshot. Some processes found no telemetry credential, and others found an outdated schema (`failed to run auth DB migrations`). That is a silent wrong read, not only a crash. + +Probe (20 rounds: 1 legacy write, then 8 concurrent new readers): 7–14 failed opens out of 160 per run across 6 runs after a legacy write. In the control, with a new-mode write instead of the legacy write, 0 of 160 failed. With an authority that is current, Turso syncs from it and never rebuilds, so this needs a preceding legacy write (or another out-of-band WAL change). + +Mitigation (`open_lock::lock_cold_open`): every `EncryptedTursoDb` connection attempt takes an exclusive `std::fs::File::lock` (`flock` on Unix, `LockFileEx` on Windows) on the sidecar `auth.db-open.lock`. It is held only around `turso::Builder::build()` and `connect()`, after the compatibility lock succeeds, so a legacy holder is still rejected immediately. The wait runs on a blocking thread, so `CREDENTIAL_LOOKUP_DEADLINE` still bounds hook lookups. Queries, writes, and migrations are not serialized. Rationale: with the compatibility lock, a stale authority can exist only while no new process holds the DB. The first serialized opener therefore rebuilds alone, and later openers sync from the rebuilt authority. The sidecar is a separate file, so the Linux `F_OFD_SETLK` lock and the macOS `flock` lock on `auth.db` are unchanged. Cost: a burst of 32 simultaneous cold opens takes about 160 ms wall time instead of about 55 ms (roughly 3 ms per serialized open), well under the 2 s hook deadline. + +Residual risks: a process stuck inside `turso::Builder::build()` delays other cold opens until it exits; hook lookups still give up at their deadline, and the kernel releases the lock when the process dies. The Agent Trace DB (`TursoDb`, also multiprocess WAL) does not take this lock. The upstream defect remains in Turso 0.8.1, and SCE only avoids triggering it. + +### Mixed-version compatibility lock + +SCE binaries from before multiprocess WAL (every release through `v0.4.0-pre-alpha-v11`, on Turso 0.8.1 or 0.7.0) open `auth.db` in legacy WAL mode. In that mode, Turso's `UnixFile::lock_file` takes a non-blocking classic POSIX record lock, `fcntl(F_SETLK, F_WRLCK)` over the whole file (`l_start = 0`, `l_len = 0`). It holds that lock while the database is open. Turso's own mixed-mode guards (`reject_live_*` in `turso_core::Database::resolve_default_storage`) never run for `turso::Builder`. `turso_sdk_kit` opens the file itself and calls `Database::open_async`, which skips them. Multiprocess mode opens the DB file with `NoLock`, so nothing stopped the two modes from overlapping. Without a guard, when an old and a new process both held the DB and both wrote, one process's committed rows were silently lost while `PRAGMA integrity_check` still reported `ok`. The regression test reproduced this: with the guard disabled, 24 of 72 committed rows were lost. + +Every `EncryptedTursoDb` open now takes a shared compatibility lock (`cli/src/services/db/open_lock.rs`) on the `auth.db` file. It is acquired inside the connection-open retry loop, before Turso opens the file, and held for the life of the `EncryptedTursoDb` value: + +- Linux: `fcntl(F_OFD_SETLK, F_RDLCK)`, an open-file-description lock on the whole file, on a descriptor the `EncryptedTursoDb` owns. Linux reports conflicts between OFD locks and classic POSIX record locks. So the new read lock and the legacy write lock exclude each other in both directions, while any number of new processes share the read lock. The lock belongs to SCE's own descriptor, so Turso opening or closing its descriptors for `auth.db` cannot release it. It is released when the `EncryptedTursoDb` is dropped or the process exits. +- macOS: `flock(LOCK_SH | LOCK_NB)` on the same SCE-owned `auth.db` descriptor. macOS has no public OFD lock, and a classic `F_SETLK` record lock is process-scoped: closing *any* descriptor for `auth.db` in the process released it, so the earlier `F_SETLK` `F_RDLCK` implementation could silently drop the guard while the `EncryptedTursoDb` was still open. A `flock` lock belongs to the open file description, so unrelated opens and closes of `auth.db` in the same process leave it in place. On macOS, `flock` and `fcntl` locks share one lock table and conflict across owners: the shared `flock` and the legacy whole-file `F_SETLK` `F_WRLCK` exclude each other in both directions (`EAGAIN`/`EWOULDBLOCK`), while any number of new processes share the `flock`. Turso's multiprocess open uses `NoLock` on the DB file and skips its legacy-WAL probe on the `turso::Builder` path, so it never requests a conflicting lock in the same process. Verified on macOS 15.4.1 (Darwin 24.4.0, arm64) with separate processes; see the macOS tests below. +- Other Unix (not Linux or macOS): `fcntl(F_SETLK, F_RDLCK)`, a classic process-scoped record lock. It conflicts with the legacy lock in the same way, but POSIX releases it when the process closes any descriptor for `auth.db`. Not built or tested on those platforms. +- Windows: no compatibility lock. Mixed versions on Windows remain unprotected. + +Resulting coexistence on Linux (and on macOS through the test suite, without the real pre-fix binary), verified with separate processes, isolated encrypted databases, a legacy child that opens with the exact pre-fix builder configuration on the pinned Turso 0.8.1, and the real pre-fix `sce` binary: + +| First holder | Second process | Result | +| --- | --- | --- | +| legacy | new read (telemetry lookup) or write (`AuthDb::new`) | Rejected after the connection-open retries (under 1 s): `auth DB database at is open in an older SCE process that does not support shared access`. Commands surface the stable `Authentication storage is unavailable` error, and hooks fall back to Noop with reason `database_error`. Hook behavior is otherwise unchanged. | +| new | legacy read or write | Rejected immediately by Turso: `Failed locking file ''. File is locked by another process` | +| new | new read or write | Shared. Writes stay serialized by Turso multiprocess WAL and the 1000 ms busy timeout. | +| legacy and new, started together | concurrent writes | Whichever opens first proceeds. The other is rejected or opens after the first exits. No committed row is lost. | +| none (all new processes exited) | legacy | Opens, decrypts, reads, and writes normally | + +`/proc/locks` shows the legacy holder as `POSIX ADVISORY WRITE … 0 EOF` and a new holder as `OFDLCK ADVISORY READ -1 … 0 EOF`. Released binaries need no change. Unsafe combinations fail explicitly instead of losing data. + +Mixed-version tests (`multiprocess_tests.rs`), all verifying persisted `telemetry_repository_bindings` rows and the `telemetry_credentials`/`auth_credentials` rows plus `integrity_check` afterwards: + +- A legacy holder makes a new telemetry lookup and a new writer fail explicitly. The legacy holder's own later write persists. +- A new holder makes a legacy open fail with the Turso lock error. +- 3 rounds of 1 legacy writer and 2 new writers start together. Every row any writer reported committed is persisted. +- `concurrent_cold_opens_after_legacy_write_all_see_committed_state`: 20 rounds of one legacy writer (8 rows), then 8 concurrent new reader processes. Every reader must resolve the credential, and every legacy row must persist with `integrity_check` `ok`. Without the cold-open lock, this test failed 20 of 20 runs. +- 4 new writers with coordinated connection lifetimes insert concurrently and commit all 32 rows; any insert failure includes the row and database error. +- After every new process exits, a legacy process reopens the DB, reads the credential, and writes. + +macOS-only tests (`#[cfg(target_os = "macos")]`, separate processes, raw `fcntl(F_SETLK, F_WRLCK)` children that take the exact legacy lock as well as legacy Turso opens): + +- `macos_new_shared_lock_rejects_legacy_exclusive_lock_until_released`: a new holder makes the raw legacy lock fail with `EAGAIN` and the legacy Turso open fail; after the holder exits, both succeed. +- `macos_legacy_exclusive_lock_rejects_new_shared_lock_until_released`: a raw legacy lock holder makes a new lookup and a new writer fail with the shared-access error; after it exits, a new reader resolves the credential. +- `macos_new_shared_lock_survives_unrelated_descriptor_close`: the new holder opens and closes a second `auth.db` descriptor after `AuthDb::new`; legacy locks and opens are still rejected. Against the old `F_SETLK` `F_RDLCK` implementation this test fails because the legacy lock is granted. +- `macos_new_processes_share_the_lock_concurrently`: two new holders (one of them after an unrelated close) and 4 new writers run together, every write commits, and the legacy lock is still rejected; after they exit, a legacy process reopens and writes. + +Remaining limits: a mixed old/new deployment is still not usable at the same time, because one side fails while the other holds the DB. Upgrade by replacing every `sce` binary used by the account. Old hook processes still in flight during the switch make new processes fail (and hooks fall back to Noop) until they exit; they no longer lose data. Turso reopens `auth.db` by path after the lock is taken, so a file that something outside SCE deletes in that window would be recreated empty. SCE never deletes `auth.db`. + ## Lifecycle integration `AuthDbLifecycle` is registered in `cli/src/services/auth_db/lifecycle.rs` following the existing DB lifecycle pattern: - `diagnose` collects auth DB path health problems. - `fix` bootstraps missing auth DB parent directory. -- `setup` calls `AuthDb::new()` to initialize the encrypted database. +- `setup` only creates the auth DB parent directory. It never creates, opens, or migrates `auth.db` and never resolves an encryption key, so `sce setup` works offline, needs no login, and leaves an existing `auth.db` byte-for-byte unchanged. A missing `auth.db` is the normal unauthenticated state; only `sce auth login` creates it. - `LifecycleProviderId::AuthDb` is the provider identifier. - The lifecycle provider is registered in deterministic order: config → local_db → auth_db → agent_trace_db → hooks. ## Token storage integration -- `cli/src/services/token_storage.rs` uses `AuthDb` through async `save_tokens`, `load_tokens`, and `delete_tokens` operations and a `tokio::sync::OnceCell>`. Completed initialization success or failure is cached; cancellation before initialization completes leaves the cell uninitialized. +- `cli/src/services/token_storage.rs` uses `AuthDb` through async `save_tokens`, `load_tokens`, and `delete_tokens` operations and a `tokio::sync::OnceCell`. Only a successful open is cached, so a failed non-interactive open never blocks a later interactive one in the same process. +- Open paths: `open_or_create_auth_db()` (`AuthDb::new()`, interactive key access) is used only to persist a new session or telemetry credential during `sce auth login`. Every other path uses `open_existing_auth_db(access)` (`EncryptedTursoDb::open_existing_at(path, access)`), which returns `None` for a missing `auth.db` and never creates the file or a key. `load_tokens(access)` and `save_tokens(token, access)` take the caller's `CredentialAccess`; `delete_tokens` is interactive and returns `false` for a missing DB. - Auth commands and the generic async control-plane credential store directly await token persistence on the application runtime. Database operations do not cross a `spawn_blocking` boundary; the OS keyring adapter retains its platform-specific synchronous behavior where required. - `token_file_path()` returns the auth DB path from `auth_db_path()` instead of a JSON file path. -- `TokenStorageError` exposes `PathResolution` and `Database` variants; former `Io`, `Serialization`, `CorruptedTokenFile`, and `Permission` variants have been removed. +- `TokenStorageError` exposes `PathResolution`, `Database`, `DatabaseLocked`, and `CredentialAccessUnavailable` variants. `CredentialAccessUnavailable` is mapped from the typed `encryption_key::CredentialAccessUnavailable` (non-interactive) or `encryption_key::CredentialAccessDenied` (interactive) errors and carries no secret. `DatabaseLocked` is mapped from the typed `db::DatabaseLocked` error. `TokenStorageError::failure()` returns the `AuthStorageFailure` cause used for user guidance. Former `Io`, `Serialization`, `CorruptedTokenFile`, and `Permission` variants have been removed. - No JSON file I/O remains in `token_storage.rs`. - The `auth_credentials` row uses constant integer ID `1` for single-row token storage. -- Encryption is required: `encryption_key::get_or_create_encryption_key()` first honors `SCE_AUTH_DB_ENCRYPTION_KEY` by deriving a deterministic 64-character hex Turso key from the non-empty env-secret text, then falls back to OS credential-store keyring get-or-create behavior when the env var is absent. No plaintext auth DB fallback exists. Missing/unavailable credential-store failures surface as `TokenStorageError::Database` and include platform-specific remediation for Linux system keyring/Secret Service, macOS Keychain, Windows Credential Store, or unsupported platforms, always mentioning `SCE_AUTH_DB_ENCRYPTION_KEY` for headless/CI use without printing the secret value. +- Encryption is required: `encryption_key::get_or_create_encryption_key()` first honors `SCE_AUTH_DB_ENCRYPTION_KEY` by deriving a deterministic 64-character hex Turso key from the non-empty env-secret text, then falls back to the OS credential store when the env var is absent. No plaintext auth DB fallback exists. +- Credential access is explicit (`encryption_key::CredentialAccess`): + - `Interactive`: used only by `sce auth login`, `logout`, and `whoami`. It may show the macOS Keychain authorization dialog. It generates and stores a new key only when the keyring reports `NoEntry` and `auth.db` does not exist. Any other read error is returned and never overwrites the stored key. A read or write failure is tagged with the typed `CredentialAccessDenied` marker only when the keyring reports `NoStorageAccess`, or, on macOS, a `PlatformFailure` whose `OSStatus` is `errSecUserCanceled`, `errSecNoAccessForItem`, `errSecAuthFailed`, `errSecInteractionNotAllowed`, or `errSecInteractionRequired`. Classification uses these codes, never error text. + - `NonInteractive`: used by Claude hooks, managed telemetry startup, the `sce doctor` telemetry health check, `sce auth telemetry-headers`, and `sce sync` (including post-commit auto-sync). It only reads, never creates. On macOS it wraps the read in `SecKeychain::disable_user_interaction()` (`SecKeychainSetUserInteractionAllowed(false)`, the UI switch for the legacy file-based keychain API that `apple-native-keyring-store`'s `keychain` feature uses through `SecKeychainFindGenericPassword`). An item the binary is not authorized for, or a locked keychain, fails with `errSecInteractionNotAllowed` instead of prompting. Every failure becomes `CredentialAccessUnavailable`. `kSecUseAuthenticationUIFail` and `LAContext.interactionNotAllowed` do not apply, because they belong to the `SecItem`/data-protection keychain and LocalAuthentication-protected items, which this adapter does not use. + - The interaction switch is process-wide, so all credential-store reads and writes in one process are serialized by a mutex. Interaction is turned back on before the next caller runs. + - Linux Secret Service and Windows have no equivalent switch in the adapter. `NonInteractive` there only guarantees no key creation. Missing/unavailable credential-store failures surface as `TokenStorageError::Database` and include platform-specific remediation for Linux system keyring/Secret Service, macOS Keychain, Windows Credential Store, or unsupported platforms, always mentioning `SCE_AUTH_DB_ENCRYPTION_KEY` for headless/CI use without printing the secret value. + +## User-facing storage guidance + +Storage failures from auth commands and `sce sync` render the `UserError` catalog entry for `auth.storage_unavailable` (runtime class, `SCE-ERR-RUNTIME`, exit code 4). Two variants in `cli/src/services/error.rs` share that key, class, and exit code. Each carries a typed `AuthStorageFailure` cause: + +- `AuthStorageUnavailable(cause)`: logout, whoami, and sync. +- `LoginAuthStorageUnavailable(cause)`: storage failures inside `sce auth login` (stored-session load, renewal save, session replacement, device-flow storage and I/O errors). + +The cause comes from typed errors only: `TokenStorageError::failure()`, and `ControlPlaneError::Storage(cause, _)` through `storage_failure()` on the sync error types. It never comes from matching message text. + +| Cause | Typed source | Guidance | +| --- | --- | --- | +| `CredentialAccess` | `CredentialAccessUnavailable` / `CredentialAccessDenied` | Platform-specific credential-store steps (below). | +| `DatabaseLocked` | `db::DatabaseLocked`: the last encrypted-open attempt failed on the compatibility-lock conflict or a Turso `Busy`/`BusySnapshot` | Another SCE process holds the auth database. Wait for other `sce` processes, including Claude Code hooks, make every installed `sce` the same version, then retry. | +| `Database` | Any other DB, path, or I/O failure | SCE could not open or read its local authentication database. Run `sce doctor`, retry, then check the log files. | + +`DatabaseLocked` and `Database` guidance is the same on every platform and for login. It never mentions the encryption key, Keychain, "Always Allow", or `sce auth login`, so it is never circular. + +`CredentialAccess` guidance is platform-specific: + +- Command, macOS: run `sce auth login` in an interactive terminal, choose "Always Allow" when macOS asks to access the SCE Keychain item, then retry the original command or run `sce doctor` to verify telemetry access. If no prompt appears, the user should check that the login Keychain is unlocked and use the same installed executable. +- Login, macOS: running login again does not help until access is allowed. Unlock the login Keychain, rerun `sce auth login`, and choose "Always Allow" to authorize this executable. +- Linux names Secret Service, Windows names Credential Manager, and other platforms name the OS credential store. None of them mention macOS or "Always Allow". + +The text never says that "Always Allow" survives a binary upgrade. Unsigned and ad-hoc-signed builds can lose Keychain trust when the executable changes. It never suggests deleting `auth.db`, regenerating the key, or resetting the Keychain. The `anyhow` source stays attached for logs. Only the fixed catalog text reaches stderr, after `redact_sensitive_text`. Typed markers are placed inside the `anyhow` chain, so rendered diagnostics, including the compatibility-lock rejection text, are unchanged. See also: [shared-turso-db.md](shared-turso-db.md), [../cli/default-path-catalog.md](../cli/default-path-catalog.md), [../context-map.md](../context-map.md) diff --git a/context/sce/claude-hook-trace-propagation.md b/context/sce/claude-hook-trace-propagation.md new file mode 100644 index 000000000..8c95f99aa --- /dev/null +++ b/context/sce/claude-hook-trace-propagation.md @@ -0,0 +1,222 @@ +# Claude Code Hook Trace-Context Propagation + +Probe evidence and verdict for whether Claude Code passes W3C trace context to command hooks, recorded by T01 of `context/plans/claude-hook-trace-propagation.md`. This file is the durable source for the parent-context decision consumed by T02–T05. Shipped hook admission and the `sce.hook` span contract are described in [cli-observability-contract.md](cli-observability-contract.md); parenting `sce.hook` on a valid `TRACEPARENT` (independent root otherwise) is shipped and described there. + +## Verdict + +**`parent_context_available`**, with two binding qualifications: + +1. **The parent is the `claude_code.interaction` span, not the tool span.** Every `PreToolUse` and `PostToolUse` command hook received a valid `TRACEPARENT` whose trace ID matched the exported Claude Code trace and whose parent span ID was the span ID of the turn's `claude_code.interaction` root span. It never matched `claude_code.tool`, `claude_code.tool.execution`, or (with detailed beta tracing) the `claude_code.hook` span that wraps the hook run. An SCE `sce.hook` span parented on this context is therefore a **sibling** of `claude_code.tool` and `claude_code.hook` under the interaction, not their child. Hook-to-tool correlation must use `tool_use_id`, which appears both in the hook stdin payload and as the `tool_use_id` attribute on Claude's `claude_code.tool` and `claude_code.tool.execution` spans. +2. **The context exists only when Claude Code tracing is active.** `TRACEPARENT` is present only when both `CLAUDE_CODE_ENABLE_TELEMETRY=1` and `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA=1` are set (so Claude Code has a live span). With telemetry off, with telemetry on but the tracing beta off, or with only `CLAUDE_CODE_PROPAGATE_TRACEPARENT=1`, hooks receive no `TRACEPARENT`. SCE must treat a missing traceparent as the normal case and export an independent root. + +`TRACESTATE` was never present. The hook stdin payload carries no trace fields. + +Deviation from plan wording: the plan's change summary assumed parenting "to Claude Code's active tool span". T03 should parent `sce.hook` on the provided interaction context as recorded here; parenting under the tool span is not possible from the hook process with Claude Code 2.1.301. + +## Claude-caller signal + +Presence-only environment markers observed in every probed hook process, compared with the Claude Code Bash tool environment of the same Claude Code version: + +| Marker | Hook process | Bash tool subprocess | +| --- | --- | --- | +| `CLAUDECODE` | present | present | +| `CLAUDE_CODE_SESSION_ID` | present | present | +| `CLAUDE_CODE_CHILD_SESSION` | present | present | +| `CLAUDE_PID` | present | present | +| `CLAUDE_CODE_ENTRYPOINT` | present | present | +| `AI_AGENT` | present | present | +| `CLAUDE_PROJECT_DIR` | present | absent | +| `CLAUDE_ENV_FILE` | absent | absent | + +Selected signal for shared subcommands (`diff-trace`, `conversation-trace`, `policy bash`): **`CLAUDECODE` and `CLAUDE_PROJECT_DIR` both present**, checked for presence only, never read for value. `CLAUDECODE` alone is not hook-specific because every Claude Code subprocess inherits it, including commands the Bash tool runs. Claude Code documents `CLAUDE_PROJECT_DIR` as set for command hooks, stdio MCP servers, and plugin LSP servers, and the Bash tool does not receive it. The Claude-only subcommands (`claude-mutation-scope`, `claude-model-state`) are admitted from argv and do not need the signal. The generated `.claude/hooks/run-sce-or-show-install-guidance.sh` helper `exec`s `sce` with the inherited environment, so these markers and `TRACEPARENT` reach the SCE process unchanged. + +Claude Code removes `OTEL_*` exporter variables from every subprocess it spawns (documented). SCE hook export must not depend on Claude's `OTEL_*` configuration; it keeps using SCE's pinned managed endpoint and stored credential. + +## Evidence + +Environment: Claude Code `2.1.301` (`service.version` resource attribute on exported spans), Linux, headless `claude -p` runs on 2026-10-17 from a scratch working directory outside this repository. The runs used `env -i` (no inherited `CLAUDECODE`/`TRACEPARENT`), `--setting-sources local`, and `--settings` pointing at a scratchpad file. That file registered a probe command hook with matcher `*` on `PreToolUse` and `PostToolUse` and pointed Claude Code's OTLP/HTTP-JSON trace exporter at a loopback receiver. User-level `~/.claude/settings.json` was not edited. Each run triggered real `Bash` and/or `Read` tool calls. + +The probe captured only `session_id`, `hook_event_name`, `tool_use_id`, `TRACEPARENT`/`TRACESTATE` presence, traceparent validity, trace ID, parent span ID, trace flags, and presence-only caller markers. One extra run captured only the top-level key names of the hook stdin payload, not their values. Exported spans were inspected for name, trace ID, span ID, parent span ID, timestamps, `span.type`, `tool_name`, `tool_use_id`, and `hook_event` only. + +### Run A — telemetry + tracing beta, Bash then Read + +Session `7d0750a6-560e-4e42-9120-ed8f14304c30`, trace `aaddb6f35b4fcaf4260a454b6a278f74`. + +| Hook | `tool_use_id` | traceparent | trace ID | parent span ID | +| --- | --- | --- | --- | --- | +| PreToolUse | `toolu_01KN1scistt4Kcj3WCWcLbEp` (Bash) | valid, flags `01` | `aaddb6f35b4fcaf4260a454b6a278f74` | `00809c8f510c99f9` | +| PostToolUse | `toolu_01KN1scistt4Kcj3WCWcLbEp` (Bash) | valid, flags `01` | `aaddb6f35b4fcaf4260a454b6a278f74` | `00809c8f510c99f9` | +| PreToolUse | `toolu_01FAZCUXyrFmYNd2R4zr28Tr` (Read) | valid, flags `01` | `aaddb6f35b4fcaf4260a454b6a278f74` | `00809c8f510c99f9` | +| PostToolUse | `toolu_01FAZCUXyrFmYNd2R4zr28Tr` (Read) | valid, flags `01` | `aaddb6f35b4fcaf4260a454b6a278f74` | `00809c8f510c99f9` | + +Exported spans (same trace): + +| Span | span ID | parent span ID | `tool_use_id` | +| --- | --- | --- | --- | +| `claude_code.interaction` | `00809c8f510c99f9` | none (`parent.source=none`) | — | +| `claude_code.llm_request` | `77020189eb8449ab` | `00809c8f510c99f9` | — | +| `claude_code.tool` (Bash) | `e385b69cbfed31e4` | `00809c8f510c99f9` | `toolu_01KN1scistt4Kcj3WCWcLbEp` | +| `claude_code.tool.blocked_on_user` | `0e4c901b3fb2b096` | `e385b69cbfed31e4` | — | +| `claude_code.tool.execution` | `242141c79233929f` | `e385b69cbfed31e4` | `toolu_01KN1scistt4Kcj3WCWcLbEp` | +| `claude_code.tool` (Read) | `9408f8abebb89bce` | `00809c8f510c99f9` | `toolu_01FAZCUXyrFmYNd2R4zr28Tr` | +| `claude_code.tool.blocked_on_user` | `abbeb106b84cd5d3` | `9408f8abebb89bce` | — | +| `claude_code.tool.execution` | `83fbb33ed6cdf6d9` | `9408f8abebb89bce` | `toolu_01FAZCUXyrFmYNd2R4zr28Tr` | +| `claude_code.llm_request` | `6661c65cfc1fc0d8` | `00809c8f510c99f9` | — | + +All four hooks shared one parent across two different tools, which equals the interaction span ID. Timing confirms that hooks run outside the tool span. The Bash `PreToolUse` probe started at `…734978746818` ns, before `claude_code.tool` started at `…734988000000`. The `PostToolUse` probe started at `…735048034958` ns, after `claude_code.tool` ended at `…735039758337`. + +### Run E — telemetry + tracing beta, Read, payload key names + +Trace `7ce9b9cb88f26f47f24496b1cf1b828e`. Both hooks for `toolu_01Qn5H38gufu6cdCHsowFzNu` received a valid traceparent with parent `a5d5939e4fd7c534`, which equals `claude_code.interaction`. The `claude_code.tool` span was `d39aebf463d29ac8`. + +Hook stdin top-level keys: `PreToolUse` had `cwd`, `effort`, `hook_event_name`, `permission_mode`, `prompt_id`, `session_id`, `tool_input`, `tool_name`, `tool_use_id`, `transcript_path`. `PostToolUse` had the same keys plus `duration_ms` and `tool_response`. Neither carried a trace field. + +### Run F2 — detailed beta tracing (`ENABLE_BETA_TRACING_DETAILED=1`, `BETA_TRACING_ENDPOINT` loopback) + +Trace `1c6afedd2f2dc795c4aaa2c94b35184b`. Both hooks for `toolu_019vSjT7iXZedtTDFXE7TucH` received parent `56d0e76fdb58fe2e`, which equals `claude_code.interaction`. + +| Span | span ID | parent span ID | start → end (ns suffix) | +| --- | --- | --- | --- | +| `claude_code.hook` (`hook_event=PreToolUse`) | `70b2f799a056a4d0` | `56d0e76fdb58fe2e` | `942778000000 → 942793833861` | +| `claude_code.tool` | `0ccef232e0abf4dc` | `56d0e76fdb58fe2e` | `942795000000 → 942803155221` | +| `claude_code.hook` (`hook_event=PostToolUse`) | `308ab39a6ca018d8` | `56d0e76fdb58fe2e` | `942804000000 → 942818720789` | + +Even when Claude Code exports its own `claude_code.hook` span, the hook process's `TRACEPARENT` names the interaction span, not the `claude_code.hook` span that wraps the hook run. + +### Gating runs (Read tool, real tool call confirmed by `num_turns=2`) + +| Run | Settings env | `TRACEPARENT` in hooks | Spans exported | +| --- | --- | --- | --- | +| B | telemetry on, tracing beta off | absent (Pre + Post) | none | +| C | telemetry off | absent (Pre + Post) | none | +| D | only `CLAUDE_CODE_PROPAGATE_TRACEPARENT=1` | absent (Pre + Post) | none | + +### Implementation cross-check + +The Claude Code 2.1.301 bundle contains a shared subprocess-environment builder. It sets `CLAUDECODE`, `CLAUDE_CODE_SESSION_ID`, `CLAUDE_CODE_CHILD_SESSION`, `CLAUDE_CODE_SESSION_ATTENDED`, `CLAUDE_PID`, `AI_AGENT` (agent source), and `CLAUDE_EFFORT`. It sets `TRACEPARENT` only when tracing is active (or `CLAUDE_CODE_PROPAGATE_TRACEPARENT` is set) and there is an active span in the current context. This is consistent with runs B–D: run D has the propagate flag but no active span, so no traceparent. + +## Documentation consulted + +- Claude Code monitoring docs, `https://code.claude.com/docs/en/monitoring-usage` (Traces (beta), TRACEPARENT propagation, span hierarchy, hook spans). The docs describe Bash/PowerShell subprocesses as inheriting the active tool execution span's context. They describe hook executions as children of `claude_code.interaction`. They say nothing about hook-process `TRACEPARENT`; the probe shows hook processes receive the interaction context. +- Claude Code hooks docs, `https://code.claude.com/docs/en/hooks` (hook environment: `CLAUDE_PROJECT_DIR`; `OTEL_*` removed from subprocesses; `prompt_id` matches the `prompt.id` event attribute). These docs do not mention `TRACEPARENT`. + +## Live verification + +Recorded by T05 on 2026-10-17. The run used an `sce` debug build of commit `eae7be8f` (T02–T04), built with `nix develop -c ./scripts/run-cli-cargo.sh build`. It used the existing stored telemetry credential: `sce auth telemetry-headers` exited 0 with non-empty output, and the output was not inspected. Both exporters sent to the canonical collector `https://otel.crocoderlab.dev`: SCE through its pinned endpoint, and Claude Code through the user-level `OTEL_EXPORTER_OTLP_ENDPOINT` with `otelHeadersHelper = sce auth telemetry-headers`. Delivery was judged only from Dash0, never from a collector response. The initial Dash0 evidence was a user-exported root-span listing for the last 30 minutes. It held 30 root spans, all with `parentSpanId` empty, ending at 12:32:23Z. A trace detail view was not available at that point. A subsequent user-provided Dash0 waterfall screenshot on 2026-10-17 independently confirms the cross-service parent-child hierarchy; see [Follow-up Dash0 waterfall evidence](#follow-up-dash0-waterfall-evidence). + +### `sce doctor` + +`sce doctor --format json` ran at 2026-10-17T12:31:35Z, exited 0, and took 0.76s. Dash0 shows one root `sce.command` span: trace `1cc8e0b1eda96df1f757fab2e3798efd`, span `3f1d92756bd5860a`, `sce.command.name=doctor`, `sce.outcome=success`, resource `service.name=sce`, `service.version=0.4.0-pre-alpha-v11`. It is an independent root, so the standalone command path is unchanged. + +Other branch-build invocations in the same window, all also visible in Dash0: + +- `sce --version` at 12:31:21Z, as `sce.command`. +- `sce auth --help` and the rejected `sce auth status` at 12:31:40Z, as two `sce.command` spans with `sce.command.name=auth`. + +The later `sce auth telemetry-headers` calls produced no span, consistent with the helper staying Noop. + +### Claude Code session with SCE hooks + +Headless Claude Code `2.1.301` ran `claude -p --model haiku --allowedTools Write,Read` in this repository. The run used the repository's project hooks, user-level telemetry settings (`CLAUDE_CODE_ENABLE_TELEMETRY=1`, `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA=1`), and the branch build first on `PATH`. The inherited parent-session `TRACEPARENT`, `CLAUDECODE`, and session variables were unset. The session made a real `Write` of a gitignored `context/tmp/` file, then a `Read` of it, and the file was deleted afterward. Session `96272753-8c39-4ec3-824d-238f5202e016`, `num_turns=3`. + +A scratchpad-only probe hook, registered beside the SCE hooks through `--settings`, recorded the context every tool hook received. All four hooks resolved `sce` to the branch build. + +| Hook | `tool_use_id` | `TRACEPARENT` trace ID | parent span ID | +| --- | --- | --- | --- | +| PreToolUse Write | `toolu_01ANo9Y6mnA876HcX4fnD7bn` | `dcd919ecc4f5d400df5b642a1a229783` | `3eec7153327c6839` | +| PostToolUse Write | `toolu_01ANo9Y6mnA876HcX4fnD7bn` | `dcd919ecc4f5d400df5b642a1a229783` | `3eec7153327c6839` | +| PreToolUse Read | `toolu_017JhhwrJk5d16ykAncy5mUy` | `dcd919ecc4f5d400df5b642a1a229783` | `3eec7153327c6839` | +| PostToolUse Read | `toolu_017JhhwrJk5d16ykAncy5mUy` | `dcd919ecc4f5d400df5b642a1a229783` | `3eec7153327c6839` | + +Observed in Dash0, all from the same Dash0 ingestion token: + +| Span | Service | Trace ID | Span ID | Parent | Attributes | +| --- | --- | --- | --- | --- | --- | +| `sce.hook` (12:32:14Z) | `sce` | `7dd41303f9bec7b68bb667b2fb39aa70` | `a9bc49c8d4a8c7ea` | none | `sce.agent=claude`, `sce.hook.event=SessionStart`, `sce.claude.session_id=96272753-…`, `sce.outcome=success` | +| `claude_code.interaction` (12:32:16Z) | `claude-code` | `dcd919ecc4f5d400df5b642a1a229783` | `3eec7153327c6839` | none | `session.id=96272753-…` | +| `sce.hook` (12:32:23Z) | `sce` | `d56f02bb11aeac1b1a2c0922d5245d05` | `7dcd68d04d9501b8` | none | `sce.agent=claude`, `sce.hook.event=SessionEnd`, `sce.claude.session_id=96272753-…`, `sce.outcome=success` | + +Neither `sce.hook` span carried `sce.hook.tool`, `sce.claude.tool_use_id`, paths, or payload content. Both are correctly independent roots: + +- `SessionStart` (`claude-model-state`) runs before the interaction span exists. +- `SessionEnd` (`claude-mutation-scope`) starts at 12:32:23.502Z, after the interaction ended at 12:32:23.257Z. + +Neither process had a parent context to join. + +### Observed hierarchy + +Confirmed in Dash0: + +- `sce doctor` → one root `sce.command`. +- Claude session-boundary hooks → independent root `sce.hook` spans. +- The Claude interaction root `claude_code.interaction` (`dcd919ec…/3eec7153327c6839`) is exactly the trace and span named by every tool hook's `TRACEPARENT`. + +Initially expected but not verified from the root-span listing for trace `dcd919ecc4f5d400df5b642a1a229783`: + +```text +claude_code.interaction (3eec7153327c6839) +├── claude_code.tool (Write / Read) +└── sce.hook ×N: PreToolUse/PostToolUse hooks (TRACEPARENT captured); UserPromptSubmit/Stop hooks not probed + └── sce.mutation_scope (claude-mutation-scope hooks) + ├── sce.git_snapshot (Write PreToolUse) + └── sce.persistence +``` + +The initial root-span listing could not show non-root spans, including Claude Code's own tool and LLM request children. The subsequent waterfall screenshot establishes that parented SCE tool-hook spans and their children do reach Dash0, but the screenshot does not display its trace ID or dataset name; therefore it cannot by itself prove that it is the specific `dcd919ecc4f5d400df5b642a1a229783` trace above. The dataset is verified separately: on 2026-10-17 the user found trace `dcd919ecc4f5d400df5b642a1a229783` in the SCE CroCoder (`sce-crocoder`) Dash0 dataset, with 6 `sce.hook` spans as direct children of `claude_code.interaction` (21 spans total; see the T05 follow-up in `context/plans/claude-hook-trace-propagation.md`). + +## Follow-up Dash0 waterfall evidence + +A user-provided Dash0 trace-detail screenshot on 2026-10-17 shows a **single** `claude_code.interaction` trace containing spans from both `claude-code` and `sce`. The UI explicitly renders: + +- Multiple `sce.hook` spans directly parented to `claude_code.interaction`, alongside `claude_code.llm_request` and `claude_code.tool` spans. This confirms live cross-service W3C parent propagation to SCE hooks (siblings of tool spans, not children of tool spans). +- Two `sce.hook → sce.mutation_scope` branches, each containing sibling `sce.git_snapshot` and `sce.persistence` children. This confirms live delivery of all three SCE operation span types and their expected parent hierarchy. +- A `claude_code.interaction` duration of 6,988 ms. The two instrumented mutation hooks shown take 154 ms and 110 ms; the first has a 120 ms mutation scope, 63 ms Git snapshot and 33 ms persistence; the second has a 100 ms mutation scope, 57 ms Git snapshot and 19 ms persistence. + +**Verification verdict:** Live parent-child propagation and SCE operation-span export are **verified visually in Dash0**, closing the hierarchy/delivery gap left by the earlier root-only listing. The screenshot itself does **not** show the trace ID, workspace/dataset (the `sce-crocoder` dataset was confirmed separately; see [Observed hierarchy](#observed-hierarchy)), per-span `tool_use_id` attribute values, or the exact time of the trace. Do not claim that the screenshot proves the prior headless session's trace ID `dcd919ecc4f5d400df5b642a1a229783`, or that it proves hook-to-tool ID equality. This observation does not measure exporter overhead or p95 hook latency. + +## Concurrent hook span loss + +A later live run expected 15 SCE hook spans and Dash0 received 8. Claude Code runs several SCE hooks for the same event at the same time, and each one opens the encrypted `auth.db` to read the telemetry credential. + +- Root cause: `EncryptedTursoDb::new()` built the Turso database without `experimental_multiprocess_wal(true)`. In that mode Turso 0.8.1 takes an exclusive, non-blocking lock on the whole `auth.db` file for the process lifetime. Concurrent hook processes failed during open with `LockingError("Failed locking file '<…>/auth.db'. File is locked by another process")` after the three connection-open retries (about 75–80 ms), well inside the 2s lookup deadline. `RuntimeTelemetry::resolve_with` then fell back to `NoopTelemetry` silently, so those hooks exported no span. +- Secondary cause: with multiprocess WAL alone, every open still ran `run_migrations()`, and `CREATE TABLE IF NOT EXISTS __sce_migrations` takes the write lock even on a current schema, which caused transient `Busy("database is locked")`. +- Reproduction at 32 concurrent processes (failures out of 256): current implementation 180; multiprocess WAL only 32; multiprocess WAL plus a 1000 ms busy timeout 0; multiprocess WAL plus skipping migrations when current 0; both remedies 0. +- Fix: multiprocess WAL on the encrypted builder, a credential lookup that never creates a missing `auth.db`, a migration-free open for a current schema, serialized and Busy-retried migrations for a missing or outdated schema, and a 1000 ms auth DB busy timeout ([auth-db.md](auth-db.md), [shared-turso-db.md](shared-turso-db.md)). Every Noop fallback is now logged locally as `sce.telemetry.fallback` with its reason. +- Correlation: `sce.hook` now carries `sce.hook.command` (`diff-trace`, `conversation-trace`, `claude-mutation-scope`, `claude-model-state`, `policy-bash`), so spans from different SCE commands on the same Claude event are distinguishable. Each entry point records it before reading stdin, so a hook whose stdin is empty, unreadable, or malformed, or that fails afterwards, still identifies itself. `tool_use_id` handling is unchanged. +- Regression evidence (`cli/src/services/auth_db/multiprocess_tests.rs`): 32 hook child processes resolve the credential through the production `load_telemetry_credential` path. They export to a loopback collector, and every `(sce.claude.tool_use_id, sce.hook.command)` pair arrives exactly once. A child with a wrong encryption key resolves Noop with reason `database_error`, and its hook output is unchanged. The suite passed 48 of 48 runs with 6 copies in parallel. Removing the multiprocess flag fails the open-holder, 32-reader, held-write, concurrent-init, and hook-export tests. Restoring unconditional migrations fails the held-write test. +- Live run on 2026-10-19, 11:08:50Z–11:09:03Z: headless Claude Code `2.1.301` (`claude -p --model haiku --allowedTools Write,Read`), session `48a276ea-ce4a-4488-8244-649f20b87a42`, `num_turns=3`, with a debug build of this fix first on `PATH` through a scratchpad shim. The shim logged only subcommand words. The stored credential was present and was not printed. The session wrote and read a gitignored `context/tmp/` file, which was then deleted. The shim recorded 13 SCE hook invocations: `claude-mutation-scope` ×7, `conversation-trace` ×4, `diff-trace` ×1, `claude-model-state` ×1. `policy bash` did not fire because its matcher is `Bash`, and `diff-trace` fired only for `Write`. Claude also ran `sce auth telemetry-headers` once. A scratchpad probe hook recorded tool hooks for `toolu_015xdt65shPcBzT4UNbbKib5` (Write) and `toolu_01VMf442BtXsW2XMvnd76YZJ` (Read), all with `TRACEPARENT` trace `132a62ceed757d0d8431011311ae76af` and parent `6a4ae45ade8a6191`. The repository log directory recorded zero `sce.telemetry.fallback` records, so none of the 13 hook processes fell back to Noop. The installed pre-fix `sce` still read the credential afterwards. +- Dash0 status for this run: **unverified**. This session had no Dash0 access, and collector responses were not used as evidence. Expected in the `sce-crocoder` dataset, in trace `132a62ceed757d0d8431011311ae76af`: one `sce.hook` per tool-hook invocation, parented to `claude_code.interaction` `6a4ae45ade8a6191`, with `sce.claude.tool_use_id` equal to the matching `claude_code.tool`, and same-event hooks distinguished by `sce.hook.command` (for example, Write `PostToolUse` → `diff-trace`, `conversation-trace`, `claude-mutation-scope`). Session-boundary hooks should appear as independent `sce.hook` roots. + +## Opt-in live smoke procedure + +This is manual, uses real credentials, Claude Code, and Dash0, and is never part of CI. + +1. Confirm a stored credential without printing it: `out=$(sce auth telemetry-headers 2>/dev/null); [ -n "$out" ] && echo present`. +2. Build the branch: `nix develop -c ./scripts/run-cli-cargo.sh build --manifest-path cli/Cargo.toml`. +3. Run `cli/target/debug/sce doctor`, and note the UTC time. +4. Optionally, add a scratchpad-only probe hook through `--settings` that records only `hook_event_name`, `tool_use_id`, and the trace ID and parent span ID parsed from `TRACEPARENT`. This captures the expected Claude trace ID. +5. From the repository root, with Claude tracing enabled (`CLAUDE_CODE_ENABLE_TELEMETRY=1`, `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA=1`), run `env -u TRACEPARENT -u TRACESTATE -u CLAUDECODE PATH="$PWD/cli/target/debug:$PATH" claude -p --allowedTools Write,Read ""`. Then delete the file. +6. In Dash0, find the `sce.command` span with `sce.command.name=doctor` at the noted time. Open the Claude trace by its trace ID, not a root-span list, and confirm: + - each tool hook's `sce.hook` has `claude_code.interaction` as its parent and a `sce.claude.tool_use_id` that matches a `claude_code.tool` span; + - the `Write` `PreToolUse` `claude-mutation-scope` hook carries `sce.mutation_scope` → `sce.git_snapshot` and `sce.persistence`; + - `SessionStart`/`SessionEnd` `sce.hook` spans are independent roots; + - every SCE hook invocation produced one `sce.hook` span, and hooks for the same Claude event differ by `sce.hook.command`; + - the branch log directory has no `sce.telemetry.fallback` warning for the run. +7. Record the trace IDs and the observed hierarchy. Record anything that was not observed as unverified, and never record a collector 2xx as delivery. + +## Not verified + +- `tool_use_id` equality between each `sce.hook` and its matching `claude_code.tool` has not been observed in Dash0. The dataset (`sce-crocoder`) and the trace ID `dcd919ecc4f5d400df5b642a1a229783` were confirmed by the user's trace lookup, not by the waterfall screenshot. The parent-child hierarchy and child-span delivery **are** visually verified; see [Follow-up Dash0 waterfall evidence](#follow-up-dash0-waterfall-evidence). +- Interactive TUI sessions were not probed for hook `TRACEPARENT`. T05 also used headless `claude -p`. Claude Code documents that only `-p`/Agent SDK sessions read an inbound `TRACEPARENT`. The outbound subprocess builder is shared, and the interactive Bash tool environment of the same version carries a valid `TRACEPARENT`. Whether interactive hooks receive the same interaction-span context is expected but unconfirmed. +- Subagent (`Agent` tool) hooks, `PostToolUseFailure`, `PermissionDenied`, and other hook events were not probed. +- Behavior behind a custom `ANTHROPIC_BASE_URL` proxy without `CLAUDE_CODE_PROPAGATE_TRACEPARENT` was not probed. Per the docs the subprocess `TRACEPARENT` follows the same switch as the API `traceparent` header. + +## Consequences for the plan + +- T02: admit the Claude-only subcommands from argv. Admit shared subcommands only when `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are both present. +- T03: the parent-context branch applies. Extract `TRACEPARENT` (and `TRACESTATE` when present) with `TraceContextPropagator` and parent `sce.hook` on it. The resulting span is a child of `claude_code.interaction` and shares its trace ID. Missing or invalid context, which is the default when Claude tracing is off, yields an independent SCE root. +- Context describes the hierarchy as `claude_code.interaction → sce.hook` (a sibling of `claude_code.tool`), with tool correlation by `tool_use_id`, not as a tool-span child. Live status is recorded in [Live verification](#live-verification). + +## Probe hygiene + +The probe hook, receiver, settings, captured payloads, and exported spans lived only in the session scratchpad and were deleted after this record was written. No probe code is in the repository, and user-level Claude settings were never modified. diff --git a/context/sce/cli-error-code-taxonomy.md b/context/sce/cli-error-code-taxonomy.md index 2ff3b4932..68382908b 100644 --- a/context/sce/cli-error-code-taxonomy.md +++ b/context/sce/cli-error-code-taxonomy.md @@ -33,10 +33,10 @@ It complements the numeric process exit-code classes documented in `context/sce/ ## Ownership - `FailureClass` in `cli/src/services/error.rs` owns class selection and stable code assignment (`FailureClass::code()`). -- `CliError::{User,Internal}` in `cli/src/services/error.rs` is the typed CLI-boundary error type; `CliError::code()`/`CliError::class()` delegate to the failure class. `CliError::User` carries a closed catalog `UserError` (`NotAuthenticated`, `NotGitRepository`, `NotGitRemote`, `AuthStorageUnavailable`, or `UnexpectedFailure`) for expected, deliberately-explained failures; `CliError::Internal` carries a live `anyhow::Error` source for every other failure, including non-classifiable setup preflight failures. `CliError::User` may also carry an optional preserved technical `source`, kept for observability only and never rendered to the terminal; the missing-remote source contains the configured remote name but no URL. -- `UserError` in `cli/src/services/error.rs` is the closed catalog of deliberately presented terminal failures. It has no arbitrary-message variant (no `Message(String)`/`Custom(...)` escape hatch): every entry returns a fixed reviewed sentence from `UserError::message()`, keyed for structured logging by `UserError::key()`. `NotAuthenticated` (`auth.not_authenticated`) covers Control Plane authentication failures from authenticated `sce auth whoami`; missing credentials from `sce auth logout`/`whoami` are successful state queries and do not enter the error catalog. `AuthStorageUnavailable` (`auth.storage_unavailable`) is used by `sce sync` and the `sce auth login`, `logout`, and `whoami` command boundary for token-storage plus `AuthError::Io`/`Storage` failures. Both auth-command mappings preserve technical sources for observability. `NotGitRepository` (`setup.not_git_repository`) is used by the setup command only when setup-owned repository-root resolution positively identifies a target as outside a Git repository; `NotGitRemote` (`setup.not_git_remote`) is used only when the configured named remote has no URL. Both preserve technical sources for observability. Other repository-root resolution, Git, and remote-lookup execution failures map to `UnexpectedFailure` (`general.unexpected_failure`). `UnexpectedFailure` is also used by `sce sync`, the config-command boundary for config execution failures, the version and doctor command boundaries for service execution failures, the auth-command boundary, and remaining setup execution failures; it renders one fixed, user-safe log-files guidance sentence and has no automatic `Try:` suffix or dynamic path input. +- `CliError::{User,Internal}` in `cli/src/services/error.rs` is the typed CLI-boundary error type; `CliError::code()`/`CliError::class()` delegate to the failure class. `CliError::User` carries a closed catalog `UserError` (`NotAuthenticated`, `NotGitRepository`, `NotGitRemote`, `AuthStorageUnavailable`, `LoginAuthStorageUnavailable`, or `UnexpectedFailure`) for expected, deliberately-explained failures; `CliError::Internal` carries a live `anyhow::Error` source for every other failure, including non-classifiable setup preflight failures. `CliError::User` may also carry an optional preserved technical `source`, kept for observability only and never rendered to the terminal; the missing-remote source contains the configured remote name but no URL. +- `UserError` in `cli/src/services/error.rs` is the closed catalog of deliberately presented terminal failures. It has no arbitrary-message variant (no `Message(String)`/`Custom(...)` escape hatch): every entry returns a fixed reviewed sentence from `UserError::message()`, keyed for structured logging by `UserError::key()`. `NotAuthenticated` (`auth.not_authenticated`) covers Control Plane authentication failures from authenticated `sce auth whoami`; missing credentials from `sce auth logout`/`whoami` are successful state queries and do not enter the error catalog. `AuthStorageUnavailable` and `LoginAuthStorageUnavailable` (both `auth.storage_unavailable`) are used by `sce sync` and the `sce auth login`, `logout`, and `whoami` command boundary for token-storage plus `AuthError::Io`/`Storage` failures; each carries a typed `AuthStorageFailure` cause (`CredentialAccess`, `DatabaseLocked`, `Database`) that selects the guidance text (see `context/sce/auth-db.md`). Both auth-command mappings preserve technical sources for observability. `NotGitRepository` (`setup.not_git_repository`) is used by the setup command only when setup-owned repository-root resolution positively identifies a target as outside a Git repository; `NotGitRemote` (`setup.not_git_remote`) is used only when the configured named remote has no URL. Both preserve technical sources for observability. Other repository-root resolution, Git, and remote-lookup execution failures map to `UnexpectedFailure` (`general.unexpected_failure`). `UnexpectedFailure` is also used by `sce sync`, the config-command boundary for config execution failures, the version and doctor command boundaries for service execution failures, the auth-command boundary, and remaining setup execution failures; it renders one fixed, user-safe log-files guidance sentence and has no automatic `Try:` suffix or dynamic path input. - Command and domain layers construct and return a `CliError`; they do not format terminal text, apply styling, or decide authentication/user-error semantics from string matching. Auth command orchestration classifies expected authentication and credential-storage failures into the existing `UserError` catalog by typed domain variants, never by string matching, and preserves the original technical chain as the optional user-error source. `app_support` is the sole owner of turning a `CliError` into the final stderr sentence. -- `UserError` in `cli/src/services/error.rs` is the closed catalog of deliberately presented terminal failures. It has no arbitrary-message variant (no `Message(String)`/`Custom(...)` escape hatch): every entry is a fixed, reviewed sentence returned by `UserError::message()`, keyed for structured logging by `UserError::key()`. `NotAuthenticated` (`auth.not_authenticated`) covers Control Plane authentication failures from authenticated `sce auth whoami`; missing credentials from `sce auth logout`/`whoami` are successful state queries and do not enter the error catalog. `AuthStorageUnavailable` (`auth.storage_unavailable`) is used by `sce sync` and the `sce auth login`, `logout`, and `whoami` command boundary for token-storage plus `AuthError::Io`/`Storage` failures. Both auth-command mappings preserve technical sources for observability. `NotGitRepository` (`setup.not_git_repository`) is used by the setup command only when setup-owned repository-root resolution positively identifies a target as outside a Git repository; it renders fixed Git-init/rerun guidance and preserves the technical source for observability. Other repository-root resolution failures, including nonexistent, inaccessible, process, and malformed-output failures, map to `UnexpectedFailure` (`general.unexpected_failure`). `UnexpectedFailure` is also used by `sce sync`, the config-command boundary for config execution failures, the version and doctor command boundaries for service execution failures, the auth-command boundary, and remaining setup execution failures; it renders one fixed, user-safe diagnostic sentence and has no automatic `Try:` suffix or dynamic path input. +- `UserError` in `cli/src/services/error.rs` is the closed catalog of deliberately presented terminal failures. It has no arbitrary-message variant (no `Message(String)`/`Custom(...)` escape hatch): every entry is a fixed, reviewed sentence returned by `UserError::message()`, keyed for structured logging by `UserError::key()`. `NotAuthenticated` (`auth.not_authenticated`) covers Control Plane authentication failures from authenticated `sce auth whoami`; missing credentials from `sce auth logout`/`whoami` are successful state queries and do not enter the error catalog. `AuthStorageUnavailable` and `LoginAuthStorageUnavailable` (both `auth.storage_unavailable`) are used by `sce sync` and the `sce auth login`, `logout`, and `whoami` command boundary for token-storage plus `AuthError::Io`/`Storage` failures; each carries a typed `AuthStorageFailure` cause (`CredentialAccess`, `DatabaseLocked`, `Database`) that selects the guidance text (see `context/sce/auth-db.md`). Both auth-command mappings preserve technical sources for observability. `NotGitRepository` (`setup.not_git_repository`) is used by the setup command only when setup-owned repository-root resolution positively identifies a target as outside a Git repository; it renders fixed Git-init/rerun guidance and preserves the technical source for observability. Other repository-root resolution failures, including nonexistent, inaccessible, process, and malformed-output failures, map to `UnexpectedFailure` (`general.unexpected_failure`). `UnexpectedFailure` is also used by `sce sync`, the config-command boundary for config execution failures, the version and doctor command boundaries for service execution failures, the auth-command boundary, and remaining setup execution failures; it renders one fixed, user-safe diagnostic sentence and has no automatic `Try:` suffix or dynamic path input. - `Logger::log_cli_error` in `cli/src/services/observability.rs` owns structured error logging with `sce.error.{code}` event IDs. - Config, version, doctor, and remaining setup execution boundaries map their unexpected failures to `UserError::UnexpectedFailure`; auth command mappings preserve their original technical sources for observability while terminal rendering remains user-safe. - `write_error_diagnostic` in `cli/src/services/app_support.rs` owns final stderr rendering: it redacts and writes the catalog variant's message without a wrapper or styling, while `CliError::Internal` retains code-bearing rendering and styles its rendered chain through `services::style::error_text_with_color_policy` under the stderr TTY/`NO_COLOR` policy (`services::style::supports_color_stderr()`), independent of stdout's TTY state. diff --git a/context/sce/cli-observability-contract.md b/context/sce/cli-observability-contract.md index d8a6d6d7d..89d95865d 100644 --- a/context/sce/cli-observability-contract.md +++ b/context/sce/cli-observability-contract.md @@ -50,6 +50,7 @@ Runtime observability consumes the shared resolved observability config from `cl - `sce.command.dispatch_start` (debug level - logged before dispatch) - `sce.command.dispatch_end` (debug level - logged after successful dispatch) - `sce.command.completed` + - `sce.telemetry.fallback` (`warn` for credential lookup failures, `debug` for a missing or expired credential - emitted once when managed telemetry resolution falls back to Noop after a lookup; fields `reason` and, for database or path errors, `error`) - Error logging uses the pattern `sce.error.{code}` where `{code}` is the classified error code (e.g., `sce.error.SCE-ERR-RUNTIME`). - All `CliError` instances are logged via `Logger::log_cli_error()` before user-facing stderr diagnostics are written; observability retains full technical detail independently of what is rendered to the terminal, and never writes a second competing stderr diagnostic for the same error. - Event records include deterministic metadata keys used by automation (`command`, `failure_class`, `component` when applicable). @@ -94,9 +95,12 @@ Runtime observability consumes the shared resolved observability config from `cl ## Managed trace export -- Defined in `cli/src/services/managed_telemetry.rs` and `cli/src/services/managed_telemetry/trace_export.rs` (plus `log_export.rs`). After startup, `RuntimeTelemetry::resolve` loads the stored telemetry credential from the encrypted auth DB in a spawned task bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s); absent, expired, malformed, unavailable, ownerless (no stored WorkOS session), or timed-out credentials select `NoopTelemetry`. Startup never issues or refreshes a credential. Hook invocations (`sce hooks ...`), `sce policy bash`, and the hidden `sce auth telemetry-headers` helper always use `NoopTelemetry` (`is_hook_invocation` in `cli/src/app.rs`). +- Defined in `cli/src/services/managed_telemetry.rs` and `cli/src/services/managed_telemetry/trace_export.rs` (plus `log_export.rs`). After startup, `RuntimeTelemetry::resolve` returns Noop (`not_configured`) without opening the auth DB unless the non-secret telemetry readiness hint is present ([telemetry-credential.md](telemetry-credential.md)). Otherwise it loads the stored telemetry credential from the encrypted auth DB, with non-interactive credential-store access, in a spawned task bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s); absent, expired, malformed, unavailable, ownerless (no stored WorkOS session), or timed-out credentials select `NoopTelemetry`. Startup never issues or refreshes a credential. The fallback is not silent: `RuntimeTelemetry::resolve` logs one `sce.telemetry.fallback` record with `reason` = `not_configured`, `credential_inaccessible`, `credential_unavailable`, `credential_expired`, `lookup_timeout`, `database_error`, `path_resolution_error`, or `exporter_init_failed`, at forced `warn` for failures and `debug` otherwise. Database and path errors add the redacted storage error text as `error`; no credential, bearer, key, or payload is logged, and the record never reaches stdout or, with `log_to_file=true`, stderr. +- Admission (`managed_span_kind` in `cli/src/app.rs`, decided from argv before credential lookup): Claude-invoked SCE hooks resolve the `sce.hook` span kind, every other hook and the hidden `sce auth telemetry-headers` helper use `NoopTelemetry` without a credential lookup, and all remaining commands use the `sce.command` span kind. `sce hooks claude-mutation-scope` and `sce hooks claude-model-state` are admitted from argv alone. The shared `sce hooks diff-trace`, `sce hooks conversation-trace`, and `sce policy bash` are admitted only when both `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are present in the environment (checked for presence only; values never read); `CLAUDE_PROJECT_DIR` is set for Claude command hooks but not for the Claude Bash tool (evidence: [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md)). Other hooks (commit hooks, Codex, OpenCode, Pi) stay Noop. Admitted hooks use the same `CREDENTIAL_LOOKUP_DEADLINE`, `SHUTDOWN_TIMEOUT`, and runtime grace bounds as commands. Decision record: [2026-10-17-claude-hook-managed-trace-admission.md](../decisions/2026-10-17-claude-hook-managed-trace-admission.md). - Export is OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev/v1/traces`. A pinned HTTP client sets `Authorization: Bearer ` (token only, never the accessor), disables redirects, rebuilds the request to the fixed endpoint, and bounds requests to 3s. Ambient `OTEL_EXPORTER_OTLP_*` values cannot redirect it. - Each command emits one `sce.command` span with `sce.command.name`, `sce.outcome` (`success`/`failure`), and `sce.error.class` (the `SCE-ERR-*` code on failure); resource attributes are `service.name=sce` and `service.version`. Raw arguments, paths, and error chains are not exported. +- An admitted Claude hook process emits exactly one `sce.hook` span instead (no `sce.command` span), with `sce.agent=claude`, `sce.hook.command`, `sce.outcome`, `sce.error.class`, and, when present as non-empty strings in the Claude hook stdin payload, `sce.hook.event` (`hook_event_name`), `sce.hook.tool` (`tool_name`), `sce.claude.session_id`, and `sce.claude.tool_use_id`. `sce.hook.command` is the stable SCE entry point (`ClaudeHookCommand`: `diff-trace`, `conversation-trace`, `claude-mutation-scope`, `claude-model-state`, `policy-bash`), so the several SCE hooks that one Claude event runs produce distinguishable spans; each Claude hook entry point records it with `trace_export::record_claude_hook_command` as its first step, before reading stdin, so the span carries it even when stdin is empty, cannot be read, or is not JSON, and when the hook then fails. `trace_export::record_claude_hook_payload(stdin)` is the only payload reader: each entry point passes its stdin to it once after a successful read, it returns immediately when the current span is disabled (Noop), and it copies only those four fields. Session and tool-use IDs are opaque correlation identifiers: span attributes only, never metric attributes, never used to derive a trace ID. Tool input and response, prompts, `cwd`, transcript paths, file contents, and the raw payload are never exported. Parent context: for the `sce.hook` kind only, `ManagedTraceTelemetry::new` reads `TRACEPARENT` (and `TRACESTATE` when present) from the process environment, extracts it with `TraceContextPropagator`, and keeps it only when the remote span context is valid; `root_span` then attaches it with `set_parent`, so `sce.hook` shares Claude Code's trace ID, its parent is the upstream span (Claude Code 2.1.301 supplies `claude_code.interaction`, making `sce.hook` a sibling of `claude_code.tool`), and its own span ID is freshly generated. Child spans created inside the instrumented hook future inherit that trace across `.await` points and worker migration. No global propagator is registered and `sce.command` never reads the variables. A missing, malformed, or all-zero traceparent (the normal case when Claude tracing is off) yields an independent SCE root. Claude `SessionStart` and `SessionEnd` hooks run outside any interaction span, so their `sce.hook` is always a root. Live status: Dash0 (dataset `sce-crocoder`) shows `sce doctor`, Claude session-boundary `sce.hook` spans, and parented tool-hook `sce.hook` spans with operation children under `claude_code.interaction`. Concurrent hook processes previously lost spans because the auth DB credential lookup failed with `LockingError` and fell back to Noop; that is fixed, and the fix is verified by multiprocess regression tests but not yet re-verified live in Dash0. The evidence and the opt-in smoke procedure are in [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md). Telemetry failures never change hook stdout, permission decisions, persisted state, or exit codes. +- Operation child spans: `trace_export::in_child_span(ManagedChildSpan, future)` wraps an existing operation in one of three named spans, each carrying only `sce.outcome` (`success`/`failure` from the operation's `Result`): `sce.mutation_scope` around every Claude adapter seam call (one coordinate or abandon boundary), `sce.git_snapshot` around the coordinator's tree capture and pin, and `sce.persistence` around the mutation-store phase of a coordinate boundary (initialize, scope/provenance registration, CAS commit loop) and of an abandon. The hierarchy is `sce.hook → sce.mutation_scope → {sce.git_snapshot, sce.persistence}`. A child span exists only when its operation runs: a read-only tool, an untracked background shell, or a close with no matching attempt exports `sce.hook` alone. Snapshot-failure taint persistence and conversation/diff-trace Agent Trace DB writes have no span. The span filter depends on the span kind: a `sce.command` dispatch exports only `sce.command` (ordinary commands that reach the coordinator, such as ref maintenance, still export one span), and a `sce.hook` dispatch exports only `sce.hook` and the three child names. Without a managed hook dispatch the child spans are disabled, and they never change ordering, locking, or errors. - The exporter uses a bounded batch queue on a dedicated thread. Final stdout/stderr rendering and classified-error logging run inside the command span (`run_command_lifecycle_with_context`), so `sce.outcome` reflects the rendered result (a stdout write failure is `failure`) and the final `sce.error.*` log is exported. Trace and log providers shut down concurrently under one shared `SHUTDOWN_TIMEOUT` (3s) deadline after rendering; failures never change command results or exit codes. - Process lifetime: `app::run_process` owns the Tokio runtime (at least two workers) and ends it with `shutdown_timeout(RUNTIME_SHUTDOWN_GRACE)` (500ms), so a stalled keychain, database, or exporter worker cannot keep the process alive. The async deadlines bound when the command proceeds; the runtime grace is the hard process-level bound. Abandoned workers are not cancelled, only no longer awaited. - Logs: `managed_telemetry/log_export.rs` bridges `target: "sce"` tracing events and their `enabled!` hints (emitted by `Logger` only when the local level is enabled) to OTLP/HTTP protobuf log records at the hardcoded `/v1/logs`, using the same pinned bearer client, batch bounds, and shutdown as traces. Records carry severity, `sce.event_id`, a body limited to path- and secret-free messages (falling back to the event ID), and only allowlisted fields (`command`, `component`, `failure_class`, `error_code`, `error_class`, `error_surface`, `user_error`) with bounded, redacted values; `error_source`, arguments, paths, and hook payloads are never exported. Local file/stderr logging is unchanged. diff --git a/context/sce/doctor-human-text-contract.md b/context/sce/doctor-human-text-contract.md index e4116634e..1ce0250d0 100644 --- a/context/sce/doctor-human-text-contract.md +++ b/context/sce/doctor-human-text-contract.md @@ -12,8 +12,11 @@ Human text output renders these sections in this exact order: 2. `Repository` 3. `Integrations` -The `Environment` domain contains `State`, `Configuration`, and `Repository -identity`. The `Repository` domain contains `Git repository`, `Post-commit Agent +The `Environment` domain contains `State`, `Configuration`, `Repository +identity`, and `Managed telemetry` (read-only credential health; rows, JSON, and +remediation in [telemetry-credential.md](telemetry-credential.md#doctor-health-check)). +A warning row expands into `Reason: `, the summary, the hook-impact line, +and a `To fix:` step list. The `Repository` domain contains `Git repository`, `Post-commit Agent Trace auto-sync`, and `Git hooks`. When snapshot-ref reconciliation is recommended (or its maintenance state is diff --git a/context/sce/shared-turso-db.md b/context/sce/shared-turso-db.md index 8b5d07143..b664a4142 100644 --- a/context/sce/shared-turso-db.md +++ b/context/sce/shared-turso-db.md @@ -12,7 +12,7 @@ - `TursoDb`: generic unencrypted adapter that owns: - no Tokio runtime ownership; all Turso futures are awaited by the application runtime - Turso local database open/connect flow using `turso::Builder::new_local()` with `experimental_multiprocess_wal(true)`, which gives concurrent `sce` processes cross-process correctness and locking on the same local database; it does not make a contended writer wait - - per-`DbSpec` Turso busy timeout, resolved by `resolve_busy_timeout::()` and installed with `Connection::busy_timeout(...)` on the connection returned by `connect()` in `open_without_migrations_at` (the open path behind `new`, `new_at`, and `open_without_migrations`). It is Turso's wait policy for `Busy`: Turso sleeps in short phases until the lock clears or the timeout elapses, and only then returns `Busy`. It is connection-wide, so it also covers `BEGIN IMMEDIATE` issued by `Transaction::new_unchecked(.., Immediate)` on the same connection. The Agent Trace DB (`db_config_key() == "agent_trace_db"`) uses `policies.database_retry.agent_trace_db.busy_timeout_ms` when configured (`0` leaves the handler unset), falling back to `AGENT_TRACE_DB_BUSY_TIMEOUT_MS = 1_000`; every other database resolves to zero, which leaves the busy handler unset, so `local_db` behavior is unchanged. `EncryptedTursoDb` (auth DB) never installs a busy handler. A single attempt may therefore spend up to about 1000 ms waiting inside Turso before returning `Busy`; the Agent Trace write-contention retry below decides whether SCE tries once more + - per-`DbSpec` Turso busy timeout, resolved by `resolve_busy_timeout::()` and installed with `Connection::busy_timeout(...)` on the connection returned by `connect()` in `open_without_migrations_at` (the open path behind `new`, `new_at`, and `open_without_migrations`). It is Turso's wait policy for `Busy`: Turso sleeps in short phases until the lock clears or the timeout elapses, and only then returns `Busy`. It is connection-wide, so it also covers `BEGIN IMMEDIATE` issued by `Transaction::new_unchecked(.., Immediate)` on the same connection. The Agent Trace DB (`db_config_key() == "agent_trace_db"`) uses `policies.database_retry.agent_trace_db.busy_timeout_ms` when configured (`0` leaves the handler unset), falling back to `AGENT_TRACE_DB_BUSY_TIMEOUT_MS = 1_000`; the auth DB (`DatabaseKind::Auth`) uses the fixed, non-configurable `AUTH_DB_BUSY_TIMEOUT_MS = 1_000`, installed by `EncryptedTursoDb::new()`; `local_db` resolves to zero, which leaves the busy handler unset, so its behavior is unchanged. A single attempt may therefore spend up to about 1000 ms waiting inside Turso before returning `Busy`; the Agent Trace write-contention retry below decides whether SCE tries once more - Agent Trace write-contention deadline, resolved by `resolve_contention_deadline::()` from `policies.database_retry.agent_trace_db.contention_deadline_ms` with fallback to `AGENT_TRACE_DB_CONTENTION_DEADLINE_MS = 2_250` (zero for every other database). It is the cutoff for starting another outer write-contention retry, not a hard operation timeout - Agent Trace write-contention retry (`write_contention_policy::()`, `run_with_write_contention_retry`), returned only for `agent_trace_db` and applied only to write units whose whole replay is safe: the two transactional primitives below (each attempt is a complete `BEGIN IMMEDIATE` → `COMMIT` unit) and `execute_idempotent_write(sql, params)`, the opt-in single-statement entrypoint used only by the repository-metadata `INSERT … ON CONFLICT DO NOTHING`, the guarded source-instance claim `UPDATE … WHERE source_instance_id = ''`, and the mutation-trace worktree/scope/scope-provenance `…_IF_ABSENT` inserts. It retries only typed `Busy`/`BusySnapshot` (classified before conversion to `anyhow`), allows at most `2` attempts, sleeps a full-jitter backoff drawn from `0..=100ms`, and admits the second attempt at two points: before the backoff sleep it requires `remaining contention deadline >= backoff + busy_timeout`, and after the sleep actually returns it re-checks `remaining >= busy_timeout`, so scheduler oversleep can never start an attempt outside the admission contract (time is measured on a monotonic clock from the operation start, so Turso's busy wait counts; nothing starts after the deadline expires). Deterministic failures return after one attempt without sleeping. On every other database these entrypoints keep the generic query retry, and `execute_idempotent_write` is plain `execute()` - config-driven connection-open retry around only the `build().await.connect()` block using async `run_with_retry` (resolved from `policies.database_retry..connection_open` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults `3` attempts, `1s` timeout, `25ms..200ms` backoff) @@ -29,8 +29,8 @@ - `count_write_contention(body) -> (T, WriteContentionCounts)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports the write-contention `attempts`, `outer_retries`, and `exhaustions` it caused on the current thread. Test instrumentation only; not production telemetry and not shared across threads or processes. - `record_write_contention_timeline(body) -> (T, Vec<(Instant, WriteContentionTimelineEvent)>)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and returns the `Instant`-stamped attempt start/end, backoff requested and backoff slept events of the write-contention retry on the current thread. Test-only measurement instrumentation used by the lock-contention suite; never logged in production. - `count_read_statements(body) -> (T, usize)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports how many `TursoDb` read statements (`query_map`) it issued on the current thread, each counted once in its synchronous prelude before the retry wrapper. A deterministic seam for tests that must prove an operation reads from a single DB snapshot — one statement — rather than several independent `SELECT`s a concurrent commit could tear across (first used by `MutationTraceStore::load_tree_roots` / `load_all_tree_roots`). Not compiled into production builds. -- `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same static shape (connection and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`, and runs embedded migrations after connect. -- `EncryptedTursoDb` exposes async `execute()`, `query_map()`, and `run_migrations()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. +- `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same static shape (connection and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, opens with `experimental_multiprocess_wal(true)` like `TursoDb` (without it Turso takes an exclusive, non-blocking lock on the whole file for the process lifetime, so a concurrent `sce` process fails with `LockingError`), installs the auth DB busy timeout, and wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`. Inside that retry loop, before Turso opens the file, it takes the shared mixed-version compatibility lock from `db/open_lock.rs` (an OFD `F_RDLCK` on Linux, classic `F_SETLK` `F_RDLCK` elsewhere on Unix) on a descriptor the `EncryptedTursoDb` owns for its lifetime. That lock excludes pre-multiprocess SCE binaries, which hold a classic `F_WRLCK` on the whole file ([auth-db.md](auth-db.md#mixed-version-compatibility-lock)). `EncryptedTursoDb::open_existing_at(path)` is the existing-only variant: it opens the file without `O_CREAT` and returns `None` for an absent file before any encryption-key resolution, file creation, or migration. After connect it migrates only when needed: a read-only readiness check (`__sce_migrations` exists and records every embedded ID) lets a current schema open with no write at all; otherwise all pending migrations run inside one `BEGIN IMMEDIATE` transaction, which serializes concurrent initializers (a waiter re-reads the metadata under the lock and applies nothing) and rolls back the whole attempt on any migration failure. Only `Busy`/`BusySnapshot` on begin or commit is retried, under `MIGRATION_RETRY_POLICY` (10 attempts, 25..200ms backoff, each attempt bounded by the 1000 ms busy timeout); a deterministic migration failure (for example a column added by a partially applied `004` whose ID was never recorded) fails after one attempt with the migration ID in the error chain. After migrating, the readiness check runs again and a schema that is still behind fails the open. +- `EncryptedTursoDb` exposes async `execute()` and `query_map()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. - Both public adapters keep concrete Turso connections and marker types; there is no shared runtime-owning core or synchronous executor bridge. - Shared lifecycle helpers: - `collect_db_path_health()` emits common parent/path health problems for DB-backed services. @@ -140,7 +140,7 @@ Retry-policy budget versus wall-clock time: the generic query retry policy keeps | Layer | Role | | --- | --- | | multiprocess WAL | cross-process correctness and locking | -| `busy_timeout_ms` | Turso-level `Busy` waiting inside one operation attempt (Agent Trace DB only) | +| `busy_timeout_ms` | Turso-level `Busy` waiting inside one operation attempt (Agent Trace DB, configurable; auth DB fixed at 1000 ms) | | outer write-contention retry | SCE retry after Turso's wait is exhausted; Agent Trace safe write units only, at most `2` attempts, full-jitter `0..=100ms` backoff | | `contention_deadline_ms` | cutoff for starting another outer write-contention retry, not a hard operation timeout | | generic query `RetryPolicy` (`query.timeout_ms`) | outer retry for every other operation, including Agent Trace reads; `timeout_ms` labels slow attempts and never interrupts them | diff --git a/context/sce/telemetry-credential.md b/context/sce/telemetry-credential.md index bf48aa36a..8226d690e 100644 --- a/context/sce/telemetry-credential.md +++ b/context/sce/telemetry-credential.md @@ -1,6 +1,6 @@ # Telemetry credential -The CLI holds one telemetry-only OpenBao credential, issued by the Control Plane and cached in the encrypted auth DB. `sce auth login` provisions it best-effort, and managed OTLP trace and log export reads it at startup as the bearer source. There is no user-facing provisioning command, refresh loop, or doctor check. +The CLI holds one telemetry-only OpenBao credential, issued by the Control Plane and cached in the encrypted auth DB. `sce auth login` provisions it best-effort, and managed OTLP trace and log export reads it at startup as the bearer source. There is no user-facing provisioning command or refresh loop. `sce doctor` reports read-only credential health ([Doctor health check](#doctor-health-check)). ## Components @@ -22,14 +22,45 @@ flowchart LR - Lookup never mints or refreshes; an expired, empty-field, negative-expiry, or identity-less (pre-`004` legacy) row is returned as `None`, so legacy rows require one `sce auth login`. - Ownership: every lookup decodes the stored WorkOS session's access-token claims and returns the credential only when `(sub, workspace_id)` equals the identity stored with it. Switching user or workspace therefore makes an older credential unusable even if it was never deleted. A new device-authorization login also deletes the stored telemetry credential before saving the new WorkOS session (`replace_session_with`), so a failed re-issuance leaves no credential; if that delete fails, the new session is not saved. Renewal of the same session keeps the credential. -- Repository bindings: `telemetry_repository_bindings` maps a repository ID (the canonical `repository_identity` hash, from `agent_trace.repository_id` or the configured remote) to the `(user_id, workspace_id)` it was enrolled under. Enrollment requires a valid, session-owned credential and records that credential's identity. `sce setup` with the Claude target enrolls the repository explicitly (`Enroll`, rebinding it to the current identity). A successful `sce auth login` enrolls the working-directory repository only if it is unbound (`EnrollIfUnbound`), so a login never silently moves a repository to another workspace. Bindings survive logout and rotation; they authorize nothing without a matching credential. +- Repository bindings: `telemetry_repository_bindings` maps a repository ID (the canonical `repository_identity` hash, from `agent_trace.repository_id` or the configured remote) to the `(user_id, workspace_id)` it was enrolled under. Enrollment requires a valid, session-owned credential and records that credential's identity. `sce setup` no longer enrolls repositories, because it must not open the encrypted auth DB or touch the OS credential store. The explicit rebinding mode (`Enroll`) currently has no production caller. A successful `sce auth login` enrolls the working-directory repository only if it is unbound (`EnrollIfUnbound`), so a login never silently moves a repository to another workspace. Bindings survive logout and rotation; they authorize nothing without a matching credential. +- Telemetry readiness hint: `cli/src/services/telemetry_readiness.rs` keeps a non-secret file, `/sce/telemetry-ready.json` (`{"expires_at_unix_seconds": N}`), next to `auth.db`. + - `sce auth login` writes it only after the credential has been issued and persisted. If provisioning, persistence, or writing the hint fails, the hint is cleared and the result is `unavailable`. + - Logout and a new device-authorization session clear it before anything else. + - It counts as ready only while the file parses, is unexpired, and `auth.db` exists. A missing, corrupt, expired, or orphaned hint means telemetry is disabled. `telemetry_readiness::state` reports the same check as `NotConfigured` (no `auth.db`, or a missing or corrupt hint), `Expired`, or `Ready`, without opening `auth.db`. `is_ready` is `state == Ready`. + - It is a fast-path hint, not authorization. The credential is still validated from the auth DB. Its purpose is that unauthenticated hooks never open `auth.db` or touch the Keychain, including existing installs that have `auth.db` but no hint, until the next `sce auth login`. - Concurrent saves are single-statement atomic replacements of one fixed row; the last complete writer wins. - `sce auth logout` deletes the WorkOS credential and the telemetry credential, attempting both even when one fails and surfacing the first storage error. `credentials_removed` is true when either record was deleted; output fields are otherwise unchanged. - Telemetry save/delete never touches the WorkOS `auth_credentials` row. ## Runtime use -- `managed_telemetry::RuntimeTelemetry::resolve` reads the stored credential at CLI startup (never issuing one); a valid unexpired `token` is used only as the `Authorization: Bearer` credential for managed OTLP trace and log export; the `accessor` is never transmitted. Absent, expired, malformed, or unavailable rows select `NoopTelemetry`. See [cli-observability-contract.md](cli-observability-contract.md). +- `managed_telemetry::RuntimeTelemetry::resolve` first checks the readiness hint. Without it, the result is Noop with reason `not_configured` (logged at `debug`), and the auth DB is not opened. With the hint, it reads the stored credential through the non-interactive credential-store path at CLI startup (never issuing one). If the store refuses silent access, the reason is `credential_inaccessible` (logged at `debug`, so a denied Keychain does not flood the log). `sce setup` is excluded from managed telemetry so that it never reaches the credential store. a valid unexpired `token` is used only as the `Authorization: Bearer` credential for managed OTLP trace and log export; the `accessor` is never transmitted. Absent, expired, malformed, or unavailable rows select `NoopTelemetry`. Every Noop fallback is classified (`not_configured`, `credential_inaccessible`, `credential_unavailable`, `credential_expired`, `lookup_timeout`, `database_error`, `path_resolution_error`, `exporter_init_failed`) and logged locally as `sce.telemetry.fallback`; failures log at `warn` (forced, so they reach the log file at the default `error` level, never stderr while `log_to_file=true`), with the redacted `TokenStorageError` text for database and path errors, and a missing hint, an inaccessible credential store, or missing or expired credentials log at `debug`. Concurrent lookups from many hook processes are supported by the auth DB's multiprocess WAL and migration-free current-schema open ([auth-db.md](auth-db.md)). Both this lookup and the header helper's repository lookup are existing-DB reads: a missing `auth.db` yields no credential and creates no DB file or encryption key. Claude-invoked SCE hooks are admitted to this lookup and export one `sce.hook` span. Every other hook and this hidden header helper skip the lookup and stay Noop. See [cli-observability-contract.md](cli-observability-contract.md). + +## Doctor health check + +`sce doctor` reports managed telemetry as the last `Environment` row and as a top-level JSON `managed_telemetry` object. `managed_telemetry::inspect_health` computes it once per invocation, and it is read-only: + +- It never issues a credential, contacts the Control Plane, or starts an OTLP exporter. +- It reads the readiness hint first. `NotConfigured` and `Expired` hints return without opening `auth.db` or touching the credential store, so a fresh install never reaches the Keychain. +- With a `Ready` hint it runs the runtime lookup: `load_telemetry_credential` (existing DB only, `CredentialAccess::NonInteractive`) under `CREDENTIAL_LOOKUP_DEADLINE`. Errors are classified by the same `TelemetryFallbackReason` mapping the runtime uses. +- `--fix` adds no repair for it. The problem is `manual_only`, so fix mode reports one `manual` fix result. Doctor never runs `sce auth login`, changes Keychain ACLs, or recreates a key. + +| Health | Text row | JSON `state` / `reason` | Problem `kind` | `next_action` | +| --- | --- | --- | --- | --- | +| No hint or no `auth.db` | `[PASS] Managed telemetry (not configured)` | `not_configured` / `not_configured` | none | none | +| Credential valid | `[PASS] Managed telemetry` | `healthy` / `null` | none | none | +| Store refuses silent access | `[WARN]` | `disabled` / `credential_inaccessible` | `managed_telemetry_credential_inaccessible` | `auth_login` | +| Hint ready, no usable credential | `[WARN]` | `disabled` / `credential_unavailable` | `managed_telemetry_credential_unavailable` | `auth_login` | +| Hint or credential expired | `[WARN]` | `disabled` / `credential_expired` | `managed_telemetry_credential_expired` | `auth_login` | +| Lookup deadline exceeded | `[WARN]` | `disabled` / `lookup_timeout` | `managed_telemetry_lookup_timeout` | `manual_steps` | +| Auth DB open/read failure | `[WARN]` | `disabled` / `database_error` | `managed_telemetry_database_error` | `manual_steps` | +| State path unresolvable | `[WARN]` | `disabled` / `path_resolution_error` | `managed_telemetry_path_resolution_error` | `manual_steps` | + +- The JSON object has `state`, `reason`, `kind`, `severity` (`warning` or `null`), and `next_action`. A warning also appears in `problems` (category `global_state`, severity `warning`, fixability `manual_only`) with the same `next_action`. It never makes doctor `not_ready`. +- Every warning says that Claude Code hooks continue working and only telemetry export is disabled. +- Only macOS `credential_inaccessible` recommends choosing "Always Allow" in the Keychain dialog shown by `sce auth login`. Other platforms name their store (Secret Service, Windows Credential Manager) instead. Expiry, missing credentials, timeouts, and DB or path errors never mention it. +- Output uses fixed text only. It never includes `TokenStorageError` details, Keychain error codes, keys, WorkOS tokens, or OpenBao tokens. The detailed reason stays in the local `sce.telemetry.fallback` log event that the same doctor invocation's startup lookup writes. +- Code: `cli/src/services/doctor/managed_telemetry.rs` (classification text and JSON) and `cli/src/services/managed_telemetry.rs` (`inspect_health`, `inspect_health_with`). ## Hidden header helper @@ -37,7 +68,8 @@ flowchart LR - Launch context (verified against Claude Code 2.1.284 with a real `claude -p` run): Claude runs the helper with no arguments, with its working directory set to the directory Claude was launched from (possibly a subdirectory), and without `CLAUDE_PROJECT_DIR`. Claude's documentation states that it does not pass `OTEL_*` variables to subprocesses. The helper therefore uses only the working directory, ignores `CLAUDE_PROJECT_DIR` (a value inherited from a parent session could name another project), and never requires an `OTEL_*` variable to be present. - Authorization: the helper walks up to the enclosing Git root, resolves the repository ID with the same precedence as Agent Trace storage, and prints headers only when that repository is bound to the identity of a valid credential that the current WorkOS session owns (see Repository bindings). An unbound repository, a repository bound to another user or workspace, a directory outside Git, or a repository without a resolvable identity gets no token. `.claude/settings.json`, SCE hook markers, and remote names are never treated as proof of ownership. - Destination tripwire: if `OTEL_EXPORTER_OTLP_ENDPOINT`, a per-signal `OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_ENDPOINT`, or `BETA_TRACING_ENDPOINT` is visible to the helper with a non-canonical value, the helper refuses. This catches visible misconfiguration only. It is not endpoint pinning (see Security boundaries). -- Repository resolution and the credential lookup together are bounded by `CREDENTIAL_LOOKUP_DEADLINE`. Resolution spawns one `git config --get` process, which is needed to compute the repository ID; there is no network call and no issuance. Every failure exits nonzero with the fixed stderr diagnostic `Telemetry credential is unavailable.` and empty stdout. The helper never initializes managed exporter telemetry. +- Without the readiness hint the helper fails closed before resolving the repository or opening the auth DB. With it, the credential lookup uses the non-interactive credential-store path and never prompts. +- Repository resolution and the credential lookup together are bounded by `CREDENTIAL_LOOKUP_DEADLINE`. Resolution spawns one `git config --get` process, which is needed to compute the repository ID; there is no network call and no issuance. Every failure exits nonzero with the fixed stderr diagnostic `Telemetry credential is unavailable.` and empty stdout. That includes a missing `auth.db`, which the helper never creates or migrates, so concurrent helper calls on a never-logged-in machine cannot race to initialize it. The helper never initializes managed exporter telemetry. - It is omitted from `sce --help`, `sce auth --help`, and shell completions. ## Security boundaries @@ -57,9 +89,9 @@ Claude Code's own OpenTelemetry export is separate from the SCE-CLI managed expo - Written keys: `otelHeadersHelper` set to `sce auth telemetry-headers`, plus an `env` block with `CLAUDE_CODE_ENABLE_TELEMETRY=1`, `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA=1` (required by Claude Code for span tracing), `OTEL_{TRACES,METRICS,LOGS}_EXPORTER=otlp`, `OTEL_EXPORTER_OTLP_PROTOCOL` / `OTEL_EXPORTER_OTLP_ENDPOINT` from `MANAGED_OTLP_PROTOCOL` / `MANAGED_OTLP_ENDPOINT`, `CLAUDE_CODE_OTEL_HEADERS_HELPER_DEBOUNCE_MS=300000`, and `OTEL_METRICS_INCLUDE_REPOSITORY=true` (unconditional because the Claude Code version cannot be detected during setup; unknown variables are ignored). - Authentication: no token is ever written to settings, generated assets, or setup output, and `OTEL_EXPORTER_OTLP_HEADERS` is never set. Claude runs the hidden helper (see Hidden header helper) and uses its `Authorization` header. - Rotation: Claude re-runs the helper at most every five minutes (debounce 300000 ms), so a credential replaced by a later `sce auth login` is picked up without rewriting settings. `sce auth logout` deletes the stored credential, after which the helper fails and Claude export stops authenticating; local deletion does not revoke the OpenBao token. -- Repository enrollment: after a successful install, setup enrolls the current repository under the stored credential's identity, bounded by `CREDENTIAL_LOOKUP_DEADLINE`. When no valid session-owned credential exists, setup succeeds and reports `authentication pending for this repository`; running `sce auth login` inside the repository enrolls it and starts export. +- Repository enrollment: setup never enrolls, because it does not open the auth DB. After a successful install it always reports `authentication pending for this repository`. Running `sce auth login` inside the repository enrolls it, if unbound, and starts export. - Failure handling: the helper's failure is nonzero with empty stdout and the fixed diagnostic `Telemetry credential is unavailable.`, and never affects other Claude behavior beyond missing telemetry auth. - Merge safety: repeated setup is byte-idempotent; unrelated settings, env, permissions, and hooks are preserved; writes are atomic via temp-file rename; an existing file's mode is kept and a new file uses `0600`; malformed JSON is never overwritten. -- Conflicts and limits: a user-owned `otelHeadersHelper`, an explicit opt-out (`CLAUDE_CODE_ENABLE_TELEMETRY`, `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA`, or `ENABLE_ENHANCED_TELEMETRY_BETA` not `1`, or any `OTEL_*_EXPORTER=none`), `OTEL_METRICS_INCLUDE_REPOSITORY` other than `true`, a different helper refresh interval, or a differing generic/per-signal OTLP endpoint, headers, protocol, or exporter in user settings or in the Linux/macOS managed-settings file (`/etc/claude-code/managed-settings.json`, `/Library/Application Support/ClaudeCode/managed-settings.json`), causes the telemetry install to be skipped and reported with the specific reason while repository setup still succeeds. Setup enrolls the repository only after a successful install. Environment variables present in a future Claude launch cannot be detected by setup and can override user settings. +- Conflicts and limits: a user-owned `otelHeadersHelper`, an explicit opt-out (`CLAUDE_CODE_ENABLE_TELEMETRY`, `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA`, or `ENABLE_ENHANCED_TELEMETRY_BETA` not `1`, or any `OTEL_*_EXPORTER=none`), `OTEL_METRICS_INCLUDE_REPOSITORY` other than `true`, a different helper refresh interval, or a differing generic/per-signal OTLP endpoint, headers, protocol, or exporter in user settings or in the Linux/macOS managed-settings file (`/etc/claude-code/managed-settings.json`, `/Library/Application Support/ClaudeCode/managed-settings.json`), causes the telemetry install to be skipped and reported with the specific reason while repository setup still succeeds. Environment variables present in a future Claude launch cannot be detected by setup and can override user settings. See also: [auth-db.md](auth-db.md), [cli-observability-contract.md](cli-observability-contract.md), [../context-map.md](../context-map.md)