From 7dc3e60d1ff52e89581524604bd1802c2bbcad55 Mon Sep 17 00:00:00 2001 From: David Abram Date: Sat, 17 Oct 2026 13:28:42 +0200 Subject: [PATCH 01/17] context: Define Claude hook trace propagation plan Capture the staged implementation plan for connecting Claude Code hook execution to managed OTLP tracing, including propagation probing, hook span admission, child spans, privacy and parity constraints, and live verification. Plan: claude-hook-trace-propagation Tasks: T01, T02, T03, T04, T05 Co-authored-by: SCE --- .../plans/claude-hook-trace-propagation.md | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 context/plans/claude-hook-trace-propagation.md diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md new file mode 100644 index 000000000..25b39cf02 --- /dev/null +++ b/context/plans/claude-hook-trace-propagation.md @@ -0,0 +1,120 @@ +# Plan: claude-hook-trace-propagation + +## Change summary + +Connect SCE's existing managed OTLP trace export (PRs #306/#325, `cli/src/services/managed_telemetry/trace_export.rs`) to Claude Code hook execution so a Claude-invoked SCE hook emits a dedicated `sce.hook` span, parented to Claude Code's active tool span when Claude Code actually passes valid W3C trace context to command hooks, and otherwise exported as an independent SCE trace carrying safe Claude session/tool correlation attributes. + +This reverses one recorded behavior: today every `sce hooks ...`, `sce policy bash`, and `sce auth telemetry-headers` invocation is forced to `NoopTelemetry` by `is_hook_invocation` in `cli/src/app.rs` (asserted by `hook_invocations_never_resolve_managed_telemetry`, and by AC4 of `claude-native-otel-instrumentation`). This plan admits only Claude-invoked hook processes to managed export; Codex, OpenCode, and Pi hook processes and `sce auth telemetry-headers` stay `Noop`. No trace-context propagator or `trace_context` module exists in the repository today; the W3C extraction is new, but it reuses the pinned `opentelemetry`/`opentelemetry_sdk` 0.33.1 and `tracing-opentelemetry` 0.34.0 crates, the existing exporter, credential lookup, endpoint pinning, and bounded shutdown. Whether parent-child propagation is possible at all is decided by a live probe in T01, not assumed. + +## Acceptance criteria + +How this plan is proven complete. Each criterion is observable and names the check that proves it. `/validate` runs these checks; no task in the stack performs final validation. + +- [ ] AC1: Durable context records, from real Claude Code execution of `PreToolUse` and `PostToolUse` around a real tool call, whether command hooks receive `TRACEPARENT`/`TRACESTATE` (or equivalent), whether the value is a valid W3C traceparent, whether its trace ID and parent span ID match a span Claude Code actually exported, the Claude Code version tested, and the documentation consulted; the probe captured only `session_id`, `hook_event_name`, `tool_use_id`, presence/validity flags, trace ID, and parent span ID, and no probe code remains in the repository. + - Validate: Inspect `context/sce/claude-hook-trace-propagation.md` for the recorded evidence table and verdict; `git ls-files` shows no probe script; grep the recorded evidence for tool input, prompt text, `Bearer`, or token-shaped values (none). +- [ ] AC2: With a valid stored telemetry credential, each Claude-invoked SCE hook process exports exactly one `sce.hook` root-of-process span (no additional `sce.command` span) to the pinned `/v1/traces` endpoint, carrying `service.name=sce`, `sce.agent=claude`, `sce.hook.event`, `sce.hook.tool` when present, and `sce.outcome`; Codex, OpenCode, and Pi hook invocations and `sce auth telemetry-headers` still resolve `NoopTelemetry`, and ordinary commands keep their unchanged `sce.command` span. + - Validate: Loopback-receiver Rust tests decode the captured OTLP payload for a Claude `PostToolUse` fixture and assert span name, count, and attributes; admission-predicate unit tests cover Claude, non-Claude, and helper invocations; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`. +- [ ] AC3: When T01 proved usable parent context: a valid upstream traceparent makes the `sce.hook` span share the upstream trace ID with its parent span ID equal to the upstream span ID, SCE span IDs are freshly generated, child spans inherit the trace across `.await` points and Tokio worker migration, and an invalid, malformed, or missing traceparent yields a new independent SCE root without panic or hook-behavior change. When T01 proved context unavailable: no extraction code ships, and the context file states the parent-child goal is blocked, why, and the smallest future bridge. + - Validate: Loopback-receiver tests for valid, invalid, all-zero, and missing traceparent asserting trace ID / parent span ID; a multi-poll async test asserting child span trace IDs; or, for the unavailable branch, inspection of `context/sce/claude-hook-trace-propagation.md` and source grep showing no propagator wiring. +- [ ] AC4: The `sce.hook` span carries Claude `session_id` and `tool_use_id` as span attributes only (never as metric attributes, never reconstructed into a trace ID), and no exported span or log contains tool input, prompts, file contents, repository paths, raw hook payloads, the OpenBao token, or the accessor. + - Validate: Loopback-receiver test injecting secret- and payload-shaped fixture values asserts their absence from the decoded payload and presence of only the two correlation IDs. +- [ ] AC5: Child spans `sce.mutation_scope`, `sce.git_snapshot`, and `sce.persistence` appear under `sce.hook` only when the corresponding operation actually executed in that hook process; a hook path that skips an operation exports no span for it. + - Validate: Loopback-receiver tests over a mutation-scope fixture that executes all three and a fixture that executes none (e.g. a non-tracked tool) assert the exact span set and parent links. +- [ ] AC6: Telemetry cannot change Claude hook behavior: with credential lookup failure, expired credential, refused or stalled receiver, or exporter timeout, hook stdout, stderr, exit code, permission decisions, Agent Trace DB rows, and mutation-trace state match the telemetry-disabled run, and process lifetime stays within the existing `CREDENTIAL_LOOKUP_DEADLINE` + `SHUTDOWN_TIMEOUT` bounds. + - Validate: Parity tests running the same Claude hook fixture with Noop and with failing managed telemetry, comparing outputs and persisted state, and asserting elapsed-time bounds. +- [ ] AC7: Standalone tracing is unchanged: `sce doctor` with a valid credential exports its `sce.command` span through the managed exporter with no Claude context required. + - Validate: Existing `managed_trace_export_posts_allowlisted_span_with_bearer_only` and related trace_export tests still pass; live evidence for `sce doctor` recorded per AC8. +- [ ] AC8: Live verification is recorded: `sce doctor` and a Claude Code session triggering SCE hooks in this repository both export through `https://otel.crocoderlab.dev` with their existing managed credentials, and the record states whether each trace was observed in the correct Dash0 workspace dataset (with trace IDs and, where proven, the parent relationship) or exactly what could not be verified; a collector 2xx alone is not recorded as delivery. + - Validate: Inspect the live-verification section of `context/sce/claude-hook-trace-propagation.md` and the T05 completion record. +- [ ] AC9: Current-state context describes hook telemetry admission, the `sce.hook` span contract and attributes, the parent-context verdict, correlation-ID privacy rules, and the opt-in live smoke procedure, without claiming distributed tracing unless AC3's parent relationship was verified live. + - Validate: Inspect the files listed under Context sync against the implemented code. + +### Full validation + +- `nix flake check` +- `nix run .#pkl-check-generated` +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry` + +### Context sync + +- `context/sce/claude-hook-trace-propagation.md` (new: probe evidence, verdict, live verification, smoke procedure) +- `context/sce/cli-observability-contract.md` +- `context/sce/telemetry-credential.md` (hook admission now differs from the helper bypass) +- `context/cli/claude-mutation-scope-integration.md` (span boundaries over the adapter) +- `context/architecture.md`, `context/glossary.md`, `context/context-map.md` + +## Task context synchronization lifecycle + +Persist this field in every plan; this is durable plan state, not chat state: + +- **Task context synchronization:** every task carries `pending | synced | blocked`. + A completed task must be `synced` before another task can start or the plan can + finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** + beside the status. Never infer `synced` from conversation history; write every + lifecycle transition to the plan file. + +## Constraints and non-goals + +- **In scope:** `cli/src/app.rs` (hook admission predicate, telemetry resolution), `cli/src/services/managed_telemetry.rs` and `managed_telemetry/trace_export.rs` (hook span, span filter, propagation), the Claude hook entry points in `cli/src/services/hooks/` (`claude_mutation_scope/`, `claude_model_state.rs`, Claude-classified paths of `diff_trace.rs`, `conversation_trace.rs`, and `sce policy bash`), span boundaries at existing call sites in `cli/src/services/mutation_trace/runtime/` (e.g. `git_snapshot.rs`) and Agent Trace DB persistence, focused tests, and the context files listed above. +- **Out of scope:** Codex, OpenCode, and Pi hooks/plugins; Pkl/generated Claude hook config; Claude settings written by `sce setup`; metrics; credential issuance, storage, or lifecycle; Control Plane or Collector changes; a collector, proxy, sidecar, daemon, or correlation engine; new public CLI commands; making `sce auth telemetry-headers` visible. +- **Constraints:** reuse the pinned OTEL crates, `TraceContextPropagator` from `opentelemetry_sdk`, `MANAGED_OTLP_*`, `load_credential_within_deadline`, and existing bounded shutdown; static dispatch, no new `dyn` abstractions or wrappers beyond what the span boundary needs; hooks stay fail-open; never derive a trace ID from Claude session or tool-use IDs; no code comments; Cargo only via `nix develop`; tests stay a minimal focused set using the existing loopback receiver; CI never needs live credentials, Claude Code, or Dash0. +- **Non-goal:** generic distributed-tracing support for every harness or every SCE command; span-instrumenting arbitrary internal functions. + +## Assumptions + +- The repository stays on Rust edition 2021 (`cli/Cargo.toml`, AGENTS.md); the request's "Rust 2024" is treated as "current Rust", since an edition migration is a separate cross-cutting change. +- `sce.hook` replaces `sce.command` for admitted Claude hook processes rather than nesting under it, so each hook process exports one root-of-process span and no redundant wrapper. +- Claude admission is decided before dispatch from argv for Claude-only subcommands (`claude-mutation-scope`, `claude-model-state`) and, for shared subcommands (`diff-trace`, `conversation-trace`, `policy bash`), from a cheap Claude-caller signal T01 confirms (presence-only check of a Claude-set hook environment marker); if no reliable signal exists, shared subcommands stay `Noop`. +- Claude `session_id` and `tool_use_id` are permitted as span attributes: they are opaque identifiers, not hook payload content, and the managed-otel-export privacy contract forbids payloads, paths, and secrets, not correlation IDs. The context sync records this explicitly. +- The "opt-in live smoke test" is a documented manual procedure in `context/sce/claude-hook-trace-propagation.md` plus recorded evidence, not a committed script or ignored Rust test, because driving Claude Code and Dash0 cannot run deterministically from the test suite. +- Probe scripts live only in the session scratchpad and user-level Claude settings changes for the probe are reverted after T01. +- The PR is opened after `/validate` passes, using the plan's recorded evidence for its description; it is not a task in this stack. + +## Task stack + +- [ ] T01: `Probe and record Claude Code hook trace-context propagation` (status:todo) + - Task ID: T01 + - Scope: In — read Claude Code's current hooks and monitoring/OTEL documentation and any inspectable implementation; run a temporary scratchpad probe registered for `PreToolUse` and `PostToolUse` in a real Claude Code session with native telemetry enabled, triggering a real tool call; capture only `session_id`, `hook_event_name`, `tool_use_id`, `TRACEPARENT`/`TRACESTATE` presence, traceparent validity, trace ID, parent span ID, and presence-only Claude caller env markers; compare captured IDs against Claude Code's exported spans to decide whether the context corresponds to an active tool span; write `context/sce/claude-hook-trace-propagation.md` with evidence, Claude Code version, and verdict. Out — any Rust change; inferring hook behavior from Bash subprocess propagation; committing the probe. + - Dependencies: none + - Done when: The context file records a verdict of `parent_context_available` or `parent_context_unavailable` backed by captured evidence for both events, names the Claude-caller signal (or its absence), the probe is removed, and the user-level Claude settings are restored. + - Verify: Inspect the context file; `git status` shows only the new context file; grep the file for prompt/tool-input/token content (none). + - Context synchronization: pending + +- [ ] T02: `Export a dedicated sce.hook span for Claude hook invocations` (status:todo) + - Task ID: T02 + - Scope: In — narrow `is_hook_invocation` into a Claude-admission decision in `cli/src/app.rs`; add the `sce.hook` span (replacing `sce.command` for admitted hooks) in `trace_export.rs` with `sce.agent`, `sce.hook.event`, `sce.hook.tool`, `sce.outcome`, `sce.error.class`, and the two correlation IDs recorded from the parsed Claude payload; loopback tests for span shape, admission, secret/payload absence, credential failure, exporter failure/stall, bounded shutdown, and hook output/state parity. Out — parent-context extraction (T03), child spans (T04), non-Claude harnesses. + - Dependencies: T01 + - Done when: AC2, AC4, AC6, and AC7 hold in local tests; `hook_invocations_never_resolve_managed_telemetry` is replaced by an admission test that keeps non-Claude hooks and `telemetry-headers` on `Noop`. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml app::tests`. + - Context synchronization: pending + +- [ ] T03: `Parent the sce.hook span on Claude-provided W3C trace context` (status:todo) + - Task ID: T03 + - Scope: In — only when T01 recorded `parent_context_available`: extract the context Claude provides using `opentelemetry_sdk::propagation::TraceContextPropagator` before the `sce.hook` span is created, attach it with `OpenTelemetrySpanExt::set_parent`, keep it scoped to the instrumented hook future, and add loopback tests for valid, invalid, all-zero, and missing traceparent plus a multi-poll async child-span test. When T01 recorded `parent_context_unavailable`: make no code change, and record in the context file that the parent-child goal is blocked, why, and the smallest future bridge (correlating Claude OTLP export records with `sce.hook` via `session_id`/`tool_use_id`). Out — reconstructing trace IDs from Claude identifiers; global propagator registration beyond the hook boundary; non-Claude callers. + - Dependencies: T02 + - Done when: AC3 holds for the branch T01 selected. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml trace_export`, or inspection of the context file for the unavailable branch. + - Context synchronization: pending + +- [ ] T04: `Add mutation-scope, git-snapshot, and persistence child spans under sce.hook` (status:todo) + - Task ID: T04 + - Scope: In — spans on the managed-trace target at the existing execution sites: the Claude mutation-scope coordinate call (`sce.mutation_scope`), the mutation-trace Git snapshot/reconciliation call (`sce.git_snapshot`), and Agent Trace DB / mutation store persistence (`sce.persistence`), each with low-cardinality outcome attributes; extend the span filter so only these named spans export; loopback tests asserting the exact span set for an executing and a non-executing hook path. Out — spans inside pure protocol functions or arbitrary helpers; changes to ordering, locking, or failure semantics. + - Dependencies: T03 + - Done when: AC5 holds, existing mutation-trace and Agent Trace tests pass unchanged, and no span is emitted for an operation that did not run. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml claude_mutation_scope`; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`. + - Context synchronization: pending + +- [ ] T05: `Record live Dash0 verification and sync hook-telemetry context` (status:todo) + - Task ID: T05 + - Scope: In — run `sce doctor` and a Claude Code session in this repository that triggers SCE hooks using a build with T02–T04; confirm both exporters use their existing managed credentials and the canonical collector, and check Dash0 for both traces in the correct workspace dataset, recording trace IDs, observed hierarchy, and anything unverified; write the opt-in live smoke procedure; update the context files listed under Context sync. Out — code changes; Collector/Control Plane changes; claiming delivery from a collector 2xx. + - Dependencies: T04 + - Done when: AC8 and AC9 hold and the context file states the exact observed hierarchy. + - Verify: Inspect the updated context files; `nix run .#pkl-check-generated` unaffected. + - Context synchronization: pending + +## Open questions + +- Admitting Claude hooks to managed export adds an encrypted `auth.db` credential lookup (up to `CREDENTIAL_LOOKUP_DEADLINE`, 2s) and an exporter shutdown (up to `SHUTDOWN_TIMEOUT`, 3s) to every Claude hook process — several per tool call, including the blocking `PreToolUse` path. `managed-otel-export` and `claude-native-otel-instrumentation` excluded hooks deliberately. Is the existing bound acceptable for hooks, or should hooks get a tighter hook-specific deadline? A smaller version is to admit only `PostToolUse` hooks, leaving the blocking `PreToolUse` path on `Noop`. +- If T01 finds Claude Code does not pass trace context to command hooks, the remaining value is correlation attributes on independent traces. Is that still worth T02–T05, or should the plan stop after T01 and T03's documented blocker? +- The request asked for Rust 2024; the plan keeps edition 2021. Confirm, or plan the edition migration as a separate change. From 3962da5235bb7e7ec35a54657058563017e7ce32 Mon Sep 17 00:00:00 2001 From: David Abram Date: Sat, 17 Oct 2026 13:55:27 +0200 Subject: [PATCH 02/17] context: Record Claude hook trace propagation evidence Capture Claude Code 2.1.301 probe results for PreToolUse and PostToolUse hooks, including the interaction-span parent context, tool_use_id correlation, caller environment markers, and telemetry gating behavior. Update the context map and complete T01 with the qualified parent_context_available verdict, while documenting that future SCE hook spans must be siblings of Claude tool spans under claude_code.interaction. Plan: claude-hook-trace-propagation (T01) Co-authored-by: SCE --- context/context-map.md | 1 + .../plans/claude-hook-trace-propagation.md | 10 +- context/sce/claude-hook-trace-propagation.md | 117 ++++++++++++++++++ 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 context/sce/claude-hook-trace-propagation.md diff --git a/context/context-map.md b/context/context-map.md index 40476a1da..023713127 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -56,6 +56,7 @@ Feature/domain context: - `context/sce/cli-version-command-contract.md` (implemented `sce version` contract for deterministic human and machine-readable runtime identification) - `context/sce/cli-shell-completion-contract.md` (implemented `sce completion` contract for deterministic Bash/Zsh/Fish completion script generation) - `context/sce/claude-raw-hook-capture.md` (removed feature: the former `sce hooks claude-capture` raw-capture route and its supporting types, replaced by the active `diff-trace` and `conversation-trace` intakes) +- `context/sce/claude-hook-trace-propagation.md` (probe evidence for Claude Code 2.1.301 command-hook trace context: `TRACEPARENT` names the `claude_code.interaction` span, not the tool span, and only when Claude tracing is active; hook-to-tool correlation by `tool_use_id`; `CLAUDECODE` + `CLAUDE_PROJECT_DIR` presence as the Claude hook-caller signal; no SCE hook telemetry ships yet) - `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary, logger and deliberately non-object-safe generic async telemetry boundaries with static capability dispatch and future-aware polling scope, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, managed OTLP trace/log export (credential-gated, hardcoded canonical collector, allowlisted fields, bounded fail-open lifecycle, reserved metrics endpoint), and the trimmed `sce config validate` status-only validation surface) - `context/sce/shared-context-code-workflow.md` (canonical `/next-task` task-synchronization lifecycle and validation-only `/validate` lifecycle, package-local phase references with single-skill control flow, and the task-synchronization-scoped `sce-decision` sibling invocation with ADR reuse/blocker propagation) - `context/sce/shared-context-plan-workflow.md` (canonical `/change-to-plan` workflow, package-local context-load/plan-authoring/template references, clarification/readiness gate contract, and one-task/one-atomic-commit task slicing) diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md index 25b39cf02..6490f717d 100644 --- a/context/plans/claude-hook-trace-propagation.md +++ b/context/plans/claude-hook-trace-propagation.md @@ -73,13 +73,19 @@ Persist this field in every plan; this is durable plan state, not chat state: ## Task stack -- [ ] T01: `Probe and record Claude Code hook trace-context propagation` (status:todo) +- [x] T01: `Probe and record Claude Code hook trace-context propagation` (status:done) - Task ID: T01 - Scope: In — read Claude Code's current hooks and monitoring/OTEL documentation and any inspectable implementation; run a temporary scratchpad probe registered for `PreToolUse` and `PostToolUse` in a real Claude Code session with native telemetry enabled, triggering a real tool call; capture only `session_id`, `hook_event_name`, `tool_use_id`, `TRACEPARENT`/`TRACESTATE` presence, traceparent validity, trace ID, parent span ID, and presence-only Claude caller env markers; compare captured IDs against Claude Code's exported spans to decide whether the context corresponds to an active tool span; write `context/sce/claude-hook-trace-propagation.md` with evidence, Claude Code version, and verdict. Out — any Rust change; inferring hook behavior from Bash subprocess propagation; committing the probe. - Dependencies: none - Done when: The context file records a verdict of `parent_context_available` or `parent_context_unavailable` backed by captured evidence for both events, names the Claude-caller signal (or its absence), the probe is removed, and the user-level Claude settings are restored. - Verify: Inspect the context file; `git status` shows only the new context file; grep the file for prompt/tool-input/token content (none). - - Context synchronization: pending + - Completed: 2026-10-17 + - Files changed: `context/sce/claude-hook-trace-propagation.md` (new) + - Result: Verdict `parent_context_available`, qualified. Headless Claude Code 2.1.301 runs with telemetry and the tracing beta on gave every `PreToolUse`/`PostToolUse` hook a valid `TRACEPARENT` with no `TRACESTATE`. Its trace ID matched the exported trace, and its parent span ID was the `claude_code.interaction` root span, never `claude_code.tool` or `claude_code.hook` (detailed beta tracing included). Hooks run outside the tool span; correlate with the tool by `tool_use_id`. Without the tracing beta, or with only `CLAUDE_CODE_PROPAGATE_TRACEPARENT=1`, no `TRACEPARENT` is present. Claude-caller signal for shared subcommands: `CLAUDECODE` and `CLAUDE_PROJECT_DIR` both present (presence only); `CLAUDE_PROJECT_DIR` is absent from the Bash tool environment. Interactive TUI hooks not probed; deferred to T05. The probe ran only in the scratchpad through `claude -p --settings`. It never edited user-level `~/.claude/settings.json` and was deleted afterward. + - Deviation: the plan assumed parenting to Claude's active tool span; the evidence shows only the interaction span is available, so T03 parents `sce.hook` as a sibling of `claude_code.tool` under `claude_code.interaction`. + - Verify outcomes: context file inspected (verdict, evidence for both events, caller signal, version, docs) — passed; `git status --porcelain -uall` shows only `?? context/sce/claude-hook-trace-propagation.md` — passed; grep of the file for `bearer|token=|sk-ant|hvs.|password|tool_input"|user_prompt"` and the probe file content returned no matches — passed; `git ls-files` contains no T01 probe code — passed. + - Context impact: new current-state context file under `context/sce/` (probe evidence and verdict); needs a `context/context-map.md` entry; no code behavior change. + - Context synchronization: synced - [ ] T02: `Export a dedicated sce.hook span for Claude hook invocations` (status:todo) - Task ID: T02 diff --git a/context/sce/claude-hook-trace-propagation.md b/context/sce/claude-hook-trace-propagation.md new file mode 100644 index 000000000..baa21d1b9 --- /dev/null +++ b/context/sce/claude-hook-trace-propagation.md @@ -0,0 +1,117 @@ +# Claude Code Hook Trace-Context Propagation + +Probe evidence and verdict for whether Claude Code passes W3C trace context to command hooks, recorded by T01 of `context/plans/claude-hook-trace-propagation.md`. This file is the durable source for the parent-context decision consumed by T02–T05; it does not describe any shipped SCE behavior yet. + +## Verdict + +**`parent_context_available`**, with two binding qualifications: + +1. **The parent is the `claude_code.interaction` span, not the tool span.** Every `PreToolUse` and `PostToolUse` command hook received a valid `TRACEPARENT` whose trace ID matched the exported Claude Code trace and whose parent span ID was the span ID of the turn's `claude_code.interaction` root span. It never matched `claude_code.tool`, `claude_code.tool.execution`, or (with detailed beta tracing) the `claude_code.hook` span that wraps the hook run. An SCE `sce.hook` span parented on this context is therefore a **sibling** of `claude_code.tool` and `claude_code.hook` under the interaction, not their child. Hook-to-tool correlation must use `tool_use_id`, which appears both in the hook stdin payload and as the `tool_use_id` attribute on Claude's `claude_code.tool` and `claude_code.tool.execution` spans. +2. **The context exists only when Claude Code tracing is active.** `TRACEPARENT` is present only when both `CLAUDE_CODE_ENABLE_TELEMETRY=1` and `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA=1` are set (so Claude Code has a live span). With telemetry off, with telemetry on but the tracing beta off, or with only `CLAUDE_CODE_PROPAGATE_TRACEPARENT=1`, hooks receive no `TRACEPARENT`. SCE must treat a missing traceparent as the normal case and export an independent root. + +`TRACESTATE` was never present. The hook stdin payload carries no trace fields. + +Deviation from plan wording: the plan's change summary assumed parenting "to Claude Code's active tool span". T03 should parent `sce.hook` on the provided interaction context as recorded here; parenting under the tool span is not possible from the hook process with Claude Code 2.1.301. + +## Claude-caller signal + +Presence-only environment markers observed in every probed hook process, compared with the Claude Code Bash tool environment of the same Claude Code version: + +| Marker | Hook process | Bash tool subprocess | +| --- | --- | --- | +| `CLAUDECODE` | present | present | +| `CLAUDE_CODE_SESSION_ID` | present | present | +| `CLAUDE_CODE_CHILD_SESSION` | present | present | +| `CLAUDE_PID` | present | present | +| `CLAUDE_CODE_ENTRYPOINT` | present | present | +| `AI_AGENT` | present | present | +| `CLAUDE_PROJECT_DIR` | present | absent | +| `CLAUDE_ENV_FILE` | absent | absent | + +Selected signal for shared subcommands (`diff-trace`, `conversation-trace`, `policy bash`): **`CLAUDECODE` and `CLAUDE_PROJECT_DIR` both present**, checked for presence only, never read for value. `CLAUDECODE` alone is not hook-specific because every Claude Code subprocess inherits it, including commands the Bash tool runs. Claude Code documents `CLAUDE_PROJECT_DIR` as set for command hooks, stdio MCP servers, and plugin LSP servers, and the Bash tool does not receive it. The Claude-only subcommands (`claude-mutation-scope`, `claude-model-state`) are admitted from argv and do not need the signal. The generated `.claude/hooks/run-sce-or-show-install-guidance.sh` helper `exec`s `sce` with the inherited environment, so these markers and `TRACEPARENT` reach the SCE process unchanged. + +Claude Code removes `OTEL_*` exporter variables from every subprocess it spawns (documented). SCE hook export must not depend on Claude's `OTEL_*` configuration; it keeps using SCE's pinned managed endpoint and stored credential. + +## Evidence + +Environment: Claude Code `2.1.301` (`service.version` resource attribute on exported spans), Linux, headless `claude -p` runs on 2026-10-17 from a scratch working directory outside this repository. The runs used `env -i` (no inherited `CLAUDECODE`/`TRACEPARENT`), `--setting-sources local`, and `--settings` pointing at a scratchpad file. That file registered a probe command hook with matcher `*` on `PreToolUse` and `PostToolUse` and pointed Claude Code's OTLP/HTTP-JSON trace exporter at a loopback receiver. User-level `~/.claude/settings.json` was not edited. Each run triggered real `Bash` and/or `Read` tool calls. + +The probe captured only `session_id`, `hook_event_name`, `tool_use_id`, `TRACEPARENT`/`TRACESTATE` presence, traceparent validity, trace ID, parent span ID, trace flags, and presence-only caller markers. One extra run captured only the top-level key names of the hook stdin payload, not their values. Exported spans were inspected for name, trace ID, span ID, parent span ID, timestamps, `span.type`, `tool_name`, `tool_use_id`, and `hook_event` only. + +### Run A — telemetry + tracing beta, Bash then Read + +Session `7d0750a6-560e-4e42-9120-ed8f14304c30`, trace `aaddb6f35b4fcaf4260a454b6a278f74`. + +| Hook | `tool_use_id` | traceparent | trace ID | parent span ID | +| --- | --- | --- | --- | --- | +| PreToolUse | `toolu_01KN1scistt4Kcj3WCWcLbEp` (Bash) | valid, flags `01` | `aaddb6f35b4fcaf4260a454b6a278f74` | `00809c8f510c99f9` | +| PostToolUse | `toolu_01KN1scistt4Kcj3WCWcLbEp` (Bash) | valid, flags `01` | `aaddb6f35b4fcaf4260a454b6a278f74` | `00809c8f510c99f9` | +| PreToolUse | `toolu_01FAZCUXyrFmYNd2R4zr28Tr` (Read) | valid, flags `01` | `aaddb6f35b4fcaf4260a454b6a278f74` | `00809c8f510c99f9` | +| PostToolUse | `toolu_01FAZCUXyrFmYNd2R4zr28Tr` (Read) | valid, flags `01` | `aaddb6f35b4fcaf4260a454b6a278f74` | `00809c8f510c99f9` | + +Exported spans (same trace): + +| Span | span ID | parent span ID | `tool_use_id` | +| --- | --- | --- | --- | +| `claude_code.interaction` | `00809c8f510c99f9` | none (`parent.source=none`) | — | +| `claude_code.llm_request` | `77020189eb8449ab` | `00809c8f510c99f9` | — | +| `claude_code.tool` (Bash) | `e385b69cbfed31e4` | `00809c8f510c99f9` | `toolu_01KN1scistt4Kcj3WCWcLbEp` | +| `claude_code.tool.blocked_on_user` | `0e4c901b3fb2b096` | `e385b69cbfed31e4` | — | +| `claude_code.tool.execution` | `242141c79233929f` | `e385b69cbfed31e4` | `toolu_01KN1scistt4Kcj3WCWcLbEp` | +| `claude_code.tool` (Read) | `9408f8abebb89bce` | `00809c8f510c99f9` | `toolu_01FAZCUXyrFmYNd2R4zr28Tr` | +| `claude_code.tool.blocked_on_user` | `abbeb106b84cd5d3` | `9408f8abebb89bce` | — | +| `claude_code.tool.execution` | `83fbb33ed6cdf6d9` | `9408f8abebb89bce` | `toolu_01FAZCUXyrFmYNd2R4zr28Tr` | +| `claude_code.llm_request` | `6661c65cfc1fc0d8` | `00809c8f510c99f9` | — | + +All four hooks shared one parent across two different tools, which equals the interaction span ID. Timing confirms that hooks run outside the tool span. The Bash `PreToolUse` probe started at `…734978746818` ns, before `claude_code.tool` started at `…734988000000`. The `PostToolUse` probe started at `…735048034958` ns, after `claude_code.tool` ended at `…735039758337`. + +### Run E — telemetry + tracing beta, Read, payload key names + +Trace `7ce9b9cb88f26f47f24496b1cf1b828e`. Both hooks for `toolu_01Qn5H38gufu6cdCHsowFzNu` received a valid traceparent with parent `a5d5939e4fd7c534`, which equals `claude_code.interaction`. The `claude_code.tool` span was `d39aebf463d29ac8`. + +Hook stdin top-level keys: `PreToolUse` had `cwd`, `effort`, `hook_event_name`, `permission_mode`, `prompt_id`, `session_id`, `tool_input`, `tool_name`, `tool_use_id`, `transcript_path`. `PostToolUse` had the same keys plus `duration_ms` and `tool_response`. Neither carried a trace field. + +### Run F2 — detailed beta tracing (`ENABLE_BETA_TRACING_DETAILED=1`, `BETA_TRACING_ENDPOINT` loopback) + +Trace `1c6afedd2f2dc795c4aaa2c94b35184b`. Both hooks for `toolu_019vSjT7iXZedtTDFXE7TucH` received parent `56d0e76fdb58fe2e`, which equals `claude_code.interaction`. + +| Span | span ID | parent span ID | start → end (ns suffix) | +| --- | --- | --- | --- | +| `claude_code.hook` (`hook_event=PreToolUse`) | `70b2f799a056a4d0` | `56d0e76fdb58fe2e` | `942778000000 → 942793833861` | +| `claude_code.tool` | `0ccef232e0abf4dc` | `56d0e76fdb58fe2e` | `942795000000 → 942803155221` | +| `claude_code.hook` (`hook_event=PostToolUse`) | `308ab39a6ca018d8` | `56d0e76fdb58fe2e` | `942804000000 → 942818720789` | + +Even when Claude Code exports its own `claude_code.hook` span, the hook process's `TRACEPARENT` names the interaction span, not the `claude_code.hook` span that wraps the hook run. + +### Gating runs (Read tool, real tool call confirmed by `num_turns=2`) + +| Run | Settings env | `TRACEPARENT` in hooks | Spans exported | +| --- | --- | --- | --- | +| B | telemetry on, tracing beta off | absent (Pre + Post) | none | +| C | telemetry off | absent (Pre + Post) | none | +| D | only `CLAUDE_CODE_PROPAGATE_TRACEPARENT=1` | absent (Pre + Post) | none | + +### Implementation cross-check + +The Claude Code 2.1.301 bundle contains a shared subprocess-environment builder. It sets `CLAUDECODE`, `CLAUDE_CODE_SESSION_ID`, `CLAUDE_CODE_CHILD_SESSION`, `CLAUDE_CODE_SESSION_ATTENDED`, `CLAUDE_PID`, `AI_AGENT` (agent source), and `CLAUDE_EFFORT`. It sets `TRACEPARENT` only when tracing is active (or `CLAUDE_CODE_PROPAGATE_TRACEPARENT` is set) and there is an active span in the current context. This is consistent with runs B–D: run D has the propagate flag but no active span, so no traceparent. + +## Documentation consulted + +- Claude Code monitoring docs, `https://code.claude.com/docs/en/monitoring-usage` (Traces (beta), TRACEPARENT propagation, span hierarchy, hook spans). The docs describe Bash/PowerShell subprocesses as inheriting the active tool execution span's context. They describe hook executions as children of `claude_code.interaction`. They say nothing about hook-process `TRACEPARENT`; the probe shows hook processes receive the interaction context. +- Claude Code hooks docs, `https://code.claude.com/docs/en/hooks` (hook environment: `CLAUDE_PROJECT_DIR`; `OTEL_*` removed from subprocesses; `prompt_id` matches the `prompt.id` event attribute). These docs do not mention `TRACEPARENT`. + +## Not verified + +- Interactive TUI sessions were not probed for hook `TRACEPARENT`. Claude Code documents that only `-p`/Agent SDK sessions read an inbound `TRACEPARENT`. The outbound subprocess builder is shared, and the interactive Bash tool environment of the same version carries a valid `TRACEPARENT` (observed during this task). Whether interactive hooks receive the same interaction-span context is expected but unconfirmed. T05 live verification must check it. +- Subagent (`Agent` tool) hooks, `PostToolUseFailure`, `PermissionDenied`, and other hook events were not probed. +- Behavior behind a custom `ANTHROPIC_BASE_URL` proxy without `CLAUDE_CODE_PROPAGATE_TRACEPARENT` was not probed. Per the docs the subprocess `TRACEPARENT` follows the same switch as the API `traceparent` header. + +## Consequences for the plan + +- T02: admit the Claude-only subcommands from argv. Admit shared subcommands only when `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are both present. +- T03: the parent-context branch applies. Extract `TRACEPARENT` (and `TRACESTATE` when present) with `TraceContextPropagator` and parent `sce.hook` on it. The resulting span is a child of `claude_code.interaction` and shares its trace ID. Missing or invalid context, which is the default when Claude tracing is off, yields an independent SCE root. +- T05 and context sync must describe the hierarchy as `claude_code.interaction → sce.hook` (sibling of `claude_code.tool`), with tool correlation by `tool_use_id`, not as a tool-span child. + +## Probe hygiene + +The probe hook, receiver, settings, captured payloads, and exported spans lived only in the session scratchpad and were deleted after this record was written. No probe code is in the repository, and user-level Claude settings were never modified. From 9d58e860f63f597a232943bf9a12d073e8178cde Mon Sep 17 00:00:00 2001 From: David Abram Date: Sat, 17 Oct 2026 14:06:12 +0200 Subject: [PATCH 03/17] runtime: Admit Claude hooks to managed trace export Enable managed OTLP telemetry for Claude-invoked hook processes while keeping non-Claude hooks and the telemetry-header helper on Noop. Classify admitted invocations before credential lookup, emit one privacy-filtered `sce.hook` span with Claude correlation IDs, and preserve fail-open hook behavior under credential or exporter failures. Add admission, payload-redaction, and parity coverage while updating the observability contract and decision record. Plan: claude-hook-trace-propagation (T02) Co-authored-by: SCE --- cli/src/app.rs | 99 +++++-- cli/src/services/bash_policy.rs | 1 + cli/src/services/hooks/claude_model_state.rs | 1 + .../hooks/claude_mutation_scope/lifecycle.rs | 103 +++++++ cli/src/services/hooks/conversation_trace.rs | 1 + cli/src/services/hooks/diff_trace.rs | 1 + cli/src/services/managed_telemetry.rs | 30 +- .../managed_telemetry/trace_export.rs | 265 +++++++++++++++++- context/architecture.md | 2 +- context/context-map.md | 4 +- ...-17-claude-hook-managed-trace-admission.md | 69 +++++ context/glossary.md | 1 + context/overview.md | 2 +- .../plans/claude-hook-trace-propagation.md | 10 +- context/sce/claude-hook-trace-propagation.md | 2 +- context/sce/cli-observability-contract.md | 4 +- 16 files changed, 547 insertions(+), 48 deletions(-) create mode 100644 context/decisions/2026-10-17-claude-hook-managed-trace-admission.md diff --git a/cli/src/app.rs b/cli/src/app.rs index e7ec4d71c..6e90c4895 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -6,6 +6,7 @@ use std::time::Duration; use crate::services; use services::app_support::{self, RunOutcome}; use services::error::CliError; +use services::managed_telemetry::trace_export::ManagedSpanKind; use services::managed_telemetry::RuntimeTelemetry; use services::observability::traits::{ Logger as LoggerTrait, NoopTelemetry, Telemetry as TelemetryTrait, @@ -15,6 +16,9 @@ const MIN_WORKER_THREADS: usize = 2; const RUNTIME_SHUTDOWN_GRACE: Duration = Duration::from_millis(500); const REPEATED_COMMAND_DISPATCH_ERROR: &str = "Command lifecycle telemetry attempted to execute command dispatch more than once"; +const CLAUDE_ONLY_HOOK_SUBCOMMANDS: [&str; 2] = ["claude-mutation-scope", "claude-model-state"]; +const SHARED_CLAUDE_HOOK_SUBCOMMANDS: [&str; 2] = ["diff-trace", "conversation-trace"]; +const CLAUDE_HOOK_CALLER_MARKERS: [&str; 2] = ["CLAUDECODE", "CLAUDE_PROJECT_DIR"]; struct StartupContext { observability_config: services::config::ResolvedObservabilityRuntimeConfig, @@ -275,7 +279,7 @@ where StderrW: Write, { let args = args.into_iter().collect::>(); - let managed_allowed = !is_hook_invocation(&args); + let span_kind = managed_span_kind(&args, is_claude_hook_caller()); let startup = perform_dependency_check(dependency_check) .and_then(|()| build_startup_context()) .and_then(initialize_runtime); @@ -293,24 +297,35 @@ where ); } }; - runtime.telemetry = RuntimeTelemetry::resolve(managed_allowed).await; + runtime.telemetry = RuntimeTelemetry::resolve(span_kind).await; let rendered = run_command_lifecycle(args, &runtime, stdout, stderr).await; runtime.telemetry.shutdown().await; app_support::exit_code_for(&rendered) } -fn is_hook_invocation(args: &[String]) -> bool { +fn managed_span_kind(args: &[String], claude_hook_caller: bool) -> Option { let mut words = args .iter() .skip(1) .map(String::as_str) .filter(|arg| !arg.starts_with('-')); - match (words.next(), words.next()) { - (Some(first), _) if first == services::hooks::NAME => true, - (Some("policy"), Some("bash")) => true, - (Some(first), Some("telemetry-headers")) if first == services::auth_command::NAME => true, - _ => false, - } + let claude_hook = match (words.next(), words.next()) { + (Some(first), Some(subcommand)) if first == services::hooks::NAME => { + CLAUDE_ONLY_HOOK_SUBCOMMANDS.contains(&subcommand) + || (claude_hook_caller && SHARED_CLAUDE_HOOK_SUBCOMMANDS.contains(&subcommand)) + } + (Some(first), _) if first == services::hooks::NAME => false, + (Some(first), Some("bash")) if first == services::bash_policy::NAME => claude_hook_caller, + (Some(first), Some("telemetry-headers")) if first == services::auth_command::NAME => false, + _ => return Some(ManagedSpanKind::Command), + }; + claude_hook.then_some(ManagedSpanKind::ClaudeHook) +} + +fn is_claude_hook_caller() -> bool { + CLAUDE_HOOK_CALLER_MARKERS + .iter() + .all(|marker| std::env::var_os(marker).is_some()) } fn perform_dependency_check anyhow::Result<()>>( @@ -924,9 +939,15 @@ pub(crate) mod tests { } let runtime = build_runtime().unwrap(); runtime.block_on(async { - let keychain = RuntimeTelemetry::resolve_with(stalled_keychain_lookup()).await; + let keychain = + RuntimeTelemetry::resolve_with(stalled_keychain_lookup(), ManagedSpanKind::Command) + .await; assert!(matches!(keychain, RuntimeTelemetry::Noop(_))); - let database = RuntimeTelemetry::resolve_with(stalled_blocking_database_lookup()).await; + let database = RuntimeTelemetry::resolve_with( + stalled_blocking_database_lookup(), + ManagedSpanKind::ClaudeHook, + ) + .await; assert!(matches!(database, RuntimeTelemetry::Noop(_))); }); finish_runtime(runtime); @@ -1073,24 +1094,52 @@ pub(crate) mod tests { } #[test] - fn hook_invocations_never_resolve_managed_telemetry() { - let hook = ["sce", "hooks", "pre-commit"].map(String::from); - let flagged_hook = ["sce", "--verbose", "hooks", "pre-commit"].map(String::from); - let other = ["sce", "config", "show"].map(String::from); - let policy = ["sce", "policy", "bash"].map(String::from); - let headers = ["sce", "auth", "telemetry-headers"].map(String::from); - let auth_login = ["sce", "auth", "login"].map(String::from); - assert!(is_hook_invocation(&policy)); - assert!(is_hook_invocation(&headers)); - assert!(!is_hook_invocation(&auth_login)); - assert!(is_hook_invocation(&hook)); - assert!(is_hook_invocation(&flagged_hook)); - assert!(!is_hook_invocation(&other)); + fn only_claude_hook_invocations_are_admitted_to_managed_hook_telemetry() { + let kind = |args: &[&str], claude_caller: bool| { + let args = args.iter().map(ToString::to_string).collect::>(); + managed_span_kind(&args, claude_caller) + }; + let claude_hook = Some(ManagedSpanKind::ClaudeHook); + let command = Some(ManagedSpanKind::Command); + for caller in [false, true] { + assert_eq!( + kind(&["sce", "hooks", "claude-mutation-scope"], caller), + claude_hook + ); + assert_eq!( + kind(&["sce", "hooks", "claude-model-state"], caller), + claude_hook + ); + assert_eq!(kind(&["sce", "hooks", "pre-commit"], caller), None); + assert_eq!(kind(&["sce", "--verbose", "hooks", "codex"], caller), None); + assert_eq!( + kind(&["sce", "hooks", "codex-mutation-scope"], caller), + None + ); + assert_eq!( + kind(&["sce", "hooks", "opencode-mutation-scope"], caller), + None + ); + assert_eq!(kind(&["sce", "hooks", "pi-mutation-scope"], caller), None); + assert_eq!(kind(&["sce", "hooks"], caller), None); + assert_eq!(kind(&["sce", "auth", "telemetry-headers"], caller), None); + assert_eq!(kind(&["sce", "auth", "login"], caller), command); + assert_eq!(kind(&["sce", "config", "show"], caller), command); + assert_eq!(kind(&["sce", "doctor"], caller), command); + } + for shared in [ + &["sce", "hooks", "diff-trace"][..], + &["sce", "hooks", "conversation-trace"], + &["sce", "policy", "bash"], + ] { + assert_eq!(kind(shared, true), claude_hook); + assert_eq!(kind(shared, false), None); + } } #[tokio::test] async fn disallowed_managed_telemetry_resolves_to_noop_without_credential_lookup() { - let telemetry = RuntimeTelemetry::resolve(false).await; + let telemetry = RuntimeTelemetry::resolve(None).await; assert!(matches!(telemetry, RuntimeTelemetry::Noop(_))); } } diff --git a/cli/src/services/bash_policy.rs b/cli/src/services/bash_policy.rs index c0712bd1a..d5abad2c0 100644 --- a/cli/src/services/bash_policy.rs +++ b/cli/src/services/bash_policy.rs @@ -457,6 +457,7 @@ struct JsonPolicyResult<'a> { pub fn run_bash_policy_request(request: &BashPolicyRequest) -> Result { let stdin_payload = read_stdin_payload()?; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); run_bash_policy_request_from_payload(request, &stdin_payload) } diff --git a/cli/src/services/hooks/claude_model_state.rs b/cli/src/services/hooks/claude_model_state.rs index 17a3e4f99..c6b64e7fb 100644 --- a/cli/src/services/hooks/claude_model_state.rs +++ b/cli/src/services/hooks/claude_model_state.rs @@ -36,6 +36,7 @@ pub(super) async fn run_claude_model_state_subcommand< return String::new(); } }; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); let session_id = fail_open_session_id(&stdin_payload); let observed_at_ms = match current_unix_time_ms() { diff --git a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs index 85f01c29b..ccfbc383e 100644 --- a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs +++ b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs @@ -47,6 +47,7 @@ pub(crate) async fn run_claude_mutation_scope_subcommand< logger: Option<&L>, ) -> Result { let stdin_payload = hooks::read_hook_stdin()?; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); run_claude_mutation_scope_from_payload(&stdin_payload, logger).await } @@ -504,3 +505,105 @@ pub(crate) async fn repair_blocked>, + seam_payloads: Vec, + state: state::AdapterState, + } + + async fn run_hook_sequence(telemetry: &T) -> HookRun { + let git_dir = tempfile::tempdir().expect("temp dir"); + let seam_payloads = Mutex::new(Vec::new()); + let resolve = async |_: &str| Ok::<_, anyhow::Error>(git_dir.path().to_path_buf()); + let model_state = async |_: &Path, _: &str, _: &str| Ok::<_, anyhow::Error>(None); + let seam = async |_: &Path, payload: &str, _: Option<&NoopLogger>| { + let mut payloads = seam_payloads.lock().expect("seam payloads"); + payloads.push(payload.to_string()); + if payloads.len() == FAILING_SEAM_CALL { + anyhow::bail!("seam failure"); + } + Ok(String::new()) + }; + let mut outputs = Vec::new(); + for payload in HOOK_SEQUENCE { + let output = telemetry + .with_default_subscriber(&mut || async { + record_claude_hook_payload(payload); + run_claude_mutation_scope_from_payload_with_resolver::( + payload, + None, + &resolve, + &model_state, + &seam, + ) + .await + .map_err(CliError::runtime) + }) + .await; + outputs.push(output.ok()); + } + HookRun { + outputs, + seam_payloads: seam_payloads.into_inner().expect("seam payloads"), + state: state::read_state(git_dir.path()).await.expect("state"), + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn failing_managed_telemetry_cannot_change_claude_hook_behavior() { + let baseline = run_hook_sequence(&NoopTelemetry).await; + assert!(baseline + .outputs + .iter() + .flatten() + .any(|output| output.contains("\"permissionDecision\":\"deny\""))); + + let refused = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let refused_address = refused.local_addr().expect("addr"); + drop(refused); + let stalled = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let stalled_address = stalled.local_addr().expect("addr"); + let holder = tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((stream, _)) = stalled.accept().await { + held.push(stream); + } + }); + + for address in [refused_address, stalled_address] { + let telemetry = loopback_hook_telemetry(address, "bao-token-value"); + let observed = run_hook_sequence(&telemetry).await; + let started = Instant::now(); + telemetry.shutdown().await; + assert!(started.elapsed() < SHUTDOWN_TIMEOUT + SHUTDOWN_MARGIN); + assert_eq!(observed, baseline); + } + holder.abort(); + } +} diff --git a/cli/src/services/hooks/conversation_trace.rs b/cli/src/services/hooks/conversation_trace.rs index 739bbd826..e93c18fc1 100644 --- a/cli/src/services/hooks/conversation_trace.rs +++ b/cli/src/services/hooks/conversation_trace.rs @@ -74,6 +74,7 @@ pub(crate) async fn run_conversation_trace_subcommand< Ok(payload) => payload, Err(error) => return log_conversation_trace_fail_open(&error, logger, None), }; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); let session_id = conversation_trace_fail_open_session_id(&stdin_payload); match run_conversation_trace_subcommand_from_payload( diff --git a/cli/src/services/hooks/diff_trace.rs b/cli/src/services/hooks/diff_trace.rs index 65ee8e892..0242e35ad 100644 --- a/cli/src/services/hooks/diff_trace.rs +++ b/cli/src/services/hooks/diff_trace.rs @@ -71,6 +71,7 @@ pub(crate) async fn run_diff_trace_subcommand payload, Err(error) => return log_diff_trace_fail_open(&error, logger, None), }; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); let session_id = diff_trace_fail_open_session_id(&stdin_payload); match run_diff_trace_subcommand_from_payload(repository_root, &stdin_payload, logger).await { diff --git a/cli/src/services/managed_telemetry.rs b/cli/src/services/managed_telemetry.rs index 74831a1ce..b2b1a3759 100644 --- a/cli/src/services/managed_telemetry.rs +++ b/cli/src/services/managed_telemetry.rs @@ -8,7 +8,7 @@ pub mod trace_export; use crate::services::error::CliError; use crate::services::observability::traits::{NoopTelemetry, Telemetry}; -use trace_export::ManagedTraceTelemetry; +use trace_export::{ManagedSpanKind, ManagedTraceTelemetry}; use crate::services::telemetry_credential_storage::{ load_telemetry_credential, TelemetryCredential, @@ -130,14 +130,14 @@ pub enum RuntimeTelemetry { } impl RuntimeTelemetry { - pub async fn resolve(managed_allowed: bool) -> Self { - if !managed_allowed { + pub async fn resolve(span_kind: Option) -> Self { + let Some(span_kind) = span_kind else { return Self::Noop(NoopTelemetry); - } - Self::resolve_with(load_telemetry_credential()).await + }; + Self::resolve_with(load_telemetry_credential(), span_kind).await } - pub(crate) async fn resolve_with(lookup: Fut) -> Self + pub(crate) async fn resolve_with(lookup: Fut, span_kind: ManagedSpanKind) -> Self where Fut: Future, TokenStorageError>> + Send @@ -149,7 +149,7 @@ impl RuntimeTelemetry { }; match select_mode(credential, now_unix_seconds()) { ManagedTelemetryMode::Managed { bearer, endpoints } => { - ManagedTraceTelemetry::new(&bearer, endpoints, runtime) + ManagedTraceTelemetry::new(&bearer, endpoints, runtime, span_kind) .map_or(Self::Noop(NoopTelemetry), Self::Managed) } ManagedTelemetryMode::Noop => Self::Noop(NoopTelemetry), @@ -240,6 +240,22 @@ mod tests { } } + #[tokio::test] + async fn claude_hook_credential_failure_or_expiry_resolves_noop() { + let failed = RuntimeTelemetry::resolve_with( + async { Err(TokenStorageError::Database("locked".to_string())) }, + ManagedSpanKind::ClaudeHook, + ) + .await; + assert!(matches!(failed, RuntimeTelemetry::Noop(_))); + let expired = RuntimeTelemetry::resolve_with( + async { Ok(Some(credential("bao-token", 1))) }, + ManagedSpanKind::ClaudeHook, + ) + .await; + assert!(matches!(expired, RuntimeTelemetry::Noop(_))); + } + #[test] fn absent_expired_or_blank_credential_selects_noop() { assert_eq!(select_mode(None, NOW), ManagedTelemetryMode::Noop); diff --git a/cli/src/services/managed_telemetry/trace_export.rs b/cli/src/services/managed_telemetry/trace_export.rs index f629d90c4..de57ed716 100644 --- a/cli/src/services/managed_telemetry/trace_export.rs +++ b/cli/src/services/managed_telemetry/trace_export.rs @@ -27,7 +27,19 @@ const SERVICE_NAME: &str = "sce"; const TRACER_NAME: &str = "sce.cli"; pub const COMMAND_SPAN_TARGET: &str = "sce.managed.trace"; const COMMAND_SPAN_NAME: &str = "sce.command"; +const HOOK_SPAN_NAME: &str = "sce.hook"; pub const COMMAND_NAME_FIELD: &str = "sce.command.name"; +const AGENT_CLAUDE: &str = "claude"; +const HOOK_EVENT_FIELD: &str = "sce.hook.event"; +const HOOK_TOOL_FIELD: &str = "sce.hook.tool"; +const CLAUDE_SESSION_ID_FIELD: &str = "sce.claude.session_id"; +const CLAUDE_TOOL_USE_ID_FIELD: &str = "sce.claude.tool_use_id"; +const CLAUDE_PAYLOAD_FIELDS: [(&str, &str); 4] = [ + ("hook_event_name", HOOK_EVENT_FIELD), + ("tool_name", HOOK_TOOL_FIELD), + ("session_id", CLAUDE_SESSION_ID_FIELD), + ("tool_use_id", CLAUDE_TOOL_USE_ID_FIELD), +]; const OUTCOME_FIELD: &str = "sce.outcome"; const ERROR_CLASS_FIELD: &str = "sce.error.class"; const OUTCOME_SUCCESS: &str = "success"; @@ -73,11 +85,18 @@ fn is_command_span(metadata: &tracing::Metadata<'_>) -> bool { metadata.is_span() && metadata.target() == COMMAND_SPAN_TARGET } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ManagedSpanKind { + Command, + ClaudeHook, +} + #[derive(Debug)] pub struct ManagedTraceTelemetry { provider: SdkTracerProvider, log_provider: SdkLoggerProvider, dispatch: Dispatch, + span_kind: ManagedSpanKind, } impl ManagedTraceTelemetry { @@ -85,12 +104,13 @@ impl ManagedTraceTelemetry { bearer: &ManagedBearerToken, endpoints: ManagedOtlpEndpoints, runtime: tokio::runtime::Handle, + span_kind: ManagedSpanKind, ) -> anyhow::Result { let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .timeout(REQUEST_TIMEOUT) .build()?; - Self::with_client(bearer, endpoints, runtime, client) + Self::with_client(bearer, endpoints, runtime, client, span_kind) } fn with_client( @@ -98,6 +118,7 @@ impl ManagedTraceTelemetry { endpoints: ManagedOtlpEndpoints, runtime: tokio::runtime::Handle, client: reqwest::Client, + span_kind: ManagedSpanKind, ) -> anyhow::Result { let mut authorization = HeaderValue::from_str(&bearer.authorization_header_value())?; authorization.set_sensitive(true); @@ -170,6 +191,7 @@ impl ManagedTraceTelemetry { provider, log_provider, dispatch, + span_kind, }) } @@ -179,6 +201,7 @@ impl ManagedTraceTelemetry { provider, log_provider, dispatch, + .. } = self; drop(dispatch); let traces = @@ -193,15 +216,26 @@ impl ManagedTraceTelemetry { .await; } - fn command_span(&self) -> Span { - tracing::dispatcher::with_default(&self.dispatch, || { - tracing::info_span!( + fn root_span(&self) -> Span { + tracing::dispatcher::with_default(&self.dispatch, || match self.span_kind { + ManagedSpanKind::Command => tracing::info_span!( target: COMMAND_SPAN_TARGET, COMMAND_SPAN_NAME, sce.command.name = field::Empty, sce.outcome = field::Empty, sce.error.class = field::Empty, - ) + ), + ManagedSpanKind::ClaudeHook => tracing::info_span!( + target: COMMAND_SPAN_TARGET, + HOOK_SPAN_NAME, + sce.agent = AGENT_CLAUDE, + sce.hook.event = field::Empty, + sce.hook.tool = field::Empty, + sce.claude.session_id = field::Empty, + sce.claude.tool_use_id = field::Empty, + sce.outcome = field::Empty, + sce.error.class = field::Empty, + ), }) } } @@ -212,7 +246,7 @@ impl Telemetry for ManagedTraceTelemetry { F: FnMut() -> Fut, Fut: Future>, { - let span = self.command_span(); + let span = self.root_span(); let result = action() .instrument(span.clone()) .with_subscriber(self.dispatch.clone()) @@ -235,6 +269,25 @@ pub fn record_command_name(name: &str) { Span::current().record(COMMAND_NAME_FIELD, name); } +pub fn record_claude_hook_payload(stdin_payload: &str) { + let span = Span::current(); + if span.is_disabled() { + return; + } + let Ok(serde_json::Value::Object(payload)) = serde_json::from_str(stdin_payload) else { + return; + }; + for (payload_key, span_field) in CLAUDE_PAYLOAD_FIELDS { + if let Some(value) = payload + .get(payload_key) + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + { + span.record(span_field, value); + } + } +} + #[cfg(test)] pub(crate) mod tests { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -308,9 +361,143 @@ pub(crate) mod tests { } } + #[derive(Debug, Default)] + pub(crate) struct DecodedSpan { + pub(crate) name: String, + pub(crate) trace_id: Vec, + pub(crate) span_id: Vec, + pub(crate) parent_span_id: Vec, + pub(crate) attributes: Vec<(String, String)>, + pub(crate) resource_attributes: Vec<(String, String)>, + } + + impl DecodedSpan { + pub(crate) fn sce_attributes(&self) -> Vec<(String, String)> { + let mut attributes = self + .attributes + .iter() + .filter(|(key, _)| key.starts_with("sce.")) + .cloned() + .collect::>(); + attributes.sort(); + attributes + } + } + + fn read_varint(bytes: &[u8], position: &mut usize) -> u64 { + let mut value = 0_u64; + let mut shift = 0; + loop { + let byte = bytes[*position]; + *position += 1; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return value; + } + shift += 7; + } + } + + fn length_delimited_fields(bytes: &[u8]) -> Vec<(u64, &[u8])> { + let mut fields = Vec::new(); + let mut position = 0; + while position < bytes.len() { + let key = read_varint(bytes, &mut position); + match key & 0x7 { + 0 => { + read_varint(bytes, &mut position); + } + 1 => position += 8, + 2 => { + let length = usize::try_from(read_varint(bytes, &mut position)).unwrap(); + fields.push((key >> 3, &bytes[position..position + length])); + position += length; + } + 5 => position += 4, + wire => panic!("unsupported protobuf wire type {wire}"), + } + } + fields + } + + fn fields_numbered(bytes: &[u8], number: u64) -> impl Iterator { + length_delimited_fields(bytes) + .into_iter() + .filter(move |(field, _)| *field == number) + .map(|(_, value)| value) + } + + fn decode_string_attribute(bytes: &[u8]) -> Option<(String, String)> { + let key = fields_numbered(bytes, 1).next()?; + let value = fields_numbered(bytes, 2).next()?; + let string = fields_numbered(value, 1).next()?; + Some(( + String::from_utf8_lossy(key).to_string(), + String::from_utf8_lossy(string).to_string(), + )) + } + + pub(crate) fn decode_spans(body: &[u8]) -> Vec { + let mut spans = Vec::new(); + for resource_spans in fields_numbered(body, 1) { + let resource_attributes = fields_numbered(resource_spans, 1) + .flat_map(|resource| fields_numbered(resource, 1)) + .filter_map(decode_string_attribute) + .collect::>(); + for scope_spans in fields_numbered(resource_spans, 2) { + for span in fields_numbered(scope_spans, 2) { + let mut decoded = DecodedSpan { + resource_attributes: resource_attributes.clone(), + ..DecodedSpan::default() + }; + for (field, value) in length_delimited_fields(span) { + match field { + 1 => decoded.trace_id = value.to_vec(), + 2 => decoded.span_id = value.to_vec(), + 4 => decoded.parent_span_id = value.to_vec(), + 5 => decoded.name = String::from_utf8_lossy(value).to_string(), + 9 => decoded.attributes.extend(decode_string_attribute(value)), + _ => {} + } + } + spans.push(decoded); + } + } + } + spans + } + + pub(crate) fn trace_spans(requests: &[CapturedRequest]) -> Vec { + requests + .iter() + .filter(|request| { + request + .head + .to_ascii_lowercase() + .starts_with("post /v1/traces ") + }) + .flat_map(|request| decode_spans(&request.body)) + .collect() + } + pub(crate) fn loopback_telemetry( address: std::net::SocketAddr, token: &str, + ) -> ManagedTraceTelemetry { + loopback_telemetry_of_kind(address, token, ManagedSpanKind::Command) + } + + pub(crate) fn loopback_hook_telemetry( + address: std::net::SocketAddr, + token: &str, + ) -> ManagedTraceTelemetry { + loopback_telemetry_of_kind(address, token, ManagedSpanKind::ClaudeHook) + } + + fn loopback_telemetry_of_kind( + address: std::net::SocketAddr, + token: &str, + span_kind: ManagedSpanKind, ) -> ManagedTraceTelemetry { let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) @@ -318,13 +505,14 @@ pub(crate) mod tests { .tls_certs_only(Vec::new()) .build() .expect("test http client"); - loopback_telemetry_with_client(address, token, client) + loopback_telemetry_with_client(address, token, client, span_kind) } fn loopback_telemetry_with_client( address: std::net::SocketAddr, token: &str, client: reqwest::Client, + span_kind: ManagedSpanKind, ) -> ManagedTraceTelemetry { let traces_endpoint: &'static str = Box::leak(format!("http://{address}/v1/traces").into_boxed_str()); @@ -357,6 +545,7 @@ pub(crate) mod tests { endpoints, tokio::runtime::Handle::current(), client, + span_kind, ) .expect("telemetry") } @@ -446,6 +635,61 @@ pub(crate) mod tests { assert!(!body.contains("bao-token-value")); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn claude_hook_export_posts_one_hook_span_with_correlation_ids_only() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr"); + let receiver = tokio::spawn(capture_requests(listener, "HTTP/1.1 200 OK", 1)); + let telemetry = loopback_hook_telemetry(address, "bao-token-value"); + let payload = r#"{"session_id":"session-7d07","transcript_path":"/home/u/private-repo/transcript.jsonl","cwd":"/home/u/private-repo","prompt":"PROMPT_TEXT_VALUE","hook_event_name":"PostToolUse","tool_name":"Write","tool_input":{"file_path":"/home/u/private-repo/src/secret.rs","content":"FILE_CONTENT_VALUE Bearer leak-value"},"tool_response":{"success":true,"output":"TOOL_RESPONSE_VALUE"},"tool_use_id":"toolu_01KN1s"}"#; + + let result = telemetry + .with_default_subscriber(&mut || async { + record_command_name("hooks"); + record_claude_hook_payload(payload); + tokio::task::yield_now().await; + Ok(String::new()) + }) + .await; + assert_eq!(result.expect("hook result"), ""); + telemetry.shutdown().await; + + let requests = receiver.await.expect("receiver join"); + let spans = trace_spans(&requests); + assert_eq!(spans.len(), 1, "{spans:?}"); + let span = &spans[0]; + assert_eq!(span.name, "sce.hook"); + assert!(span.parent_span_id.is_empty()); + assert!(span + .resource_attributes + .contains(&("service.name".to_string(), "sce".to_string()))); + let expected = [ + ("sce.agent", "claude"), + ("sce.claude.session_id", "session-7d07"), + ("sce.claude.tool_use_id", "toolu_01KN1s"), + ("sce.hook.event", "PostToolUse"), + ("sce.hook.tool", "Write"), + ("sce.outcome", "success"), + ] + .map(|(key, value)| (key.to_string(), value.to_string())); + assert_eq!(span.sce_attributes(), expected); + for request in &requests { + let body = String::from_utf8_lossy(&request.body).to_string(); + for leaked in [ + "private-repo", + "PROMPT_TEXT_VALUE", + "FILE_CONTENT_VALUE", + "TOOL_RESPONSE_VALUE", + "leak-value", + "bao-token-value", + "local-accessor-secret", + "sce.command", + ] { + assert!(!body.contains(leaked), "leaked {leaked}"); + } + } + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn managed_trace_export_does_not_follow_redirects_or_alter_results() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); @@ -647,7 +891,12 @@ pub(crate) mod tests { .build() .expect("test http client"); let telemetry = - loopback_telemetry_with_client(address, "bao-token-value", unbounded_client); + loopback_telemetry_with_client( + address, + "bao-token-value", + unbounded_client, + ManagedSpanKind::Command, + ); let result = telemetry .with_default_subscriber(&mut || async { record_command_name("config"); diff --git a/context/architecture.md b/context/architecture.md index c13ca8749..08f752e9b 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -53,7 +53,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - Command parse-time conversion and run-time handling are separated by an internal static `RuntimeCommand` seam. `cli/src/services/command_registry.rs` defines the `RuntimeCommand` enum with variants for help/help-text, version, completion, auth, config, setup, doctor, hooks, policy, and sync, plus a deterministic `CommandRegistry` name catalog populated by `build_default_registry()`. `parse_command_phase` in `cli/src/app.rs` delegates clap-output conversion to `cli/src/services/parse/command_runtime.rs`, which owns clap error classification, help rendering bridges, longest-valid-parent traversal for unknown command paths, and parsed-request-to-enum conversion while returning concrete enum values. Service-owned `command.rs` modules define command payload structs and generic execution methods with narrow context requirements: context-free commands accept any context, hooks requires logger access, setup/doctor require repo-root scoping, and central dispatch requires the union of logger plus repo-root-scoping capabilities. `services::app_support::execute_command_phase` emits lifecycle logs around `RuntimeCommand::execute_with_stderr(...)`; the enum performs the only central dispatch match and delegates business behavior to the service-owned command structs, with sync receiving the app-owned stderr writer for format-gated progress. - Startup observability bootstrapping in `cli/src/app.rs` still tolerates invalid default-discovered config files by continuing with degraded defaults plus `sce.config.invalid_config` warn-level logs, but the warning/logging work is now isolated behind the startup-context and runtime-initialization phases rather than one inline startup function. - `cli/src/services/observability.rs` provides deterministic runtime observability controls and rendering for app lifecycle logs, including shared config-resolved threshold/format, explicit config-file/default `log_to_file`, and `log_dir` inputs with precedence `env > config file > defaults` for non-flag observability keys, stable event identifiers, severity filtering, the forced-emission warning path used for invalid discovered config startup diagnostics, error-specific stderr suppression when file logging is enabled while non-error records and file-write diagnostics remain on stderr, redaction-safe emission through the shared security helper, and log-directory writes with bounded retention. Config resolution also carries a positive config-file/default-only `log_file_retention_limit` (`10` by default) into startup observability config and `sce config show`; the concrete logger stores that resolved value and threads it through primary and v2 cleanup. When `log_dir` resolves from `SCE_LOG_DIR`, config, or the `/sce/logs` default, each enabled or forced log operation selects `/sce-.log` or `/sce--.log` using the machine-local date and optional logger session context, with deterministic percent-encoding for unsafe session filename bytes; after successfully writing a newly created selected file, retention keeps the configured number of newest direct regular `*.log` files by mtime plus path/name tie-break and fails open on cleanup errors. Its `observability::traits` submodule exposes the current `Logger` API with `Option<&str>` session context plus the generic async, deliberately non-object-safe `Telemetry` capability and `NoopLogger`; the concrete observability `Logger` and `NoopTelemetry` implement those capability traits; `NoopTelemetry` installs no tracing subscriber. `services::app_support::render_run_outcome` consumes the logger through that trait boundary when logging classified errors and stdout-write failures. -- `cli/src/services/managed_telemetry.rs` (+ `managed_telemetry/trace_export.rs`, `managed_telemetry/log_export.rs`) owns hardcoded managed OTLP defaults, credential-backed runtime telemetry selection from the encrypted auth DB, and bounded fail-open OTLP/HTTP protobuf export of `sce.command` spans and allowlisted `Logger` events to the pinned canonical collector (metrics endpoint reserved, not exported); see `context/sce/cli-observability-contract.md`. +- `cli/src/services/managed_telemetry.rs` (+ `managed_telemetry/trace_export.rs`, `managed_telemetry/log_export.rs`) owns hardcoded managed OTLP defaults, credential-backed runtime telemetry selection from the encrypted auth DB, and bounded fail-open OTLP/HTTP protobuf export of `sce.command` spans (or one `sce.hook` span for Claude-invoked hooks admitted by `managed_span_kind` in `cli/src/app.rs`) and allowlisted `Logger` events to the pinned canonical collector (metrics endpoint reserved, not exported); see `context/sce/cli-observability-contract.md`. - `cli/src/services/observability.rs` no longer owns duplicate log enums or parsing helpers; it consumes the canonical primitive seam from `cli/src/services/config/mod.rs` and stays focused on concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary. - `cli/src/cli_schema.rs` is now the canonical owner for top-level command metadata for the real clap-backed command set (`auth`, `config`, `setup`, `doctor`, `hooks`, `policy`, `sync`, `version`, `completion`), including the slim top-level help purpose text and per-command visibility on `sce`, `sce help`, and `sce --help`; `cli/src/command_surface.rs` remains the custom top-level help renderer and known-command classifier, adding the synthetic `help` row plus the ASCII banner while consuming that shared metadata instead of maintaining a parallel command catalog. - `cli/src/services/default_paths.rs` is the canonical production path catalog for the CLI: it resolves config/state/cache roots with platform-aware XDG or `dirs` fallbacks through an internal `roots` seam, exposes named default paths for current persisted artifacts and database/log files (global config, auth tokens, auth DB, local DB, default observability log directory, and the sole Agent Trace DB path helper `agent_trace_db_path_for_repository` under `repos//agent-trace.db`; the former global-sentinel and per-checkout Agent Trace path helpers were removed by the `retire-legacy-agent-trace-db` plan), and owns canonical repo-relative, embedded-asset, install, hook, and context-path accessors so non-test production path definitions have one shared owner. Compile-time generated payload paths are owned by `build.rs` under `OUT_DIR`, not by the default-path catalog. Current production consumers such as config discovery, observability config resolution, doctor reporting, setup/install flows, database adapters, checkout identity, Agent Trace storage resolution, and local hook runtime path resolution consume this shared catalog rather than defining owned path literals in their own modules. diff --git a/context/context-map.md b/context/context-map.md index 023713127..a17e8a208 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -56,8 +56,8 @@ Feature/domain context: - `context/sce/cli-version-command-contract.md` (implemented `sce version` contract for deterministic human and machine-readable runtime identification) - `context/sce/cli-shell-completion-contract.md` (implemented `sce completion` contract for deterministic Bash/Zsh/Fish completion script generation) - `context/sce/claude-raw-hook-capture.md` (removed feature: the former `sce hooks claude-capture` raw-capture route and its supporting types, replaced by the active `diff-trace` and `conversation-trace` intakes) -- `context/sce/claude-hook-trace-propagation.md` (probe evidence for Claude Code 2.1.301 command-hook trace context: `TRACEPARENT` names the `claude_code.interaction` span, not the tool span, and only when Claude tracing is active; hook-to-tool correlation by `tool_use_id`; `CLAUDECODE` + `CLAUDE_PROJECT_DIR` presence as the Claude hook-caller signal; no SCE hook telemetry ships yet) -- `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary, logger and deliberately non-object-safe generic async telemetry boundaries with static capability dispatch and future-aware polling scope, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, managed OTLP trace/log export (credential-gated, hardcoded canonical collector, allowlisted fields, bounded fail-open lifecycle, reserved metrics endpoint), and the trimmed `sce config validate` status-only validation surface) +- `context/sce/claude-hook-trace-propagation.md` (probe evidence for Claude Code 2.1.301 command-hook trace context: `TRACEPARENT` names the `claude_code.interaction` span, not the tool span, and only when Claude tracing is active; hook-to-tool correlation by `tool_use_id`; `CLAUDECODE` + `CLAUDE_PROJECT_DIR` presence as the Claude hook-caller signal; the shipped `sce.hook` contract lives in `cli-observability-contract.md`, admission decision `context/decisions/2026-10-17-claude-hook-managed-trace-admission.md`) +- `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary, logger and deliberately non-object-safe generic async telemetry boundaries with static capability dispatch and future-aware polling scope, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, managed OTLP trace/log export (credential-gated, hardcoded canonical collector, Claude-only hook admission with one `sce.hook` span and correlation-only session/tool-use IDs, allowlisted fields, bounded fail-open lifecycle, reserved metrics endpoint), and the trimmed `sce config validate` status-only validation surface) - `context/sce/shared-context-code-workflow.md` (canonical `/next-task` task-synchronization lifecycle and validation-only `/validate` lifecycle, package-local phase references with single-skill control flow, and the task-synchronization-scoped `sce-decision` sibling invocation with ADR reuse/blocker propagation) - `context/sce/shared-context-plan-workflow.md` (canonical `/change-to-plan` workflow, package-local context-load/plan-authoring/template references, clarification/readiness gate contract, and one-task/one-atomic-commit task slicing) - [Context workflow rules](sce/context-workflow-rules.md) (canonical bootstrap, ongoing context maintenance, task synchronization, hygiene, discoverability, and feature-existence rules) diff --git a/context/decisions/2026-10-17-claude-hook-managed-trace-admission.md b/context/decisions/2026-10-17-claude-hook-managed-trace-admission.md new file mode 100644 index 000000000..b03919f2b --- /dev/null +++ b/context/decisions/2026-10-17-claude-hook-managed-trace-admission.md @@ -0,0 +1,69 @@ +# Decision: Admit Claude-invoked SCE hooks to managed trace export as one `sce.hook` span + +Date: 2026-10-17 +Status: Accepted +Plan: `context/plans/claude-hook-trace-propagation.md` +Task: `T02` + +## Context + +Managed OTLP export previously forced every `sce hooks ...` invocation, `sce policy bash`, and the hidden `sce auth telemetry-headers` helper to `NoopTelemetry`. That kept hook processes free of credential lookup and exporter shutdown latency, and it kept hook data off the network. Claude Code runs SCE hooks around every tool call. Correlating those hook runs with Claude Code's own exported traces requires SCE hook processes to export something. Probe evidence for Claude Code 2.1.301 showed two things. First, the Claude-only hook subcommands can be recognized from argv. Second, Claude command hooks can be told apart from the Claude Bash tool by the presence of both `CLAUDECODE` and `CLAUDE_PROJECT_DIR`. + +## Decision + +Claude-invoked SCE hook processes are admitted to managed trace export. Each admitted process exports exactly one root-of-process `sce.hook` span in place of `sce.command`. + +- **Admitted from argv:** `sce hooks claude-mutation-scope` and `sce hooks claude-model-state`. +- **Admitted only when `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are both present** (checked for presence only): `sce hooks diff-trace`, `sce hooks conversation-trace`, and `sce policy bash`. +- **Never admitted:** every other hook (commit, Codex, OpenCode, and Pi) and `sce auth telemetry-headers`. These stay on `NoopTelemetry`. +- **Exported attributes:** `sce.agent=claude`, `sce.hook.event`, `sce.hook.tool`, `sce.outcome`, `sce.error.class`, and the Claude `session_id` and `tool_use_id`. The two IDs are correlation-only span attributes. + +## Rationale + +- Admitting only Claude callers delivers the correlation value without changing the latency or privacy profile of other harnesses' hooks. +- Replacing `sce.command` instead of nesting under it keeps one span per hook process. +- Session and tool-use IDs are opaque identifiers, not payload content. They are the only fields that join an SCE hook run to Claude's `claude_code.tool` span. +- Presence-only environment checks avoid reading values. + +## Alternatives considered + +- **Keep all hooks on Noop** — rejected because it gives no way to correlate SCE hook work with Claude Code traces. +- **Admit every hook process** — rejected because it widens latency and data exposure to Codex, OpenCode, Pi, and commit hooks, which have no proven correlation need. +- **Use `CLAUDECODE` alone as the caller signal** — rejected because the Claude Bash tool also inherits it, so commands run by the agent would be misclassified as hooks. +- **Nest `sce.hook` under `sce.command`** — rejected because it adds a redundant wrapper span per hook process. + +## Compatibility and risks + +- This reverses the earlier guarantee that hooks never resolve managed telemetry. +- Admitted Claude hook processes, including the blocking `PreToolUse` path, can wait up to the existing `CREDENTIAL_LOOKUP_DEADLINE` (2s) plus `SHUTDOWN_TIMEOUT` (3s), under the runtime grace bound. Whether hooks need a tighter, hook-specific deadline remains an open question in the plan. +- Claude session and tool-use IDs now leave the machine when a valid telemetry credential is stored. They go only to the pinned managed collector. + +## Guardrails + +- Only the four string fields `hook_event_name`, `tool_name`, `session_id`, and `tool_use_id` are read from the hook payload for telemetry. Nothing is read when the current span is disabled. +- Tool input and response, prompts, `cwd`, transcript paths, file contents, raw payloads, the OpenBao token, and the accessor are never exported. +- Session and tool-use IDs are never metric attributes and are never used to derive a trace ID. +- Telemetry stays fail-open. Credential, exporter, or collector failures cannot change hook stdout, permission decisions, persisted state, or exit codes. +- Endpoint pinning, the bearer-only header, disabled redirects, and the existing bounded shutdown are unchanged. + +## Consequences + +- Claude-invoked SCE hook runs become visible as `sce.hook` spans. They can be joined to Claude Code tool spans by `tool_use_id`. +- Non-Claude hook processes keep their previous Noop behavior and cost. +- The admission predicate in `cli/src/app.rs` is now part of the telemetry privacy contract and is covered by an admission test matrix. + +## Follow-up + +- Parenting `sce.hook` on the Claude-provided `TRACEPARENT` and adding child spans are tracked by later tasks of the plan. + +## References + +- Plan: [`claude-hook-trace-propagation`](../plans/claude-hook-trace-propagation.md) +- Task: `T02` +- Current-state context: [`cli-observability-contract.md`](../sce/cli-observability-contract.md) +- Current-state context: [`claude-hook-trace-propagation.md`](../sce/claude-hook-trace-propagation.md) +- Current-state context: [`telemetry-credential.md`](../sce/telemetry-credential.md) +- Evidence: [`cli/src/app.rs`](../../cli/src/app.rs) (`only_claude_hook_invocations_are_admitted_to_managed_hook_telemetry`) +- Evidence: [`cli/src/services/managed_telemetry/trace_export.rs`](../../cli/src/services/managed_telemetry/trace_export.rs) (`claude_hook_export_posts_one_hook_span_with_correlation_ids_only`) +- Evidence: [`cli/src/services/hooks/claude_mutation_scope/lifecycle.rs`](../../cli/src/services/hooks/claude_mutation_scope/lifecycle.rs) (`failing_managed_telemetry_cannot_change_claude_hook_behavior`) +- Related decision: [`Application-owned async command runtime`](2026-10-07-application-owned-async-command-runtime.md) diff --git a/context/glossary.md b/context/glossary.md index 1bd61e54c..02f22843d 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -80,6 +80,7 @@ - `encrypted Turso adapter`: Generic native async adapter seam in `cli/src/services/db/mod.rs` exposed as `EncryptedTursoDb`, structurally parallel to `TursoDb` and owning no Tokio runtime. Its constructor resolves the encryption key via `encryption_key::get_or_create_encryption_key(&db_path, db_name)`, which derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text before falling back to OS credential-store keyring get-or-create behavior; credential-store default registration is guarded by stable `OnceLock` plus an atomic in-progress flag so errors or panics leave initialization retryable without mutex poisoning. The adapter enables Turso local encryption with strict `aegis256` cipher selection through `turso::EncryptionOpts`, wraps encrypted local open/connect in the default DB connection-open retry policy, and runs embedded migrations after retry has produced a connection; execute, query, query_map, and migration operations are async and preserve `__sce_migrations` tracking parity. - `auth DB adapter`: Module in `cli/src/services/auth_db/mod.rs` that defines `AuthDbSpec` and exposes `AuthDb` as an `EncryptedTursoDb` alias. It resolves the canonical `/sce/auth.db` path with `auth_db_path()`, keeps encryption mandatory with `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret precedence before OS keyring fallback and no plaintext mode, and embeds ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. Auth runtime token-storage is now wired through `cli/src/services/token_storage.rs`, which persists tokens via the `auth_credentials` table in the encrypted auth DB instead of a JSON file. - `telemetry credential`: Telemetry-only OpenBao token and accessor issued by the Control Plane `POST /auth/telemetry/credential`, stored as one fixed row in the encrypted auth DB `telemetry_credentials` table by `cli/src/services/telemetry_credential_storage.rs`, returned only while unexpired, removed by `sce auth logout`, not revoked server-side by local deletion, and used as the sole bearer secret for managed OTLP export (never the accessor); see `context/sce/telemetry-credential.md`. +- `sce.hook span`: The single root-of-process managed trace span a Claude-invoked SCE hook exports in place of `sce.command`, carrying `sce.agent=claude`, the hook event, tool name, outcome, and the Claude `session_id`/`tool_use_id` as correlation-only attributes. Admission is Claude-only (Claude-only hook subcommands by argv; shared hook subcommands and `sce policy bash` only with `CLAUDECODE` and `CLAUDE_PROJECT_DIR` present); see `context/sce/cli-observability-contract.md`. - `AuthDbLifecycle`: Lifecycle provider in `cli/src/services/auth_db/lifecycle.rs` that implements `ServiceLifecycle` for encrypted auth DB setup/doctor integration. `diagnose` collects auth DB path health problems, `fix` bootstraps missing auth DB parent directory, and `setup` calls `AuthDb::new()`. Registered as `LifecycleProviderId::AuthDb` in the shared lifecycle catalog. - `agent trace DB adapter`: Modules under `cli/src/services/agent_trace_db/` that define the sole repository-scoped `RepositoryAgentTraceDb = TursoDb` adapter (the checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter and its 15-file migration chain were removed by the `retire-legacy-agent-trace-db` plan). The repository adapter uses the `agent-trace-repository` migration set (fresh baseline schema plus additive `source_instance_id` and `claude_model_state` migrations) with `repository_metadata`, repository-level `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts` tables, no row-level `checkout_id`, typed parameterized insert helpers, the non-exported `claude_model_state` latest-locally-observed register with exact `(session_id, agent_id)` reads and guarded writes, and chronological recent `diff_traces` query/parse support. `AgentTraceDbLifecycle` initializes/checks repository-scoped storage through `agent_trace_storage`. - `post-commit Agent Trace auto-sync readiness`: The doctor report fact that explains whether the enabled post-commit trigger is ready without invoking it. Doctor compares the installed `post-commit` hook's SCE managed block using the same currency semantics as setup and resolves config-file-only `agent_trace.auto_sync` with source metadata. JSON states are `ready`, `disabled`, `not_ready`, and `not_applicable`; explicit disable is healthy, while existing hook problems continue to own overall readiness and remediation. See [automatic Agent Trace synchronization](cli/agent-trace-auto-sync.md) and [doctor human text](sce/doctor-human-text-contract.md). diff --git a/context/overview.md b/context/overview.md index ddc104775..47cf66fc2 100644 --- a/context/overview.md +++ b/context/overview.md @@ -65,7 +65,7 @@ Its command loop is implemented with `clap` derive-based argument parsing and `a The current doctor presentation contract supersedes the earlier output-shape scaffolding wording above: human text uses the compact Environment/Repository/Integrations hierarchy with healthy rows collapsed and unhealthy branches expanded, while JSON retains complete path, identity, problem, and fix-result detail. See `context/sce/doctor-human-text-contract.md`. The command loop now enforces a stable exit-code contract in `cli/src/app.rs`: `2` parse failures, `3` invocation validation failures, `4` runtime failures, and `5` dependency startup failures. Auth command orchestration routes authenticated Control Plane authentication failures, token-storage failures, and approved user-facing fallback failures through typed `CliError::User` catalog entries (`NotAuthenticated`, `AuthStorageUnavailable`, `NotGitRepository`, and `UnexpectedFailure`) while retaining technical sources for observability; missing-credential logout/whoami state queries succeed with their documented text/JSON reports, and internal auth failures remain runtime errors. The same runtime also emits stable CLI stderr diagnostics: internal failures use `SCE-ERR-PARSE`, `SCE-ERR-VALIDATION`, `SCE-ERR-RUNTIME`, or `SCE-ERR-DEPENDENCY` in deterministic `Error []: ...` diagnostics with class-default `Try:` remediation appended when missing, while expected failures emit only their redacted, unstyled catalog messages. The command boundary's former flat, string-only `ClassifiedError` has been replaced by typed `CliError` in `cli/src/services/error.rs`: `CliError::User` carries a closed catalog (`NotAuthenticated`, `NotGitRepository`, `NotGitRemote`, authentication-storage `AuthStorageUnavailable`, and general `UnexpectedFailure`) for expected, deliberately-explained failures rendered without the technical source, wrapper, styling, or automatic `Try:` guidance; the general entry renders one fixed static log-files guidance sentence without dynamic path text. `CliError::Internal` carries a live `anyhow::Error` source rendered as the real error chain with the existing styled wrapper and class-default remediation; `app_support` is the sole owner of the distinct terminal paths, and `sce sync` classifies authentication, credential-storage, and all other sync failures as cataloged user errors while preserving their technical sources for observability. Setup preflight errors preserve technical sources, including the configured remote name, while raw remote URLs remain out of user-facing diagnostics. See `context/sce/cli-error-code-taxonomy.md` for the full contract. -The app runtime now also includes a structured observability baseline in `cli/src/services/observability.rs`: deterministic env-controlled log threshold/format (`SCE_LOG_LEVEL` defaults to `error`; `SCE_LOG_FORMAT` defaults to `text`), default-backed log-directory routing (`SCE_LOG_DIR` / config-file `log_dir` / `/sce/logs`) with per-operation machine-local dated file selection, optional session filename partitioning, and creation-triggered retention of direct regular `*.log` files to 10 entries, stable lifecycle event IDs, normal logger records routed only to files when file logging is enabled and only error-level logger records routed to `stderr` when file logging is disabled, direct file-write diagnostics remaining on `stderr`, and `observability::traits` boundaries for logger and telemetry behavior. When the encrypted auth DB holds a valid unexpired telemetry credential, non-hook commands also export `sce.command` spans and allowlisted logger events as OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev` `/v1/traces` and `/v1/logs` paths (bearer token only, redirects disabled, ambient `OTEL_EXPORTER_OTLP_*` and repo config cannot redirect it, fail-open and bounded); without a credential nothing is exported, and `/v1/metrics` is reserved with no metrics pipeline. +The app runtime now also includes a structured observability baseline in `cli/src/services/observability.rs`: deterministic env-controlled log threshold/format (`SCE_LOG_LEVEL` defaults to `error`; `SCE_LOG_FORMAT` defaults to `text`), default-backed log-directory routing (`SCE_LOG_DIR` / config-file `log_dir` / `/sce/logs`) with per-operation machine-local dated file selection, optional session filename partitioning, and creation-triggered retention of direct regular `*.log` files to 10 entries, stable lifecycle event IDs, normal logger records routed only to files when file logging is enabled and only error-level logger records routed to `stderr` when file logging is disabled, direct file-write diagnostics remaining on `stderr`, and `observability::traits` boundaries for logger and telemetry behavior. When the encrypted auth DB holds a valid unexpired telemetry credential, commands also export `sce.command` spans (Claude-invoked SCE hooks export one `sce.hook` span instead; other hooks export nothing) and allowlisted logger events as OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev` `/v1/traces` and `/v1/logs` paths (bearer token only, redirects disabled, ambient `OTEL_EXPORTER_OTLP_*` and repo config cannot redirect it, fail-open and bounded); without a credential nothing is exported, and `/v1/metrics` is reserved with no metrics pipeline. One application-level multi-thread Tokio runtime directly awaits static command dispatch through async credentials, lifecycle providers, hooks, sync/export, and Turso persistence. The shared Turso adapters own no runtime or executor bridge, and foreground hooks return after required persistence completes. See [application execution runtime](architecture.md#application-execution-runtime). The app command dispatcher now enforces a centralized stdout/stderr stream contract in `cli/src/app.rs`: command success payloads are emitted on stdout only, while redacted user-facing diagnostics and text-mode sync progress are emitted on stderr; JSON sync remains silent. `cli/src/app.rs` also now runs through explicit startup phases — dependency check, observability config resolution, runtime initialization, command parse/execute, and output rendering — with the app runtime carrying logger/telemetry plus static command-catalog state across those phases while preserving the existing exit-code and degraded-startup contracts. Within that lifecycle, `parse_command_phase` delegates clap-to-runtime conversion to `cli/src/services/parse/command_runtime.rs`, which returns a static `RuntimeCommand` enum, `services::app_support::execute_command_phase` logs around enum-owned `execute_with_stderr(...)` dispatch, and generic `RunOutcome` rendering logs classified errors through the logger trait boundary without coupling render support to the production logger type. Command payload structs for `help`, `version`, `completion`, `auth`, `config`, `setup`, `doctor`, `hooks`, and `sync` live in service-owned `command.rs` files; `cli/src/services/command_registry.rs` owns the deterministic static command-name catalog and enum variants instead of boxed command trait objects. The CLI now also enforces a shared output-format parser contract in `cli/src/services/output_format.rs`, with canonical `--format ` parsing and command-specific actionable invalid-value guidance reused by `config` and `version` services. A compile-safe service lifecycle seam also exists in `cli/src/services/lifecycle.rs`: `ServiceLifecycle` exposes default no-op `diagnose`, `fix`, and `setup` methods against the narrow `HasRepoRoot` accessor, uses lifecycle-owned health/fix/setup result types, and owns the shared static `LifecycleProvider` enum catalog/factory with deterministic config → local*db → auth_db → agent_trace_db → hooks ordering and no boxed provider aggregation. Hooks has a `services/hooks/lifecycle.rs` provider for hook rollout diagnosis/fix/setup, config has a `services/config/lifecycle.rs` provider for global/repo-local config validation plus repo-local config bootstrap, local_db has a `services/local_db/lifecycle.rs` provider for canonical local DB path health, parent-directory readiness/bootstrap, and `LocalDb::new()` setup, auth_db has a `services/auth_db/lifecycle.rs` provider for canonical auth DB path health, parent-directory readiness/bootstrap, and `AuthDb::new()` setup, and agent_trace_db has a `services/agent_trace_db/lifecycle.rs` provider for repository-scoped Agent Trace DB setup and repository DB path health/parent readiness from resolved repository identity, returning an actionable "requires a Git repository" diagnostic outside repository context (no global/checkout fallback path). Doctor runtime aggregates the full shared provider catalog for `diagnose` and `fix` and adapts lifecycle records into doctor-owned output records; setup command aggregates the shared provider catalog for `setup` with hooks included only when requested and adapts lifecycle setup outcomes before rendering setup-owned messages. diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md index 6490f717d..ec97578cb 100644 --- a/context/plans/claude-hook-trace-propagation.md +++ b/context/plans/claude-hook-trace-propagation.md @@ -87,13 +87,19 @@ Persist this field in every plan; this is durable plan state, not chat state: - Context impact: new current-state context file under `context/sce/` (probe evidence and verdict); needs a `context/context-map.md` entry; no code behavior change. - Context synchronization: synced -- [ ] T02: `Export a dedicated sce.hook span for Claude hook invocations` (status:todo) +- [x] T02: `Export a dedicated sce.hook span for Claude hook invocations` (status:done) - Task ID: T02 - Scope: In — narrow `is_hook_invocation` into a Claude-admission decision in `cli/src/app.rs`; add the `sce.hook` span (replacing `sce.command` for admitted hooks) in `trace_export.rs` with `sce.agent`, `sce.hook.event`, `sce.hook.tool`, `sce.outcome`, `sce.error.class`, and the two correlation IDs recorded from the parsed Claude payload; loopback tests for span shape, admission, secret/payload absence, credential failure, exporter failure/stall, bounded shutdown, and hook output/state parity. Out — parent-context extraction (T03), child spans (T04), non-Claude harnesses. - Dependencies: T01 - Done when: AC2, AC4, AC6, and AC7 hold in local tests; `hook_invocations_never_resolve_managed_telemetry` is replaced by an admission test that keeps non-Claude hooks and `telemetry-headers` on `Noop`. - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml app::tests`. - - Context synchronization: pending + - Completed: 2026-10-17 + - Files changed: `cli/src/app.rs`, `cli/src/services/managed_telemetry.rs`, `cli/src/services/managed_telemetry/trace_export.rs`, `cli/src/services/hooks/claude_mutation_scope/lifecycle.rs`, `cli/src/services/hooks/claude_model_state.rs`, `cli/src/services/hooks/diff_trace.rs`, `cli/src/services/hooks/conversation_trace.rs`, `cli/src/services/bash_policy.rs` + - Result: `is_hook_invocation` replaced by `managed_span_kind(args, claude_hook_caller) -> Option`. `hooks claude-mutation-scope` and `hooks claude-model-state` are admitted from argv. `hooks diff-trace`, `hooks conversation-trace`, and `policy bash` are admitted only when `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are both present (presence only). Every other hook subcommand and `auth telemetry-headers` resolve `None` (Noop); every other command resolves `Command`. `RuntimeTelemetry::resolve`/`resolve_with` and `ManagedTraceTelemetry::new` take the span kind. For `ClaudeHook`, the root span is `sce.hook` (same `sce.managed.trace` target and filter, no `sce.command`), with `sce.agent=claude`, `sce.hook.event`, `sce.hook.tool`, `sce.claude.session_id`, `sce.claude.tool_use_id`, `sce.outcome`, and `sce.error.class`. `record_claude_hook_payload` returns early on a disabled span and copies only the four string fields `hook_event_name`, `tool_name`, `session_id`, and `tool_use_id`; it is called once after stdin is read in the five Claude entry points. Tests added: an admission matrix; a loopback test that decodes the OTLP protobuf (new test-only decoder `decode_spans`/`trace_spans`, reusable by T03/T04) and asserts exactly one root `sce.hook` span, its exact `sce.*` attribute set, `service.name=sce`, and the absence of paths, prompt, file content, tool response, `Bearer` value, token, accessor, and `sce.command`; credential failure and expiry resolve Noop for `ClaudeHook`; and a parity test that runs a four-event Claude mutation-scope fixture sequence (including a deny decision) under Noop, refused-receiver, and stalled-receiver managed telemetry, asserting identical outputs, seam (persistence) payloads, and adapter state, with shutdown under `SHUTDOWN_TIMEOUT` + 1s. + - Assumptions: hooks reuse the existing `CREDENTIAL_LOOKUP_DEADLINE` and `SHUTDOWN_TIMEOUT` bounds per AC6, and the open question on a tighter hook deadline stays open; correlation attributes are named `sce.claude.session_id` and `sce.claude.tool_use_id`. AC6 persistence parity is proven at the mutation-scope seam (the Agent Trace DB write boundary), not by opening a real Agent Trace DB. + - Verify outcomes: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`: 22 passed, 0 failed (passed); `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml app::tests`: 12 passed, 0 failed (passed); additionally `nix flake check`: all checks passed (cli-tests, cli-clippy, cli-fmt, actionlint). + - Context impact: important change. Hook telemetry admission reverses the recorded "hooks are always Noop" contract (`context/sce/cli-observability-contract.md`, `context/sce/telemetry-credential.md`, AC4 of `claude-native-otel-instrumentation` context), and the new `sce.hook` span contract and correlation-ID privacy rule affect `context/architecture.md` and `context/glossary.md`. + - Context synchronization: synced - [ ] T03: `Parent the sce.hook span on Claude-provided W3C trace context` (status:todo) - Task ID: T03 diff --git a/context/sce/claude-hook-trace-propagation.md b/context/sce/claude-hook-trace-propagation.md index baa21d1b9..2afb4dd27 100644 --- a/context/sce/claude-hook-trace-propagation.md +++ b/context/sce/claude-hook-trace-propagation.md @@ -1,6 +1,6 @@ # Claude Code Hook Trace-Context Propagation -Probe evidence and verdict for whether Claude Code passes W3C trace context to command hooks, recorded by T01 of `context/plans/claude-hook-trace-propagation.md`. This file is the durable source for the parent-context decision consumed by T02–T05; it does not describe any shipped SCE behavior yet. +Probe evidence and verdict for whether Claude Code passes W3C trace context to command hooks, recorded by T01 of `context/plans/claude-hook-trace-propagation.md`. This file is the durable source for the parent-context decision consumed by T02–T05. Shipped hook admission and the `sce.hook` span contract are described in [cli-observability-contract.md](cli-observability-contract.md); parenting `sce.hook` on `TRACEPARENT` is not shipped yet, so the span is currently an independent root. ## Verdict diff --git a/context/sce/cli-observability-contract.md b/context/sce/cli-observability-contract.md index d8a6d6d7d..30550cadd 100644 --- a/context/sce/cli-observability-contract.md +++ b/context/sce/cli-observability-contract.md @@ -94,9 +94,11 @@ Runtime observability consumes the shared resolved observability config from `cl ## Managed trace export -- Defined in `cli/src/services/managed_telemetry.rs` and `cli/src/services/managed_telemetry/trace_export.rs` (plus `log_export.rs`). After startup, `RuntimeTelemetry::resolve` loads the stored telemetry credential from the encrypted auth DB in a spawned task bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s); absent, expired, malformed, unavailable, ownerless (no stored WorkOS session), or timed-out credentials select `NoopTelemetry`. Startup never issues or refreshes a credential. Hook invocations (`sce hooks ...`), `sce policy bash`, and the hidden `sce auth telemetry-headers` helper always use `NoopTelemetry` (`is_hook_invocation` in `cli/src/app.rs`). +- Defined in `cli/src/services/managed_telemetry.rs` and `cli/src/services/managed_telemetry/trace_export.rs` (plus `log_export.rs`). After startup, `RuntimeTelemetry::resolve` loads the stored telemetry credential from the encrypted auth DB in a spawned task bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s); absent, expired, malformed, unavailable, ownerless (no stored WorkOS session), or timed-out credentials select `NoopTelemetry`. Startup never issues or refreshes a credential. +- Admission (`managed_span_kind` in `cli/src/app.rs`, decided from argv before credential lookup): Claude-invoked SCE hooks resolve the `sce.hook` span kind, every other hook and the hidden `sce auth telemetry-headers` helper use `NoopTelemetry` without a credential lookup, and all remaining commands use the `sce.command` span kind. `sce hooks claude-mutation-scope` and `sce hooks claude-model-state` are admitted from argv alone. The shared `sce hooks diff-trace`, `sce hooks conversation-trace`, and `sce policy bash` are admitted only when both `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are present in the environment (checked for presence only; values never read); `CLAUDE_PROJECT_DIR` is set for Claude command hooks but not for the Claude Bash tool (evidence: [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md)). Other hooks (commit hooks, Codex, OpenCode, Pi) stay Noop. Admitted hooks use the same `CREDENTIAL_LOOKUP_DEADLINE`, `SHUTDOWN_TIMEOUT`, and runtime grace bounds as commands. Decision record: [2026-10-17-claude-hook-managed-trace-admission.md](../decisions/2026-10-17-claude-hook-managed-trace-admission.md). - Export is OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev/v1/traces`. A pinned HTTP client sets `Authorization: Bearer ` (token only, never the accessor), disables redirects, rebuilds the request to the fixed endpoint, and bounds requests to 3s. Ambient `OTEL_EXPORTER_OTLP_*` values cannot redirect it. - Each command emits one `sce.command` span with `sce.command.name`, `sce.outcome` (`success`/`failure`), and `sce.error.class` (the `SCE-ERR-*` code on failure); resource attributes are `service.name=sce` and `service.version`. Raw arguments, paths, and error chains are not exported. +- An admitted Claude hook process emits exactly one `sce.hook` span instead (no `sce.command` span), with `sce.agent=claude`, `sce.outcome`, `sce.error.class`, and, when present as non-empty strings in the Claude hook stdin payload, `sce.hook.event` (`hook_event_name`), `sce.hook.tool` (`tool_name`), `sce.claude.session_id`, and `sce.claude.tool_use_id`. `trace_export::record_claude_hook_payload` is the only reader: each Claude hook entry point passes its stdin to it once, it returns immediately when the current span is disabled (Noop), and it copies only those four fields. Session and tool-use IDs are opaque correlation identifiers: span attributes only, never metric attributes, never used to derive a trace ID. Tool input and response, prompts, `cwd`, transcript paths, file contents, and the raw payload are never exported. The span is currently an independent root; telemetry failures never change hook stdout, permission decisions, persisted state, or exit codes. - The exporter uses a bounded batch queue on a dedicated thread. Final stdout/stderr rendering and classified-error logging run inside the command span (`run_command_lifecycle_with_context`), so `sce.outcome` reflects the rendered result (a stdout write failure is `failure`) and the final `sce.error.*` log is exported. Trace and log providers shut down concurrently under one shared `SHUTDOWN_TIMEOUT` (3s) deadline after rendering; failures never change command results or exit codes. - Process lifetime: `app::run_process` owns the Tokio runtime (at least two workers) and ends it with `shutdown_timeout(RUNTIME_SHUTDOWN_GRACE)` (500ms), so a stalled keychain, database, or exporter worker cannot keep the process alive. The async deadlines bound when the command proceeds; the runtime grace is the hard process-level bound. Abandoned workers are not cancelled, only no longer awaited. - Logs: `managed_telemetry/log_export.rs` bridges `target: "sce"` tracing events and their `enabled!` hints (emitted by `Logger` only when the local level is enabled) to OTLP/HTTP protobuf log records at the hardcoded `/v1/logs`, using the same pinned bearer client, batch bounds, and shutdown as traces. Records carry severity, `sce.event_id`, a body limited to path- and secret-free messages (falling back to the event ID), and only allowlisted fields (`command`, `component`, `failure_class`, `error_code`, `error_class`, `error_surface`, `user_error`) with bounded, redacted values; `error_source`, arguments, paths, and hook payloads are never exported. Local file/stderr logging is unchanged. From c3e67d29d841ef7044b977ebffa748405d69a5e4 Mon Sep 17 00:00:00 2001 From: David Abram Date: Sat, 17 Oct 2026 14:14:43 +0200 Subject: [PATCH 04/17] telemetry: Parent Claude hook spans on propagated trace context Join Claude hook telemetry to Claude Code's W3C trace when a valid parent is available, while preserving independent roots for missing or invalid context. Extract TRACEPARENT/TRACESTATE only for Claude hooks, attach the parent to the root span, and verify child-span inheritance across async polls and worker migration. Plan: claude-hook-trace-propagation (T03) Co-authored-by: SCE --- .../managed_telemetry/trace_export.rs | 178 +++++++++++++++++- context/glossary.md | 2 +- .../plans/claude-hook-trace-propagation.md | 13 +- context/sce/claude-hook-trace-propagation.md | 2 +- context/sce/cli-observability-contract.md | 2 +- 5 files changed, 189 insertions(+), 8 deletions(-) diff --git a/cli/src/services/managed_telemetry/trace_export.rs b/cli/src/services/managed_telemetry/trace_export.rs index de57ed716..216059e05 100644 --- a/cli/src/services/managed_telemetry/trace_export.rs +++ b/cli/src/services/managed_telemetry/trace_export.rs @@ -1,19 +1,23 @@ +use std::collections::HashMap; use std::future::Future; use std::time::Duration; use opentelemetry::logs::LoggerProvider as _; -use opentelemetry::trace::TracerProvider as _; -use opentelemetry::KeyValue; +use opentelemetry::propagation::TextMapPropagator as _; +use opentelemetry::trace::{TraceContextExt as _, TracerProvider as _}; +use opentelemetry::{Context, KeyValue}; use opentelemetry_http::{Bytes, HttpClient, HttpError, Request, Response}; use opentelemetry_otlp::{LogExporter, SpanExporter, WithExportConfig, WithHttpConfig}; use opentelemetry_sdk::logs::{ BatchConfigBuilder as LogBatchConfigBuilder, BatchLogProcessor, SdkLoggerProvider, }; +use opentelemetry_sdk::propagation::TraceContextPropagator; use opentelemetry_sdk::trace::{BatchConfigBuilder, BatchSpanProcessor, SdkTracerProvider}; use opentelemetry_sdk::Resource; use reqwest::header::{HeaderValue, AUTHORIZATION, CONTENT_TYPE}; use tracing::instrument::WithSubscriber; use tracing::{field, Dispatch, Instrument, Span}; +use tracing_opentelemetry::OpenTelemetrySpanExt as _; use tracing_subscriber::filter::filter_fn; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::{Layer, Registry}; @@ -40,6 +44,10 @@ const CLAUDE_PAYLOAD_FIELDS: [(&str, &str); 4] = [ ("session_id", CLAUDE_SESSION_ID_FIELD), ("tool_use_id", CLAUDE_TOOL_USE_ID_FIELD), ]; +const TRACEPARENT_ENV: &str = "TRACEPARENT"; +const TRACESTATE_ENV: &str = "TRACESTATE"; +const TRACEPARENT_HEADER: &str = "traceparent"; +const TRACESTATE_HEADER: &str = "tracestate"; const OUTCOME_FIELD: &str = "sce.outcome"; const ERROR_CLASS_FIELD: &str = "sce.error.class"; const OUTCOME_SUCCESS: &str = "success"; @@ -97,6 +105,7 @@ pub struct ManagedTraceTelemetry { log_provider: SdkLoggerProvider, dispatch: Dispatch, span_kind: ManagedSpanKind, + parent_context: Option, } impl ManagedTraceTelemetry { @@ -110,7 +119,14 @@ impl ManagedTraceTelemetry { .redirect(reqwest::redirect::Policy::none()) .timeout(REQUEST_TIMEOUT) .build()?; - Self::with_client(bearer, endpoints, runtime, client, span_kind) + let mut telemetry = Self::with_client(bearer, endpoints, runtime, client, span_kind)?; + if span_kind == ManagedSpanKind::ClaudeHook { + telemetry.parent_context = extract_parent_context( + std::env::var(TRACEPARENT_ENV).ok(), + std::env::var(TRACESTATE_ENV).ok(), + ); + } + Ok(telemetry) } fn with_client( @@ -192,6 +208,7 @@ impl ManagedTraceTelemetry { log_provider, dispatch, span_kind, + parent_context: None, }) } @@ -217,6 +234,14 @@ impl ManagedTraceTelemetry { } fn root_span(&self) -> Span { + let span = self.unparented_root_span(); + if let Some(parent_context) = &self.parent_context { + let _ = span.set_parent(parent_context.clone()); + } + span + } + + fn unparented_root_span(&self) -> Span { tracing::dispatcher::with_default(&self.dispatch, || match self.span_kind { ManagedSpanKind::Command => tracing::info_span!( target: COMMAND_SPAN_TARGET, @@ -265,6 +290,19 @@ impl Telemetry for ManagedTraceTelemetry { } } +fn extract_parent_context( + traceparent: Option, + tracestate: Option, +) -> Option { + let traceparent = traceparent?; + let mut carrier = HashMap::from([(TRACEPARENT_HEADER.to_string(), traceparent)]); + if let Some(tracestate) = tracestate { + carrier.insert(TRACESTATE_HEADER.to_string(), tracestate); + } + let context = TraceContextPropagator::new().extract(&carrier); + context.span().span_context().is_valid().then_some(context) +} + pub fn record_command_name(name: &str) { Span::current().record(COMMAND_NAME_FIELD, name); } @@ -494,6 +532,48 @@ pub(crate) mod tests { loopback_telemetry_of_kind(address, token, ManagedSpanKind::ClaudeHook) } + fn loopback_hook_telemetry_with_traceparent( + address: std::net::SocketAddr, + traceparent: Option<&str>, + ) -> ManagedTraceTelemetry { + let mut telemetry = loopback_hook_telemetry(address, "bao-token-value"); + telemetry.parent_context = extract_parent_context( + traceparent.map(str::to_string), + Some("vendor=opaque".to_string()), + ); + telemetry + } + + const UPSTREAM_TRACE_ID: [u8; 16] = [ + 0x4b, 0xf9, 0x2f, 0x35, 0x77, 0xb3, 0x4d, 0xa6, 0xa3, 0xce, 0x92, 0x9d, 0x0e, 0x0e, 0x47, + 0x36, + ]; + const UPSTREAM_SPAN_ID: [u8; 8] = [0x00, 0xf0, 0x67, 0xaa, 0x0b, 0xa9, 0x02, 0xb7]; + const UPSTREAM_TRACEPARENT: &str = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + + async fn export_hook_spans( + traceparent: Option<&str>, + action: impl Future + Send + 'static, + ) -> Vec { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr"); + let receiver = tokio::spawn(capture_requests(listener, "HTTP/1.1 200 OK", 1)); + let telemetry = loopback_hook_telemetry_with_traceparent(address, traceparent); + let mut action = Some(action); + let result = telemetry + .with_default_subscriber(&mut || { + let action = action.take().expect("single invocation"); + async move { + action.await; + Ok(String::new()) + } + }) + .await; + assert_eq!(result.expect("hook result"), ""); + telemetry.shutdown().await; + trace_spans(&receiver.await.expect("receiver join")) + } + fn loopback_telemetry_of_kind( address: std::net::SocketAddr, token: &str, @@ -690,6 +770,98 @@ pub(crate) mod tests { } } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn claude_hook_span_joins_valid_upstream_trace_with_fresh_span_id() { + let spans = export_hook_spans(Some(UPSTREAM_TRACEPARENT), async { + tokio::task::yield_now().await; + }) + .await; + assert_eq!(spans.len(), 1, "{spans:?}"); + let span = &spans[0]; + assert_eq!(span.name, "sce.hook"); + assert_eq!(span.trace_id, UPSTREAM_TRACE_ID); + assert_eq!(span.parent_span_id, UPSTREAM_SPAN_ID); + assert_eq!(span.span_id.len(), 8); + assert_ne!(span.span_id, UPSTREAM_SPAN_ID); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn claude_hook_span_starts_independent_root_without_usable_traceparent() { + for traceparent in [ + None, + Some(""), + Some("not-a-traceparent"), + Some("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7"), + Some("00-4BF92F3577B34DA6A3CE929D0E0E4736-00f067aa0ba902b7-01"), + Some("ff-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"), + Some("00-00000000000000000000000000000000-00f067aa0ba902b7-01"), + Some("00-4bf92f3577b34da6a3ce929d0e0e4736-0000000000000000-01"), + ] { + let spans = export_hook_spans(traceparent, async {}).await; + assert_eq!(spans.len(), 1, "{traceparent:?}: {spans:?}"); + let span = &spans[0]; + assert_eq!(span.name, "sce.hook"); + assert!(span.parent_span_id.is_empty(), "{traceparent:?}"); + assert_eq!(span.trace_id.len(), 16, "{traceparent:?}"); + assert_ne!(span.trace_id, UPSTREAM_TRACE_ID, "{traceparent:?}"); + assert!( + span.trace_id.iter().any(|byte| *byte != 0), + "{traceparent:?}" + ); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn claude_hook_child_spans_inherit_upstream_trace_across_polls_and_workers() { + let spans = export_hook_spans(Some(UPSTREAM_TRACEPARENT), async { + for _ in 0..8 { + tokio::task::yield_now().await; + } + let child = tracing::info_span!(target: COMMAND_SPAN_TARGET, "sce.test.child"); + async { + tokio::task::yield_now().await; + tokio::spawn( + async { + tokio::task::yield_now().await; + let grandchild = + tracing::info_span!(target: COMMAND_SPAN_TARGET, "sce.test.grandchild"); + async { + for _ in 0..4 { + tokio::task::yield_now().await; + } + } + .instrument(grandchild) + .await; + } + .in_current_span() + .with_current_subscriber(), + ) + .await + .expect("spawned child join"); + tokio::task::yield_now().await; + } + .instrument(child) + .await; + }) + .await; + assert_eq!(spans.len(), 3, "{spans:?}"); + let by_name = |name: &str| { + spans + .iter() + .find(|span| span.name == name) + .unwrap_or_else(|| panic!("missing {name}")) + }; + let hook = by_name("sce.hook"); + let child = by_name("sce.test.child"); + let grandchild = by_name("sce.test.grandchild"); + for span in [hook, child, grandchild] { + assert_eq!(span.trace_id, UPSTREAM_TRACE_ID, "{}", span.name); + } + assert_eq!(hook.parent_span_id, UPSTREAM_SPAN_ID); + assert_eq!(child.parent_span_id, hook.span_id); + assert_eq!(grandchild.parent_span_id, child.span_id); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn managed_trace_export_does_not_follow_redirects_or_alter_results() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); diff --git a/context/glossary.md b/context/glossary.md index 02f22843d..01608cf96 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -80,7 +80,7 @@ - `encrypted Turso adapter`: Generic native async adapter seam in `cli/src/services/db/mod.rs` exposed as `EncryptedTursoDb`, structurally parallel to `TursoDb` and owning no Tokio runtime. Its constructor resolves the encryption key via `encryption_key::get_or_create_encryption_key(&db_path, db_name)`, which derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text before falling back to OS credential-store keyring get-or-create behavior; credential-store default registration is guarded by stable `OnceLock` plus an atomic in-progress flag so errors or panics leave initialization retryable without mutex poisoning. The adapter enables Turso local encryption with strict `aegis256` cipher selection through `turso::EncryptionOpts`, wraps encrypted local open/connect in the default DB connection-open retry policy, and runs embedded migrations after retry has produced a connection; execute, query, query_map, and migration operations are async and preserve `__sce_migrations` tracking parity. - `auth DB adapter`: Module in `cli/src/services/auth_db/mod.rs` that defines `AuthDbSpec` and exposes `AuthDb` as an `EncryptedTursoDb` alias. It resolves the canonical `/sce/auth.db` path with `auth_db_path()`, keeps encryption mandatory with `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret precedence before OS keyring fallback and no plaintext mode, and embeds ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. Auth runtime token-storage is now wired through `cli/src/services/token_storage.rs`, which persists tokens via the `auth_credentials` table in the encrypted auth DB instead of a JSON file. - `telemetry credential`: Telemetry-only OpenBao token and accessor issued by the Control Plane `POST /auth/telemetry/credential`, stored as one fixed row in the encrypted auth DB `telemetry_credentials` table by `cli/src/services/telemetry_credential_storage.rs`, returned only while unexpired, removed by `sce auth logout`, not revoked server-side by local deletion, and used as the sole bearer secret for managed OTLP export (never the accessor); see `context/sce/telemetry-credential.md`. -- `sce.hook span`: The single root-of-process managed trace span a Claude-invoked SCE hook exports in place of `sce.command`, carrying `sce.agent=claude`, the hook event, tool name, outcome, and the Claude `session_id`/`tool_use_id` as correlation-only attributes. Admission is Claude-only (Claude-only hook subcommands by argv; shared hook subcommands and `sce policy bash` only with `CLAUDECODE` and `CLAUDE_PROJECT_DIR` present); see `context/sce/cli-observability-contract.md`. +- `sce.hook span`: The single root-of-process managed trace span a Claude-invoked SCE hook exports in place of `sce.command`, carrying `sce.agent=claude`, the hook event, tool name, outcome, and the Claude `session_id`/`tool_use_id` as correlation-only attributes. It is parented on a valid Claude-provided `TRACEPARENT` (the `claude_code.interaction` span, so it sits beside `claude_code.tool`) and is otherwise an independent root. Admission is Claude-only (Claude-only hook subcommands by argv; shared hook subcommands and `sce policy bash` only with `CLAUDECODE` and `CLAUDE_PROJECT_DIR` present); see `context/sce/cli-observability-contract.md`. - `AuthDbLifecycle`: Lifecycle provider in `cli/src/services/auth_db/lifecycle.rs` that implements `ServiceLifecycle` for encrypted auth DB setup/doctor integration. `diagnose` collects auth DB path health problems, `fix` bootstraps missing auth DB parent directory, and `setup` calls `AuthDb::new()`. Registered as `LifecycleProviderId::AuthDb` in the shared lifecycle catalog. - `agent trace DB adapter`: Modules under `cli/src/services/agent_trace_db/` that define the sole repository-scoped `RepositoryAgentTraceDb = TursoDb` adapter (the checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter and its 15-file migration chain were removed by the `retire-legacy-agent-trace-db` plan). The repository adapter uses the `agent-trace-repository` migration set (fresh baseline schema plus additive `source_instance_id` and `claude_model_state` migrations) with `repository_metadata`, repository-level `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts` tables, no row-level `checkout_id`, typed parameterized insert helpers, the non-exported `claude_model_state` latest-locally-observed register with exact `(session_id, agent_id)` reads and guarded writes, and chronological recent `diff_traces` query/parse support. `AgentTraceDbLifecycle` initializes/checks repository-scoped storage through `agent_trace_storage`. - `post-commit Agent Trace auto-sync readiness`: The doctor report fact that explains whether the enabled post-commit trigger is ready without invoking it. Doctor compares the installed `post-commit` hook's SCE managed block using the same currency semantics as setup and resolves config-file-only `agent_trace.auto_sync` with source metadata. JSON states are `ready`, `disabled`, `not_ready`, and `not_applicable`; explicit disable is healthy, while existing hook problems continue to own overall readiness and remediation. See [automatic Agent Trace synchronization](cli/agent-trace-auto-sync.md) and [doctor human text](sce/doctor-human-text-contract.md). diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md index ec97578cb..f6e25e336 100644 --- a/context/plans/claude-hook-trace-propagation.md +++ b/context/plans/claude-hook-trace-propagation.md @@ -101,13 +101,22 @@ Persist this field in every plan; this is durable plan state, not chat state: - Context impact: important change. Hook telemetry admission reverses the recorded "hooks are always Noop" contract (`context/sce/cli-observability-contract.md`, `context/sce/telemetry-credential.md`, AC4 of `claude-native-otel-instrumentation` context), and the new `sce.hook` span contract and correlation-ID privacy rule affect `context/architecture.md` and `context/glossary.md`. - Context synchronization: synced -- [ ] T03: `Parent the sce.hook span on Claude-provided W3C trace context` (status:todo) +- [x] T03: `Parent the sce.hook span on Claude-provided W3C trace context` (status:done) - Task ID: T03 - Scope: In — only when T01 recorded `parent_context_available`: extract the context Claude provides using `opentelemetry_sdk::propagation::TraceContextPropagator` before the `sce.hook` span is created, attach it with `OpenTelemetrySpanExt::set_parent`, keep it scoped to the instrumented hook future, and add loopback tests for valid, invalid, all-zero, and missing traceparent plus a multi-poll async child-span test. When T01 recorded `parent_context_unavailable`: make no code change, and record in the context file that the parent-child goal is blocked, why, and the smallest future bridge (correlating Claude OTLP export records with `sce.hook` via `session_id`/`tool_use_id`). Out — reconstructing trace IDs from Claude identifiers; global propagator registration beyond the hook boundary; non-Claude callers. - Dependencies: T02 - Done when: AC3 holds for the branch T01 selected. - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml trace_export`, or inspection of the context file for the unavailable branch. - - Context synchronization: pending + - Completed: 2026-10-17 + - Files changed: `cli/src/services/managed_telemetry/trace_export.rs` + - Result: `parent_context_available` branch implemented. `ManagedTraceTelemetry` gained a `parent_context: Option` field. Only for `ManagedSpanKind::ClaudeHook`, `ManagedTraceTelemetry::new` fills it from process env `TRACEPARENT`/`TRACESTATE`. The pure `extract_parent_context` helper does this by running `opentelemetry_sdk::propagation::TraceContextPropagator::extract` over a `HashMap` carrier, and it keeps the context only when the remote span context is valid. `root_span` calls `OpenTelemetrySpanExt::set_parent` on the `sce.hook` span and ignores its result (fail-open). The span is entered only through `.instrument(...)` on the hook future, so no global propagator is registered and the parent never leaks outside the hook boundary. `sce.command` and non-Claude paths never read the env. Tests added (loopback receiver plus the existing OTLP decoder): + - A valid traceparent gives `sce.hook` the upstream trace ID, its parent span ID is the upstream span ID, and its own span ID is new. + - A table of missing, empty, malformed, truncated, uppercase, version `ff`, all-zero trace ID, and all-zero span ID values each yields one independent root `sce.hook` with a new non-zero trace ID and no parent. + - On a 4-worker runtime, a child span created after 8 polls and a grandchild inside a `tokio::spawn`ed task (`in_current_span` + `with_current_subscriber`) share the upstream trace ID with a correct parent chain. + - Deviation: per T01, the upstream parent is `claude_code.interaction`, so `sce.hook` is a sibling of `claude_code.tool`, not its child. + - Verify outcomes: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml trace_export`: 13 passed, 0 failed (passed); additionally `nix flake check`: all checks passed (cli-tests, cli-clippy, cli-fmt). + - Context impact: important change. `sce.hook` now joins Claude's trace when Claude Code's tracing beta supplies `TRACEPARENT`, otherwise it stays an independent root. This affects the span contract in `context/sce/cli-observability-contract.md` and the verdict/implementation notes in `context/sce/claude-hook-trace-propagation.md`; the glossary/architecture entries for `sce.hook` gain the parent-context rule. + - Context synchronization: synced - [ ] T04: `Add mutation-scope, git-snapshot, and persistence child spans under sce.hook` (status:todo) - Task ID: T04 diff --git a/context/sce/claude-hook-trace-propagation.md b/context/sce/claude-hook-trace-propagation.md index 2afb4dd27..fee7be183 100644 --- a/context/sce/claude-hook-trace-propagation.md +++ b/context/sce/claude-hook-trace-propagation.md @@ -1,6 +1,6 @@ # Claude Code Hook Trace-Context Propagation -Probe evidence and verdict for whether Claude Code passes W3C trace context to command hooks, recorded by T01 of `context/plans/claude-hook-trace-propagation.md`. This file is the durable source for the parent-context decision consumed by T02–T05. Shipped hook admission and the `sce.hook` span contract are described in [cli-observability-contract.md](cli-observability-contract.md); parenting `sce.hook` on `TRACEPARENT` is not shipped yet, so the span is currently an independent root. +Probe evidence and verdict for whether Claude Code passes W3C trace context to command hooks, recorded by T01 of `context/plans/claude-hook-trace-propagation.md`. This file is the durable source for the parent-context decision consumed by T02–T05. Shipped hook admission and the `sce.hook` span contract are described in [cli-observability-contract.md](cli-observability-contract.md); parenting `sce.hook` on a valid `TRACEPARENT` (independent root otherwise) is shipped and described there. ## Verdict diff --git a/context/sce/cli-observability-contract.md b/context/sce/cli-observability-contract.md index 30550cadd..b85720ac8 100644 --- a/context/sce/cli-observability-contract.md +++ b/context/sce/cli-observability-contract.md @@ -98,7 +98,7 @@ Runtime observability consumes the shared resolved observability config from `cl - Admission (`managed_span_kind` in `cli/src/app.rs`, decided from argv before credential lookup): Claude-invoked SCE hooks resolve the `sce.hook` span kind, every other hook and the hidden `sce auth telemetry-headers` helper use `NoopTelemetry` without a credential lookup, and all remaining commands use the `sce.command` span kind. `sce hooks claude-mutation-scope` and `sce hooks claude-model-state` are admitted from argv alone. The shared `sce hooks diff-trace`, `sce hooks conversation-trace`, and `sce policy bash` are admitted only when both `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are present in the environment (checked for presence only; values never read); `CLAUDE_PROJECT_DIR` is set for Claude command hooks but not for the Claude Bash tool (evidence: [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md)). Other hooks (commit hooks, Codex, OpenCode, Pi) stay Noop. Admitted hooks use the same `CREDENTIAL_LOOKUP_DEADLINE`, `SHUTDOWN_TIMEOUT`, and runtime grace bounds as commands. Decision record: [2026-10-17-claude-hook-managed-trace-admission.md](../decisions/2026-10-17-claude-hook-managed-trace-admission.md). - Export is OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev/v1/traces`. A pinned HTTP client sets `Authorization: Bearer ` (token only, never the accessor), disables redirects, rebuilds the request to the fixed endpoint, and bounds requests to 3s. Ambient `OTEL_EXPORTER_OTLP_*` values cannot redirect it. - Each command emits one `sce.command` span with `sce.command.name`, `sce.outcome` (`success`/`failure`), and `sce.error.class` (the `SCE-ERR-*` code on failure); resource attributes are `service.name=sce` and `service.version`. Raw arguments, paths, and error chains are not exported. -- An admitted Claude hook process emits exactly one `sce.hook` span instead (no `sce.command` span), with `sce.agent=claude`, `sce.outcome`, `sce.error.class`, and, when present as non-empty strings in the Claude hook stdin payload, `sce.hook.event` (`hook_event_name`), `sce.hook.tool` (`tool_name`), `sce.claude.session_id`, and `sce.claude.tool_use_id`. `trace_export::record_claude_hook_payload` is the only reader: each Claude hook entry point passes its stdin to it once, it returns immediately when the current span is disabled (Noop), and it copies only those four fields. Session and tool-use IDs are opaque correlation identifiers: span attributes only, never metric attributes, never used to derive a trace ID. Tool input and response, prompts, `cwd`, transcript paths, file contents, and the raw payload are never exported. The span is currently an independent root; telemetry failures never change hook stdout, permission decisions, persisted state, or exit codes. +- An admitted Claude hook process emits exactly one `sce.hook` span instead (no `sce.command` span), with `sce.agent=claude`, `sce.outcome`, `sce.error.class`, and, when present as non-empty strings in the Claude hook stdin payload, `sce.hook.event` (`hook_event_name`), `sce.hook.tool` (`tool_name`), `sce.claude.session_id`, and `sce.claude.tool_use_id`. `trace_export::record_claude_hook_payload` is the only reader: each Claude hook entry point passes its stdin to it once, it returns immediately when the current span is disabled (Noop), and it copies only those four fields. Session and tool-use IDs are opaque correlation identifiers: span attributes only, never metric attributes, never used to derive a trace ID. Tool input and response, prompts, `cwd`, transcript paths, file contents, and the raw payload are never exported. Parent context: for the `sce.hook` kind only, `ManagedTraceTelemetry::new` reads `TRACEPARENT` (and `TRACESTATE` when present) from the process environment, extracts it with `TraceContextPropagator`, and keeps it only when the remote span context is valid; `root_span` then attaches it with `set_parent`, so `sce.hook` shares Claude Code's trace ID, its parent is the upstream span (Claude Code 2.1.301 supplies `claude_code.interaction`, making `sce.hook` a sibling of `claude_code.tool`), and its own span ID is freshly generated. Child spans created inside the instrumented hook future inherit that trace across `.await` points and worker migration. No global propagator is registered and `sce.command` never reads the variables. A missing, malformed, or all-zero traceparent (the normal case when Claude tracing is off) yields an independent SCE root. Telemetry failures never change hook stdout, permission decisions, persisted state, or exit codes. - The exporter uses a bounded batch queue on a dedicated thread. Final stdout/stderr rendering and classified-error logging run inside the command span (`run_command_lifecycle_with_context`), so `sce.outcome` reflects the rendered result (a stdout write failure is `failure`) and the final `sce.error.*` log is exported. Trace and log providers shut down concurrently under one shared `SHUTDOWN_TIMEOUT` (3s) deadline after rendering; failures never change command results or exit codes. - Process lifetime: `app::run_process` owns the Tokio runtime (at least two workers) and ends it with `shutdown_timeout(RUNTIME_SHUTDOWN_GRACE)` (500ms), so a stalled keychain, database, or exporter worker cannot keep the process alive. The async deadlines bound when the command proceeds; the runtime grace is the hard process-level bound. Abandoned workers are not cancelled, only no longer awaited. - Logs: `managed_telemetry/log_export.rs` bridges `target: "sce"` tracing events and their `enabled!` hints (emitted by `Logger` only when the local level is enabled) to OTLP/HTTP protobuf log records at the hardcoded `/v1/logs`, using the same pinned bearer client, batch bounds, and shutdown as traces. Records carry severity, `sce.event_id`, a body limited to path- and secret-free messages (falling back to the event ID), and only allowlisted fields (`command`, `component`, `failure_class`, `error_code`, `error_class`, `error_surface`, `user_error`) with bounded, redacted values; `error_source`, arguments, paths, and hook payloads are never exported. Local file/stderr logging is unchanged. From d67518b113d5679ac0fbac01a9fde933fd044760 Mon Sep 17 00:00:00 2001 From: David Abram Date: Sat, 17 Oct 2026 14:29:50 +0200 Subject: [PATCH 05/17] observability: Export Claude hook operation child spans Expose the mutation-scope, Git snapshot, and persistence phases of admitted Claude hooks as low-cardinality OTLP child spans so hook traces show which operations actually ran without changing command behavior. Add reusable span instrumentation at the Claude adapter, mutation-trace coordinator, and scope runtime boundaries, and filter child spans out of ordinary command exports. Ref: context/plans/claude-hook-trace-propagation.md (T04) Co-authored-by: SCE --- .../hooks/claude_mutation_scope/lifecycle.rs | 171 +++++++++++++++++- .../managed_telemetry/trace_export.rs | 143 +++++++++++++-- .../mutation_trace/runtime/coordinator.rs | 42 ++++- .../mutation_trace/runtime/scope_runtime.rs | 7 +- context/architecture.md | 2 +- .../cli/claude-mutation-scope-integration.md | 2 +- context/context-map.md | 2 +- context/glossary.md | 2 +- context/overview.md | 2 +- .../plans/claude-hook-trace-propagation.md | 22 ++- context/sce/cli-observability-contract.md | 1 + 11 files changed, 363 insertions(+), 33 deletions(-) diff --git a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs index ccfbc383e..91c8c8168 100644 --- a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs +++ b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::Result; use crate::services::hooks; +use crate::services::managed_telemetry::trace_export::{in_child_span, ManagedChildSpan}; use crate::services::mutation_trace::runtime::resolve_git_dir; use super::events::BASH_TOOL_NAME; @@ -94,7 +95,21 @@ pub(super) async fn run_claude_mutation_scope_from_payload_with_resolver< seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { let event = parse_claude_hook_event(stdin_payload)?; - dispatch_claude_hook_event(event, logger, resolve_git_dir, model_state_resolver, seam).await + let traced_seam = async |repository_root: &Path, payload: &str, logger: Option<&L>| { + in_child_span( + ManagedChildSpan::MutationScope, + seam(repository_root, payload, logger), + ) + .await + }; + dispatch_claude_hook_event( + event, + logger, + resolve_git_dir, + model_state_resolver, + &traced_seam, + ) + .await } pub(super) async fn dispatch_claude_hook_event< @@ -508,17 +523,27 @@ pub(crate) async fn repair_blocked (Option, Vec) { + let dir = tempfile::tempdir().expect("temp dir"); + let root = dir.path().join("repo"); + init_repo(&root); + let git_dir = root.join(".git"); + let db_path = dir.path().join("agent-trace.db"); + RepositoryAgentTraceDb::new_at(&db_path) + .await + .expect("create db") + .verify_or_initialize_repository_metadata(TEST_REPOSITORY_ID) + .await + .expect("initialize metadata"); + let resolve = async |_: &str| Ok::<_, anyhow::Error>(git_dir.clone()); + let model_state = async |_: &Path, _: &str, _: &str| Ok::<_, anyhow::Error>(None); + let seam = async |_: &Path, payload: &str, _: Option<&NoopLogger>| { + drive_mutation_scope( + &root, + parse_mutation_scope_payload(payload)?, + None::<&NoopLogger>, + &mut Vec::new(), + async |root: &Path, boundary| { + coordinate(root, boundary, async || { + RepositoryAgentTraceDb::open_without_migrations_at(&db_path).await + }) + .await + }, + async |root: &Path, scope| { + abandon_scope(root, scope, async || { + RepositoryAgentTraceDb::open_without_migrations_at(&db_path).await + }) + .await + }, + advise_after_completed_boundary, + ) + .await + }; + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr"); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let receiver = tokio::spawn(capture_requests_until(listener, stopped)); + let telemetry = loopback_hook_telemetry(address, "bao-token-value"); + let output = telemetry + .with_default_subscriber(&mut || async { + record_claude_hook_payload(payload); + run_claude_mutation_scope_from_payload_with_resolver::( + payload, + None, + &resolve, + &model_state, + &seam, + ) + .await + .map_err(CliError::runtime) + }) + .await; + telemetry.shutdown().await; + stop.send(()).expect("stop receiver"); + let spans = trace_spans(&receiver.await.expect("receiver join")); + (output.ok(), spans) + } + + fn span_named<'a>(spans: &'a [DecodedSpan], name: &str) -> &'a DecodedSpan { + spans + .iter() + .find(|span| span.name == name) + .unwrap_or_else(|| panic!("missing {name}: {spans:?}")) + } + + fn sorted_names(spans: &[DecodedSpan]) -> Vec<&str> { + let mut names = spans + .iter() + .map(|span| span.name.as_str()) + .collect::>(); + names.sort_unstable(); + names + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn tracked_claude_hook_exports_mutation_snapshot_and_persistence_children() { + let (output, spans) = export_claude_hook_spans(WRITE_PRE_TOOL_USE).await; + + assert_eq!(output.as_deref(), Some("")); + assert_eq!( + sorted_names(&spans), + [ + "sce.git_snapshot", + "sce.hook", + "sce.mutation_scope", + "sce.persistence" + ] + ); + let hook = span_named(&spans, "sce.hook"); + let mutation_scope = span_named(&spans, "sce.mutation_scope"); + let git_snapshot = span_named(&spans, "sce.git_snapshot"); + let persistence = span_named(&spans, "sce.persistence"); + assert!(hook.parent_span_id.is_empty()); + assert_eq!(mutation_scope.parent_span_id, hook.span_id); + assert_eq!(git_snapshot.parent_span_id, mutation_scope.span_id); + assert_eq!(persistence.parent_span_id, mutation_scope.span_id); + let success = vec![("sce.outcome".to_string(), "success".to_string())]; + for child in [mutation_scope, git_snapshot, persistence] { + assert_eq!(child.trace_id, hook.trace_id, "{}", child.name); + assert_eq!(child.sce_attributes(), success, "{}", child.name); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn read_only_claude_hook_exports_no_operation_children() { + let payload = + WRITE_PRE_TOOL_USE.replace("\"tool_name\":\"Write\"", "\"tool_name\":\"Read\""); + assert_ne!(payload, WRITE_PRE_TOOL_USE); + + let (output, spans) = export_claude_hook_spans(&payload).await; + + assert_eq!(output.as_deref(), Some("")); + assert_eq!(sorted_names(&spans), ["sce.hook"]); + } } diff --git a/cli/src/services/managed_telemetry/trace_export.rs b/cli/src/services/managed_telemetry/trace_export.rs index 216059e05..59d35ebee 100644 --- a/cli/src/services/managed_telemetry/trace_export.rs +++ b/cli/src/services/managed_telemetry/trace_export.rs @@ -32,6 +32,15 @@ const TRACER_NAME: &str = "sce.cli"; pub const COMMAND_SPAN_TARGET: &str = "sce.managed.trace"; const COMMAND_SPAN_NAME: &str = "sce.command"; const HOOK_SPAN_NAME: &str = "sce.hook"; +const MUTATION_SCOPE_SPAN_NAME: &str = "sce.mutation_scope"; +const GIT_SNAPSHOT_SPAN_NAME: &str = "sce.git_snapshot"; +const PERSISTENCE_SPAN_NAME: &str = "sce.persistence"; +const HOOK_EXPORTED_SPAN_NAMES: [&str; 4] = [ + HOOK_SPAN_NAME, + MUTATION_SCOPE_SPAN_NAME, + GIT_SNAPSHOT_SPAN_NAME, + PERSISTENCE_SPAN_NAME, +]; pub const COMMAND_NAME_FIELD: &str = "sce.command.name"; const AGENT_CLAUDE: &str = "claude"; const HOOK_EVENT_FIELD: &str = "sce.hook.event"; @@ -90,7 +99,15 @@ impl HttpClient for PinnedBearerClient { } fn is_command_span(metadata: &tracing::Metadata<'_>) -> bool { - metadata.is_span() && metadata.target() == COMMAND_SPAN_TARGET + metadata.is_span() + && metadata.target() == COMMAND_SPAN_TARGET + && metadata.name() == COMMAND_SPAN_NAME +} + +fn is_hook_span(metadata: &tracing::Metadata<'_>) -> bool { + metadata.is_span() + && metadata.target() == COMMAND_SPAN_TARGET + && HOOK_EXPORTED_SPAN_NAMES.contains(&metadata.name()) } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -99,6 +116,13 @@ pub enum ManagedSpanKind { ClaudeHook, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ManagedChildSpan { + MutationScope, + GitSnapshot, + Persistence, +} + #[derive(Debug)] pub struct ManagedTraceTelemetry { provider: SdkTracerProvider, @@ -194,11 +218,13 @@ impl ManagedTraceTelemetry { .build(), ) .build(); + let span_filter: fn(&tracing::Metadata<'_>) -> bool = match span_kind { + ManagedSpanKind::Command => is_command_span, + ManagedSpanKind::ClaudeHook => is_hook_span, + }; let span_layer = tracing_opentelemetry::layer() .with_tracer(provider.tracer(TRACER_NAME)) - .with_filter(filter_fn( - is_command_span as fn(&tracing::Metadata<'_>) -> bool, - )); + .with_filter(filter_fn(span_filter)); let log_layer = LogBridgeLayer::new(log_provider.logger(TRACER_NAME)).with_filter( filter_fn(is_exportable_event as fn(&tracing::Metadata<'_>) -> bool), ); @@ -303,6 +329,41 @@ fn extract_parent_context( context.span().span_context().is_valid().then_some(context) } +fn child_span(kind: ManagedChildSpan) -> Span { + match kind { + ManagedChildSpan::MutationScope => tracing::info_span!( + target: COMMAND_SPAN_TARGET, + MUTATION_SCOPE_SPAN_NAME, + sce.outcome = field::Empty, + ), + ManagedChildSpan::GitSnapshot => tracing::info_span!( + target: COMMAND_SPAN_TARGET, + GIT_SNAPSHOT_SPAN_NAME, + sce.outcome = field::Empty, + ), + ManagedChildSpan::Persistence => tracing::info_span!( + target: COMMAND_SPAN_TARGET, + PERSISTENCE_SPAN_NAME, + sce.outcome = field::Empty, + ), + } +} + +pub async fn in_child_span( + kind: ManagedChildSpan, + operation: impl Future>, +) -> Result { + let span = child_span(kind); + let result = Box::pin(operation.instrument(span.clone())).await; + let outcome = if result.is_ok() { + OUTCOME_SUCCESS + } else { + OUTCOME_FAILURE + }; + span.record(OUTCOME_FIELD, outcome); + result +} + pub fn record_command_name(name: &str) { Span::current().record(COMMAND_NAME_FIELD, name); } @@ -358,6 +419,19 @@ pub(crate) mod tests { captured } + pub(crate) async fn capture_requests_until( + listener: TcpListener, + mut stop: tokio::sync::oneshot::Receiver<()>, + ) -> Vec { + let mut captured = Vec::new(); + loop { + tokio::select! { + request = capture_one_request(&listener, "HTTP/1.1 200 OK") => captured.push(request), + _ = &mut stop => return captured, + } + } + } + async fn capture_one_request( listener: &TcpListener, status_line: &'static str, @@ -817,31 +891,27 @@ pub(crate) mod tests { for _ in 0..8 { tokio::task::yield_now().await; } - let child = tracing::info_span!(target: COMMAND_SPAN_TARGET, "sce.test.child"); - async { + in_child_span(ManagedChildSpan::MutationScope, async { tokio::task::yield_now().await; tokio::spawn( async { tokio::task::yield_now().await; - let grandchild = - tracing::info_span!(target: COMMAND_SPAN_TARGET, "sce.test.grandchild"); - async { + in_child_span(ManagedChildSpan::GitSnapshot, async { for _ in 0..4 { tokio::task::yield_now().await; } - } - .instrument(grandchild) - .await; + Ok::<(), ()>(()) + }) + .await } .in_current_span() .with_current_subscriber(), ) .await - .expect("spawned child join"); - tokio::task::yield_now().await; - } - .instrument(child) - .await; + .expect("spawned child join") + }) + .await + .expect("child span result"); }) .await; assert_eq!(spans.len(), 3, "{spans:?}"); @@ -852,8 +922,8 @@ pub(crate) mod tests { .unwrap_or_else(|| panic!("missing {name}")) }; let hook = by_name("sce.hook"); - let child = by_name("sce.test.child"); - let grandchild = by_name("sce.test.grandchild"); + let child = by_name("sce.mutation_scope"); + let grandchild = by_name("sce.git_snapshot"); for span in [hook, child, grandchild] { assert_eq!(span.trace_id, UPSTREAM_TRACE_ID, "{}", span.name); } @@ -862,6 +932,41 @@ pub(crate) mod tests { assert_eq!(grandchild.parent_span_id, child.span_id); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn command_export_keeps_single_command_span_when_child_operations_run() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr"); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let receiver = tokio::spawn(capture_requests_until(listener, stopped)); + let telemetry = loopback_telemetry(address, "bao-token-value"); + + let result = telemetry + .with_default_subscriber(&mut || async { + record_command_name("doctor"); + for kind in [ + ManagedChildSpan::MutationScope, + ManagedChildSpan::GitSnapshot, + ManagedChildSpan::Persistence, + ] { + in_child_span(kind, async { Ok::<(), ()>(()) }) + .await + .expect("child operation"); + } + Ok(String::new()) + }) + .await; + assert_eq!(result.expect("command result"), ""); + telemetry.shutdown().await; + stop.send(()).expect("stop receiver"); + + let spans = trace_spans(&receiver.await.expect("receiver join")); + let names = spans + .iter() + .map(|span| span.name.as_str()) + .collect::>(); + assert_eq!(names, ["sce.command"]); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn managed_trace_export_does_not_follow_redirects_or_alter_results() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); diff --git a/cli/src/services/mutation_trace/runtime/coordinator.rs b/cli/src/services/mutation_trace/runtime/coordinator.rs index 94eee0341..ccb3c5aec 100644 --- a/cli/src/services/mutation_trace/runtime/coordinator.rs +++ b/cli/src/services/mutation_trace/runtime/coordinator.rs @@ -5,6 +5,7 @@ use anyhow::Result; use uuid::Uuid; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::managed_telemetry::trace_export::{in_child_span, ManagedChildSpan}; use crate::services::mutation_trace::protocol; use crate::services::mutation_trace::store::{ CasResult, DurableTransition, MutationTraceStore, ScopeProvenance, @@ -353,8 +354,8 @@ async fn coordinate_boundary_inner( worktree_id: &WorktreeId, boundary: &RuntimeBoundary, inherited_external_taint: bool, - mut after_load: AfterLoad, - mut after_recovery: AfterRecovery, + after_load: AfterLoad, + after_recovery: AfterRecovery, ) -> Result where C: SnapshotCapture, @@ -363,11 +364,44 @@ where { let mut store = MutationTraceStore::new(db); - let observed_tree = match capture_and_pin(capture, pin_lease, worktree_id).await { + let observed_tree = match in_child_span( + ManagedChildSpan::GitSnapshot, + capture_and_pin(capture, pin_lease, worktree_id), + ) + .await + { Ok(tree) => tree, Err(source) => return Err(handle_snapshot_failure(&mut store, worktree_id, source).await), }; + in_child_span( + ManagedChildSpan::Persistence, + persist_boundary( + &mut store, + worktree_id, + boundary, + observed_tree, + inherited_external_taint, + after_load, + after_recovery, + ), + ) + .await +} + +async fn persist_boundary( + store: &mut MutationTraceStore<'_>, + worktree_id: &WorktreeId, + boundary: &RuntimeBoundary, + observed_tree: TreeId, + inherited_external_taint: bool, + mut after_load: AfterLoad, + mut after_recovery: AfterRecovery, +) -> Result +where + AfterLoad: FnMut(u32), + AfterRecovery: FnMut(u32) -> Result<()>, +{ store .initialize_worktree(worktree_id, &observed_tree) .await @@ -383,7 +417,7 @@ where None => None, }; - register_start_provenance(&store, boundary, registered_scope.as_ref()).await?; + register_start_provenance(store, boundary, registered_scope.as_ref()).await?; let type_boundary = into_protocol_boundary(boundary, worktree_id); let scope_ref = types::boundary_scope(&type_boundary); diff --git a/cli/src/services/mutation_trace/runtime/scope_runtime.rs b/cli/src/services/mutation_trace/runtime/scope_runtime.rs index fb3397452..11bf84547 100644 --- a/cli/src/services/mutation_trace/runtime/scope_runtime.rs +++ b/cli/src/services/mutation_trace/runtime/scope_runtime.rs @@ -1,6 +1,7 @@ use std::path::Path; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::managed_telemetry::trace_export::{in_child_span, ManagedChildSpan}; use crate::services::mutation_trace::protocol; use crate::services::mutation_trace::store::{CasResult, DurableTransition, MutationTraceStore}; use crate::services::mutation_trace::types::{ProtocolState, ScopeId, ScopeStatus, WorktreeId}; @@ -147,7 +148,11 @@ where }); } - let outcome = abandon_protected(protected.worktree_id(), scope, open_db, after_load).await?; + let outcome = in_child_span( + ManagedChildSpan::Persistence, + abandon_protected(protected.worktree_id(), scope, open_db, after_load), + ) + .await?; if matches!(outcome, AbandonScopeOutcome::RecoveryRequired { .. }) { return Ok(outcome); diff --git a/context/architecture.md b/context/architecture.md index 08f752e9b..6c6d2bbca 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -53,7 +53,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - Command parse-time conversion and run-time handling are separated by an internal static `RuntimeCommand` seam. `cli/src/services/command_registry.rs` defines the `RuntimeCommand` enum with variants for help/help-text, version, completion, auth, config, setup, doctor, hooks, policy, and sync, plus a deterministic `CommandRegistry` name catalog populated by `build_default_registry()`. `parse_command_phase` in `cli/src/app.rs` delegates clap-output conversion to `cli/src/services/parse/command_runtime.rs`, which owns clap error classification, help rendering bridges, longest-valid-parent traversal for unknown command paths, and parsed-request-to-enum conversion while returning concrete enum values. Service-owned `command.rs` modules define command payload structs and generic execution methods with narrow context requirements: context-free commands accept any context, hooks requires logger access, setup/doctor require repo-root scoping, and central dispatch requires the union of logger plus repo-root-scoping capabilities. `services::app_support::execute_command_phase` emits lifecycle logs around `RuntimeCommand::execute_with_stderr(...)`; the enum performs the only central dispatch match and delegates business behavior to the service-owned command structs, with sync receiving the app-owned stderr writer for format-gated progress. - Startup observability bootstrapping in `cli/src/app.rs` still tolerates invalid default-discovered config files by continuing with degraded defaults plus `sce.config.invalid_config` warn-level logs, but the warning/logging work is now isolated behind the startup-context and runtime-initialization phases rather than one inline startup function. - `cli/src/services/observability.rs` provides deterministic runtime observability controls and rendering for app lifecycle logs, including shared config-resolved threshold/format, explicit config-file/default `log_to_file`, and `log_dir` inputs with precedence `env > config file > defaults` for non-flag observability keys, stable event identifiers, severity filtering, the forced-emission warning path used for invalid discovered config startup diagnostics, error-specific stderr suppression when file logging is enabled while non-error records and file-write diagnostics remain on stderr, redaction-safe emission through the shared security helper, and log-directory writes with bounded retention. Config resolution also carries a positive config-file/default-only `log_file_retention_limit` (`10` by default) into startup observability config and `sce config show`; the concrete logger stores that resolved value and threads it through primary and v2 cleanup. When `log_dir` resolves from `SCE_LOG_DIR`, config, or the `/sce/logs` default, each enabled or forced log operation selects `/sce-.log` or `/sce--.log` using the machine-local date and optional logger session context, with deterministic percent-encoding for unsafe session filename bytes; after successfully writing a newly created selected file, retention keeps the configured number of newest direct regular `*.log` files by mtime plus path/name tie-break and fails open on cleanup errors. Its `observability::traits` submodule exposes the current `Logger` API with `Option<&str>` session context plus the generic async, deliberately non-object-safe `Telemetry` capability and `NoopLogger`; the concrete observability `Logger` and `NoopTelemetry` implement those capability traits; `NoopTelemetry` installs no tracing subscriber. `services::app_support::render_run_outcome` consumes the logger through that trait boundary when logging classified errors and stdout-write failures. -- `cli/src/services/managed_telemetry.rs` (+ `managed_telemetry/trace_export.rs`, `managed_telemetry/log_export.rs`) owns hardcoded managed OTLP defaults, credential-backed runtime telemetry selection from the encrypted auth DB, and bounded fail-open OTLP/HTTP protobuf export of `sce.command` spans (or one `sce.hook` span for Claude-invoked hooks admitted by `managed_span_kind` in `cli/src/app.rs`) and allowlisted `Logger` events to the pinned canonical collector (metrics endpoint reserved, not exported); see `context/sce/cli-observability-contract.md`. +- `cli/src/services/managed_telemetry.rs` (+ `managed_telemetry/trace_export.rs`, `managed_telemetry/log_export.rs`) owns hardcoded managed OTLP defaults, credential-backed runtime telemetry selection from the encrypted auth DB, and bounded fail-open OTLP/HTTP protobuf export of `sce.command` spans (or one `sce.hook` span, with `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` operation children opened through `trace_export::in_child_span` at the Claude adapter seam and in `mutation_trace::runtime`, for Claude-invoked hooks admitted by `managed_span_kind` in `cli/src/app.rs`) and allowlisted `Logger` events to the pinned canonical collector (metrics endpoint reserved, not exported); see `context/sce/cli-observability-contract.md`. - `cli/src/services/observability.rs` no longer owns duplicate log enums or parsing helpers; it consumes the canonical primitive seam from `cli/src/services/config/mod.rs` and stays focused on concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary. - `cli/src/cli_schema.rs` is now the canonical owner for top-level command metadata for the real clap-backed command set (`auth`, `config`, `setup`, `doctor`, `hooks`, `policy`, `sync`, `version`, `completion`), including the slim top-level help purpose text and per-command visibility on `sce`, `sce help`, and `sce --help`; `cli/src/command_surface.rs` remains the custom top-level help renderer and known-command classifier, adding the synthetic `help` row plus the ASCII banner while consuming that shared metadata instead of maintaining a parallel command catalog. - `cli/src/services/default_paths.rs` is the canonical production path catalog for the CLI: it resolves config/state/cache roots with platform-aware XDG or `dirs` fallbacks through an internal `roots` seam, exposes named default paths for current persisted artifacts and database/log files (global config, auth tokens, auth DB, local DB, default observability log directory, and the sole Agent Trace DB path helper `agent_trace_db_path_for_repository` under `repos//agent-trace.db`; the former global-sentinel and per-checkout Agent Trace path helpers were removed by the `retire-legacy-agent-trace-db` plan), and owns canonical repo-relative, embedded-asset, install, hook, and context-path accessors so non-test production path definitions have one shared owner. Compile-time generated payload paths are owned by `build.rs` under `OUT_DIR`, not by the default-path catalog. Current production consumers such as config discovery, observability config resolution, doctor reporting, setup/install flows, database adapters, checkout identity, Agent Trace storage resolution, and local hook runtime path resolution consume this shared catalog rather than defining owned path literals in their own modules. diff --git a/context/cli/claude-mutation-scope-integration.md b/context/cli/claude-mutation-scope-integration.md index cd37b8c3e..47daf9ccf 100644 --- a/context/cli/claude-mutation-scope-integration.md +++ b/context/cli/claude-mutation-scope-integration.md @@ -238,7 +238,7 @@ mutation-stack dependency is the reused `RuntimeBoundary` path or spawned `sce` subprocess. The production resolver uses the hooks-layer Agent Trace DB opener only for the exact `claude_model_state` read; the seam retains strict parsing, lazy DB acquisition, -durable-completion classification, and empty-stdout semantics. +durable-completion classification, and empty-stdout semantics. The adapter also wraps each seam call in the managed `sce.mutation_scope` trace span ([cli-observability-contract.md](../sce/cli-observability-contract.md)), which is disabled without a managed hook dispatch. ## Related context diff --git a/context/context-map.md b/context/context-map.md index a17e8a208..5256aa42d 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -57,7 +57,7 @@ Feature/domain context: - `context/sce/cli-shell-completion-contract.md` (implemented `sce completion` contract for deterministic Bash/Zsh/Fish completion script generation) - `context/sce/claude-raw-hook-capture.md` (removed feature: the former `sce hooks claude-capture` raw-capture route and its supporting types, replaced by the active `diff-trace` and `conversation-trace` intakes) - `context/sce/claude-hook-trace-propagation.md` (probe evidence for Claude Code 2.1.301 command-hook trace context: `TRACEPARENT` names the `claude_code.interaction` span, not the tool span, and only when Claude tracing is active; hook-to-tool correlation by `tool_use_id`; `CLAUDECODE` + `CLAUDE_PROJECT_DIR` presence as the Claude hook-caller signal; the shipped `sce.hook` contract lives in `cli-observability-contract.md`, admission decision `context/decisions/2026-10-17-claude-hook-managed-trace-admission.md`) -- `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary, logger and deliberately non-object-safe generic async telemetry boundaries with static capability dispatch and future-aware polling scope, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, managed OTLP trace/log export (credential-gated, hardcoded canonical collector, Claude-only hook admission with one `sce.hook` span and correlation-only session/tool-use IDs, allowlisted fields, bounded fail-open lifecycle, reserved metrics endpoint), and the trimmed `sce config validate` status-only validation surface) +- `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary, logger and deliberately non-object-safe generic async telemetry boundaries with static capability dispatch and future-aware polling scope, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, managed OTLP trace/log export (credential-gated, hardcoded canonical collector, Claude-only hook admission with one `sce.hook` span and correlation-only session/tool-use IDs, its `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` operation child spans with a span-kind-specific export filter, allowlisted fields, bounded fail-open lifecycle, reserved metrics endpoint), and the trimmed `sce config validate` status-only validation surface) - `context/sce/shared-context-code-workflow.md` (canonical `/next-task` task-synchronization lifecycle and validation-only `/validate` lifecycle, package-local phase references with single-skill control flow, and the task-synchronization-scoped `sce-decision` sibling invocation with ADR reuse/blocker propagation) - `context/sce/shared-context-plan-workflow.md` (canonical `/change-to-plan` workflow, package-local context-load/plan-authoring/template references, clarification/readiness gate contract, and one-task/one-atomic-commit task slicing) - [Context workflow rules](sce/context-workflow-rules.md) (canonical bootstrap, ongoing context maintenance, task synchronization, hygiene, discoverability, and feature-existence rules) diff --git a/context/glossary.md b/context/glossary.md index 01608cf96..f42dfe56b 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -80,7 +80,7 @@ - `encrypted Turso adapter`: Generic native async adapter seam in `cli/src/services/db/mod.rs` exposed as `EncryptedTursoDb`, structurally parallel to `TursoDb` and owning no Tokio runtime. Its constructor resolves the encryption key via `encryption_key::get_or_create_encryption_key(&db_path, db_name)`, which derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text before falling back to OS credential-store keyring get-or-create behavior; credential-store default registration is guarded by stable `OnceLock` plus an atomic in-progress flag so errors or panics leave initialization retryable without mutex poisoning. The adapter enables Turso local encryption with strict `aegis256` cipher selection through `turso::EncryptionOpts`, wraps encrypted local open/connect in the default DB connection-open retry policy, and runs embedded migrations after retry has produced a connection; execute, query, query_map, and migration operations are async and preserve `__sce_migrations` tracking parity. - `auth DB adapter`: Module in `cli/src/services/auth_db/mod.rs` that defines `AuthDbSpec` and exposes `AuthDb` as an `EncryptedTursoDb` alias. It resolves the canonical `/sce/auth.db` path with `auth_db_path()`, keeps encryption mandatory with `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret precedence before OS keyring fallback and no plaintext mode, and embeds ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. Auth runtime token-storage is now wired through `cli/src/services/token_storage.rs`, which persists tokens via the `auth_credentials` table in the encrypted auth DB instead of a JSON file. - `telemetry credential`: Telemetry-only OpenBao token and accessor issued by the Control Plane `POST /auth/telemetry/credential`, stored as one fixed row in the encrypted auth DB `telemetry_credentials` table by `cli/src/services/telemetry_credential_storage.rs`, returned only while unexpired, removed by `sce auth logout`, not revoked server-side by local deletion, and used as the sole bearer secret for managed OTLP export (never the accessor); see `context/sce/telemetry-credential.md`. -- `sce.hook span`: The single root-of-process managed trace span a Claude-invoked SCE hook exports in place of `sce.command`, carrying `sce.agent=claude`, the hook event, tool name, outcome, and the Claude `session_id`/`tool_use_id` as correlation-only attributes. It is parented on a valid Claude-provided `TRACEPARENT` (the `claude_code.interaction` span, so it sits beside `claude_code.tool`) and is otherwise an independent root. Admission is Claude-only (Claude-only hook subcommands by argv; shared hook subcommands and `sce policy bash` only with `CLAUDECODE` and `CLAUDE_PROJECT_DIR` present); see `context/sce/cli-observability-contract.md`. +- `sce.hook span`: The single root-of-process managed trace span a Claude-invoked SCE hook exports in place of `sce.command`, carrying `sce.agent=claude`, the hook event, tool name, outcome, and the Claude `session_id`/`tool_use_id` as correlation-only attributes. It is parented on a valid Claude-provided `TRACEPARENT` (the `claude_code.interaction` span, so it sits beside `claude_code.tool`) and is otherwise an independent root. Its only exported children are `sce.mutation_scope` (one per adapter seam call) and, beneath it, `sce.git_snapshot` and `sce.persistence`, each emitted only when that operation actually runs and carrying only `sce.outcome`. Admission is Claude-only (Claude-only hook subcommands by argv; shared hook subcommands and `sce policy bash` only with `CLAUDECODE` and `CLAUDE_PROJECT_DIR` present); see `context/sce/cli-observability-contract.md`. - `AuthDbLifecycle`: Lifecycle provider in `cli/src/services/auth_db/lifecycle.rs` that implements `ServiceLifecycle` for encrypted auth DB setup/doctor integration. `diagnose` collects auth DB path health problems, `fix` bootstraps missing auth DB parent directory, and `setup` calls `AuthDb::new()`. Registered as `LifecycleProviderId::AuthDb` in the shared lifecycle catalog. - `agent trace DB adapter`: Modules under `cli/src/services/agent_trace_db/` that define the sole repository-scoped `RepositoryAgentTraceDb = TursoDb` adapter (the checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter and its 15-file migration chain were removed by the `retire-legacy-agent-trace-db` plan). The repository adapter uses the `agent-trace-repository` migration set (fresh baseline schema plus additive `source_instance_id` and `claude_model_state` migrations) with `repository_metadata`, repository-level `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts` tables, no row-level `checkout_id`, typed parameterized insert helpers, the non-exported `claude_model_state` latest-locally-observed register with exact `(session_id, agent_id)` reads and guarded writes, and chronological recent `diff_traces` query/parse support. `AgentTraceDbLifecycle` initializes/checks repository-scoped storage through `agent_trace_storage`. - `post-commit Agent Trace auto-sync readiness`: The doctor report fact that explains whether the enabled post-commit trigger is ready without invoking it. Doctor compares the installed `post-commit` hook's SCE managed block using the same currency semantics as setup and resolves config-file-only `agent_trace.auto_sync` with source metadata. JSON states are `ready`, `disabled`, `not_ready`, and `not_applicable`; explicit disable is healthy, while existing hook problems continue to own overall readiness and remediation. See [automatic Agent Trace synchronization](cli/agent-trace-auto-sync.md) and [doctor human text](sce/doctor-human-text-contract.md). diff --git a/context/overview.md b/context/overview.md index 47cf66fc2..cef3d0f8b 100644 --- a/context/overview.md +++ b/context/overview.md @@ -65,7 +65,7 @@ Its command loop is implemented with `clap` derive-based argument parsing and `a The current doctor presentation contract supersedes the earlier output-shape scaffolding wording above: human text uses the compact Environment/Repository/Integrations hierarchy with healthy rows collapsed and unhealthy branches expanded, while JSON retains complete path, identity, problem, and fix-result detail. See `context/sce/doctor-human-text-contract.md`. The command loop now enforces a stable exit-code contract in `cli/src/app.rs`: `2` parse failures, `3` invocation validation failures, `4` runtime failures, and `5` dependency startup failures. Auth command orchestration routes authenticated Control Plane authentication failures, token-storage failures, and approved user-facing fallback failures through typed `CliError::User` catalog entries (`NotAuthenticated`, `AuthStorageUnavailable`, `NotGitRepository`, and `UnexpectedFailure`) while retaining technical sources for observability; missing-credential logout/whoami state queries succeed with their documented text/JSON reports, and internal auth failures remain runtime errors. The same runtime also emits stable CLI stderr diagnostics: internal failures use `SCE-ERR-PARSE`, `SCE-ERR-VALIDATION`, `SCE-ERR-RUNTIME`, or `SCE-ERR-DEPENDENCY` in deterministic `Error []: ...` diagnostics with class-default `Try:` remediation appended when missing, while expected failures emit only their redacted, unstyled catalog messages. The command boundary's former flat, string-only `ClassifiedError` has been replaced by typed `CliError` in `cli/src/services/error.rs`: `CliError::User` carries a closed catalog (`NotAuthenticated`, `NotGitRepository`, `NotGitRemote`, authentication-storage `AuthStorageUnavailable`, and general `UnexpectedFailure`) for expected, deliberately-explained failures rendered without the technical source, wrapper, styling, or automatic `Try:` guidance; the general entry renders one fixed static log-files guidance sentence without dynamic path text. `CliError::Internal` carries a live `anyhow::Error` source rendered as the real error chain with the existing styled wrapper and class-default remediation; `app_support` is the sole owner of the distinct terminal paths, and `sce sync` classifies authentication, credential-storage, and all other sync failures as cataloged user errors while preserving their technical sources for observability. Setup preflight errors preserve technical sources, including the configured remote name, while raw remote URLs remain out of user-facing diagnostics. See `context/sce/cli-error-code-taxonomy.md` for the full contract. -The app runtime now also includes a structured observability baseline in `cli/src/services/observability.rs`: deterministic env-controlled log threshold/format (`SCE_LOG_LEVEL` defaults to `error`; `SCE_LOG_FORMAT` defaults to `text`), default-backed log-directory routing (`SCE_LOG_DIR` / config-file `log_dir` / `/sce/logs`) with per-operation machine-local dated file selection, optional session filename partitioning, and creation-triggered retention of direct regular `*.log` files to 10 entries, stable lifecycle event IDs, normal logger records routed only to files when file logging is enabled and only error-level logger records routed to `stderr` when file logging is disabled, direct file-write diagnostics remaining on `stderr`, and `observability::traits` boundaries for logger and telemetry behavior. When the encrypted auth DB holds a valid unexpired telemetry credential, commands also export `sce.command` spans (Claude-invoked SCE hooks export one `sce.hook` span instead; other hooks export nothing) and allowlisted logger events as OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev` `/v1/traces` and `/v1/logs` paths (bearer token only, redirects disabled, ambient `OTEL_EXPORTER_OTLP_*` and repo config cannot redirect it, fail-open and bounded); without a credential nothing is exported, and `/v1/metrics` is reserved with no metrics pipeline. +The app runtime now also includes a structured observability baseline in `cli/src/services/observability.rs`: deterministic env-controlled log threshold/format (`SCE_LOG_LEVEL` defaults to `error`; `SCE_LOG_FORMAT` defaults to `text`), default-backed log-directory routing (`SCE_LOG_DIR` / config-file `log_dir` / `/sce/logs`) with per-operation machine-local dated file selection, optional session filename partitioning, and creation-triggered retention of direct regular `*.log` files to 10 entries, stable lifecycle event IDs, normal logger records routed only to files when file logging is enabled and only error-level logger records routed to `stderr` when file logging is disabled, direct file-write diagnostics remaining on `stderr`, and `observability::traits` boundaries for logger and telemetry behavior. When the encrypted auth DB holds a valid unexpired telemetry credential, commands also export `sce.command` spans (Claude-invoked SCE hooks export one root `sce.hook` span instead, with `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` children only for operations that ran; other hooks export nothing) and allowlisted logger events as OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev` `/v1/traces` and `/v1/logs` paths (bearer token only, redirects disabled, ambient `OTEL_EXPORTER_OTLP_*` and repo config cannot redirect it, fail-open and bounded); without a credential nothing is exported, and `/v1/metrics` is reserved with no metrics pipeline. One application-level multi-thread Tokio runtime directly awaits static command dispatch through async credentials, lifecycle providers, hooks, sync/export, and Turso persistence. The shared Turso adapters own no runtime or executor bridge, and foreground hooks return after required persistence completes. See [application execution runtime](architecture.md#application-execution-runtime). The app command dispatcher now enforces a centralized stdout/stderr stream contract in `cli/src/app.rs`: command success payloads are emitted on stdout only, while redacted user-facing diagnostics and text-mode sync progress are emitted on stderr; JSON sync remains silent. `cli/src/app.rs` also now runs through explicit startup phases — dependency check, observability config resolution, runtime initialization, command parse/execute, and output rendering — with the app runtime carrying logger/telemetry plus static command-catalog state across those phases while preserving the existing exit-code and degraded-startup contracts. Within that lifecycle, `parse_command_phase` delegates clap-to-runtime conversion to `cli/src/services/parse/command_runtime.rs`, which returns a static `RuntimeCommand` enum, `services::app_support::execute_command_phase` logs around enum-owned `execute_with_stderr(...)` dispatch, and generic `RunOutcome` rendering logs classified errors through the logger trait boundary without coupling render support to the production logger type. Command payload structs for `help`, `version`, `completion`, `auth`, `config`, `setup`, `doctor`, `hooks`, and `sync` live in service-owned `command.rs` files; `cli/src/services/command_registry.rs` owns the deterministic static command-name catalog and enum variants instead of boxed command trait objects. The CLI now also enforces a shared output-format parser contract in `cli/src/services/output_format.rs`, with canonical `--format ` parsing and command-specific actionable invalid-value guidance reused by `config` and `version` services. A compile-safe service lifecycle seam also exists in `cli/src/services/lifecycle.rs`: `ServiceLifecycle` exposes default no-op `diagnose`, `fix`, and `setup` methods against the narrow `HasRepoRoot` accessor, uses lifecycle-owned health/fix/setup result types, and owns the shared static `LifecycleProvider` enum catalog/factory with deterministic config → local*db → auth_db → agent_trace_db → hooks ordering and no boxed provider aggregation. Hooks has a `services/hooks/lifecycle.rs` provider for hook rollout diagnosis/fix/setup, config has a `services/config/lifecycle.rs` provider for global/repo-local config validation plus repo-local config bootstrap, local_db has a `services/local_db/lifecycle.rs` provider for canonical local DB path health, parent-directory readiness/bootstrap, and `LocalDb::new()` setup, auth_db has a `services/auth_db/lifecycle.rs` provider for canonical auth DB path health, parent-directory readiness/bootstrap, and `AuthDb::new()` setup, and agent_trace_db has a `services/agent_trace_db/lifecycle.rs` provider for repository-scoped Agent Trace DB setup and repository DB path health/parent readiness from resolved repository identity, returning an actionable "requires a Git repository" diagnostic outside repository context (no global/checkout fallback path). Doctor runtime aggregates the full shared provider catalog for `diagnose` and `fix` and adapts lifecycle records into doctor-owned output records; setup command aggregates the shared provider catalog for `setup` with hooks included only when requested and adapts lifecycle setup outcomes before rendering setup-owned messages. diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md index f6e25e336..d3c4c1e77 100644 --- a/context/plans/claude-hook-trace-propagation.md +++ b/context/plans/claude-hook-trace-propagation.md @@ -118,13 +118,31 @@ Persist this field in every plan; this is durable plan state, not chat state: - Context impact: important change. `sce.hook` now joins Claude's trace when Claude Code's tracing beta supplies `TRACEPARENT`, otherwise it stays an independent root. This affects the span contract in `context/sce/cli-observability-contract.md` and the verdict/implementation notes in `context/sce/claude-hook-trace-propagation.md`; the glossary/architecture entries for `sce.hook` gain the parent-context rule. - Context synchronization: synced -- [ ] T04: `Add mutation-scope, git-snapshot, and persistence child spans under sce.hook` (status:todo) +- [x] T04: `Add mutation-scope, git-snapshot, and persistence child spans under sce.hook` (status:done) - Task ID: T04 - Scope: In — spans on the managed-trace target at the existing execution sites: the Claude mutation-scope coordinate call (`sce.mutation_scope`), the mutation-trace Git snapshot/reconciliation call (`sce.git_snapshot`), and Agent Trace DB / mutation store persistence (`sce.persistence`), each with low-cardinality outcome attributes; extend the span filter so only these named spans export; loopback tests asserting the exact span set for an executing and a non-executing hook path. Out — spans inside pure protocol functions or arbitrary helpers; changes to ordering, locking, or failure semantics. - Dependencies: T03 - Done when: AC5 holds, existing mutation-trace and Agent Trace tests pass unchanged, and no span is emitted for an operation that did not run. - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml claude_mutation_scope`; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`. - - Context synchronization: pending + - Completed: 2026-10-17 + - Files changed: `cli/src/services/managed_telemetry/trace_export.rs`, `cli/src/services/hooks/claude_mutation_scope/lifecycle.rs`, `cli/src/services/mutation_trace/runtime/coordinator.rs`, `cli/src/services/mutation_trace/runtime/scope_runtime.rs` + - Result: Added `ManagedChildSpan { MutationScope, GitSnapshot, Persistence }` and `in_child_span(kind, future)`. The helper wraps the future in a span named `sce.mutation_scope`, `sce.git_snapshot`, or `sce.persistence` on the `sce.managed.trace` target, boxes the instrumented future, and records `sce.outcome=success|failure` from the `Result`. The span filter is now selected per span kind: a `Command` dispatch exports only `sce.command`, so ordinary commands that reach the coordinator still export exactly one span; a `ClaudeHook` dispatch exports only `sce.hook` and the three child names. Span sites: + - `run_claude_mutation_scope_from_payload_with_resolver` wraps every seam (coordinate/abandon boundary) call in `sce.mutation_scope`. + - `coordinate_boundary_inner` wraps `capture_and_pin` in `sce.git_snapshot`. + - The post-snapshot store phase (initialize, register, provenance, CAS loop) moved unchanged into `persist_boundary`, which is wrapped in `sce.persistence`. + - `abandon_scope_inner` wraps `abandon_protected` in `sce.persistence`. + + Spans are disabled without a managed hook dispatch. Ordering, locking, and errors are unchanged. Tests added: + - A real temporary git repo and Agent Trace DB, with a real `coordinate` seam, run a Claude `Write` PreToolUse. It exports exactly `sce.hook` → `sce.mutation_scope` → {`sce.git_snapshot`, `sce.persistence`} in one trace, all with `success`. + - A read-only `Read` PreToolUse exports only `sce.hook`. + - A `Command` dispatch that runs all three child operations exports only `sce.command`. + + The T03 multi-poll test now uses the real child span names. New test helper `capture_requests_until` collects requests until shutdown, so batch timing cannot change the captured set. + - Assumptions: `sce.persistence` covers the mutation-store phase of coordinate and abandon only. Taint persistence after a snapshot failure, and conversation/diff-trace Agent Trace DB writes, are not wrapped. `sce.git_snapshot` and `sce.persistence` are grandchildren of `sce.hook` through `sce.mutation_scope`. Child spans carry only `sce.outcome`. + - Deviation: The first `nix flake check` failed `cli-clippy` with `large_futures` at 15 call sites, because the instrumented chain exceeded the 16 KiB future limit. Fixed in scope by `Box::pin`-ing the instrumented future inside `in_child_span`. + - Verify outcomes: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml claude_mutation_scope`: 7 passed, 0 failed (passed); `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`: 26 passed, 0 failed (passed); additionally `nix flake check` after the fix: all checks passed (cli-tests incl. existing mutation-trace and Agent Trace suites, cli-clippy, cli-fmt). + - Context impact: important change. The `sce.hook` span contract gains three operation child spans, plus the rule that spans are exported per span kind (`sce.command` alone for commands; `sce.hook` plus children for Claude hooks). This affects `context/sce/cli-observability-contract.md`, `context/sce/claude-hook-trace-propagation.md`, `context/cli/claude-mutation-scope-integration.md` (span boundaries over the adapter seam and coordinator), and the `context/glossary.md` `sce.hook` entry. + - Context synchronization: synced - [ ] T05: `Record live Dash0 verification and sync hook-telemetry context` (status:todo) - Task ID: T05 diff --git a/context/sce/cli-observability-contract.md b/context/sce/cli-observability-contract.md index b85720ac8..40df94050 100644 --- a/context/sce/cli-observability-contract.md +++ b/context/sce/cli-observability-contract.md @@ -99,6 +99,7 @@ Runtime observability consumes the shared resolved observability config from `cl - Export is OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev/v1/traces`. A pinned HTTP client sets `Authorization: Bearer ` (token only, never the accessor), disables redirects, rebuilds the request to the fixed endpoint, and bounds requests to 3s. Ambient `OTEL_EXPORTER_OTLP_*` values cannot redirect it. - Each command emits one `sce.command` span with `sce.command.name`, `sce.outcome` (`success`/`failure`), and `sce.error.class` (the `SCE-ERR-*` code on failure); resource attributes are `service.name=sce` and `service.version`. Raw arguments, paths, and error chains are not exported. - An admitted Claude hook process emits exactly one `sce.hook` span instead (no `sce.command` span), with `sce.agent=claude`, `sce.outcome`, `sce.error.class`, and, when present as non-empty strings in the Claude hook stdin payload, `sce.hook.event` (`hook_event_name`), `sce.hook.tool` (`tool_name`), `sce.claude.session_id`, and `sce.claude.tool_use_id`. `trace_export::record_claude_hook_payload` is the only reader: each Claude hook entry point passes its stdin to it once, it returns immediately when the current span is disabled (Noop), and it copies only those four fields. Session and tool-use IDs are opaque correlation identifiers: span attributes only, never metric attributes, never used to derive a trace ID. Tool input and response, prompts, `cwd`, transcript paths, file contents, and the raw payload are never exported. Parent context: for the `sce.hook` kind only, `ManagedTraceTelemetry::new` reads `TRACEPARENT` (and `TRACESTATE` when present) from the process environment, extracts it with `TraceContextPropagator`, and keeps it only when the remote span context is valid; `root_span` then attaches it with `set_parent`, so `sce.hook` shares Claude Code's trace ID, its parent is the upstream span (Claude Code 2.1.301 supplies `claude_code.interaction`, making `sce.hook` a sibling of `claude_code.tool`), and its own span ID is freshly generated. Child spans created inside the instrumented hook future inherit that trace across `.await` points and worker migration. No global propagator is registered and `sce.command` never reads the variables. A missing, malformed, or all-zero traceparent (the normal case when Claude tracing is off) yields an independent SCE root. Telemetry failures never change hook stdout, permission decisions, persisted state, or exit codes. +- Operation child spans: `trace_export::in_child_span(ManagedChildSpan, future)` wraps an existing operation in one of three named spans, each carrying only `sce.outcome` (`success`/`failure` from the operation's `Result`): `sce.mutation_scope` around every Claude adapter seam call (one coordinate or abandon boundary), `sce.git_snapshot` around the coordinator's tree capture and pin, and `sce.persistence` around the mutation-store phase of a coordinate boundary (initialize, scope/provenance registration, CAS commit loop) and of an abandon. The hierarchy is `sce.hook → sce.mutation_scope → {sce.git_snapshot, sce.persistence}`. A child span exists only when its operation runs: a read-only tool, an untracked background shell, or a close with no matching attempt exports `sce.hook` alone. Snapshot-failure taint persistence and conversation/diff-trace Agent Trace DB writes have no span. The span filter depends on the span kind: a `sce.command` dispatch exports only `sce.command` (ordinary commands that reach the coordinator, such as ref maintenance, still export one span), and a `sce.hook` dispatch exports only `sce.hook` and the three child names. Without a managed hook dispatch the child spans are disabled, and they never change ordering, locking, or errors. - The exporter uses a bounded batch queue on a dedicated thread. Final stdout/stderr rendering and classified-error logging run inside the command span (`run_command_lifecycle_with_context`), so `sce.outcome` reflects the rendered result (a stdout write failure is `failure`) and the final `sce.error.*` log is exported. Trace and log providers shut down concurrently under one shared `SHUTDOWN_TIMEOUT` (3s) deadline after rendering; failures never change command results or exit codes. - Process lifetime: `app::run_process` owns the Tokio runtime (at least two workers) and ends it with `shutdown_timeout(RUNTIME_SHUTDOWN_GRACE)` (500ms), so a stalled keychain, database, or exporter worker cannot keep the process alive. The async deadlines bound when the command proceeds; the runtime grace is the hard process-level bound. Abandoned workers are not cancelled, only no longer awaited. - Logs: `managed_telemetry/log_export.rs` bridges `target: "sce"` tracing events and their `enabled!` hints (emitted by `Logger` only when the local level is enabled) to OTLP/HTTP protobuf log records at the hardcoded `/v1/logs`, using the same pinned bearer client, batch bounds, and shutdown as traces. Records carry severity, `sce.event_id`, a body limited to path- and secret-free messages (falling back to the event ID), and only allowlisted fields (`command`, `component`, `failure_class`, `error_code`, `error_class`, `error_surface`, `user_error`) with bounded, redacted values; `error_source`, arguments, paths, and hook payloads are never exported. Local file/stderr logging is unchanged. From 62fbe5a7425e73a48a05d0a01191af5202a204fc Mon Sep 17 00:00:00 2001 From: David Abram Date: Sat, 17 Oct 2026 18:00:38 +0200 Subject: [PATCH 06/17] context: Record Claude hook telemetry verification Complete T05 for the claude-hook-trace-propagation plan by recording Dash0 evidence for `sce doctor` and a Claude Code session, documenting the verified and unverified span hierarchy, and adding an opt-in live smoke procedure. Sync the current admission, session-boundary behavior, credential, and context-map documentation. The live evidence confirms standalone and session-boundary exports while leaving parented tool-hook delivery unverified; local validation and smoke results are recorded without claiming collector delivery. Plan: claude-hook-trace-propagation, task T05 Co-authored-by: SCE --- context/context-map.md | 2 +- .../plans/claude-hook-trace-propagation.md | 67 ++++++++++++--- context/sce/claude-hook-trace-propagation.md | 84 ++++++++++++++++++- context/sce/cli-observability-contract.md | 2 +- context/sce/telemetry-credential.md | 2 +- 5 files changed, 141 insertions(+), 16 deletions(-) diff --git a/context/context-map.md b/context/context-map.md index 5256aa42d..4544c0f36 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -56,7 +56,7 @@ Feature/domain context: - `context/sce/cli-version-command-contract.md` (implemented `sce version` contract for deterministic human and machine-readable runtime identification) - `context/sce/cli-shell-completion-contract.md` (implemented `sce completion` contract for deterministic Bash/Zsh/Fish completion script generation) - `context/sce/claude-raw-hook-capture.md` (removed feature: the former `sce hooks claude-capture` raw-capture route and its supporting types, replaced by the active `diff-trace` and `conversation-trace` intakes) -- `context/sce/claude-hook-trace-propagation.md` (probe evidence for Claude Code 2.1.301 command-hook trace context: `TRACEPARENT` names the `claude_code.interaction` span, not the tool span, and only when Claude tracing is active; hook-to-tool correlation by `tool_use_id`; `CLAUDECODE` + `CLAUDE_PROJECT_DIR` presence as the Claude hook-caller signal; the shipped `sce.hook` contract lives in `cli-observability-contract.md`, admission decision `context/decisions/2026-10-17-claude-hook-managed-trace-admission.md`) +- `context/sce/claude-hook-trace-propagation.md` (probe evidence for Claude Code 2.1.301 command-hook trace context: `TRACEPARENT` names the `claude_code.interaction` span, not the tool span, and only when Claude tracing is active; hook-to-tool correlation by `tool_use_id`; `CLAUDECODE` + `CLAUDE_PROJECT_DIR` presence as the Claude hook-caller signal; T05 live Dash0 verification (doctor and session-boundary hook spans observed, parented tool-hook hierarchy unverified) and the opt-in live smoke procedure; the shipped `sce.hook` contract lives in `cli-observability-contract.md`, admission decision `context/decisions/2026-10-17-claude-hook-managed-trace-admission.md`) - `context/sce/cli-observability-contract.md` (implemented config-backed runtime observability contract for the flat logging config-file shape with explicit config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` env-over-config-over-`/sce/logs` fallback, append-only local-date/session log file routing with a one-time complete-record `-v2.log` fallback on primary open/append/flush failure, creation-triggered retention of direct regular `*.log` files to 10 entries, reliable producer-native diff-trace/conversation-trace session routing and hook-specific non-duplicated Agent Trace DB-open error events, deterministic session filename sanitization, concrete `Logger` behavior plus the `NoopTelemetry`-backed telemetry capability boundary, logger and deliberately non-object-safe generic async telemetry boundaries with static capability dispatch and future-aware polling scope, AppContext observability wiring, generic `RunOutcome` final rendering, runtime-classified repeated telemetry action protection, operator-facing `sce config show` observability reporting, managed OTLP trace/log export (credential-gated, hardcoded canonical collector, Claude-only hook admission with one `sce.hook` span and correlation-only session/tool-use IDs, its `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` operation child spans with a span-kind-specific export filter, allowlisted fields, bounded fail-open lifecycle, reserved metrics endpoint), and the trimmed `sce config validate` status-only validation surface) - `context/sce/shared-context-code-workflow.md` (canonical `/next-task` task-synchronization lifecycle and validation-only `/validate` lifecycle, package-local phase references with single-skill control flow, and the task-synchronization-scoped `sce-decision` sibling invocation with ADR reuse/blocker propagation) - `context/sce/shared-context-plan-workflow.md` (canonical `/change-to-plan` workflow, package-local context-load/plan-authoring/template references, clarification/readiness gate contract, and one-task/one-atomic-commit task slicing) diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md index d3c4c1e77..a66a9a02d 100644 --- a/context/plans/claude-hook-trace-propagation.md +++ b/context/plans/claude-hook-trace-propagation.md @@ -10,23 +10,23 @@ This reverses one recorded behavior: today every `sce hooks ...`, `sce policy ba How this plan is proven complete. Each criterion is observable and names the check that proves it. `/validate` runs these checks; no task in the stack performs final validation. -- [ ] AC1: Durable context records, from real Claude Code execution of `PreToolUse` and `PostToolUse` around a real tool call, whether command hooks receive `TRACEPARENT`/`TRACESTATE` (or equivalent), whether the value is a valid W3C traceparent, whether its trace ID and parent span ID match a span Claude Code actually exported, the Claude Code version tested, and the documentation consulted; the probe captured only `session_id`, `hook_event_name`, `tool_use_id`, presence/validity flags, trace ID, and parent span ID, and no probe code remains in the repository. +- [x] AC1: Durable context records, from real Claude Code execution of `PreToolUse` and `PostToolUse` around a real tool call, whether command hooks receive `TRACEPARENT`/`TRACESTATE` (or equivalent), whether the value is a valid W3C traceparent, whether its trace ID and parent span ID match a span Claude Code actually exported, the Claude Code version tested, and the documentation consulted; the probe captured only `session_id`, `hook_event_name`, `tool_use_id`, presence/validity flags, trace ID, and parent span ID, and no probe code remains in the repository. - Validate: Inspect `context/sce/claude-hook-trace-propagation.md` for the recorded evidence table and verdict; `git ls-files` shows no probe script; grep the recorded evidence for tool input, prompt text, `Bearer`, or token-shaped values (none). -- [ ] AC2: With a valid stored telemetry credential, each Claude-invoked SCE hook process exports exactly one `sce.hook` root-of-process span (no additional `sce.command` span) to the pinned `/v1/traces` endpoint, carrying `service.name=sce`, `sce.agent=claude`, `sce.hook.event`, `sce.hook.tool` when present, and `sce.outcome`; Codex, OpenCode, and Pi hook invocations and `sce auth telemetry-headers` still resolve `NoopTelemetry`, and ordinary commands keep their unchanged `sce.command` span. +- [x] AC2: With a valid stored telemetry credential, each Claude-invoked SCE hook process exports exactly one `sce.hook` root-of-process span (no additional `sce.command` span) to the pinned `/v1/traces` endpoint, carrying `service.name=sce`, `sce.agent=claude`, `sce.hook.event`, `sce.hook.tool` when present, and `sce.outcome`; Codex, OpenCode, and Pi hook invocations and `sce auth telemetry-headers` still resolve `NoopTelemetry`, and ordinary commands keep their unchanged `sce.command` span. - Validate: Loopback-receiver Rust tests decode the captured OTLP payload for a Claude `PostToolUse` fixture and assert span name, count, and attributes; admission-predicate unit tests cover Claude, non-Claude, and helper invocations; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry`. -- [ ] AC3: When T01 proved usable parent context: a valid upstream traceparent makes the `sce.hook` span share the upstream trace ID with its parent span ID equal to the upstream span ID, SCE span IDs are freshly generated, child spans inherit the trace across `.await` points and Tokio worker migration, and an invalid, malformed, or missing traceparent yields a new independent SCE root without panic or hook-behavior change. When T01 proved context unavailable: no extraction code ships, and the context file states the parent-child goal is blocked, why, and the smallest future bridge. +- [x] AC3: When T01 proved usable parent context: a valid upstream traceparent makes the `sce.hook` span share the upstream trace ID with its parent span ID equal to the upstream span ID, SCE span IDs are freshly generated, child spans inherit the trace across `.await` points and Tokio worker migration, and an invalid, malformed, or missing traceparent yields a new independent SCE root without panic or hook-behavior change. When T01 proved context unavailable: no extraction code ships, and the context file states the parent-child goal is blocked, why, and the smallest future bridge. - Validate: Loopback-receiver tests for valid, invalid, all-zero, and missing traceparent asserting trace ID / parent span ID; a multi-poll async test asserting child span trace IDs; or, for the unavailable branch, inspection of `context/sce/claude-hook-trace-propagation.md` and source grep showing no propagator wiring. -- [ ] AC4: The `sce.hook` span carries Claude `session_id` and `tool_use_id` as span attributes only (never as metric attributes, never reconstructed into a trace ID), and no exported span or log contains tool input, prompts, file contents, repository paths, raw hook payloads, the OpenBao token, or the accessor. +- [x] AC4: The `sce.hook` span carries Claude `session_id` and `tool_use_id` as span attributes only (never as metric attributes, never reconstructed into a trace ID), and no exported span or log contains tool input, prompts, file contents, repository paths, raw hook payloads, the OpenBao token, or the accessor. - Validate: Loopback-receiver test injecting secret- and payload-shaped fixture values asserts their absence from the decoded payload and presence of only the two correlation IDs. -- [ ] AC5: Child spans `sce.mutation_scope`, `sce.git_snapshot`, and `sce.persistence` appear under `sce.hook` only when the corresponding operation actually executed in that hook process; a hook path that skips an operation exports no span for it. +- [x] AC5: Child spans `sce.mutation_scope`, `sce.git_snapshot`, and `sce.persistence` appear under `sce.hook` only when the corresponding operation actually executed in that hook process; a hook path that skips an operation exports no span for it. - Validate: Loopback-receiver tests over a mutation-scope fixture that executes all three and a fixture that executes none (e.g. a non-tracked tool) assert the exact span set and parent links. -- [ ] AC6: Telemetry cannot change Claude hook behavior: with credential lookup failure, expired credential, refused or stalled receiver, or exporter timeout, hook stdout, stderr, exit code, permission decisions, Agent Trace DB rows, and mutation-trace state match the telemetry-disabled run, and process lifetime stays within the existing `CREDENTIAL_LOOKUP_DEADLINE` + `SHUTDOWN_TIMEOUT` bounds. +- [x] AC6: Telemetry cannot change Claude hook behavior: with credential lookup failure, expired credential, refused or stalled receiver, or exporter timeout, hook stdout, stderr, exit code, permission decisions, Agent Trace DB rows, and mutation-trace state match the telemetry-disabled run, and process lifetime stays within the existing `CREDENTIAL_LOOKUP_DEADLINE` + `SHUTDOWN_TIMEOUT` bounds. - Validate: Parity tests running the same Claude hook fixture with Noop and with failing managed telemetry, comparing outputs and persisted state, and asserting elapsed-time bounds. -- [ ] AC7: Standalone tracing is unchanged: `sce doctor` with a valid credential exports its `sce.command` span through the managed exporter with no Claude context required. +- [x] AC7: Standalone tracing is unchanged: `sce doctor` with a valid credential exports its `sce.command` span through the managed exporter with no Claude context required. - Validate: Existing `managed_trace_export_posts_allowlisted_span_with_bearer_only` and related trace_export tests still pass; live evidence for `sce doctor` recorded per AC8. -- [ ] AC8: Live verification is recorded: `sce doctor` and a Claude Code session triggering SCE hooks in this repository both export through `https://otel.crocoderlab.dev` with their existing managed credentials, and the record states whether each trace was observed in the correct Dash0 workspace dataset (with trace IDs and, where proven, the parent relationship) or exactly what could not be verified; a collector 2xx alone is not recorded as delivery. +- [x] AC8: Live verification is recorded: `sce doctor` and a Claude Code session triggering SCE hooks in this repository both export through `https://otel.crocoderlab.dev` with their existing managed credentials, and the record states whether each trace was observed in the correct Dash0 workspace dataset (with trace IDs and, where proven, the parent relationship) or exactly what could not be verified; a collector 2xx alone is not recorded as delivery. - Validate: Inspect the live-verification section of `context/sce/claude-hook-trace-propagation.md` and the T05 completion record. -- [ ] AC9: Current-state context describes hook telemetry admission, the `sce.hook` span contract and attributes, the parent-context verdict, correlation-ID privacy rules, and the opt-in live smoke procedure, without claiming distributed tracing unless AC3's parent relationship was verified live. +- [x] AC9: Current-state context describes hook telemetry admission, the `sce.hook` span contract and attributes, the parent-context verdict, correlation-ID privacy rules, and the opt-in live smoke procedure, without claiming distributed tracing unless AC3's parent relationship was verified live. - Validate: Inspect the files listed under Context sync against the implemented code. ### Full validation @@ -144,16 +144,61 @@ Persist this field in every plan; this is durable plan state, not chat state: - Context impact: important change. The `sce.hook` span contract gains three operation child spans, plus the rule that spans are exported per span kind (`sce.command` alone for commands; `sce.hook` plus children for Claude hooks). This affects `context/sce/cli-observability-contract.md`, `context/sce/claude-hook-trace-propagation.md`, `context/cli/claude-mutation-scope-integration.md` (span boundaries over the adapter seam and coordinator), and the `context/glossary.md` `sce.hook` entry. - Context synchronization: synced -- [ ] T05: `Record live Dash0 verification and sync hook-telemetry context` (status:todo) +- [x] T05: `Record live Dash0 verification and sync hook-telemetry context` (status:done) - Task ID: T05 - Scope: In — run `sce doctor` and a Claude Code session in this repository that triggers SCE hooks using a build with T02–T04; confirm both exporters use their existing managed credentials and the canonical collector, and check Dash0 for both traces in the correct workspace dataset, recording trace IDs, observed hierarchy, and anything unverified; write the opt-in live smoke procedure; update the context files listed under Context sync. Out — code changes; Collector/Control Plane changes; claiming delivery from a collector 2xx. - Dependencies: T04 - Done when: AC8 and AC9 hold and the context file states the exact observed hierarchy. - Verify: Inspect the updated context files; `nix run .#pkl-check-generated` unaffected. - - Context synchronization: pending + - Completed: 2026-10-17 + - Files changed: `context/sce/claude-hook-trace-propagation.md`, `context/sce/cli-observability-contract.md`, `context/sce/telemetry-credential.md`, `context/context-map.md` + - Result: Live run on 2026-10-17 used a debug build of `eae7be8f` and the existing stored credential (presence checked, value never printed). Dash0 evidence was a user-exported 30-span root-span listing; no trace detail view was provided. + - Observed in Dash0: `sce doctor` exported one root `sce.command` (trace `1cc8e0b1eda96df1f757fab2e3798efd`). Headless Claude Code 2.1.301 session `96272753-…` (real `Write` + `Read` in this repo, branch `sce` on `PATH`, tracing beta on) exported root `claude_code.interaction` `dcd919ecc4f5d400df5b642a1a229783`/`3eec7153327c6839`, which is exactly the trace/span every tool hook's `TRACEPARENT` named (scratchpad probe). The session's `SessionStart` and `SessionEnd` `sce.hook` spans are independent roots with `sce.agent=claude`, the session ID, and `success`, because they run outside the interaction span. `sce auth telemetry-headers` produced no span. + - Not verified: the parented tool-hook `sce.hook` spans and their `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` children inside trace `dcd919ec…`. A root-span listing cannot show them (it also omits Claude's own `claude_code.tool` children). No unparented tool-hook `sce.hook` roots appear in the window. Also not verified: the Dash0 dataset name and interactive TUI hooks. + - Follow-up (2026-10-17, user found the trace in Dash0): trace `dcd919ecc4f5d400df5b642a1a229783` is in the SCE CroCoder (`sce-crocoder`) dataset. Root `claude_code.interaction` (service `claude-code`, session `96272753-8c39-4ec3-824d-238f5202e016`, repo `crocoder-dev/shared-context-engineering`), started 12:32:16 UTC, about 7.0s, status UNSET, 0 descendants in error, 0 error or warning logs. It has 11 direct children (21 spans total): 3 `claude_code.llm_request`, 2 `claude_code.tool`, and 6 `sce.hook` spans from service `sce`. Most `sce.hook` spans took 1–26ms; the two with children took 111–154ms. This confirms the parented `sce.hook` spans and the dataset live. The names of the `sce.hook` children and which hook events the six spans belong to were not given in the evidence. + - Recorded the evidence, observed vs. expected hierarchy, and an opt-in live smoke procedure in `claude-hook-trace-propagation.md`. Added session-boundary root behavior and live status to the observability contract, hook admission vs. helper bypass to `telemetry-credential.md`, and a context-map entry update. + - Deviation: AC8's parent relationship was first recorded as unverified; the follow-up above now records it as observed in Dash0. The other context files still describe distributed tracing as locally tested, not live-verified. + - Verify outcomes: context files inspected against T02–T04 code and Dash0 evidence — passed; secret grep of the context file for bearer values, the Dash0 token identifier, prompt/tool-input keys — no matches (passed); `nix run .#pkl-check-generated`: ephemeral generation passed, 142 files (passed); smoke file removed and `git status` shows only the four context files — passed. + - Context impact: localized context-only change; the live-verification record and smoke procedure live in `context/sce/claude-hook-trace-propagation.md` with cross-references from the observability contract, telemetry credential doc, and context map; no code behavior change. + - Context synchronization: synced ## Open questions - Admitting Claude hooks to managed export adds an encrypted `auth.db` credential lookup (up to `CREDENTIAL_LOOKUP_DEADLINE`, 2s) and an exporter shutdown (up to `SHUTDOWN_TIMEOUT`, 3s) to every Claude hook process — several per tool call, including the blocking `PreToolUse` path. `managed-otel-export` and `claude-native-otel-instrumentation` excluded hooks deliberately. Is the existing bound acceptable for hooks, or should hooks get a tighter hook-specific deadline? A smaller version is to admit only `PostToolUse` hooks, leaving the blocking `PreToolUse` path on `Noop`. - If T01 finds Claude Code does not pass trace context to command hooks, the remaining value is correlation attributes on independent traces. Is that still worth T02–T05, or should the plan stop after T01 and T03's documented blocker? - The request asked for Rust 2024; the plan keeps edition 2021. Confirm, or plan the edition migration as a separate change. + +## Validation Report + +**Status:** validated +**Date:** 2026-10-17 + +### Commands run + +- `nix flake check` -> exit 0 (all checks passed for x86_64-linux, including cli-tests, cli-clippy, cli-fmt, cli-generated-input, pkl-generated, config-lib and npm Bun/Biome checks, workflow-actionlint) +- `nix run .#pkl-check-generated` -> exit 0 (ephemeral Pkl generation passed: 142 files) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml managed_telemetry` -> exit 0 (26 passed, 0 failed) +- `git diff --name-only main...HEAD` plus `git status --porcelain -uall` -> exit 0 (branch changes limited to the planned code and context files; no probe files added; working tree holds only the T05 context edits) +- `grep -inE "bearer [A-Za-z0-9]|tool_input\"|user_prompt|prompt\"|sk-ant|hvs\.|token=|password" context/sce/claude-hook-trace-propagation.md` -> exit 1 (no matches) + +### Success-criteria verification + +- [x] AC1: Probe evidence recorded from real Claude Code hooks -> `context/sce/claude-hook-trace-propagation.md` has the verdict, Claude Code 2.1.301, per-hook traceparent tables for `PreToolUse`/`PostToolUse` matched against exported spans, gating runs, and the docs consulted; the secret/payload grep found nothing; the branch diff adds no probe files (the `probe*` fixtures in `git ls-files` predate this branch and belong to earlier adapter plans). +- [x] AC2: One `sce.hook` span per Claude hook; non-Claude hooks and the helper stay Noop -> `claude_hook_export_posts_one_hook_span_with_correlation_ids_only`, the `app::tests` admission matrix, and `managed_trace_export_posts_allowlisted_span_with_bearer_only` passed in the `managed_telemetry` and flake runs. +- [x] AC3: Parent-context branch -> `claude_hook_span_joins_valid_upstream_trace_with_fresh_span_id`, `claude_hook_span_starts_independent_root_without_usable_traceparent` (missing/malformed/all-zero table), and `claude_hook_child_spans_inherit_upstream_trace_across_polls_and_workers` passed. +- [x] AC4: Only correlation IDs, no payload or secrets -> `claude_hook_export_posts_one_hook_span_with_correlation_ids_only` asserts the exact `sce.*` attribute set and that paths, prompt, file content, tool response, Bearer value, token, and accessor are absent; passed. +- [x] AC5: Child spans only when the operation ran -> the mutation-scope loopback test (`sce.hook` → `sce.mutation_scope` → {`sce.git_snapshot`, `sce.persistence`}) and `read_only_claude_hook_exports_no_operation_children` passed in `nix flake check` cli-tests. +- [x] AC6: Telemetry cannot change hook behavior -> `failing_managed_telemetry_cannot_change_claude_hook_behavior` (Noop vs. refused vs. stalled receiver, output/seam/state parity, bounded shutdown) and `claude_hook_credential_failure_or_expiry_resolves_noop` passed. +- [x] AC7: Standalone tracing unchanged -> `managed_trace_export_posts_allowlisted_span_with_bearer_only` and the related trace_export tests passed; live `sce doctor` `sce.command` span (trace `1cc8e0b1eda96df1f757fab2e3798efd`) recorded. +- [x] AC8: Live verification recorded -> the Live verification section of the context file and the T05 record state the observed Dash0 spans and trace IDs, and say exactly what was not verified: the parented tool-hook hierarchy, the Dash0 dataset name, and interactive TUI hooks. Delivery is not inferred from a collector 2xx. Follow-up 2026-10-17: the user found trace `dcd919ecc4f5d400df5b642a1a229783` in the `sce-crocoder` dataset with 6 parented `sce.hook` spans under `claude_code.interaction` (see the T05 follow-up), so the parent relationship and dataset are now observed live; interactive TUI hooks remain unverified. +- [x] AC9: Current-state context synced -> `cli-observability-contract.md`, `telemetry-credential.md`, `claude-mutation-scope-integration.md`, `architecture.md`, `glossary.md`, and `context-map.md` describe admission, the `sce.hook` contract and children, the parent-context verdict, the correlation-ID privacy rule, and the smoke procedure. They describe distributed tracing as locally tested and not live-verified, which matches the code inspected. + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- Resolved 2026-10-17: the user found trace `dcd919ecc4f5d400df5b642a1a229783` in the `sce-crocoder` Dash0 dataset with 6 `sce.hook` spans as direct children of `claude_code.interaction`, two of them with child spans. The child span names were not listed, so live delivery of `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` by name is still proven only by local loopback tests. +- Interactive TUI sessions, subagent hooks, and other hook events were not probed for `TRACEPARENT`. +- Each Claude hook process can now take up to `CREDENTIAL_LOOKUP_DEADLINE` + `SHUTDOWN_TIMEOUT` longer, including the blocking `PreToolUse` path; the plan's open question on a tighter hook deadline is still unresolved. diff --git a/context/sce/claude-hook-trace-propagation.md b/context/sce/claude-hook-trace-propagation.md index fee7be183..95cf37a05 100644 --- a/context/sce/claude-hook-trace-propagation.md +++ b/context/sce/claude-hook-trace-propagation.md @@ -100,9 +100,89 @@ The Claude Code 2.1.301 bundle contains a shared subprocess-environment builder. - Claude Code monitoring docs, `https://code.claude.com/docs/en/monitoring-usage` (Traces (beta), TRACEPARENT propagation, span hierarchy, hook spans). The docs describe Bash/PowerShell subprocesses as inheriting the active tool execution span's context. They describe hook executions as children of `claude_code.interaction`. They say nothing about hook-process `TRACEPARENT`; the probe shows hook processes receive the interaction context. - Claude Code hooks docs, `https://code.claude.com/docs/en/hooks` (hook environment: `CLAUDE_PROJECT_DIR`; `OTEL_*` removed from subprocesses; `prompt_id` matches the `prompt.id` event attribute). These docs do not mention `TRACEPARENT`. +## Live verification + +Recorded by T05 on 2026-10-17. The run used an `sce` debug build of commit `eae7be8f` (T02–T04), built with `nix develop -c ./scripts/run-cli-cargo.sh build`. It used the existing stored telemetry credential: `sce auth telemetry-headers` exited 0 with non-empty output, and the output was not inspected. Both exporters sent to the canonical collector `https://otel.crocoderlab.dev`: SCE through its pinned endpoint, and Claude Code through the user-level `OTEL_EXPORTER_OTLP_ENDPOINT` with `otelHeadersHelper = sce auth telemetry-headers`. Delivery was judged only from Dash0, never from a collector response. The Dash0 evidence is a user-exported root-span listing for the last 30 minutes. It held 30 root spans, all with `parentSpanId` empty, ending at 12:32:23Z. A trace detail view was not provided. + +### `sce doctor` + +`sce doctor --format json` ran at 2026-10-17T12:31:35Z, exited 0, and took 0.76s. Dash0 shows one root `sce.command` span: trace `1cc8e0b1eda96df1f757fab2e3798efd`, span `3f1d92756bd5860a`, `sce.command.name=doctor`, `sce.outcome=success`, resource `service.name=sce`, `service.version=0.4.0-pre-alpha-v11`. It is an independent root, so the standalone command path is unchanged. + +Other branch-build invocations in the same window, all also visible in Dash0: + +- `sce --version` at 12:31:21Z, as `sce.command`. +- `sce auth --help` and the rejected `sce auth status` at 12:31:40Z, as two `sce.command` spans with `sce.command.name=auth`. + +The later `sce auth telemetry-headers` calls produced no span, consistent with the helper staying Noop. + +### Claude Code session with SCE hooks + +Headless Claude Code `2.1.301` ran `claude -p --model haiku --allowedTools Write,Read` in this repository. The run used the repository's project hooks, user-level telemetry settings (`CLAUDE_CODE_ENABLE_TELEMETRY=1`, `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA=1`), and the branch build first on `PATH`. The inherited parent-session `TRACEPARENT`, `CLAUDECODE`, and session variables were unset. The session made a real `Write` of a gitignored `context/tmp/` file, then a `Read` of it, and the file was deleted afterward. Session `96272753-8c39-4ec3-824d-238f5202e016`, `num_turns=3`. + +A scratchpad-only probe hook, registered beside the SCE hooks through `--settings`, recorded the context every tool hook received. All four hooks resolved `sce` to the branch build. + +| Hook | `tool_use_id` | `TRACEPARENT` trace ID | parent span ID | +| --- | --- | --- | --- | +| PreToolUse Write | `toolu_01ANo9Y6mnA876HcX4fnD7bn` | `dcd919ecc4f5d400df5b642a1a229783` | `3eec7153327c6839` | +| PostToolUse Write | `toolu_01ANo9Y6mnA876HcX4fnD7bn` | `dcd919ecc4f5d400df5b642a1a229783` | `3eec7153327c6839` | +| PreToolUse Read | `toolu_017JhhwrJk5d16ykAncy5mUy` | `dcd919ecc4f5d400df5b642a1a229783` | `3eec7153327c6839` | +| PostToolUse Read | `toolu_017JhhwrJk5d16ykAncy5mUy` | `dcd919ecc4f5d400df5b642a1a229783` | `3eec7153327c6839` | + +Observed in Dash0, all from the same Dash0 ingestion token: + +| Span | Service | Trace ID | Span ID | Parent | Attributes | +| --- | --- | --- | --- | --- | --- | +| `sce.hook` (12:32:14Z) | `sce` | `7dd41303f9bec7b68bb667b2fb39aa70` | `a9bc49c8d4a8c7ea` | none | `sce.agent=claude`, `sce.hook.event=SessionStart`, `sce.claude.session_id=96272753-…`, `sce.outcome=success` | +| `claude_code.interaction` (12:32:16Z) | `claude-code` | `dcd919ecc4f5d400df5b642a1a229783` | `3eec7153327c6839` | none | `session.id=96272753-…` | +| `sce.hook` (12:32:23Z) | `sce` | `d56f02bb11aeac1b1a2c0922d5245d05` | `7dcd68d04d9501b8` | none | `sce.agent=claude`, `sce.hook.event=SessionEnd`, `sce.claude.session_id=96272753-…`, `sce.outcome=success` | + +Neither `sce.hook` span carried `sce.hook.tool`, `sce.claude.tool_use_id`, paths, or payload content. Both are correctly independent roots: + +- `SessionStart` (`claude-model-state`) runs before the interaction span exists. +- `SessionEnd` (`claude-mutation-scope`) starts at 12:32:23.502Z, after the interaction ended at 12:32:23.257Z. + +Neither process had a parent context to join. + +### Observed hierarchy + +Confirmed in Dash0: + +- `sce doctor` → one root `sce.command`. +- Claude session-boundary hooks → independent root `sce.hook` spans. +- The Claude interaction root `claude_code.interaction` (`dcd919ec…/3eec7153327c6839`) is exactly the trace and span named by every tool hook's `TRACEPARENT`. + +Expected but not verified in Dash0 for trace `dcd919ecc4f5d400df5b642a1a229783`: + +```text +claude_code.interaction (3eec7153327c6839) +├── claude_code.tool (Write / Read) +└── sce.hook ×N: PreToolUse/PostToolUse hooks (TRACEPARENT captured); UserPromptSubmit/Stop hooks not probed + └── sce.mutation_scope (claude-mutation-scope hooks) + ├── sce.git_snapshot (Write PreToolUse) + └── sce.persistence +``` + +The root-span listing cannot show non-root spans: even Claude Code's own `claude_code.tool` and in-trace `claude_code.llm_request` children for this session are absent from it. So the parented `sce.hook` spans and their children could not be confirmed or ruled out. The listing does contain every root span in the 12:31–12:33 window, and it has no unparented `sce.hook` for `UserPromptSubmit`, `PreToolUse`, `PostToolUse`, or `Stop`. So these hook spans either joined the interaction trace, as the local loopback tests prove they do, or were not delivered. Confirming the parent relationship in Dash0 requires opening trace `dcd919ecc4f5d400df5b642a1a229783` in the trace view. Distributed tracing between Claude Code and SCE is therefore implemented and locally tested, but not yet verified live. The Dash0 dataset name was not reported: SCE and Claude Code resources share one ingestion token, but which named dataset that token writes to was not checked. + +## Opt-in live smoke procedure + +This is manual, uses real credentials, Claude Code, and Dash0, and is never part of CI. + +1. Confirm a stored credential without printing it: `out=$(sce auth telemetry-headers 2>/dev/null); [ -n "$out" ] && echo present`. +2. Build the branch: `nix develop -c ./scripts/run-cli-cargo.sh build --manifest-path cli/Cargo.toml`. +3. Run `cli/target/debug/sce doctor`, and note the UTC time. +4. Optionally, add a scratchpad-only probe hook through `--settings` that records only `hook_event_name`, `tool_use_id`, and the trace ID and parent span ID parsed from `TRACEPARENT`. This captures the expected Claude trace ID. +5. From the repository root, with Claude tracing enabled (`CLAUDE_CODE_ENABLE_TELEMETRY=1`, `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA=1`), run `env -u TRACEPARENT -u TRACESTATE -u CLAUDECODE PATH="$PWD/cli/target/debug:$PATH" claude -p --allowedTools Write,Read ""`. Then delete the file. +6. In Dash0, find the `sce.command` span with `sce.command.name=doctor` at the noted time. Open the Claude trace by its trace ID, not a root-span list, and confirm: + - each tool hook's `sce.hook` has `claude_code.interaction` as its parent and a `sce.claude.tool_use_id` that matches a `claude_code.tool` span; + - the `Write` `PreToolUse` `claude-mutation-scope` hook carries `sce.mutation_scope` → `sce.git_snapshot` and `sce.persistence`; + - `SessionStart`/`SessionEnd` `sce.hook` spans are independent roots. +7. Record the trace IDs and the observed hierarchy. Record anything that was not observed as unverified, and never record a collector 2xx as delivery. + ## Not verified -- Interactive TUI sessions were not probed for hook `TRACEPARENT`. Claude Code documents that only `-p`/Agent SDK sessions read an inbound `TRACEPARENT`. The outbound subprocess builder is shared, and the interactive Bash tool environment of the same version carries a valid `TRACEPARENT` (observed during this task). Whether interactive hooks receive the same interaction-span context is expected but unconfirmed. T05 live verification must check it. +- The parent-child relationship in Dash0 between `claude_code.interaction` and the tool-hook `sce.hook` spans, and the delivery of `sce.mutation_scope`, `sce.git_snapshot`, and `sce.persistence` spans. See [Live verification](#live-verification). +- Interactive TUI sessions were not probed for hook `TRACEPARENT`. T05 also used headless `claude -p`. Claude Code documents that only `-p`/Agent SDK sessions read an inbound `TRACEPARENT`. The outbound subprocess builder is shared, and the interactive Bash tool environment of the same version carries a valid `TRACEPARENT`. Whether interactive hooks receive the same interaction-span context is expected but unconfirmed. - Subagent (`Agent` tool) hooks, `PostToolUseFailure`, `PermissionDenied`, and other hook events were not probed. - Behavior behind a custom `ANTHROPIC_BASE_URL` proxy without `CLAUDE_CODE_PROPAGATE_TRACEPARENT` was not probed. Per the docs the subprocess `TRACEPARENT` follows the same switch as the API `traceparent` header. @@ -110,7 +190,7 @@ The Claude Code 2.1.301 bundle contains a shared subprocess-environment builder. - T02: admit the Claude-only subcommands from argv. Admit shared subcommands only when `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are both present. - T03: the parent-context branch applies. Extract `TRACEPARENT` (and `TRACESTATE` when present) with `TraceContextPropagator` and parent `sce.hook` on it. The resulting span is a child of `claude_code.interaction` and shares its trace ID. Missing or invalid context, which is the default when Claude tracing is off, yields an independent SCE root. -- T05 and context sync must describe the hierarchy as `claude_code.interaction → sce.hook` (sibling of `claude_code.tool`), with tool correlation by `tool_use_id`, not as a tool-span child. +- Context describes the hierarchy as `claude_code.interaction → sce.hook` (a sibling of `claude_code.tool`), with tool correlation by `tool_use_id`, not as a tool-span child. Live status is recorded in [Live verification](#live-verification). ## Probe hygiene diff --git a/context/sce/cli-observability-contract.md b/context/sce/cli-observability-contract.md index 40df94050..5652b3d62 100644 --- a/context/sce/cli-observability-contract.md +++ b/context/sce/cli-observability-contract.md @@ -98,7 +98,7 @@ Runtime observability consumes the shared resolved observability config from `cl - Admission (`managed_span_kind` in `cli/src/app.rs`, decided from argv before credential lookup): Claude-invoked SCE hooks resolve the `sce.hook` span kind, every other hook and the hidden `sce auth telemetry-headers` helper use `NoopTelemetry` without a credential lookup, and all remaining commands use the `sce.command` span kind. `sce hooks claude-mutation-scope` and `sce hooks claude-model-state` are admitted from argv alone. The shared `sce hooks diff-trace`, `sce hooks conversation-trace`, and `sce policy bash` are admitted only when both `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are present in the environment (checked for presence only; values never read); `CLAUDE_PROJECT_DIR` is set for Claude command hooks but not for the Claude Bash tool (evidence: [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md)). Other hooks (commit hooks, Codex, OpenCode, Pi) stay Noop. Admitted hooks use the same `CREDENTIAL_LOOKUP_DEADLINE`, `SHUTDOWN_TIMEOUT`, and runtime grace bounds as commands. Decision record: [2026-10-17-claude-hook-managed-trace-admission.md](../decisions/2026-10-17-claude-hook-managed-trace-admission.md). - Export is OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev/v1/traces`. A pinned HTTP client sets `Authorization: Bearer ` (token only, never the accessor), disables redirects, rebuilds the request to the fixed endpoint, and bounds requests to 3s. Ambient `OTEL_EXPORTER_OTLP_*` values cannot redirect it. - Each command emits one `sce.command` span with `sce.command.name`, `sce.outcome` (`success`/`failure`), and `sce.error.class` (the `SCE-ERR-*` code on failure); resource attributes are `service.name=sce` and `service.version`. Raw arguments, paths, and error chains are not exported. -- An admitted Claude hook process emits exactly one `sce.hook` span instead (no `sce.command` span), with `sce.agent=claude`, `sce.outcome`, `sce.error.class`, and, when present as non-empty strings in the Claude hook stdin payload, `sce.hook.event` (`hook_event_name`), `sce.hook.tool` (`tool_name`), `sce.claude.session_id`, and `sce.claude.tool_use_id`. `trace_export::record_claude_hook_payload` is the only reader: each Claude hook entry point passes its stdin to it once, it returns immediately when the current span is disabled (Noop), and it copies only those four fields. Session and tool-use IDs are opaque correlation identifiers: span attributes only, never metric attributes, never used to derive a trace ID. Tool input and response, prompts, `cwd`, transcript paths, file contents, and the raw payload are never exported. Parent context: for the `sce.hook` kind only, `ManagedTraceTelemetry::new` reads `TRACEPARENT` (and `TRACESTATE` when present) from the process environment, extracts it with `TraceContextPropagator`, and keeps it only when the remote span context is valid; `root_span` then attaches it with `set_parent`, so `sce.hook` shares Claude Code's trace ID, its parent is the upstream span (Claude Code 2.1.301 supplies `claude_code.interaction`, making `sce.hook` a sibling of `claude_code.tool`), and its own span ID is freshly generated. Child spans created inside the instrumented hook future inherit that trace across `.await` points and worker migration. No global propagator is registered and `sce.command` never reads the variables. A missing, malformed, or all-zero traceparent (the normal case when Claude tracing is off) yields an independent SCE root. Telemetry failures never change hook stdout, permission decisions, persisted state, or exit codes. +- An admitted Claude hook process emits exactly one `sce.hook` span instead (no `sce.command` span), with `sce.agent=claude`, `sce.outcome`, `sce.error.class`, and, when present as non-empty strings in the Claude hook stdin payload, `sce.hook.event` (`hook_event_name`), `sce.hook.tool` (`tool_name`), `sce.claude.session_id`, and `sce.claude.tool_use_id`. `trace_export::record_claude_hook_payload` is the only reader: each Claude hook entry point passes its stdin to it once, it returns immediately when the current span is disabled (Noop), and it copies only those four fields. Session and tool-use IDs are opaque correlation identifiers: span attributes only, never metric attributes, never used to derive a trace ID. Tool input and response, prompts, `cwd`, transcript paths, file contents, and the raw payload are never exported. Parent context: for the `sce.hook` kind only, `ManagedTraceTelemetry::new` reads `TRACEPARENT` (and `TRACESTATE` when present) from the process environment, extracts it with `TraceContextPropagator`, and keeps it only when the remote span context is valid; `root_span` then attaches it with `set_parent`, so `sce.hook` shares Claude Code's trace ID, its parent is the upstream span (Claude Code 2.1.301 supplies `claude_code.interaction`, making `sce.hook` a sibling of `claude_code.tool`), and its own span ID is freshly generated. Child spans created inside the instrumented hook future inherit that trace across `.await` points and worker migration. No global propagator is registered and `sce.command` never reads the variables. A missing, malformed, or all-zero traceparent (the normal case when Claude tracing is off) yields an independent SCE root. Claude `SessionStart` and `SessionEnd` hooks run outside any interaction span, so their `sce.hook` is always a root. Live status: Dash0 shows `sce doctor` and Claude session-boundary `sce.hook` spans. The parented tool-hook hierarchy is tested locally but not yet confirmed in Dash0. The evidence and the opt-in smoke procedure are in [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md). Telemetry failures never change hook stdout, permission decisions, persisted state, or exit codes. - Operation child spans: `trace_export::in_child_span(ManagedChildSpan, future)` wraps an existing operation in one of three named spans, each carrying only `sce.outcome` (`success`/`failure` from the operation's `Result`): `sce.mutation_scope` around every Claude adapter seam call (one coordinate or abandon boundary), `sce.git_snapshot` around the coordinator's tree capture and pin, and `sce.persistence` around the mutation-store phase of a coordinate boundary (initialize, scope/provenance registration, CAS commit loop) and of an abandon. The hierarchy is `sce.hook → sce.mutation_scope → {sce.git_snapshot, sce.persistence}`. A child span exists only when its operation runs: a read-only tool, an untracked background shell, or a close with no matching attempt exports `sce.hook` alone. Snapshot-failure taint persistence and conversation/diff-trace Agent Trace DB writes have no span. The span filter depends on the span kind: a `sce.command` dispatch exports only `sce.command` (ordinary commands that reach the coordinator, such as ref maintenance, still export one span), and a `sce.hook` dispatch exports only `sce.hook` and the three child names. Without a managed hook dispatch the child spans are disabled, and they never change ordering, locking, or errors. - The exporter uses a bounded batch queue on a dedicated thread. Final stdout/stderr rendering and classified-error logging run inside the command span (`run_command_lifecycle_with_context`), so `sce.outcome` reflects the rendered result (a stdout write failure is `failure`) and the final `sce.error.*` log is exported. Trace and log providers shut down concurrently under one shared `SHUTDOWN_TIMEOUT` (3s) deadline after rendering; failures never change command results or exit codes. - Process lifetime: `app::run_process` owns the Tokio runtime (at least two workers) and ends it with `shutdown_timeout(RUNTIME_SHUTDOWN_GRACE)` (500ms), so a stalled keychain, database, or exporter worker cannot keep the process alive. The async deadlines bound when the command proceeds; the runtime grace is the hard process-level bound. Abandoned workers are not cancelled, only no longer awaited. diff --git a/context/sce/telemetry-credential.md b/context/sce/telemetry-credential.md index bf48aa36a..4638ac6fe 100644 --- a/context/sce/telemetry-credential.md +++ b/context/sce/telemetry-credential.md @@ -29,7 +29,7 @@ flowchart LR ## Runtime use -- `managed_telemetry::RuntimeTelemetry::resolve` reads the stored credential at CLI startup (never issuing one); a valid unexpired `token` is used only as the `Authorization: Bearer` credential for managed OTLP trace and log export; the `accessor` is never transmitted. Absent, expired, malformed, or unavailable rows select `NoopTelemetry`. See [cli-observability-contract.md](cli-observability-contract.md). +- `managed_telemetry::RuntimeTelemetry::resolve` reads the stored credential at CLI startup (never issuing one); a valid unexpired `token` is used only as the `Authorization: Bearer` credential for managed OTLP trace and log export; the `accessor` is never transmitted. Absent, expired, malformed, or unavailable rows select `NoopTelemetry`. Claude-invoked SCE hooks are admitted to this lookup and export one `sce.hook` span. Every other hook and this hidden header helper skip the lookup and stay Noop. See [cli-observability-contract.md](cli-observability-contract.md). ## Hidden header helper From 368f9268fb8f6836d2ce3443c53f17ec4dc76a47 Mon Sep 17 00:00:00 2001 From: David Abram Date: Sat, 17 Oct 2026 20:05:03 +0200 Subject: [PATCH 07/17] docs: verify live Claude-to-SCE trace hierarchy in Dash0 --- context/sce/claude-hook-trace-propagation.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/context/sce/claude-hook-trace-propagation.md b/context/sce/claude-hook-trace-propagation.md index 95cf37a05..64671ec6a 100644 --- a/context/sce/claude-hook-trace-propagation.md +++ b/context/sce/claude-hook-trace-propagation.md @@ -102,7 +102,7 @@ The Claude Code 2.1.301 bundle contains a shared subprocess-environment builder. ## Live verification -Recorded by T05 on 2026-10-17. The run used an `sce` debug build of commit `eae7be8f` (T02–T04), built with `nix develop -c ./scripts/run-cli-cargo.sh build`. It used the existing stored telemetry credential: `sce auth telemetry-headers` exited 0 with non-empty output, and the output was not inspected. Both exporters sent to the canonical collector `https://otel.crocoderlab.dev`: SCE through its pinned endpoint, and Claude Code through the user-level `OTEL_EXPORTER_OTLP_ENDPOINT` with `otelHeadersHelper = sce auth telemetry-headers`. Delivery was judged only from Dash0, never from a collector response. The Dash0 evidence is a user-exported root-span listing for the last 30 minutes. It held 30 root spans, all with `parentSpanId` empty, ending at 12:32:23Z. A trace detail view was not provided. +Recorded by T05 on 2026-10-17. The run used an `sce` debug build of commit `eae7be8f` (T02–T04), built with `nix develop -c ./scripts/run-cli-cargo.sh build`. It used the existing stored telemetry credential: `sce auth telemetry-headers` exited 0 with non-empty output, and the output was not inspected. Both exporters sent to the canonical collector `https://otel.crocoderlab.dev`: SCE through its pinned endpoint, and Claude Code through the user-level `OTEL_EXPORTER_OTLP_ENDPOINT` with `otelHeadersHelper = sce auth telemetry-headers`. Delivery was judged only from Dash0, never from a collector response. The initial Dash0 evidence was a user-exported root-span listing for the last 30 minutes. It held 30 root spans, all with `parentSpanId` empty, ending at 12:32:23Z. A trace detail view was not available at that point. A subsequent user-provided Dash0 waterfall screenshot on 2026-10-17 independently confirms the cross-service parent-child hierarchy; see [Follow-up Dash0 waterfall evidence](#follow-up-dash0-waterfall-evidence). ### `sce doctor` @@ -151,7 +151,7 @@ Confirmed in Dash0: - Claude session-boundary hooks → independent root `sce.hook` spans. - The Claude interaction root `claude_code.interaction` (`dcd919ec…/3eec7153327c6839`) is exactly the trace and span named by every tool hook's `TRACEPARENT`. -Expected but not verified in Dash0 for trace `dcd919ecc4f5d400df5b642a1a229783`: +Initially expected but not verified from the root-span listing for trace `dcd919ecc4f5d400df5b642a1a229783`: ```text claude_code.interaction (3eec7153327c6839) @@ -162,7 +162,17 @@ claude_code.interaction (3eec7153327c6839) └── sce.persistence ``` -The root-span listing cannot show non-root spans: even Claude Code's own `claude_code.tool` and in-trace `claude_code.llm_request` children for this session are absent from it. So the parented `sce.hook` spans and their children could not be confirmed or ruled out. The listing does contain every root span in the 12:31–12:33 window, and it has no unparented `sce.hook` for `UserPromptSubmit`, `PreToolUse`, `PostToolUse`, or `Stop`. So these hook spans either joined the interaction trace, as the local loopback tests prove they do, or were not delivered. Confirming the parent relationship in Dash0 requires opening trace `dcd919ecc4f5d400df5b642a1a229783` in the trace view. Distributed tracing between Claude Code and SCE is therefore implemented and locally tested, but not yet verified live. The Dash0 dataset name was not reported: SCE and Claude Code resources share one ingestion token, but which named dataset that token writes to was not checked. +The initial root-span listing could not show non-root spans, including Claude Code's own tool and LLM request children. The subsequent waterfall screenshot establishes that parented SCE tool-hook spans and their children do reach Dash0, but the screenshot does not display its trace ID or dataset name; therefore it cannot by itself prove that it is the specific `dcd919ecc4f5d400df5b642a1a229783` trace above. The Dash0 dataset name remains unverified. + +## Follow-up Dash0 waterfall evidence + +A user-provided Dash0 trace-detail screenshot on 2026-10-17 shows a **single** `claude_code.interaction` trace containing spans from both `claude-code` and `sce`. The UI explicitly renders: + +- Multiple `sce.hook` spans directly parented to `claude_code.interaction`, alongside `claude_code.llm_request` and `claude_code.tool` spans. This confirms live cross-service W3C parent propagation to SCE hooks (siblings of tool spans, not children of tool spans). +- Two `sce.hook → sce.mutation_scope` branches, each containing sibling `sce.git_snapshot` and `sce.persistence` children. This confirms live delivery of all three SCE operation span types and their expected parent hierarchy. +- A `claude_code.interaction` duration of 6,988 ms. The two instrumented mutation hooks shown take 154 ms and 110 ms; the first has a 120 ms mutation scope, 63 ms Git snapshot and 33 ms persistence; the second has a 100 ms mutation scope, 57 ms Git snapshot and 19 ms persistence. + +**Verification verdict:** Live parent-child propagation and SCE operation-span export are **verified visually in Dash0**, closing the hierarchy/delivery gap left by the earlier root-only listing. The screenshot does **not** show the trace ID, workspace/dataset, per-span `tool_use_id` attribute values, or the exact time of the trace. Do not claim that the screenshot proves the prior headless session's trace ID `dcd919ecc4f5d400df5b642a1a229783`, or that it proves hook-to-tool ID equality. This observation does not measure exporter overhead or p95 hook latency. ## Opt-in live smoke procedure @@ -181,7 +191,7 @@ This is manual, uses real credentials, Claude Code, and Dash0, and is never part ## Not verified -- The parent-child relationship in Dash0 between `claude_code.interaction` and the tool-hook `sce.hook` spans, and the delivery of `sce.mutation_scope`, `sce.git_snapshot`, and `sce.persistence` spans. See [Live verification](#live-verification). +- The exact trace ID, dataset name, and `tool_use_id` equality between each `sce.hook` and its matching `claude_code.tool` were not exposed in the follow-up waterfall screenshot. The parent-child hierarchy and child-span delivery **are** visually verified; see [Follow-up Dash0 waterfall evidence](#follow-up-dash0-waterfall-evidence). - Interactive TUI sessions were not probed for hook `TRACEPARENT`. T05 also used headless `claude -p`. Claude Code documents that only `-p`/Agent SDK sessions read an inbound `TRACEPARENT`. The outbound subprocess builder is shared, and the interactive Bash tool environment of the same version carries a valid `TRACEPARENT`. Whether interactive hooks receive the same interaction-span context is expected but unconfirmed. - Subagent (`Agent` tool) hooks, `PostToolUseFailure`, `PermissionDenied`, and other hook events were not probed. - Behavior behind a custom `ANTHROPIC_BASE_URL` proxy without `CLAUDE_CODE_PROPAGATE_TRACEPARENT` was not probed. Per the docs the subprocess `TRACEPARENT` follows the same switch as the API `traceparent` header. From f5b97f80114ef9e00f1b55565229f76711ee382f Mon Sep 17 00:00:00 2001 From: David Abram Date: Mon, 19 Oct 2026 13:12:48 +0200 Subject: [PATCH 08/17] runtime: Harden concurrent hook telemetry and diagnose fallbacks Prevent concurrent Claude hook processes from silently losing managed telemetry when encrypted auth.db opens contend. Enable multiprocess WAL and an auth DB busy timeout, avoid migration writes on current schemas, serialize and retry pending migrations, and skip database creation for missing-credential reads. Classify and log Noop fallbacks, identify hook entry points with `sce.hook.command`, and add multiprocess coverage for readers, migrations, and correlated hook spans. Document rollout risks and validation evidence in the `claude-hook-trace-propagation` plan. Ref: #330 Co-authored-by: SCE --- cli/src/app.rs | 17 +- cli/src/services/auth_db/mod.rs | 3 + .../services/auth_db/multiprocess_tests.rs | 579 ++++++++++++++++++ cli/src/services/bash_policy.rs | 5 +- cli/src/services/db/mod.rs | 152 +++-- cli/src/services/hooks/claude_model_state.rs | 5 +- .../hooks/claude_mutation_scope/lifecycle.rs | 11 +- cli/src/services/hooks/conversation_trace.rs | 5 +- cli/src/services/hooks/diff_trace.rs | 5 +- cli/src/services/managed_telemetry.rs | 178 +++++- .../managed_telemetry/trace_export.rs | 29 +- .../services/telemetry_credential_storage.rs | 10 + .../plans/claude-hook-trace-propagation.md | 31 + context/sce/auth-db.md | 18 + context/sce/claude-hook-trace-propagation.md | 23 +- context/sce/cli-observability-contract.md | 5 +- context/sce/shared-turso-db.md | 8 +- context/sce/telemetry-credential.md | 2 +- 18 files changed, 1005 insertions(+), 81 deletions(-) create mode 100644 cli/src/services/auth_db/multiprocess_tests.rs diff --git a/cli/src/app.rs b/cli/src/app.rs index 6e90c4895..36996e35a 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -297,7 +297,7 @@ where ); } }; - runtime.telemetry = RuntimeTelemetry::resolve(span_kind).await; + runtime.telemetry = RuntimeTelemetry::resolve(span_kind, &runtime.logger).await; let rendered = run_command_lifecycle(args, &runtime, stdout, stderr).await; runtime.telemetry.shutdown().await; app_support::exit_code_for(&rendered) @@ -939,16 +939,24 @@ pub(crate) mod tests { } let runtime = build_runtime().unwrap(); runtime.block_on(async { - let keychain = + let (keychain, fallback) = RuntimeTelemetry::resolve_with(stalled_keychain_lookup(), ManagedSpanKind::Command) .await; assert!(matches!(keychain, RuntimeTelemetry::Noop(_))); - let database = RuntimeTelemetry::resolve_with( + assert_eq!( + fallback.map(|fallback| fallback.reason), + Some(services::managed_telemetry::TelemetryFallbackReason::LookupTimeout) + ); + let (database, fallback) = RuntimeTelemetry::resolve_with( stalled_blocking_database_lookup(), ManagedSpanKind::ClaudeHook, ) .await; assert!(matches!(database, RuntimeTelemetry::Noop(_))); + assert_eq!( + fallback.map(|fallback| fallback.reason), + Some(services::managed_telemetry::TelemetryFallbackReason::LookupTimeout) + ); }); finish_runtime(runtime); } @@ -1139,7 +1147,8 @@ pub(crate) mod tests { #[tokio::test] async fn disallowed_managed_telemetry_resolves_to_noop_without_credential_lookup() { - let telemetry = RuntimeTelemetry::resolve(None).await; + let telemetry = + RuntimeTelemetry::resolve(None, &services::observability::traits::NoopLogger).await; assert!(matches!(telemetry, RuntimeTelemetry::Noop(_))); } } diff --git a/cli/src/services/auth_db/mod.rs b/cli/src/services/auth_db/mod.rs index 5e2934bb0..5e464738d 100644 --- a/cli/src/services/auth_db/mod.rs +++ b/cli/src/services/auth_db/mod.rs @@ -35,3 +35,6 @@ impl DbSpec for AuthDbSpec { pub type AuthDb = EncryptedTursoDb; pub mod lifecycle; + +#[cfg(test)] +mod multiprocess_tests; diff --git a/cli/src/services/auth_db/multiprocess_tests.rs b/cli/src/services/auth_db/multiprocess_tests.rs new file mode 100644 index 000000000..179b4c4e1 --- /dev/null +++ b/cli/src/services/auth_db/multiprocess_tests.rs @@ -0,0 +1,579 @@ +use std::collections::BTreeSet; +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::PathBuf; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use anyhow::Result; + +use super::{AuthDb, AuthDbSpec}; +use crate::generated_migrations; +use crate::services::db::encryption_key::AUTH_DB_ENCRYPTION_KEY_ENV; +use crate::services::db::{DatabaseKind, DbSpec, EncryptedTursoDb}; +use crate::services::default_paths::auth_db_path; +use crate::services::managed_telemetry::trace_export::tests::{ + capture_requests_until, loopback_hook_telemetry, trace_spans, +}; +use crate::services::managed_telemetry::trace_export::{ + record_claude_hook_payload, ClaudeHookCommand, ManagedSpanKind, +}; +use crate::services::managed_telemetry::{ + load_credential_within_deadline, select_mode, ManagedTelemetryMode, RuntimeTelemetry, + TelemetryFallbackReason, +}; +use crate::services::observability::traits::Telemetry; +use crate::services::telemetry_credential_storage::{ + load_telemetry_credential, save_telemetry_credential, test_access_token, TelemetryCredential, + TelemetryIdentity, +}; + +const CHILD_TEST: &str = "services::auth_db::multiprocess_tests::auth_db_child_process"; +const ROLE_ENV: &str = "SCE_AUTH_DB_TEST_CHILD_ROLE"; +const COLLECTOR_ENV: &str = "SCE_AUTH_DB_TEST_COLLECTOR"; +const HOOK_INDEX_ENV: &str = "SCE_AUTH_DB_TEST_HOOK_INDEX"; +const TEST_SECRET: &str = "auth-db-multiprocess-test-secret"; +const WRONG_SECRET: &str = "auth-db-multiprocess-wrong-secret"; +const TOKEN: &str = "tok-multiprocess"; +const USER_ID: &str = "user_mp"; +const WORKSPACE_ID: &str = "workspace_mp"; +const READY: &str = "SCE_AUTH_DB_CHILD_READY"; +const TOKEN_PREFIX: &str = "SCE_AUTH_DB_CHILD_TOKEN="; +const CONCURRENT_PROCESSES: usize = 32; +const CONCURRENT_MIGRATORS: usize = 4; +const READER_ROUNDS: usize = 3; +const CHILD_KILL_AFTER: Duration = Duration::from_mins(1); +const CHILD_POLL_INTERVAL: Duration = Duration::from_millis(10); +const FAR_FUTURE: u64 = 4_102_444_800; +const HOOK_COMMANDS: [ClaudeHookCommand; 5] = [ + ClaudeHookCommand::DiffTrace, + ClaudeHookCommand::ConversationTrace, + ClaudeHookCommand::ClaudeMutationScope, + ClaudeHookCommand::ClaudeModelState, + ClaudeHookCommand::PolicyBash, +]; +const PRE_BINDING_MIGRATIONS: usize = 3; +const BINDING_MIGRATION_ID: &str = "004_bind_telemetry_credentials"; + +struct PreBindingAuthDbSpec; + +impl DbSpec for PreBindingAuthDbSpec { + fn db_name() -> &'static str { + AuthDbSpec::db_name() + } + + fn db_path() -> Result { + auth_db_path() + } + + fn migrations() -> &'static [(&'static str, &'static str)] { + &generated_migrations::AUTH_MIGRATIONS[..PRE_BINDING_MIGRATIONS] + } + + const KIND: DatabaseKind = DatabaseKind::Auth; +} + +static FAN_OUT_LOCK: Mutex<()> = Mutex::new(()); + +fn serialize_fan_out() -> MutexGuard<'static, ()> { + FAN_OUT_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +struct SchemalessAuthDbSpec; + +impl DbSpec for SchemalessAuthDbSpec { + fn db_name() -> &'static str { + AuthDbSpec::db_name() + } + + fn db_path() -> Result { + auth_db_path() + } + + fn migrations() -> &'static [(&'static str, &'static str)] { + &[] + } + + const KIND: DatabaseKind = DatabaseKind::Auth; +} + +struct Sandbox { + dir: tempfile::TempDir, +} + +impl Sandbox { + fn new() -> Self { + Self { + dir: tempfile::tempdir().expect("sandbox"), + } + } + + fn command(&self, role: &str) -> Command { + let root = self.dir.path(); + let mut command = Command::new(std::env::current_exe().expect("test binary")); + command + .args(["--exact", CHILD_TEST, "--nocapture", "--test-threads=1"]) + .env(ROLE_ENV, role) + .env("HOME", root) + .env("XDG_STATE_HOME", root.join("state")) + .env("XDG_CONFIG_HOME", root.join("config")) + .env("XDG_CACHE_HOME", root.join("cache")) + .env("XDG_DATA_HOME", root.join("data")) + .env(AUTH_DB_ENCRYPTION_KEY_ENV, TEST_SECRET) + .env_remove("TRACEPARENT") + .env_remove("TRACESTATE") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + command + } + + fn run(&self, role: &str) -> String { + wait_success(role, self.command(role).spawn().expect("spawn child")) + } + + fn hold(&self, role: &str) -> HeldChild { + let mut child = self + .command(role) + .stdin(Stdio::piped()) + .spawn() + .expect("spawn holder"); + let stdin = child.stdin.take().expect("holder stdin"); + let mut stdout = BufReader::new(child.stdout.take().expect("holder stdout")); + let mut line = String::new(); + loop { + line.clear(); + let read = stdout.read_line(&mut line).expect("holder stdout line"); + if read == 0 { + let _ = child.kill(); + let output = child.wait_with_output().expect("holder output"); + panic!( + "{role} holder exited before ready: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + if line.trim_end().ends_with(READY) { + break; + } + } + HeldChild { + role: role.to_string(), + child, + stdin, + stdout, + } + } + + fn seed(&self) { + self.run("seed"); + } +} + +struct HeldChild { + role: String, + child: Child, + stdin: ChildStdin, + stdout: BufReader, +} + +impl HeldChild { + fn release(mut self) { + writeln!(self.stdin).expect("release holder"); + drop(self.stdin); + let mut rest = String::new(); + self.stdout + .read_to_string(&mut rest) + .expect("holder stdout"); + let output = self.child.wait_with_output().expect("holder output"); + assert!( + output.status.success() && rest.contains("1 passed"), + "{} holder failed:\n{rest}\n{}", + self.role, + String::from_utf8_lossy(&output.stderr) + ); + } +} + +fn wait_success(role: &str, mut child: Child) -> String { + let started = Instant::now(); + while child.try_wait().expect("child status").is_none() { + if started.elapsed() > CHILD_KILL_AFTER { + let _ = child.kill(); + panic!("{role} child did not exit within {CHILD_KILL_AFTER:?}"); + } + std::thread::sleep(CHILD_POLL_INTERVAL); + } + let output = child.wait_with_output().expect("child output"); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + assert!( + output.status.success() && stdout.contains("1 passed"), + "{role} child failed:\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + stdout +} + +fn read_token(stdout: &str) -> Option<&str> { + stdout + .lines() + .find_map(|line| line.split_once(TOKEN_PREFIX).map(|(_, token)| token)) +} + +fn spawn_readers(sandbox: &Sandbox) -> Vec { + (0..CONCURRENT_PROCESSES) + .map(|_| sandbox.command("read").spawn().expect("spawn reader")) + .collect() +} + +fn wait_readers(readers: Vec) -> usize { + readers + .into_iter() + .map(|child| wait_success("read", child)) + .filter(|stdout| read_token(stdout) == Some(TOKEN)) + .count() +} + +#[test] +fn reader_succeeds_while_another_process_holds_auth_db_open() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let holder = sandbox.hold("hold-open"); + let reader = sandbox.run("read"); + assert_eq!(read_token(&reader), Some(TOKEN)); + holder.release(); +} + +#[test] +fn thirty_two_reader_processes_all_resolve_the_credential() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + for round in 0..READER_ROUNDS { + let started = Instant::now(); + let succeeded = wait_readers(spawn_readers(&sandbox)); + println!( + "round {round}: {succeeded}/{CONCURRENT_PROCESSES} readers in {:?}", + started.elapsed() + ); + assert_eq!(succeeded, CONCURRENT_PROCESSES, "round {round}"); + } +} + +#[test] +fn current_schema_reader_performs_no_migration_write_under_held_write_lock() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let writer = sandbox.hold("hold-write"); + let reader = sandbox.run("read"); + assert_eq!(read_token(&reader), Some(TOKEN)); + writer.release(); + sandbox.run("verify-schema"); +} + +#[test] +fn telemetry_lookup_never_creates_a_missing_auth_db() { + let _fan_out = serialize_fan_out(); + Sandbox::new().run("lookup-missing"); +} + +#[test] +fn concurrent_migration_of_schemaless_db_applies_each_migration_once() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.run("create-schemaless"); + + let initializers = (0..CONCURRENT_MIGRATORS) + .map(|_| sandbox.command("init").spawn().expect("spawn initializer")) + .collect::>(); + for child in initializers { + wait_success("init", child); + } + sandbox.run("verify-schema"); +} + +#[test] +fn outdated_schema_is_migrated_and_partial_schema_fails_clearly() { + let _fan_out = serialize_fan_out(); + Sandbox::new().run("upgrade"); + Sandbox::new().run("partial"); +} + +#[test] +fn concurrent_hook_processes_export_correlated_hook_spans() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .expect("test runtime") + .block_on(export_concurrent_hook_spans(&sandbox)); +} + +async fn export_concurrent_hook_spans(sandbox: &Sandbox) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let address = listener.local_addr().expect("addr"); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let receiver = tokio::spawn(capture_requests_until(listener, stopped)); + + let hooks = (0..CONCURRENT_PROCESSES) + .map(|index| { + sandbox + .command("hook") + .env(COLLECTOR_ENV, address.to_string()) + .env(HOOK_INDEX_ENV, index.to_string()) + .spawn() + .expect("spawn hook") + }) + .collect::>(); + let mut failing = sandbox.command("hook-credential-failure"); + failing.env(AUTH_DB_ENCRYPTION_KEY_ENV, WRONG_SECRET); + let failing = failing.spawn().expect("spawn failing hook"); + tokio::task::spawn_blocking(move || { + for child in hooks { + wait_success("hook", child); + } + wait_success("hook-credential-failure", failing); + }) + .await + .expect("hook children join"); + stop.send(()).expect("stop collector"); + + let spans = trace_spans(&receiver.await.expect("collector join")); + let observed = spans + .iter() + .filter(|span| span.name == "sce.hook") + .map(|span| { + let attribute = |key: &str| { + span.attributes + .iter() + .find(|(name, _)| name == key) + .map(|(_, value)| value.clone()) + .unwrap_or_default() + }; + ( + attribute("sce.claude.tool_use_id"), + attribute("sce.hook.command"), + ) + }) + .collect::>(); + let expected = (0..CONCURRENT_PROCESSES) + .map(|index| { + ( + hook_tool_use_id(index), + hook_command(index).as_str().to_string(), + ) + }) + .collect::>(); + assert_eq!(observed, expected); +} + +fn hook_command(index: usize) -> ClaudeHookCommand { + HOOK_COMMANDS[index % HOOK_COMMANDS.len()] +} + +fn hook_tool_use_id(index: usize) -> String { + format!("toolu_mp_{index:02}") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn auth_db_child_process() { + let Ok(role) = std::env::var(ROLE_ENV) else { + return; + }; + match role.as_str() { + "seed" => seed().await, + "hold-open" => { + let _db = AuthDb::new().await.expect("holder opens auth DB"); + wait_for_release(); + } + "hold-write" => { + let db = AuthDb::new().await.expect("writer opens auth DB"); + db.execute("BEGIN IMMEDIATE", ()) + .await + .expect("writer takes write lock"); + wait_for_release(); + db.execute("ROLLBACK", ()).await.expect("writer rollback"); + } + "read" => { + let credential = load_telemetry_credential() + .await + .expect("credential lookup succeeds") + .expect("credential present"); + println!("{TOKEN_PREFIX}{}", credential.token); + } + "lookup-missing" => { + assert_eq!( + load_telemetry_credential() + .await + .expect("missing auth DB is not an error"), + None + ); + assert!(!auth_db_path().expect("auth DB path").exists()); + } + "create-schemaless" => { + let db = EncryptedTursoDb::::new() + .await + .expect("schemaless auth DB opens"); + assert!(applied_migrations(&db).await.is_empty()); + } + "init" => { + let db = AuthDb::new().await.expect("initializer opens auth DB"); + assert_eq!(applied_migrations(&db).await.len(), expected_migrations()); + } + "verify-schema" => { + let db = AuthDb::new().await.expect("verifier opens auth DB"); + assert_eq!(applied_migrations(&db).await.len(), expected_migrations()); + let check = db + .query_map("PRAGMA integrity_check", (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("integrity check"); + assert_eq!(check, vec!["ok".to_string()]); + } + "upgrade" => upgrade_outdated_schema().await, + "partial" => reject_partial_schema().await, + "hook" => run_hook_child().await, + "hook-credential-failure" => { + let (telemetry, fallback) = RuntimeTelemetry::resolve_with( + load_telemetry_credential(), + ManagedSpanKind::ClaudeHook, + ) + .await; + assert!(matches!(telemetry, RuntimeTelemetry::Noop(_))); + let fallback = fallback.expect("credential failure is diagnosed"); + assert_eq!(fallback.reason, TelemetryFallbackReason::DatabaseError); + let output = telemetry + .with_default_subscriber(&mut || async { Ok("hook-output".to_string()) }) + .await; + assert_eq!(output.expect("hook result unaffected"), "hook-output"); + } + other => panic!("unknown child role {other}"), + } +} + +async fn seed() { + let db = AuthDb::new().await.expect("seed opens auth DB"); + db.execute( + "INSERT OR REPLACE INTO auth_credentials \ + (id, access_token, token_type, expires_in, refresh_token, scope, stored_at_unix_seconds) \ + VALUES (1, ?1, 'Bearer', 3600, 'refresh', NULL, 1)", + (test_access_token(USER_ID, Some(WORKSPACE_ID)),), + ) + .await + .expect("session seeds"); + drop(db); + save_telemetry_credential(&TelemetryCredential { + token: TOKEN.to_string(), + accessor: "accessor-multiprocess".to_string(), + expires_at_unix_seconds: FAR_FUTURE, + identity: TelemetryIdentity { + user_id: USER_ID.to_string(), + workspace_id: WORKSPACE_ID.to_string(), + }, + }) + .await + .expect("telemetry credential seeds"); +} + +async fn run_hook_child() { + let index = std::env::var(HOOK_INDEX_ENV) + .expect("hook index") + .parse::() + .expect("numeric hook index"); + let address = std::env::var(COLLECTOR_ENV) + .expect("collector address") + .parse() + .expect("socket address"); + let credential = load_credential_within_deadline(load_telemetry_credential()) + .await + .expect("managed credential resolves"); + let ManagedTelemetryMode::Managed { .. } = select_mode(Some(credential.clone()), 0) else { + panic!("credential selects managed telemetry"); + }; + let telemetry = loopback_hook_telemetry(address, &credential.token); + let payload = serde_json::json!({ + "hook_event_name": "PostToolUse", + "tool_name": "Write", + "session_id": "session-mp", + "tool_use_id": hook_tool_use_id(index), + }) + .to_string(); + let output = telemetry + .with_default_subscriber(&mut || async { + record_claude_hook_payload(hook_command(index), &payload); + Ok(String::new()) + }) + .await; + assert_eq!(output.expect("hook result"), ""); + telemetry.shutdown().await; +} + +fn wait_for_release() { + println!("{READY}"); + let mut line = String::new(); + let _ = std::io::stdin().read_line(&mut line); +} + +fn expected_migrations() -> usize { + generated_migrations::AUTH_MIGRATIONS.len() +} + +async fn applied_migrations(db: &EncryptedTursoDb) -> Vec { + db.query_map("SELECT id FROM __sce_migrations ORDER BY id", (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("migration ids") +} + +async fn upgrade_outdated_schema() { + let old = EncryptedTursoDb::::new() + .await + .expect("pre-binding schema opens"); + assert_eq!(applied_migrations(&old).await.len(), PRE_BINDING_MIGRATIONS); + drop(old); + let db = AuthDb::new().await.expect("outdated schema upgrades"); + assert_eq!(applied_migrations(&db).await.len(), expected_migrations()); + db.query_map( + "SELECT user_id, workspace_id FROM telemetry_credentials", + (), + |_| Ok(()), + ) + .await + .expect("binding columns exist"); +} + +async fn reject_partial_schema() { + let old = EncryptedTursoDb::::new() + .await + .expect("pre-binding schema opens"); + old.execute( + "ALTER TABLE telemetry_credentials ADD COLUMN user_id TEXT NOT NULL DEFAULT ''", + (), + ) + .await + .expect("partial migration applies"); + drop(old); + let error = AuthDb::new() + .await + .err() + .expect("partially migrated schema must not open"); + let chain = format!("{error:#}"); + assert!(chain.contains(BINDING_MIGRATION_ID), "{chain}"); + let old = EncryptedTursoDb::::new() + .await + .expect("pre-binding schema reopens"); + assert_eq!(applied_migrations(&old).await.len(), PRE_BINDING_MIGRATIONS); +} diff --git a/cli/src/services/bash_policy.rs b/cli/src/services/bash_policy.rs index d5abad2c0..a1edff3b9 100644 --- a/cli/src/services/bash_policy.rs +++ b/cli/src/services/bash_policy.rs @@ -457,7 +457,10 @@ struct JsonPolicyResult<'a> { pub fn run_bash_policy_request(request: &BashPolicyRequest) -> Result { let stdin_payload = read_stdin_payload()?; - crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); + crate::services::managed_telemetry::trace_export::record_claude_hook_payload( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::PolicyBash, + &stdin_payload, + ); run_bash_policy_request_from_payload(request, &stdin_payload) } diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index 05f9d76d1..517253fec 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -29,8 +29,13 @@ const ENCRYPTION_CIPHER_AEGIS256: &str = "aegis256"; const CONNECTION_OPEN_RETRY_POLICY: RetryPolicy = RetryPolicy::builtin(3, 1_000, 25, 200); const CONNECTION_OPEN_RETRY_HINT: &str = "retry after the database lock clears; if the issue persists, stop other SCE processes using this database and rerun the command"; const QUERY_RETRY_POLICY: RetryPolicy = RetryPolicy::builtin(5, 200, 25, 100); +const MIGRATION_RETRY_POLICY: RetryPolicy = RetryPolicy::builtin(10, 1_000, 25, 200); const QUERY_RETRY_HINT: &str = "retry after the database lock clears; if the issue persists, stop other SCE processes using this database and rerun the command"; const AGENT_TRACE_DB_BUSY_TIMEOUT_MS: u64 = 1_000; +const AUTH_DB_BUSY_TIMEOUT_MS: u64 = 1_000; +const MIGRATIONS_TABLE_EXISTS_SQL: &str = + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = '__sce_migrations' LIMIT 1"; +const SELECT_APPLIED_MIGRATIONS_SQL: &str = "SELECT id FROM __sce_migrations"; const AGENT_TRACE_DB_CONTENTION_DEADLINE_MS: u64 = 2_250; const AGENT_TRACE_DB_WRITE_CONTENTION_MAX_ATTEMPTS: u32 = 2; const AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS: u64 = 100; @@ -532,6 +537,99 @@ impl TursoConnectionCore { async fn run_migrations(&self) -> Result<()> { run_embedded_migrations(&self.conn, M::db_name(), M::migrations()).await } + + async fn has_pending_migrations(&self) -> Result { + let db_name = M::db_name(); + let mut tables = self + .conn + .query(MIGRATIONS_TABLE_EXISTS_SQL, ()) + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration readiness query failed: {e}"))?; + let table_exists = tables + .next() + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration readiness row fetch failed: {e}"))? + .is_some(); + drop(tables); + if !table_exists { + return Ok(true); + } + + let mut rows = self + .conn + .query(SELECT_APPLIED_MIGRATIONS_SQL, ()) + .await + .map_err(|e| anyhow::anyhow!("{db_name} applied migration query failed: {e}"))?; + let mut applied = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| anyhow::anyhow!("{db_name} applied migration row fetch failed: {e}"))? + { + applied.push( + row.get::(0) + .map_err(|e| anyhow::anyhow!("{db_name} applied migration id invalid: {e}"))?, + ); + } + + Ok(M::migrations() + .iter() + .any(|(id, _)| !applied.iter().any(|applied_id| applied_id == id))) + } + + async fn run_migrations_serialized(&mut self) -> std::result::Result<(), WriteAttemptFailure> { + let db_name = M::db_name(); + let tx = turso::transaction::Transaction::new( + &mut self.conn, + turso::transaction::TransactionBehavior::Immediate, + ) + .await + .map_err(|e| classify_turso_error(db_name, "failed to begin migration transaction", &e))?; + + match run_embedded_migrations(&tx, db_name, M::migrations()).await { + Ok(()) => tx.commit().await.map_err(|e| { + classify_turso_error(db_name, "failed to commit migration transaction", &e) + }), + Err(error) => { + let _ = tx.rollback().await; + Err(WriteAttemptFailure::Deterministic(error)) + } + } + } + + async fn ensure_migrations_current(&mut self) -> Result<()> { + if !self.has_pending_migrations().await? { + return Ok(()); + } + + let db_name = M::db_name(); + let operation_name = format!("migrate {db_name} database"); + run_with_retry_elapsed( + MIGRATION_RETRY_POLICY, + &operation_name, + QUERY_RETRY_HINT, + MigrationAttempt(self), + ) + .await??; + + if self.has_pending_migrations().await? { + anyhow::bail!("{db_name} schema is incomplete after running migrations"); + } + + Ok(()) + } +} + +struct MigrationAttempt<'a, M: DbSpec>(&'a mut TursoConnectionCore); + +impl RetryOperation> for MigrationAttempt<'_, M> { + async fn run(&mut self, _attempt: u32) -> Result> { + match self.0.run_migrations_serialized().await { + Ok(()) => Ok(Ok(())), + Err(WriteAttemptFailure::Retryable(error)) => Err(error), + Err(WriteAttemptFailure::Deterministic(error)) => Ok(Err(error)), + } + } } fn resolve_connection_open_retry_policy() -> RetryPolicy { @@ -566,20 +664,6 @@ fn resolve_query_retry_policy() -> RetryPolicy { QUERY_RETRY_POLICY } -/// Resolve the Turso busy timeout applied to every connection opened for `M`. -/// -/// Multiprocess WAL provides cross-process correctness and locking; it does -/// not wait for a contended lock. The busy timeout is Turso's own wait policy -/// for a `Busy` result: while another process holds the write lock, Turso -/// sleeps in short phases until the lock clears or the timeout elapses, and -/// only then returns `Busy`. Because `Transaction::new(.., Immediate)` -/// runs `BEGIN IMMEDIATE` through `Connection::execute` on the same connection, -/// the wait also covers writer-lock acquisition. The timeout is connection-wide. -/// -/// Only the Agent Trace DB has a busy timeout, taken from -/// `policies.database_retry.agent_trace_db.busy_timeout_ms` when configured; -/// every other database resolves to zero, which leaves Turso's busy handler -/// unset. fn resolve_busy_timeout() -> std::time::Duration { busy_timeout_from_config::(crate::services::config::get_database_retry_config()) } @@ -587,6 +671,9 @@ fn resolve_busy_timeout() -> std::time::Duration { fn busy_timeout_from_config( config: Option<&DatabaseRetryConfig>, ) -> std::time::Duration { + if M::KIND == DatabaseKind::Auth { + return std::time::Duration::from_millis(AUTH_DB_BUSY_TIMEOUT_MS); + } agent_trace_db_millis::( config, |db| db.busy_timeout_ms, @@ -792,10 +879,6 @@ pub struct TursoDb { core: TursoConnectionCore, } -/// Generic encrypted Turso database adapter. -/// -/// Mirrors the structural seams of [`TursoDb`] while reserving encrypted local -/// database initialization for services that require at-rest encryption. pub struct EncryptedTursoDb { core: TursoConnectionCore, } @@ -1266,11 +1349,6 @@ impl TursoDb { } impl EncryptedTursoDb { - /// Open or create the encrypted database at the spec-provided canonical - /// path. - /// - /// This constructor is the encrypted counterpart to [`TursoDb::new`] and - /// uses a strict encrypted local-builder path. pub async fn new() -> Result { let db_name = M::db_name(); let db_path = M::db_path().with_context(|| format!("failed to resolve {db_name} path"))?; @@ -1285,6 +1363,7 @@ impl EncryptedTursoDb { ensure_db_parent_dir(db_name, &db_path)?; let retry_policy = resolve_connection_open_retry_policy::(); + let busy_timeout = resolve_busy_timeout::(); let operation_name = format!("open encrypted {db_name} database connection"); let conn = run_with_retry_elapsed( @@ -1302,6 +1381,7 @@ impl EncryptedTursoDb { }; let db = turso::Builder::new_local(path_str) + .experimental_multiprocess_wal(true) .experimental_encryption(true) .with_encryption(encryption_opts) .build() @@ -1313,17 +1393,20 @@ impl EncryptedTursoDb { ) })?; - db.connect().map_err(|e| { + let conn = db.connect().map_err(|e| { anyhow::anyhow!("failed to connect to encrypted {db_name} database: {e}") - }) + })?; + apply_busy_timeout(&conn, db_name, busy_timeout)?; + Ok(conn) }, ).await?; - let db = Self { + let mut db = Self { core: TursoConnectionCore::new(conn), }; - db.run_migrations() + db.core + .ensure_migrations_current() .await .with_context(|| format!("failed to run {db_name} migrations"))?; @@ -1417,16 +1500,6 @@ impl EncryptedTursoDb { Ok(results) } - - /// Run all embedded migrations in order. - /// - /// Applied migration IDs are recorded in `__sce_migrations` so later - /// initializations apply only migrations that were not already recorded. - /// Existing databases without migration metadata are brought forward by - /// re-applying the current idempotent migration set and recording each ID. - pub async fn run_migrations(&self) -> Result<()> { - self.core.run_migrations().await - } } #[cfg(test)] @@ -1458,7 +1531,10 @@ mod database_kind_tests { busy_timeout_from_config::(None), Duration::ZERO ); - assert_eq!(busy_timeout_from_config::(None), Duration::ZERO); + assert_eq!( + busy_timeout_from_config::(None), + Duration::from_secs(1) + ); assert_eq!( busy_timeout_from_config::(None), Duration::from_secs(1) diff --git a/cli/src/services/hooks/claude_model_state.rs b/cli/src/services/hooks/claude_model_state.rs index c6b64e7fb..21cafe1d2 100644 --- a/cli/src/services/hooks/claude_model_state.rs +++ b/cli/src/services/hooks/claude_model_state.rs @@ -36,7 +36,10 @@ pub(super) async fn run_claude_model_state_subcommand< return String::new(); } }; - crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); + crate::services::managed_telemetry::trace_export::record_claude_hook_payload( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::ClaudeModelState, + &stdin_payload, + ); let session_id = fail_open_session_id(&stdin_payload); let observed_at_ms = match current_unix_time_ms() { diff --git a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs index 91c8c8168..cfb97469b 100644 --- a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs +++ b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs @@ -48,7 +48,10 @@ pub(crate) async fn run_claude_mutation_scope_subcommand< logger: Option<&L>, ) -> Result { let stdin_payload = hooks::read_hook_stdin()?; - crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); + crate::services::managed_telemetry::trace_export::record_claude_hook_payload( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::ClaudeMutationScope, + &stdin_payload, + ); run_claude_mutation_scope_from_payload(&stdin_payload, logger).await } @@ -539,7 +542,7 @@ mod tests { capture_requests_until, loopback_hook_telemetry, trace_spans, DecodedSpan, }; use crate::services::managed_telemetry::trace_export::{ - record_claude_hook_payload, SHUTDOWN_TIMEOUT, + record_claude_hook_payload, ClaudeHookCommand, SHUTDOWN_TIMEOUT, }; use crate::services::mutation_trace::runtime::{ abandon_scope, advise_after_completed_boundary, coordinate, @@ -582,7 +585,7 @@ mod tests { for payload in HOOK_SEQUENCE { let output = telemetry .with_default_subscriber(&mut || async { - record_claude_hook_payload(payload); + record_claude_hook_payload(ClaudeHookCommand::ClaudeMutationScope, payload); run_claude_mutation_scope_from_payload_with_resolver::( payload, None, @@ -699,7 +702,7 @@ mod tests { let telemetry = loopback_hook_telemetry(address, "bao-token-value"); let output = telemetry .with_default_subscriber(&mut || async { - record_claude_hook_payload(payload); + record_claude_hook_payload(ClaudeHookCommand::ClaudeMutationScope, payload); run_claude_mutation_scope_from_payload_with_resolver::( payload, None, diff --git a/cli/src/services/hooks/conversation_trace.rs b/cli/src/services/hooks/conversation_trace.rs index e93c18fc1..88c358cac 100644 --- a/cli/src/services/hooks/conversation_trace.rs +++ b/cli/src/services/hooks/conversation_trace.rs @@ -74,7 +74,10 @@ pub(crate) async fn run_conversation_trace_subcommand< Ok(payload) => payload, Err(error) => return log_conversation_trace_fail_open(&error, logger, None), }; - crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); + crate::services::managed_telemetry::trace_export::record_claude_hook_payload( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::ConversationTrace, + &stdin_payload, + ); let session_id = conversation_trace_fail_open_session_id(&stdin_payload); match run_conversation_trace_subcommand_from_payload( diff --git a/cli/src/services/hooks/diff_trace.rs b/cli/src/services/hooks/diff_trace.rs index 0242e35ad..33780be5f 100644 --- a/cli/src/services/hooks/diff_trace.rs +++ b/cli/src/services/hooks/diff_trace.rs @@ -71,7 +71,10 @@ pub(crate) async fn run_diff_trace_subcommand payload, Err(error) => return log_diff_trace_fail_open(&error, logger, None), }; - crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); + crate::services::managed_telemetry::trace_export::record_claude_hook_payload( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::DiffTrace, + &stdin_payload, + ); let session_id = diff_trace_fail_open_session_id(&stdin_payload); match run_diff_trace_subcommand_from_payload(repository_root, &stdin_payload, logger).await { diff --git a/cli/src/services/managed_telemetry.rs b/cli/src/services/managed_telemetry.rs index b2b1a3759..25f0a9a9d 100644 --- a/cli/src/services/managed_telemetry.rs +++ b/cli/src/services/managed_telemetry.rs @@ -7,7 +7,7 @@ pub mod log_export; pub mod trace_export; use crate::services::error::CliError; -use crate::services::observability::traits::{NoopTelemetry, Telemetry}; +use crate::services::observability::traits::{Logger, NoopTelemetry, Telemetry}; use trace_export::{ManagedSpanKind, ManagedTraceTelemetry}; use crate::services::telemetry_credential_storage::{ @@ -26,6 +26,11 @@ const TRACES_PATH: &str = "/v1/traces"; #[cfg(test)] const LOGS_PATH: &str = "/v1/logs"; const REDACTED: &str = ""; +const TELEMETRY_FALLBACK_EVENT_ID: &str = "sce.telemetry.fallback"; +const TELEMETRY_FALLBACK_MESSAGE: &str = + "Managed telemetry disabled for this invocation; continuing without export"; +const FALLBACK_REASON_FIELD: &str = "reason"; +const FALLBACK_ERROR_FIELD: &str = "error"; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ManagedOtlpEndpoints { @@ -108,14 +113,97 @@ where joined.ok()?.ok() } -pub(crate) async fn load_credential_within_deadline(lookup: Fut) -> Option +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum TelemetryFallbackReason { + CredentialUnavailable, + CredentialExpired, + LookupTimeout, + DatabaseError, + PathResolutionError, + ExporterInitFailed, +} + +impl TelemetryFallbackReason { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::CredentialUnavailable => "credential_unavailable", + Self::CredentialExpired => "credential_expired", + Self::LookupTimeout => "lookup_timeout", + Self::DatabaseError => "database_error", + Self::PathResolutionError => "path_resolution_error", + Self::ExporterInitFailed => "exporter_init_failed", + } + } + + const fn is_failure(self) -> bool { + !matches!(self, Self::CredentialUnavailable | Self::CredentialExpired) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TelemetryFallback { + pub(crate) reason: TelemetryFallbackReason, + detail: Option, +} + +impl TelemetryFallback { + const fn new(reason: TelemetryFallbackReason) -> Self { + Self { + reason, + detail: None, + } + } + + fn from_storage_error(error: &TokenStorageError) -> Self { + let reason = match error { + TokenStorageError::Database(_) => TelemetryFallbackReason::DatabaseError, + TokenStorageError::PathResolution(_) => TelemetryFallbackReason::PathResolutionError, + }; + Self { + reason, + detail: Some(error.to_string()), + } + } + + fn log(&self, logger: &impl Logger) { + let mut fields = vec![(FALLBACK_REASON_FIELD, self.reason.as_str())]; + if let Some(detail) = &self.detail { + fields.push((FALLBACK_ERROR_FIELD, detail.as_str())); + } + if self.reason.is_failure() { + logger.warn( + TELEMETRY_FALLBACK_EVENT_ID, + TELEMETRY_FALLBACK_MESSAGE, + &fields, + None, + ); + } else { + logger.debug( + TELEMETRY_FALLBACK_EVENT_ID, + TELEMETRY_FALLBACK_MESSAGE, + &fields, + None, + ); + } + } +} + +pub(crate) async fn load_credential_within_deadline( + lookup: Fut, +) -> Result where Fut: Future, TokenStorageError>> + Send + 'static, { - run_with_deadline(CREDENTIAL_LOOKUP_DEADLINE, lookup) - .await - .and_then(Result::ok) - .flatten() + match run_with_deadline(CREDENTIAL_LOOKUP_DEADLINE, lookup).await { + None => Err(TelemetryFallback::new( + TelemetryFallbackReason::LookupTimeout, + )), + Some(Err(error)) => Err(TelemetryFallback::from_storage_error(&error)), + Some(Ok(None)) => Err(TelemetryFallback::new( + TelemetryFallbackReason::CredentialUnavailable, + )), + Some(Ok(Some(credential))) => Ok(credential), + } } fn now_unix_seconds() -> u64 { @@ -130,29 +218,52 @@ pub enum RuntimeTelemetry { } impl RuntimeTelemetry { - pub async fn resolve(span_kind: Option) -> Self { + pub async fn resolve(span_kind: Option, logger: &impl Logger) -> Self { let Some(span_kind) = span_kind else { return Self::Noop(NoopTelemetry); }; - Self::resolve_with(load_telemetry_credential(), span_kind).await + let (telemetry, fallback) = + Self::resolve_with(load_telemetry_credential(), span_kind).await; + if let Some(fallback) = fallback { + fallback.log(logger); + } + telemetry } - pub(crate) async fn resolve_with(lookup: Fut, span_kind: ManagedSpanKind) -> Self + pub(crate) async fn resolve_with( + lookup: Fut, + span_kind: ManagedSpanKind, + ) -> (Self, Option) where Fut: Future, TokenStorageError>> + Send + 'static, { - let credential = load_credential_within_deadline(lookup).await; + let credential = match load_credential_within_deadline(lookup).await { + Ok(credential) => credential, + Err(fallback) => return (Self::Noop(NoopTelemetry), Some(fallback)), + }; let Ok(runtime) = tokio::runtime::Handle::try_current() else { - return Self::Noop(NoopTelemetry); + return (Self::Noop(NoopTelemetry), None); }; - match select_mode(credential, now_unix_seconds()) { + match select_mode(Some(credential), now_unix_seconds()) { ManagedTelemetryMode::Managed { bearer, endpoints } => { - ManagedTraceTelemetry::new(&bearer, endpoints, runtime, span_kind) - .map_or(Self::Noop(NoopTelemetry), Self::Managed) + match ManagedTraceTelemetry::new(&bearer, endpoints, runtime, span_kind) { + Ok(telemetry) => (Self::Managed(telemetry), None), + Err(_) => ( + Self::Noop(NoopTelemetry), + Some(TelemetryFallback::new( + TelemetryFallbackReason::ExporterInitFailed, + )), + ), + } } - ManagedTelemetryMode::Noop => Self::Noop(NoopTelemetry), + ManagedTelemetryMode::Noop => ( + Self::Noop(NoopTelemetry), + Some(TelemetryFallback::new( + TelemetryFallbackReason::CredentialExpired, + )), + ), } } @@ -241,19 +352,50 @@ mod tests { } #[tokio::test] - async fn claude_hook_credential_failure_or_expiry_resolves_noop() { - let failed = RuntimeTelemetry::resolve_with( + async fn claude_hook_credential_failure_or_expiry_resolves_noop_with_reason() { + let (failed, fallback) = RuntimeTelemetry::resolve_with( async { Err(TokenStorageError::Database("locked".to_string())) }, ManagedSpanKind::ClaudeHook, ) .await; assert!(matches!(failed, RuntimeTelemetry::Noop(_))); - let expired = RuntimeTelemetry::resolve_with( + let fallback = fallback.expect("database failure is diagnosed"); + assert_eq!(fallback.reason, TelemetryFallbackReason::DatabaseError); + assert!(fallback.reason.is_failure()); + assert!(fallback + .detail + .as_deref() + .is_some_and(|detail| detail.contains("locked"))); + + let (path, fallback) = RuntimeTelemetry::resolve_with( + async { Err(TokenStorageError::PathResolution("no home".to_string())) }, + ManagedSpanKind::ClaudeHook, + ) + .await; + assert!(matches!(path, RuntimeTelemetry::Noop(_))); + assert_eq!( + fallback.map(|fallback| fallback.reason), + Some(TelemetryFallbackReason::PathResolutionError) + ); + + let (expired, fallback) = RuntimeTelemetry::resolve_with( async { Ok(Some(credential("bao-token", 1))) }, ManagedSpanKind::ClaudeHook, ) .await; assert!(matches!(expired, RuntimeTelemetry::Noop(_))); + let fallback = fallback.expect("expiry is diagnosed"); + assert_eq!(fallback.reason, TelemetryFallbackReason::CredentialExpired); + assert!(!fallback.reason.is_failure()); + assert_eq!(fallback.detail, None); + + let (missing, fallback) = + RuntimeTelemetry::resolve_with(async { Ok(None) }, ManagedSpanKind::ClaudeHook).await; + assert!(matches!(missing, RuntimeTelemetry::Noop(_))); + assert_eq!( + fallback.map(|fallback| fallback.reason), + Some(TelemetryFallbackReason::CredentialUnavailable) + ); } #[test] diff --git a/cli/src/services/managed_telemetry/trace_export.rs b/cli/src/services/managed_telemetry/trace_export.rs index 59d35ebee..491531fe5 100644 --- a/cli/src/services/managed_telemetry/trace_export.rs +++ b/cli/src/services/managed_telemetry/trace_export.rs @@ -45,6 +45,7 @@ pub const COMMAND_NAME_FIELD: &str = "sce.command.name"; const AGENT_CLAUDE: &str = "claude"; const HOOK_EVENT_FIELD: &str = "sce.hook.event"; const HOOK_TOOL_FIELD: &str = "sce.hook.tool"; +const HOOK_COMMAND_FIELD: &str = "sce.hook.command"; const CLAUDE_SESSION_ID_FIELD: &str = "sce.claude.session_id"; const CLAUDE_TOOL_USE_ID_FIELD: &str = "sce.claude.tool_use_id"; const CLAUDE_PAYLOAD_FIELDS: [(&str, &str); 4] = [ @@ -116,6 +117,27 @@ pub enum ManagedSpanKind { ClaudeHook, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ClaudeHookCommand { + DiffTrace, + ConversationTrace, + ClaudeMutationScope, + ClaudeModelState, + PolicyBash, +} + +impl ClaudeHookCommand { + pub const fn as_str(self) -> &'static str { + match self { + Self::DiffTrace => "diff-trace", + Self::ConversationTrace => "conversation-trace", + Self::ClaudeMutationScope => "claude-mutation-scope", + Self::ClaudeModelState => "claude-model-state", + Self::PolicyBash => "policy-bash", + } + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ManagedChildSpan { MutationScope, @@ -280,6 +302,7 @@ impl ManagedTraceTelemetry { target: COMMAND_SPAN_TARGET, HOOK_SPAN_NAME, sce.agent = AGENT_CLAUDE, + sce.hook.command = field::Empty, sce.hook.event = field::Empty, sce.hook.tool = field::Empty, sce.claude.session_id = field::Empty, @@ -368,11 +391,12 @@ pub fn record_command_name(name: &str) { Span::current().record(COMMAND_NAME_FIELD, name); } -pub fn record_claude_hook_payload(stdin_payload: &str) { +pub fn record_claude_hook_payload(command: ClaudeHookCommand, stdin_payload: &str) { let span = Span::current(); if span.is_disabled() { return; } + span.record(HOOK_COMMAND_FIELD, command.as_str()); let Ok(serde_json::Value::Object(payload)) = serde_json::from_str(stdin_payload) else { return; }; @@ -800,7 +824,7 @@ pub(crate) mod tests { let result = telemetry .with_default_subscriber(&mut || async { record_command_name("hooks"); - record_claude_hook_payload(payload); + record_claude_hook_payload(ClaudeHookCommand::DiffTrace, payload); tokio::task::yield_now().await; Ok(String::new()) }) @@ -821,6 +845,7 @@ pub(crate) mod tests { ("sce.agent", "claude"), ("sce.claude.session_id", "session-7d07"), ("sce.claude.tool_use_id", "toolu_01KN1s"), + ("sce.hook.command", "diff-trace"), ("sce.hook.event", "PostToolUse"), ("sce.hook.tool", "Write"), ("sce.outcome", "success"), diff --git a/cli/src/services/telemetry_credential_storage.rs b/cli/src/services/telemetry_credential_storage.rs index 3110e029c..a42e7537e 100644 --- a/cli/src/services/telemetry_credential_storage.rs +++ b/cli/src/services/telemetry_credential_storage.rs @@ -6,6 +6,7 @@ use base64::Engine; use serde::Deserialize; use crate::services::db::{DbSpec, EncryptedTursoDb}; +use crate::services::default_paths::auth_db_path; use crate::services::token_storage::{get_auth_db, TokenStorageError}; const TELEMETRY_CREDENTIAL_ROW_ID: i64 = 1; @@ -86,9 +87,18 @@ pub async fn save_telemetry_credential( } pub async fn load_telemetry_credential() -> Result, TokenStorageError> { + if !auth_db_exists()? { + return Ok(None); + } load_with(get_auth_db().await?, current_unix_timestamp_seconds()?).await } +fn auth_db_exists() -> Result { + auth_db_path() + .map(|path| path.exists()) + .map_err(|error| TokenStorageError::PathResolution(error.to_string())) +} + pub async fn load_repository_telemetry_credential( repository_id: String, ) -> Result, TokenStorageError> { diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md index a66a9a02d..35dbcb27e 100644 --- a/context/plans/claude-hook-trace-propagation.md +++ b/context/plans/claude-hook-trace-propagation.md @@ -202,3 +202,34 @@ Persist this field in every plan; this is durable plan state, not chat state: - Resolved 2026-10-17: the user found trace `dcd919ecc4f5d400df5b642a1a229783` in the `sce-crocoder` Dash0 dataset with 6 `sce.hook` spans as direct children of `claude_code.interaction`, two of them with child spans. The child span names were not listed, so live delivery of `sce.mutation_scope`/`sce.git_snapshot`/`sce.persistence` by name is still proven only by local loopback tests. - Interactive TUI sessions, subagent hooks, and other hook events were not probed for `TRACEPARENT`. - Each Claude hook process can now take up to `CREDENTIAL_LOOKUP_DEADLINE` + `SHUTDOWN_TIMEOUT` longer, including the blocking `PreToolUse` path; the plan's open question on a tighter hook deadline is still unresolved. + +## Follow-up validation: concurrent hook telemetry reliability (PR #330) + +**Date:** 2026-10-19 + +### Change + +- Root cause of missing live `sce.hook` spans (8 of 15 expected in an earlier run): the encrypted `auth.db` opened without Turso multiprocess WAL, so concurrent hook processes failed with `LockingError` and fell back silently to Noop. Fix and evidence: [Concurrent hook span loss](../sce/claude-hook-trace-propagation.md#concurrent-hook-span-loss), [auth-db.md](../sce/auth-db.md#concurrency-and-schema-readiness). +- Added `sce.hook.command`, `sce.telemetry.fallback` diagnostics, migration-free current-schema opens, serialized Busy-retried migrations, a 1000 ms auth DB busy timeout, and a telemetry lookup that never creates a missing `auth.db`. + +### Commands run + +- `nix flake check` -> exit 0 (all checks, including cli-tests, cli-clippy, cli-fmt, pkl-generated) +- `nix build .#checks.x86_64-linux.cli-tests --rebuild` ×2 -> all tests passed both times; the command exits 1 only on Nix's output-determinism comparison +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml auth_db::multiprocess_tests` -> 8 passed +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml telemetry_credential` -> 31 passed +- Multiprocess suite stress, 6 parallel copies × 8 rounds -> 48 of 48 passed +- Mutation checks: without `experimental_multiprocess_wal(true)`, 5 of 7 multiprocess tests fail; with unconditional migrations, the held-write-lock reader test fails + +### Results + +- 32 reader processes × 3 rounds: 32/32 each round (about 60–190 ms per round, debug build). +- 32 concurrent hook processes: every `(sce.claude.tool_use_id, sce.hook.command)` pair exported exactly once to a loopback collector. A wrong-key child resolves Noop with reason `database_error` and unchanged hook output. +- Mixed versions: unsafe for concurrent writers (silent lost update); sequential use and legacy reads are safe. See [Mixed-version rollout risk](../sce/auth-db.md#mixed-version-rollout-risk). +- Live: headless session `48a276ea-ce4a-4488-8244-649f20b87a42`, trace `132a62ceed757d0d8431011311ae76af`, 13 SCE hook invocations, zero local telemetry fallbacks. Dash0 delivery is **unverified**. + +### Residual risks + +- Dash0 confirmation of the 13 spans, their `sce.hook.command` values, and `tool_use_id` equality is still needed. +- Mixed old/new `sce` writers can lose updates; rollout guidance is in `auth-db.md`. +- Turso 0.8.1 has a multiprocess-WAL race when many processes concurrently create or migrate a DB under heavy load. It was not observed on read-only hook paths. diff --git a/context/sce/auth-db.md b/context/sce/auth-db.md index 08e1163e2..5542c548e 100644 --- a/context/sce/auth-db.md +++ b/context/sce/auth-db.md @@ -48,6 +48,24 @@ Current migration baseline: Storage operations live in `cli/src/services/telemetry_credential_storage.rs`: `save_telemetry_credential` is a single-statement `INSERT OR REPLACE` of the fixed row (last complete writer wins), `load_telemetry_credential` returns the row only when token, accessor, and identity are non-empty, the expiry is non-negative and in the future, and the stored WorkOS session's access-token `(sub, workspace_id)` claims equal the row's identity (it never mints or refreshes), `load_repository_telemetry_credential` additionally requires a matching repository binding, `bind_repository_telemetry` records a binding for the current credential identity, and `delete_telemetry_credential` removes the row. `TelemetryCredential` has a redacted `Debug`, and storage errors use fixed messages without secret values. Saving or deleting never touches `auth_credentials`. Managed OTLP export reads this row at CLI startup through `load_telemetry_credential` and uses only the token as bearer. The shared `AuthDb` singleton comes from `token_storage::get_auth_db`. `sce auth logout` deletes both the `auth_credentials` row and the telemetry row; see [telemetry-credential.md](telemetry-credential.md) for lifecycle and security boundaries. +## Concurrency and schema readiness + +- Many short-lived `sce` processes read `auth.db` at once: every admitted Claude hook resolves the telemetry credential at startup. `EncryptedTursoDb::new()` therefore opens with Turso multiprocess WAL, alongside encryption, so concurrent readers share the file. Multiprocess WAL creates `auth.db-wal` and the `auth.db-tshm` coordination file next to the database. +- A current schema is opened read-only: no `CREATE TABLE IF NOT EXISTS __sce_migrations` or other DDL runs when every embedded migration ID is already recorded. A missing or outdated DB is migrated in one `BEGIN IMMEDIATE` transaction, with a retry only for genuine `Busy`. A partially migrated schema fails with the failing migration ID rather than being treated as current. Details: [shared-turso-db.md](shared-turso-db.md). +- `load_telemetry_credential` (the managed-telemetry lookup used by every command and admitted hook) is an existing-DB read path: when `auth.db` does not exist it returns no credential without creating the file or an OS-keyring key. Only `sce auth`, `sce setup`, and other writers create the DB. This also keeps concurrent hooks on a never-logged-in machine from racing to create it. +- The auth DB busy timeout is 1000 ms. Hook reads stay bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s), which aborts the lookup regardless of busy waits or retries. +- Regression coverage (`cli/src/services/auth_db/multiprocess_tests.rs`, child processes of the test binary with sandboxed `HOME`/`XDG_*` and an env-provided `SCE_AUTH_DB_ENCRYPTION_KEY`; never the real DB or keyring): a reader succeeds while another process holds the DB open; 32 reader processes × 3 rounds all resolve the credential; a current-schema reader succeeds while another process holds `BEGIN IMMEDIATE`, which proves it performs no migration write; a telemetry lookup on a missing DB returns none and creates nothing; 4 processes concurrently migrating an existing schemaless DB record each migration once and pass `PRAGMA integrity_check`; an outdated schema upgrades and a partial one fails clearly and rolls back; 32 concurrent hook processes export correlated `sce.hook` spans. + +### Upstream Turso multiprocess race + +Under heavy load, Turso 0.8.1 multiprocess WAL has an internal race on concurrent cold opens that involve writes. Stress runs (6 parallel copies of the suite, with 32 processes creating or migrating the same DB) occasionally failed: `short read on page 3`, a `turso_assert` panic `reader slot released by non-owner` in `shared_wal_coordination.rs`, or, in the Nix sandbox, a writer that stayed `database is locked` through all 10 migration attempts (about 49 s). The read-only paths never failed across about 3,800 cold-opening reader and hook processes, and that is all hooks do on a current schema. The migration regression therefore uses 4 concurrent migrators, which was 0 failures in 48 parallel stress runs. Concurrent creation and migration by many writers is left as a documented upstream limitation. It affects first-time initialization only, and the Agent Trace DB already runs in the same Turso mode. + +### Mixed-version rollout risk + +Binaries from before this change open `auth.db` in legacy (exclusive-lock) WAL mode. A probe on the pinned Turso 0.8.1 with the same encrypted builder showed that Turso's mixed-mode guards do not fire on this path. With a legacy holder, a multiprocess process opens and writes, and with a multiprocess holder, a legacy process opens and writes; neither gets a `LockingError`. When both modes hold the DB and both write, the last committer's stale WAL view wins and the other process's committed rows are silently lost, while `PRAGMA integrity_check` still reports `ok`. A legacy holder running only the no-op `CREATE TABLE IF NOT EXISTS` lost nothing, and sequential use is safe: after all multiprocess processes exit, the legacy `sce` binary opens, decrypts, reads, and writes the DB, and integrity stays `ok`. + +Hooks never write `auth.db`. Writers are `sce auth login`/`logout`, `sce setup` repository enrollment, telemetry-credential issuance, and the `sce sync` token refresh. So the hazard needs an old-version and a new-version `sce` process alive at the same time with at least one of them writing. Rollout: replace the binary while no `sce auth`, `sce setup`, or `sce sync` command is running, and do not keep two `sce` versions on `PATH` used by the same account. Short-lived old hook processes that are mid-flight during an upgrade only read. Closing the hazard in code would need a cross-mode lock guard (for example a shared `fcntl` lock held by new processes, which conflicts with the legacy exclusive lock). That needs a direct `rustix`/`libc` dependency and is not part of this change. + ## Lifecycle integration `AuthDbLifecycle` is registered in `cli/src/services/auth_db/lifecycle.rs` following the existing DB lifecycle pattern: diff --git a/context/sce/claude-hook-trace-propagation.md b/context/sce/claude-hook-trace-propagation.md index 64671ec6a..c77876e90 100644 --- a/context/sce/claude-hook-trace-propagation.md +++ b/context/sce/claude-hook-trace-propagation.md @@ -162,7 +162,7 @@ claude_code.interaction (3eec7153327c6839) └── sce.persistence ``` -The initial root-span listing could not show non-root spans, including Claude Code's own tool and LLM request children. The subsequent waterfall screenshot establishes that parented SCE tool-hook spans and their children do reach Dash0, but the screenshot does not display its trace ID or dataset name; therefore it cannot by itself prove that it is the specific `dcd919ecc4f5d400df5b642a1a229783` trace above. The Dash0 dataset name remains unverified. +The initial root-span listing could not show non-root spans, including Claude Code's own tool and LLM request children. The subsequent waterfall screenshot establishes that parented SCE tool-hook spans and their children do reach Dash0, but the screenshot does not display its trace ID or dataset name; therefore it cannot by itself prove that it is the specific `dcd919ecc4f5d400df5b642a1a229783` trace above. The dataset is verified separately: on 2026-10-17 the user found trace `dcd919ecc4f5d400df5b642a1a229783` in the SCE CroCoder (`sce-crocoder`) Dash0 dataset, with 6 `sce.hook` spans as direct children of `claude_code.interaction` (21 spans total; see the T05 follow-up in `context/plans/claude-hook-trace-propagation.md`). ## Follow-up Dash0 waterfall evidence @@ -172,7 +172,20 @@ A user-provided Dash0 trace-detail screenshot on 2026-10-17 shows a **single** ` - Two `sce.hook → sce.mutation_scope` branches, each containing sibling `sce.git_snapshot` and `sce.persistence` children. This confirms live delivery of all three SCE operation span types and their expected parent hierarchy. - A `claude_code.interaction` duration of 6,988 ms. The two instrumented mutation hooks shown take 154 ms and 110 ms; the first has a 120 ms mutation scope, 63 ms Git snapshot and 33 ms persistence; the second has a 100 ms mutation scope, 57 ms Git snapshot and 19 ms persistence. -**Verification verdict:** Live parent-child propagation and SCE operation-span export are **verified visually in Dash0**, closing the hierarchy/delivery gap left by the earlier root-only listing. The screenshot does **not** show the trace ID, workspace/dataset, per-span `tool_use_id` attribute values, or the exact time of the trace. Do not claim that the screenshot proves the prior headless session's trace ID `dcd919ecc4f5d400df5b642a1a229783`, or that it proves hook-to-tool ID equality. This observation does not measure exporter overhead or p95 hook latency. +**Verification verdict:** Live parent-child propagation and SCE operation-span export are **verified visually in Dash0**, closing the hierarchy/delivery gap left by the earlier root-only listing. The screenshot itself does **not** show the trace ID, workspace/dataset (the `sce-crocoder` dataset was confirmed separately; see [Observed hierarchy](#observed-hierarchy)), per-span `tool_use_id` attribute values, or the exact time of the trace. Do not claim that the screenshot proves the prior headless session's trace ID `dcd919ecc4f5d400df5b642a1a229783`, or that it proves hook-to-tool ID equality. This observation does not measure exporter overhead or p95 hook latency. + +## Concurrent hook span loss + +A later live run expected 15 SCE hook spans and Dash0 received 8. Claude Code runs several SCE hooks for the same event at the same time, and each one opens the encrypted `auth.db` to read the telemetry credential. + +- Root cause: `EncryptedTursoDb::new()` built the Turso database without `experimental_multiprocess_wal(true)`. In that mode Turso 0.8.1 takes an exclusive, non-blocking lock on the whole `auth.db` file for the process lifetime. Concurrent hook processes failed during open with `LockingError("Failed locking file '<…>/auth.db'. File is locked by another process")` after the three connection-open retries (about 75–80 ms), well inside the 2s lookup deadline. `RuntimeTelemetry::resolve_with` then fell back to `NoopTelemetry` silently, so those hooks exported no span. +- Secondary cause: with multiprocess WAL alone, every open still ran `run_migrations()`, and `CREATE TABLE IF NOT EXISTS __sce_migrations` takes the write lock even on a current schema, which caused transient `Busy("database is locked")`. +- Reproduction at 32 concurrent processes (failures out of 256): current implementation 180; multiprocess WAL only 32; multiprocess WAL plus a 1000 ms busy timeout 0; multiprocess WAL plus skipping migrations when current 0; both remedies 0. +- Fix: multiprocess WAL on the encrypted builder, a credential lookup that never creates a missing `auth.db`, a migration-free open for a current schema, serialized and Busy-retried migrations for a missing or outdated schema, and a 1000 ms auth DB busy timeout ([auth-db.md](auth-db.md), [shared-turso-db.md](shared-turso-db.md)). Every Noop fallback is now logged locally as `sce.telemetry.fallback` with its reason. +- Correlation: `sce.hook` now carries `sce.hook.command` (`diff-trace`, `conversation-trace`, `claude-mutation-scope`, `claude-model-state`, `policy-bash`), so spans from different SCE commands on the same Claude event are distinguishable. `tool_use_id` handling is unchanged. +- Regression evidence (`cli/src/services/auth_db/multiprocess_tests.rs`): 32 hook child processes resolve the credential through the production `load_telemetry_credential` path. They export to a loopback collector, and every `(sce.claude.tool_use_id, sce.hook.command)` pair arrives exactly once. A child with a wrong encryption key resolves Noop with reason `database_error`, and its hook output is unchanged. The suite passed 48 of 48 runs with 6 copies in parallel. Removing the multiprocess flag fails the open-holder, 32-reader, held-write, concurrent-init, and hook-export tests. Restoring unconditional migrations fails the held-write test. +- Live run on 2026-10-19, 11:08:50Z–11:09:03Z: headless Claude Code `2.1.301` (`claude -p --model haiku --allowedTools Write,Read`), session `48a276ea-ce4a-4488-8244-649f20b87a42`, `num_turns=3`, with a debug build of this fix first on `PATH` through a scratchpad shim. The shim logged only subcommand words. The stored credential was present and was not printed. The session wrote and read a gitignored `context/tmp/` file, which was then deleted. The shim recorded 13 SCE hook invocations: `claude-mutation-scope` ×7, `conversation-trace` ×4, `diff-trace` ×1, `claude-model-state` ×1. `policy bash` did not fire because its matcher is `Bash`, and `diff-trace` fired only for `Write`. Claude also ran `sce auth telemetry-headers` once. A scratchpad probe hook recorded tool hooks for `toolu_015xdt65shPcBzT4UNbbKib5` (Write) and `toolu_01VMf442BtXsW2XMvnd76YZJ` (Read), all with `TRACEPARENT` trace `132a62ceed757d0d8431011311ae76af` and parent `6a4ae45ade8a6191`. The repository log directory recorded zero `sce.telemetry.fallback` records, so none of the 13 hook processes fell back to Noop. The installed pre-fix `sce` still read the credential afterwards. +- Dash0 status for this run: **unverified**. This session had no Dash0 access, and collector responses were not used as evidence. Expected in the `sce-crocoder` dataset, in trace `132a62ceed757d0d8431011311ae76af`: one `sce.hook` per tool-hook invocation, parented to `claude_code.interaction` `6a4ae45ade8a6191`, with `sce.claude.tool_use_id` equal to the matching `claude_code.tool`, and same-event hooks distinguished by `sce.hook.command` (for example, Write `PostToolUse` → `diff-trace`, `conversation-trace`, `claude-mutation-scope`). Session-boundary hooks should appear as independent `sce.hook` roots. ## Opt-in live smoke procedure @@ -186,12 +199,14 @@ This is manual, uses real credentials, Claude Code, and Dash0, and is never part 6. In Dash0, find the `sce.command` span with `sce.command.name=doctor` at the noted time. Open the Claude trace by its trace ID, not a root-span list, and confirm: - each tool hook's `sce.hook` has `claude_code.interaction` as its parent and a `sce.claude.tool_use_id` that matches a `claude_code.tool` span; - the `Write` `PreToolUse` `claude-mutation-scope` hook carries `sce.mutation_scope` → `sce.git_snapshot` and `sce.persistence`; - - `SessionStart`/`SessionEnd` `sce.hook` spans are independent roots. + - `SessionStart`/`SessionEnd` `sce.hook` spans are independent roots; + - every SCE hook invocation produced one `sce.hook` span, and hooks for the same Claude event differ by `sce.hook.command`; + - the branch log directory has no `sce.telemetry.fallback` warning for the run. 7. Record the trace IDs and the observed hierarchy. Record anything that was not observed as unverified, and never record a collector 2xx as delivery. ## Not verified -- The exact trace ID, dataset name, and `tool_use_id` equality between each `sce.hook` and its matching `claude_code.tool` were not exposed in the follow-up waterfall screenshot. The parent-child hierarchy and child-span delivery **are** visually verified; see [Follow-up Dash0 waterfall evidence](#follow-up-dash0-waterfall-evidence). +- `tool_use_id` equality between each `sce.hook` and its matching `claude_code.tool` has not been observed in Dash0. The dataset (`sce-crocoder`) and the trace ID `dcd919ecc4f5d400df5b642a1a229783` were confirmed by the user's trace lookup, not by the waterfall screenshot. The parent-child hierarchy and child-span delivery **are** visually verified; see [Follow-up Dash0 waterfall evidence](#follow-up-dash0-waterfall-evidence). - Interactive TUI sessions were not probed for hook `TRACEPARENT`. T05 also used headless `claude -p`. Claude Code documents that only `-p`/Agent SDK sessions read an inbound `TRACEPARENT`. The outbound subprocess builder is shared, and the interactive Bash tool environment of the same version carries a valid `TRACEPARENT`. Whether interactive hooks receive the same interaction-span context is expected but unconfirmed. - Subagent (`Agent` tool) hooks, `PostToolUseFailure`, `PermissionDenied`, and other hook events were not probed. - Behavior behind a custom `ANTHROPIC_BASE_URL` proxy without `CLAUDE_CODE_PROPAGATE_TRACEPARENT` was not probed. Per the docs the subprocess `TRACEPARENT` follows the same switch as the API `traceparent` header. diff --git a/context/sce/cli-observability-contract.md b/context/sce/cli-observability-contract.md index 5652b3d62..1d2391d96 100644 --- a/context/sce/cli-observability-contract.md +++ b/context/sce/cli-observability-contract.md @@ -50,6 +50,7 @@ Runtime observability consumes the shared resolved observability config from `cl - `sce.command.dispatch_start` (debug level - logged before dispatch) - `sce.command.dispatch_end` (debug level - logged after successful dispatch) - `sce.command.completed` + - `sce.telemetry.fallback` (`warn` for credential lookup failures, `debug` for a missing or expired credential - emitted once when managed telemetry resolution falls back to Noop after a lookup; fields `reason` and, for database or path errors, `error`) - Error logging uses the pattern `sce.error.{code}` where `{code}` is the classified error code (e.g., `sce.error.SCE-ERR-RUNTIME`). - All `CliError` instances are logged via `Logger::log_cli_error()` before user-facing stderr diagnostics are written; observability retains full technical detail independently of what is rendered to the terminal, and never writes a second competing stderr diagnostic for the same error. - Event records include deterministic metadata keys used by automation (`command`, `failure_class`, `component` when applicable). @@ -94,11 +95,11 @@ Runtime observability consumes the shared resolved observability config from `cl ## Managed trace export -- Defined in `cli/src/services/managed_telemetry.rs` and `cli/src/services/managed_telemetry/trace_export.rs` (plus `log_export.rs`). After startup, `RuntimeTelemetry::resolve` loads the stored telemetry credential from the encrypted auth DB in a spawned task bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s); absent, expired, malformed, unavailable, ownerless (no stored WorkOS session), or timed-out credentials select `NoopTelemetry`. Startup never issues or refreshes a credential. +- Defined in `cli/src/services/managed_telemetry.rs` and `cli/src/services/managed_telemetry/trace_export.rs` (plus `log_export.rs`). After startup, `RuntimeTelemetry::resolve` loads the stored telemetry credential from the encrypted auth DB in a spawned task bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s); absent, expired, malformed, unavailable, ownerless (no stored WorkOS session), or timed-out credentials select `NoopTelemetry`. Startup never issues or refreshes a credential. The fallback is not silent: `RuntimeTelemetry::resolve` logs one `sce.telemetry.fallback` record with `reason` = `credential_unavailable`, `credential_expired`, `lookup_timeout`, `database_error`, `path_resolution_error`, or `exporter_init_failed`, at forced `warn` for failures and `debug` otherwise. Database and path errors add the redacted storage error text as `error`; no credential, bearer, key, or payload is logged, and the record never reaches stdout or, with `log_to_file=true`, stderr. - Admission (`managed_span_kind` in `cli/src/app.rs`, decided from argv before credential lookup): Claude-invoked SCE hooks resolve the `sce.hook` span kind, every other hook and the hidden `sce auth telemetry-headers` helper use `NoopTelemetry` without a credential lookup, and all remaining commands use the `sce.command` span kind. `sce hooks claude-mutation-scope` and `sce hooks claude-model-state` are admitted from argv alone. The shared `sce hooks diff-trace`, `sce hooks conversation-trace`, and `sce policy bash` are admitted only when both `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are present in the environment (checked for presence only; values never read); `CLAUDE_PROJECT_DIR` is set for Claude command hooks but not for the Claude Bash tool (evidence: [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md)). Other hooks (commit hooks, Codex, OpenCode, Pi) stay Noop. Admitted hooks use the same `CREDENTIAL_LOOKUP_DEADLINE`, `SHUTDOWN_TIMEOUT`, and runtime grace bounds as commands. Decision record: [2026-10-17-claude-hook-managed-trace-admission.md](../decisions/2026-10-17-claude-hook-managed-trace-admission.md). - Export is OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev/v1/traces`. A pinned HTTP client sets `Authorization: Bearer ` (token only, never the accessor), disables redirects, rebuilds the request to the fixed endpoint, and bounds requests to 3s. Ambient `OTEL_EXPORTER_OTLP_*` values cannot redirect it. - Each command emits one `sce.command` span with `sce.command.name`, `sce.outcome` (`success`/`failure`), and `sce.error.class` (the `SCE-ERR-*` code on failure); resource attributes are `service.name=sce` and `service.version`. Raw arguments, paths, and error chains are not exported. -- An admitted Claude hook process emits exactly one `sce.hook` span instead (no `sce.command` span), with `sce.agent=claude`, `sce.outcome`, `sce.error.class`, and, when present as non-empty strings in the Claude hook stdin payload, `sce.hook.event` (`hook_event_name`), `sce.hook.tool` (`tool_name`), `sce.claude.session_id`, and `sce.claude.tool_use_id`. `trace_export::record_claude_hook_payload` is the only reader: each Claude hook entry point passes its stdin to it once, it returns immediately when the current span is disabled (Noop), and it copies only those four fields. Session and tool-use IDs are opaque correlation identifiers: span attributes only, never metric attributes, never used to derive a trace ID. Tool input and response, prompts, `cwd`, transcript paths, file contents, and the raw payload are never exported. Parent context: for the `sce.hook` kind only, `ManagedTraceTelemetry::new` reads `TRACEPARENT` (and `TRACESTATE` when present) from the process environment, extracts it with `TraceContextPropagator`, and keeps it only when the remote span context is valid; `root_span` then attaches it with `set_parent`, so `sce.hook` shares Claude Code's trace ID, its parent is the upstream span (Claude Code 2.1.301 supplies `claude_code.interaction`, making `sce.hook` a sibling of `claude_code.tool`), and its own span ID is freshly generated. Child spans created inside the instrumented hook future inherit that trace across `.await` points and worker migration. No global propagator is registered and `sce.command` never reads the variables. A missing, malformed, or all-zero traceparent (the normal case when Claude tracing is off) yields an independent SCE root. Claude `SessionStart` and `SessionEnd` hooks run outside any interaction span, so their `sce.hook` is always a root. Live status: Dash0 shows `sce doctor` and Claude session-boundary `sce.hook` spans. The parented tool-hook hierarchy is tested locally but not yet confirmed in Dash0. The evidence and the opt-in smoke procedure are in [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md). Telemetry failures never change hook stdout, permission decisions, persisted state, or exit codes. +- An admitted Claude hook process emits exactly one `sce.hook` span instead (no `sce.command` span), with `sce.agent=claude`, `sce.hook.command`, `sce.outcome`, `sce.error.class`, and, when present as non-empty strings in the Claude hook stdin payload, `sce.hook.event` (`hook_event_name`), `sce.hook.tool` (`tool_name`), `sce.claude.session_id`, and `sce.claude.tool_use_id`. `sce.hook.command` is the stable SCE entry point (`ClaudeHookCommand`: `diff-trace`, `conversation-trace`, `claude-mutation-scope`, `claude-model-state`, `policy-bash`), so the several SCE hooks that one Claude event runs produce distinguishable spans; it is recorded even when the payload is not JSON. `trace_export::record_claude_hook_payload(command, stdin)` is the only reader: each Claude hook entry point passes its command and stdin to it once, it returns immediately when the current span is disabled (Noop), and it copies only those four fields. Session and tool-use IDs are opaque correlation identifiers: span attributes only, never metric attributes, never used to derive a trace ID. Tool input and response, prompts, `cwd`, transcript paths, file contents, and the raw payload are never exported. Parent context: for the `sce.hook` kind only, `ManagedTraceTelemetry::new` reads `TRACEPARENT` (and `TRACESTATE` when present) from the process environment, extracts it with `TraceContextPropagator`, and keeps it only when the remote span context is valid; `root_span` then attaches it with `set_parent`, so `sce.hook` shares Claude Code's trace ID, its parent is the upstream span (Claude Code 2.1.301 supplies `claude_code.interaction`, making `sce.hook` a sibling of `claude_code.tool`), and its own span ID is freshly generated. Child spans created inside the instrumented hook future inherit that trace across `.await` points and worker migration. No global propagator is registered and `sce.command` never reads the variables. A missing, malformed, or all-zero traceparent (the normal case when Claude tracing is off) yields an independent SCE root. Claude `SessionStart` and `SessionEnd` hooks run outside any interaction span, so their `sce.hook` is always a root. Live status: Dash0 (dataset `sce-crocoder`) shows `sce doctor`, Claude session-boundary `sce.hook` spans, and parented tool-hook `sce.hook` spans with operation children under `claude_code.interaction`. Concurrent hook processes previously lost spans because the auth DB credential lookup failed with `LockingError` and fell back to Noop; that is fixed, and the fix is verified by multiprocess regression tests but not yet re-verified live in Dash0. The evidence and the opt-in smoke procedure are in [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md). Telemetry failures never change hook stdout, permission decisions, persisted state, or exit codes. - Operation child spans: `trace_export::in_child_span(ManagedChildSpan, future)` wraps an existing operation in one of three named spans, each carrying only `sce.outcome` (`success`/`failure` from the operation's `Result`): `sce.mutation_scope` around every Claude adapter seam call (one coordinate or abandon boundary), `sce.git_snapshot` around the coordinator's tree capture and pin, and `sce.persistence` around the mutation-store phase of a coordinate boundary (initialize, scope/provenance registration, CAS commit loop) and of an abandon. The hierarchy is `sce.hook → sce.mutation_scope → {sce.git_snapshot, sce.persistence}`. A child span exists only when its operation runs: a read-only tool, an untracked background shell, or a close with no matching attempt exports `sce.hook` alone. Snapshot-failure taint persistence and conversation/diff-trace Agent Trace DB writes have no span. The span filter depends on the span kind: a `sce.command` dispatch exports only `sce.command` (ordinary commands that reach the coordinator, such as ref maintenance, still export one span), and a `sce.hook` dispatch exports only `sce.hook` and the three child names. Without a managed hook dispatch the child spans are disabled, and they never change ordering, locking, or errors. - The exporter uses a bounded batch queue on a dedicated thread. Final stdout/stderr rendering and classified-error logging run inside the command span (`run_command_lifecycle_with_context`), so `sce.outcome` reflects the rendered result (a stdout write failure is `failure`) and the final `sce.error.*` log is exported. Trace and log providers shut down concurrently under one shared `SHUTDOWN_TIMEOUT` (3s) deadline after rendering; failures never change command results or exit codes. - Process lifetime: `app::run_process` owns the Tokio runtime (at least two workers) and ends it with `shutdown_timeout(RUNTIME_SHUTDOWN_GRACE)` (500ms), so a stalled keychain, database, or exporter worker cannot keep the process alive. The async deadlines bound when the command proceeds; the runtime grace is the hard process-level bound. Abandoned workers are not cancelled, only no longer awaited. diff --git a/context/sce/shared-turso-db.md b/context/sce/shared-turso-db.md index 8b5d07143..ca5b6136e 100644 --- a/context/sce/shared-turso-db.md +++ b/context/sce/shared-turso-db.md @@ -12,7 +12,7 @@ - `TursoDb`: generic unencrypted adapter that owns: - no Tokio runtime ownership; all Turso futures are awaited by the application runtime - Turso local database open/connect flow using `turso::Builder::new_local()` with `experimental_multiprocess_wal(true)`, which gives concurrent `sce` processes cross-process correctness and locking on the same local database; it does not make a contended writer wait - - per-`DbSpec` Turso busy timeout, resolved by `resolve_busy_timeout::()` and installed with `Connection::busy_timeout(...)` on the connection returned by `connect()` in `open_without_migrations_at` (the open path behind `new`, `new_at`, and `open_without_migrations`). It is Turso's wait policy for `Busy`: Turso sleeps in short phases until the lock clears or the timeout elapses, and only then returns `Busy`. It is connection-wide, so it also covers `BEGIN IMMEDIATE` issued by `Transaction::new_unchecked(.., Immediate)` on the same connection. The Agent Trace DB (`db_config_key() == "agent_trace_db"`) uses `policies.database_retry.agent_trace_db.busy_timeout_ms` when configured (`0` leaves the handler unset), falling back to `AGENT_TRACE_DB_BUSY_TIMEOUT_MS = 1_000`; every other database resolves to zero, which leaves the busy handler unset, so `local_db` behavior is unchanged. `EncryptedTursoDb` (auth DB) never installs a busy handler. A single attempt may therefore spend up to about 1000 ms waiting inside Turso before returning `Busy`; the Agent Trace write-contention retry below decides whether SCE tries once more + - per-`DbSpec` Turso busy timeout, resolved by `resolve_busy_timeout::()` and installed with `Connection::busy_timeout(...)` on the connection returned by `connect()` in `open_without_migrations_at` (the open path behind `new`, `new_at`, and `open_without_migrations`). It is Turso's wait policy for `Busy`: Turso sleeps in short phases until the lock clears or the timeout elapses, and only then returns `Busy`. It is connection-wide, so it also covers `BEGIN IMMEDIATE` issued by `Transaction::new_unchecked(.., Immediate)` on the same connection. The Agent Trace DB (`db_config_key() == "agent_trace_db"`) uses `policies.database_retry.agent_trace_db.busy_timeout_ms` when configured (`0` leaves the handler unset), falling back to `AGENT_TRACE_DB_BUSY_TIMEOUT_MS = 1_000`; the auth DB (`DatabaseKind::Auth`) uses the fixed, non-configurable `AUTH_DB_BUSY_TIMEOUT_MS = 1_000`, installed by `EncryptedTursoDb::new()`; `local_db` resolves to zero, which leaves the busy handler unset, so its behavior is unchanged. A single attempt may therefore spend up to about 1000 ms waiting inside Turso before returning `Busy`; the Agent Trace write-contention retry below decides whether SCE tries once more - Agent Trace write-contention deadline, resolved by `resolve_contention_deadline::()` from `policies.database_retry.agent_trace_db.contention_deadline_ms` with fallback to `AGENT_TRACE_DB_CONTENTION_DEADLINE_MS = 2_250` (zero for every other database). It is the cutoff for starting another outer write-contention retry, not a hard operation timeout - Agent Trace write-contention retry (`write_contention_policy::()`, `run_with_write_contention_retry`), returned only for `agent_trace_db` and applied only to write units whose whole replay is safe: the two transactional primitives below (each attempt is a complete `BEGIN IMMEDIATE` → `COMMIT` unit) and `execute_idempotent_write(sql, params)`, the opt-in single-statement entrypoint used only by the repository-metadata `INSERT … ON CONFLICT DO NOTHING`, the guarded source-instance claim `UPDATE … WHERE source_instance_id = ''`, and the mutation-trace worktree/scope/scope-provenance `…_IF_ABSENT` inserts. It retries only typed `Busy`/`BusySnapshot` (classified before conversion to `anyhow`), allows at most `2` attempts, sleeps a full-jitter backoff drawn from `0..=100ms`, and admits the second attempt at two points: before the backoff sleep it requires `remaining contention deadline >= backoff + busy_timeout`, and after the sleep actually returns it re-checks `remaining >= busy_timeout`, so scheduler oversleep can never start an attempt outside the admission contract (time is measured on a monotonic clock from the operation start, so Turso's busy wait counts; nothing starts after the deadline expires). Deterministic failures return after one attempt without sleeping. On every other database these entrypoints keep the generic query retry, and `execute_idempotent_write` is plain `execute()` - config-driven connection-open retry around only the `build().await.connect()` block using async `run_with_retry` (resolved from `policies.database_retry..connection_open` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults `3` attempts, `1s` timeout, `25ms..200ms` backoff) @@ -29,8 +29,8 @@ - `count_write_contention(body) -> (T, WriteContentionCounts)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports the write-contention `attempts`, `outer_retries`, and `exhaustions` it caused on the current thread. Test instrumentation only; not production telemetry and not shared across threads or processes. - `record_write_contention_timeline(body) -> (T, Vec<(Instant, WriteContentionTimelineEvent)>)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and returns the `Instant`-stamped attempt start/end, backoff requested and backoff slept events of the write-contention retry on the current thread. Test-only measurement instrumentation used by the lock-contention suite; never logged in production. - `count_read_statements(body) -> (T, usize)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports how many `TursoDb` read statements (`query_map`) it issued on the current thread, each counted once in its synchronous prelude before the retry wrapper. A deterministic seam for tests that must prove an operation reads from a single DB snapshot — one statement — rather than several independent `SELECT`s a concurrent commit could tear across (first used by `MutationTraceStore::load_tree_roots` / `load_all_tree_roots`). Not compiled into production builds. -- `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same static shape (connection and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`, and runs embedded migrations after connect. -- `EncryptedTursoDb` exposes async `execute()`, `query_map()`, and `run_migrations()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. +- `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same static shape (connection and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, opens with `experimental_multiprocess_wal(true)` like `TursoDb` (without it Turso takes an exclusive, non-blocking lock on the whole file for the process lifetime, so a concurrent `sce` process fails with `LockingError`), installs the auth DB busy timeout, and wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`. After connect it migrates only when needed: a read-only readiness check (`__sce_migrations` exists and records every embedded ID) lets a current schema open with no write at all; otherwise all pending migrations run inside one `BEGIN IMMEDIATE` transaction, which serializes concurrent initializers (a waiter re-reads the metadata under the lock and applies nothing) and rolls back the whole attempt on any migration failure. Only `Busy`/`BusySnapshot` on begin or commit is retried, under `MIGRATION_RETRY_POLICY` (10 attempts, 25..200ms backoff, each attempt bounded by the 1000 ms busy timeout); a deterministic migration failure (for example a column added by a partially applied `004` whose ID was never recorded) fails after one attempt with the migration ID in the error chain. After migrating, the readiness check runs again and a schema that is still behind fails the open. +- `EncryptedTursoDb` exposes async `execute()` and `query_map()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. - Both public adapters keep concrete Turso connections and marker types; there is no shared runtime-owning core or synchronous executor bridge. - Shared lifecycle helpers: - `collect_db_path_health()` emits common parent/path health problems for DB-backed services. @@ -140,7 +140,7 @@ Retry-policy budget versus wall-clock time: the generic query retry policy keeps | Layer | Role | | --- | --- | | multiprocess WAL | cross-process correctness and locking | -| `busy_timeout_ms` | Turso-level `Busy` waiting inside one operation attempt (Agent Trace DB only) | +| `busy_timeout_ms` | Turso-level `Busy` waiting inside one operation attempt (Agent Trace DB, configurable; auth DB fixed at 1000 ms) | | outer write-contention retry | SCE retry after Turso's wait is exhausted; Agent Trace safe write units only, at most `2` attempts, full-jitter `0..=100ms` backoff | | `contention_deadline_ms` | cutoff for starting another outer write-contention retry, not a hard operation timeout | | generic query `RetryPolicy` (`query.timeout_ms`) | outer retry for every other operation, including Agent Trace reads; `timeout_ms` labels slow attempts and never interrupts them | diff --git a/context/sce/telemetry-credential.md b/context/sce/telemetry-credential.md index 4638ac6fe..c60c8cbc6 100644 --- a/context/sce/telemetry-credential.md +++ b/context/sce/telemetry-credential.md @@ -29,7 +29,7 @@ flowchart LR ## Runtime use -- `managed_telemetry::RuntimeTelemetry::resolve` reads the stored credential at CLI startup (never issuing one); a valid unexpired `token` is used only as the `Authorization: Bearer` credential for managed OTLP trace and log export; the `accessor` is never transmitted. Absent, expired, malformed, or unavailable rows select `NoopTelemetry`. Claude-invoked SCE hooks are admitted to this lookup and export one `sce.hook` span. Every other hook and this hidden header helper skip the lookup and stay Noop. See [cli-observability-contract.md](cli-observability-contract.md). +- `managed_telemetry::RuntimeTelemetry::resolve` reads the stored credential at CLI startup (never issuing one); a valid unexpired `token` is used only as the `Authorization: Bearer` credential for managed OTLP trace and log export; the `accessor` is never transmitted. Absent, expired, malformed, or unavailable rows select `NoopTelemetry`. Every Noop fallback after a lookup is classified (`credential_unavailable`, `credential_expired`, `lookup_timeout`, `database_error`, `path_resolution_error`, `exporter_init_failed`) and logged locally as `sce.telemetry.fallback`; failures log at `warn` (forced, so they reach the log file at the default `error` level, never stderr while `log_to_file=true`), with the redacted `TokenStorageError` text for database and path errors, and missing or expired credentials log at `debug`. Concurrent lookups from many hook processes are supported by the auth DB's multiprocess WAL and migration-free current-schema open ([auth-db.md](auth-db.md)). Claude-invoked SCE hooks are admitted to this lookup and export one `sce.hook` span. Every other hook and this hidden header helper skip the lookup and stay Noop. See [cli-observability-contract.md](cli-observability-contract.md). ## Hidden header helper From 23e9806123db363af6e068bddc6ca5abd5e3355e Mon Sep 17 00:00:00 2001 From: David Abram Date: Mon, 19 Oct 2026 13:53:40 +0200 Subject: [PATCH 09/17] runtime: Harden auth DB compatibility and Claude hook telemetry Prevent mixed-version auth DB writers from overlapping by holding a shared compatibility lock, and make telemetry credential lookups open existing databases without creating files or encryption keys. Record each Claude hook command before reading stdin so malformed or unreadable payloads still identify the hook span. Update regression coverage and the related auth DB and observability documentation. Co-authored-by: SCE --- cli/Cargo.lock | 1 + cli/Cargo.toml | 3 + .../services/auth_db/multiprocess_tests.rs | 633 +++++++++++++++++- cli/src/services/bash_policy.rs | 6 +- cli/src/services/db/mod.rs | 44 +- cli/src/services/db/open_lock.rs | 76 +++ cli/src/services/hooks/claude_model_state.rs | 8 +- .../hooks/claude_mutation_scope/lifecycle.rs | 14 +- cli/src/services/hooks/conversation_trace.rs | 8 +- cli/src/services/hooks/diff_trace.rs | 8 +- .../managed_telemetry/trace_export.rs | 10 +- .../services/telemetry_credential_storage.rs | 25 +- cli/src/services/token_storage.rs | 16 + .../plans/claude-hook-trace-propagation.md | 37 +- context/sce/auth-db.md | 36 +- context/sce/claude-hook-trace-propagation.md | 2 +- context/sce/cli-observability-contract.md | 2 +- context/sce/shared-turso-db.md | 2 +- context/sce/telemetry-credential.md | 4 +- 19 files changed, 868 insertions(+), 67 deletions(-) create mode 100644 cli/src/services/db/open_lock.rs diff --git a/cli/Cargo.lock b/cli/Cargo.lock index c364cdce0..2179dc7f2 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -3823,6 +3823,7 @@ dependencies = [ "inquire", "jsonschema", "keyring-core", + "libc", "murmur3", "opentelemetry", "opentelemetry-http", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index a842db904..2cd7f2d45 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -58,6 +58,9 @@ opentelemetry = { version = "=0.33.1", default-features = false, features = ["tr async-trait = "0.1" opentelemetry_sdk = { version = "=0.33.1", default-features = false, features = ["trace", "logs"] } +[target.'cfg(unix)'.dependencies] +libc = "0.2.186" + [target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies] zbus-secret-service-keyring-store = { version = "1.0.0", features = ["rt-tokio-crypto-rust"] } diff --git a/cli/src/services/auth_db/multiprocess_tests.rs b/cli/src/services/auth_db/multiprocess_tests.rs index 179b4c4e1..81f72d03a 100644 --- a/cli/src/services/auth_db/multiprocess_tests.rs +++ b/cli/src/services/auth_db/multiprocess_tests.rs @@ -1,6 +1,6 @@ use std::collections::BTreeSet; use std::io::{BufRead, BufReader, Read, Write}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; use std::sync::{Mutex, MutexGuard}; use std::time::{Duration, Instant}; @@ -9,22 +9,32 @@ use anyhow::Result; use super::{AuthDb, AuthDbSpec}; use crate::generated_migrations; -use crate::services::db::encryption_key::AUTH_DB_ENCRYPTION_KEY_ENV; +use crate::services::bash_policy::{ + run_bash_policy_request, BashPolicyRequest, PolicyInputMode, PolicyOutputMode, +}; +use crate::services::db::encryption_key::{ + get_or_create_encryption_key, AUTH_DB_ENCRYPTION_KEY_ENV, +}; use crate::services::db::{DatabaseKind, DbSpec, EncryptedTursoDb}; use crate::services::default_paths::auth_db_path; +use crate::services::error::CliError; +use crate::services::hooks::{run_hooks_subcommand, HookSubcommand}; +use crate::services::managed_telemetry::claude_helper::repository_id_for; +use crate::services::managed_telemetry::claude_helper::tests::git_repository_with_remote; use crate::services::managed_telemetry::trace_export::tests::{ capture_requests_until, loopback_hook_telemetry, trace_spans, }; use crate::services::managed_telemetry::trace_export::{ - record_claude_hook_payload, ClaudeHookCommand, ManagedSpanKind, + record_claude_hook_command, record_claude_hook_payload, ClaudeHookCommand, ManagedSpanKind, }; use crate::services::managed_telemetry::{ load_credential_within_deadline, select_mode, ManagedTelemetryMode, RuntimeTelemetry, TelemetryFallbackReason, }; -use crate::services::observability::traits::Telemetry; +use crate::services::observability::traits::{NoopLogger, Telemetry}; use crate::services::telemetry_credential_storage::{ - load_telemetry_credential, save_telemetry_credential, test_access_token, TelemetryCredential, + bind_repository_telemetry, load_repository_telemetry_credential, load_telemetry_credential, + save_telemetry_credential, test_access_token, RepositoryBindingMode, TelemetryCredential, TelemetryIdentity, }; @@ -54,6 +64,30 @@ const HOOK_COMMANDS: [ClaudeHookCommand; 5] = [ ]; const PRE_BINDING_MIGRATIONS: usize = 3; const BINDING_MIGRATION_ID: &str = "004_bind_telemetry_credentials"; +const WRITER_ENV: &str = "SCE_AUTH_DB_TEST_WRITER"; +const COMMITTED_PREFIX: &str = "SCE_AUTH_DB_CHILD_COMMITTED="; +const ROW_PREFIX: &str = "SCE_AUTH_DB_CHILD_ROW="; +const REJECTED: &str = "SCE_AUTH_DB_CHILD_REJECTED"; +const WRITER_ROWS: usize = 8; +const WRITER_PAUSE: Duration = Duration::from_millis(20); +const MIXED_WRITER_ROUNDS: usize = 3; +const NEW_WRITERS: usize = 4; +const SHARED_ACCESS_REJECTION: &str = "does not support shared access"; +const LEGACY_LOCK_REJECTION: &str = "File is locked by another process"; +const LEGACY_CIPHER: &str = "aegis256"; +const INSERT_BINDING_SQL: &str = "INSERT INTO telemetry_repository_bindings \ + (repository_id, user_id, workspace_id, bound_at_unix_seconds) VALUES (?1, ?2, ?3, 1)"; +const AUTH_DB_FILES: [&str; 3] = ["auth.db", "auth.db-wal", "auth.db-tshm"]; +const HEADERS_REPOSITORY: &str = "bound-repo"; +const UNBOUND_REPOSITORY: &str = "unbound-repo"; +const HEADERS_AUTHORIZATION: &str = r#"{"Authorization":"Bearer tok-multiprocess"}"#; +const HEADERS_UNAVAILABLE: &str = "Telemetry credential is unavailable."; +const STDIN_SECRET: &str = "SECRET_STDIN_VALUE"; +const FAIL_OPEN_HOOK_COMMANDS: [ClaudeHookCommand; 3] = [ + ClaudeHookCommand::DiffTrace, + ClaudeHookCommand::ConversationTrace, + ClaudeHookCommand::ClaudeModelState, +]; struct PreBindingAuthDbSpec; @@ -127,9 +161,34 @@ impl Sandbox { .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("OTEL_") { + command.env_remove(key); + } + } command } + fn writer(&self, role: &str, writer: &str) -> Child { + self.command(role) + .env(WRITER_ENV, writer) + .spawn() + .expect("spawn writer") + } + + fn rows(&self) -> BTreeSet { + prefixed_lines(&self.run("verify-rows"), ROW_PREFIX) + } + + fn auth_db_files(&self) -> Vec { + let state = self.dir.path().join("state").join("sce"); + AUTH_DB_FILES + .iter() + .filter(|name| state.join(name).exists()) + .map(ToString::to_string) + .collect() + } + fn run(&self, role: &str) -> String { wait_success(role, self.command(role).spawn().expect("spawn child")) } @@ -215,6 +274,24 @@ fn wait_success(role: &str, mut child: Child) -> String { stdout } +fn prefixed_lines(stdout: &str, prefix: &str) -> BTreeSet { + stdout + .lines() + .filter_map(|line| line.split_once(prefix).map(|(_, value)| value.to_string())) + .collect() +} + +fn committed_rows(children: Vec<(String, Child)>) -> (BTreeSet, usize) { + let mut committed = BTreeSet::new(); + let mut rejected = 0; + for (role, child) in children { + let stdout = wait_success(&role, child); + committed.extend(prefixed_lines(&stdout, COMMITTED_PREFIX)); + rejected += usize::from(stdout.contains(REJECTED)); + } + (committed, rejected) +} + fn read_token(stdout: &str) -> Option<&str> { stdout .lines() @@ -318,6 +395,319 @@ fn concurrent_hook_processes_export_correlated_hook_spans() { .block_on(export_concurrent_hook_spans(&sandbox)); } +#[test] +fn legacy_holder_rejects_new_readers_and_writers_explicitly() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let legacy = sandbox.hold("legacy-hold"); + sandbox.run("expect-rejected-read"); + sandbox.run("expect-rejected-write"); + legacy.release(); + + assert_eq!(sandbox.rows(), BTreeSet::from(["legacy-held".to_string()])); + sandbox.run("legacy-roundtrip"); +} + +#[test] +fn new_holder_rejects_legacy_writer_explicitly() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let holder = sandbox.hold("hold-open"); + sandbox.run("legacy-expect-rejected"); + holder.release(); + + sandbox.run("legacy-roundtrip"); + assert_eq!( + sandbox.rows(), + BTreeSet::from(["legacy-roundtrip".to_string()]) + ); +} + +#[test] +fn concurrent_legacy_and_new_writers_never_lose_committed_rows() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let mut committed = BTreeSet::new(); + let mut rejected = 0; + for round in 0..MIXED_WRITER_ROUNDS { + let writers = vec![ + ( + "legacy-writer".to_string(), + sandbox.writer("legacy-writer", &format!("legacy-{round}")), + ), + ( + "new-writer".to_string(), + sandbox.writer("new-writer", &format!("new-{round}-a")), + ), + ( + "new-writer".to_string(), + sandbox.writer("new-writer", &format!("new-{round}-b")), + ), + ]; + let (round_committed, round_rejected) = committed_rows(writers); + committed.extend(round_committed); + rejected += round_rejected; + } + println!( + "mixed writers: {} committed rows, {rejected} rejected opens", + committed.len() + ); + + let persisted = sandbox.rows(); + let lost = committed.difference(&persisted).collect::>(); + assert!(lost.is_empty(), "committed rows lost: {lost:?}"); + sandbox.run("legacy-roundtrip"); +} + +#[test] +fn concurrent_new_writers_all_commit_and_legacy_reopens_afterwards() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let writers = (0..NEW_WRITERS) + .map(|index| { + ( + "new-writer".to_string(), + sandbox.writer("new-writer", &format!("new-{index}")), + ) + }) + .collect(); + let (committed, rejected) = committed_rows(writers); + assert_eq!(rejected, 0); + assert_eq!(committed.len(), NEW_WRITERS * WRITER_ROWS); + assert_eq!(sandbox.rows(), committed); + + sandbox.run("legacy-roundtrip"); + let mut expected = committed; + expected.insert("legacy-roundtrip".to_string()); + assert_eq!(sandbox.rows(), expected); +} + +#[test] +fn repository_telemetry_lookup_never_creates_a_missing_auth_db() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + let mut lookup = sandbox.command("repository-lookup-missing"); + lookup.env(AUTH_DB_ENCRYPTION_KEY_ENV, ""); + wait_success( + "repository-lookup-missing", + lookup.spawn().expect("spawn lookup"), + ); + assert!(sandbox.auth_db_files().is_empty()); +} + +#[test] +fn concurrent_telemetry_headers_on_missing_auth_db_fail_closed_without_creating_it() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + let repository = git_repository_with_remote( + sandbox.dir.path(), + HEADERS_REPOSITORY, + "https://github.com/acme/bound-repo", + ); + + let helpers = (0..CONCURRENT_PROCESSES) + .map(|_| spawn_headers(&sandbox, &repository)) + .collect::>(); + for helper in helpers { + assert_headers_unavailable(helper); + } + assert!(sandbox.auth_db_files().is_empty()); +} + +#[test] +fn telemetry_headers_serve_only_the_bound_repository() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + let bound = git_repository_with_remote( + sandbox.dir.path(), + HEADERS_REPOSITORY, + "https://github.com/acme/bound-repo", + ); + let unbound = git_repository_with_remote( + sandbox.dir.path(), + UNBOUND_REPOSITORY, + "https://github.com/acme/unbound-repo", + ); + sandbox.seed(); + let mut bind = sandbox.command("bind"); + bind.current_dir(&bound); + wait_success("bind", bind.spawn().expect("spawn bind")); + + let output = spawn_headers(&sandbox, &bound) + .wait_with_output() + .expect("bound headers output"); + assert!(output.status.success(), "bound headers failed"); + assert!(String::from_utf8_lossy(&output.stdout).contains(HEADERS_AUTHORIZATION)); + assert_headers_unavailable(spawn_headers(&sandbox, &unbound)); +} + +fn spawn_headers(sandbox: &Sandbox, repository: &Path) -> Child { + let mut helper = sandbox.command("headers"); + helper.current_dir(repository); + helper.spawn().expect("spawn headers") +} + +fn assert_headers_unavailable(helper: Child) { + let output = helper.wait_with_output().expect("headers output"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "headers child failed: {stderr}"); + assert!(stdout.contains("1 passed"), "{stdout}"); + assert!(!stdout.contains("Authorization"), "{stdout}"); + assert!(stderr.contains(HEADERS_UNAVAILABLE), "{stderr}"); +} + +#[test] +fn every_claude_hook_entry_point_records_its_command_identity() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + std::fs::create_dir_all(sandbox.dir.path().join("work")).expect("work dir"); + tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .expect("test runtime") + .block_on(export_hook_entry_spans(&sandbox)); +} + +fn hook_stdin_cases(command: ClaudeHookCommand) -> [(Vec, String); 4] { + let tool_use_id = format!("toolu_entry_{}", command.as_str()); + let valid = serde_json::json!({ + "hook_event_name": "PreToolUse", + "tool_name": "Bash", + "session_id": "session-entry", + "tool_use_id": tool_use_id, + "tool_input": { "command": "true" }, + }) + .to_string(); + [ + (valid.into_bytes(), tool_use_id), + (Vec::new(), String::new()), + ( + format!("{{\"tool_use_id\":\"{STDIN_SECRET}").into_bytes(), + String::new(), + ), + ( + [b"\xff\xfe".as_slice(), STDIN_SECRET.as_bytes()].concat(), + String::new(), + ), + ] +} + +async fn export_hook_entry_spans(sandbox: &Sandbox) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let address = listener.local_addr().expect("addr"); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let receiver = tokio::spawn(capture_requests_until(listener, stopped)); + + let mut expected = Vec::new(); + let mut hooks = Vec::new(); + for (index, command) in HOOK_COMMANDS.iter().enumerate() { + for (stdin, tool_use_id) in hook_stdin_cases(*command) { + let mut child = sandbox + .command("hook-entry") + .current_dir(sandbox.dir.path().join("work")) + .env(COLLECTOR_ENV, address.to_string()) + .env(HOOK_INDEX_ENV, index.to_string()) + .stdin(Stdio::piped()) + .spawn() + .expect("spawn hook entry"); + let mut child_stdin = child.stdin.take().expect("hook entry stdin"); + child_stdin.write_all(&stdin).expect("write hook stdin"); + drop(child_stdin); + hooks.push(child); + expected.push((command.as_str().to_string(), tool_use_id)); + } + } + tokio::task::spawn_blocking(move || { + for child in hooks { + wait_success("hook-entry", child); + } + }) + .await + .expect("hook entry children join"); + stop.send(()).expect("stop collector"); + + let requests = receiver.await.expect("collector join"); + for request in &requests { + let body = String::from_utf8_lossy(&request.body); + assert!(!body.contains(STDIN_SECRET), "hook payload leaked"); + assert!(!body.contains("bao-token-value"), "token leaked"); + } + let mut observed = Vec::new(); + for span in trace_spans(&requests) + .iter() + .filter(|span| span.name == "sce.hook") + { + let attribute = |key: &str| { + span.attributes + .iter() + .find(|(name, _)| name == key) + .map(|(_, value)| value.clone()) + .unwrap_or_default() + }; + let command = attribute("sce.hook.command"); + let tool_use_id = attribute("sce.claude.tool_use_id"); + let outcome = attribute("sce.outcome"); + if FAIL_OPEN_HOOK_COMMANDS + .iter() + .any(|hook| hook.as_str() == command) + { + assert_eq!(outcome, "success", "{command} must fail open"); + } + if command == ClaudeHookCommand::PolicyBash.as_str() && tool_use_id.is_empty() { + assert_eq!(outcome, "failure", "malformed policy-bash input must fail"); + } + observed.push((command, tool_use_id)); + } + observed.sort(); + expected.sort(); + assert_eq!(observed, expected); +} + +async fn run_hook_entry_child() { + let index = std::env::var(HOOK_INDEX_ENV) + .expect("hook index") + .parse::() + .expect("numeric hook index"); + let address = std::env::var(COLLECTOR_ENV) + .expect("collector address") + .parse() + .expect("socket address"); + let telemetry = loopback_hook_telemetry(address, "bao-token-value"); + let command = HOOK_COMMANDS[index]; + let _ = telemetry + .with_default_subscriber(&mut || async move { + let subcommand = match command { + ClaudeHookCommand::DiffTrace => HookSubcommand::DiffTrace, + ClaudeHookCommand::ConversationTrace => HookSubcommand::ConversationTrace, + ClaudeHookCommand::ClaudeMutationScope => HookSubcommand::ClaudeMutationScope, + ClaudeHookCommand::ClaudeModelState => HookSubcommand::ClaudeModelState, + ClaudeHookCommand::PolicyBash => { + return run_bash_policy_request(&BashPolicyRequest { + input: PolicyInputMode::ClaudePreToolUse, + output: PolicyOutputMode::ClaudeHook, + }); + } + }; + run_hooks_subcommand(&subcommand, None::<&NoopLogger>) + .await + .map_err(CliError::runtime) + }) + .await; + telemetry.shutdown().await; +} + async fn export_concurrent_hook_spans(sandbox: &Sandbox) { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await @@ -445,6 +835,131 @@ async fn auth_db_child_process() { "upgrade" => upgrade_outdated_schema().await, "partial" => reject_partial_schema().await, "hook" => run_hook_child().await, + "hook-entry" => run_hook_entry_child().await, + other => run_compatibility_child(other).await, + } +} + +async fn run_compatibility_child(role: &str) { + match role { + "legacy-hold" => { + let conn = open_legacy_auth_db().await.expect("legacy holder opens"); + wait_for_release(); + insert_binding(&conn, "legacy-held") + .await + .expect("legacy holder writes after release"); + } + "legacy-expect-rejected" => { + let Err(error) = open_legacy_auth_db().await else { + panic!("legacy open must be rejected while a new process holds the DB"); + }; + assert!( + format!("{error:#}").contains(LEGACY_LOCK_REJECTION), + "{error:#}" + ); + } + "legacy-roundtrip" => legacy_roundtrip().await, + "legacy-writer" => match open_legacy_auth_db().await { + Ok(conn) => { + for row in writer_rows() { + if insert_binding(&conn, &row).await.is_err() { + break; + } + println!("{COMMITTED_PREFIX}{row}"); + tokio::time::sleep(WRITER_PAUSE).await; + } + } + Err(error) => { + assert!( + format!("{error:#}").contains(LEGACY_LOCK_REJECTION), + "{error:#}" + ); + println!("{REJECTED}"); + } + }, + other => run_new_version_child(other).await, + } +} + +async fn run_new_version_child(role: &str) { + match role { + "new-writer" => match AuthDb::new().await { + Ok(db) => { + for row in writer_rows() { + if db + .execute(INSERT_BINDING_SQL, (row.as_str(), USER_ID, WORKSPACE_ID)) + .await + .is_err() + { + break; + } + println!("{COMMITTED_PREFIX}{row}"); + tokio::time::sleep(WRITER_PAUSE).await; + } + } + Err(error) => { + assert!( + format!("{error:#}").contains(SHARED_ACCESS_REJECTION), + "{error:#}" + ); + println!("{REJECTED}"); + } + }, + "expect-rejected-read" => { + let error = load_telemetry_credential() + .await + .expect_err("lookup must fail while a legacy process holds the DB"); + assert!( + error.to_string().contains(SHARED_ACCESS_REJECTION), + "{error}" + ); + } + "expect-rejected-write" => { + let error = AuthDb::new() + .await + .err() + .expect("writer open must fail while a legacy process holds the DB"); + assert!( + format!("{error:#}").contains(SHARED_ACCESS_REJECTION), + "{error:#}" + ); + } + "verify-rows" => print_verified_rows().await, + "repository-lookup-missing" => { + assert_eq!( + load_repository_telemetry_credential("repo-missing".to_string()) + .await + .expect("missing auth DB is not an error"), + None + ); + assert_eq!( + load_telemetry_credential() + .await + .expect("missing auth DB is not an error"), + None + ); + } + "bind" => { + let repository_id = repository_id_for(&std::env::current_dir().expect("cwd")) + .expect("repository identity"); + assert!( + bind_repository_telemetry(repository_id, RepositoryBindingMode::Enroll) + .await + .expect("binding succeeds") + ); + } + "headers" => { + let code = Box::pin(crate::app::run( + ["sce", "auth", "telemetry-headers"] + .into_iter() + .map(String::from), + )) + .await; + println!(); + if code != std::process::ExitCode::SUCCESS { + assert_eq!(code, std::process::ExitCode::from(4)); + } + } "hook-credential-failure" => { let (telemetry, fallback) = RuntimeTelemetry::resolve_with( load_telemetry_credential(), @@ -512,7 +1027,8 @@ async fn run_hook_child() { .to_string(); let output = telemetry .with_default_subscriber(&mut || async { - record_claude_hook_payload(hook_command(index), &payload); + record_claude_hook_command(hook_command(index)); + record_claude_hook_payload(&payload); Ok(String::new()) }) .await; @@ -520,6 +1036,111 @@ async fn run_hook_child() { telemetry.shutdown().await; } +fn writer_rows() -> Vec { + let writer = std::env::var(WRITER_ENV).expect("writer id"); + (0..WRITER_ROWS) + .map(|row| format!("{writer}-{row:02}")) + .collect() +} + +async fn open_legacy_auth_db() -> Result { + let path = auth_db_path()?; + let hexkey = get_or_create_encryption_key(&path, AuthDbSpec::db_name())?; + let db = turso::Builder::new_local(path.to_str().expect("UTF-8 auth DB path")) + .experimental_encryption(true) + .with_encryption(turso::EncryptionOpts { + hexkey, + cipher: LEGACY_CIPHER.to_string(), + }) + .build() + .await?; + Ok(db.connect()?) +} + +async fn insert_binding(conn: &turso::Connection, row: &str) -> Result<()> { + conn.execute(INSERT_BINDING_SQL, (row, USER_ID, WORKSPACE_ID)) + .await?; + Ok(()) +} + +async fn legacy_roundtrip() { + let conn = open_legacy_auth_db() + .await + .expect("legacy binary reopens the auth DB"); + let mut rows = conn + .query("SELECT token FROM telemetry_credentials WHERE id = 1", ()) + .await + .expect("legacy reads telemetry credential"); + let row = rows + .next() + .await + .expect("legacy credential row fetch") + .expect("legacy sees credential row"); + assert_eq!(row.get::(0).expect("token"), TOKEN); + drop(rows); + insert_binding(&conn, "legacy-roundtrip") + .await + .expect("legacy writes after new processes exit"); +} + +async fn print_verified_rows() { + let db = AuthDb::new().await.expect("verifier opens auth DB"); + assert_credential_rows_intact(&db).await; + for row in db + .query_map( + "SELECT repository_id FROM telemetry_repository_bindings ORDER BY repository_id", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("binding rows") + { + println!("{ROW_PREFIX}{row}"); + } +} + +async fn assert_credential_rows_intact(db: &AuthDb) { + let tokens = db + .query_map( + "SELECT token, user_id, workspace_id FROM telemetry_credentials", + (), + |row| { + Ok(( + row.get::(0)?, + row.get::(1)?, + row.get::(2)?, + )) + }, + ) + .await + .expect("telemetry credential rows"); + assert_eq!( + tokens, + vec![( + TOKEN.to_string(), + USER_ID.to_string(), + WORKSPACE_ID.to_string() + )] + ); + let sessions = db + .query_map("SELECT access_token FROM auth_credentials", (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("session rows"); + assert_eq!( + sessions, + vec![test_access_token(USER_ID, Some(WORKSPACE_ID))] + ); + let check = db + .query_map("PRAGMA integrity_check", (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("integrity check"); + assert_eq!(check, vec!["ok".to_string()]); +} + fn wait_for_release() { println!("{READY}"); let mut line = String::new(); diff --git a/cli/src/services/bash_policy.rs b/cli/src/services/bash_policy.rs index a1edff3b9..c4a53b88d 100644 --- a/cli/src/services/bash_policy.rs +++ b/cli/src/services/bash_policy.rs @@ -456,11 +456,11 @@ struct JsonPolicyResult<'a> { } pub fn run_bash_policy_request(request: &BashPolicyRequest) -> Result { - let stdin_payload = read_stdin_payload()?; - crate::services::managed_telemetry::trace_export::record_claude_hook_payload( + crate::services::managed_telemetry::trace_export::record_claude_hook_command( crate::services::managed_telemetry::trace_export::ClaudeHookCommand::PolicyBash, - &stdin_payload, ); + let stdin_payload = read_stdin_payload()?; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); run_bash_policy_request_from_payload(request, &stdin_payload) } diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index 517253fec..9f48718f4 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -41,6 +41,7 @@ const AGENT_TRACE_DB_WRITE_CONTENTION_MAX_ATTEMPTS: u32 = 2; const AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS: u64 = 100; pub mod encryption_key; +mod open_lock; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum DatabaseKind { @@ -875,12 +876,23 @@ fn apply_busy_timeout( .map_err(|e| anyhow::anyhow!("failed to set {db_name} database busy timeout: {e}")) } +async fn resolve_encryption_key(db_path: &Path, db_name: &str) -> Result { + let key_path = db_path.to_path_buf(); + let key_name = db_name.to_owned(); + tokio::task::spawn_blocking(move || { + encryption_key::get_or_create_encryption_key(&key_path, &key_name) + }) + .await + .context("encryption-key resolution worker failed")? +} + pub struct TursoDb { core: TursoConnectionCore, } pub struct EncryptedTursoDb { core: TursoConnectionCore, + _open_lock: fs::File, } impl TursoDb { @@ -1352,16 +1364,30 @@ impl EncryptedTursoDb { pub async fn new() -> Result { let db_name = M::db_name(); let db_path = M::db_path().with_context(|| format!("failed to resolve {db_name} path"))?; - let key_path = db_path.clone(); - let key_name = db_name.to_owned(); - let encryption_key = tokio::task::spawn_blocking(move || { - encryption_key::get_or_create_encryption_key(&key_path, &key_name) - }) - .await - .context("encryption-key resolution worker failed")??; - + let encryption_key = resolve_encryption_key(&db_path, db_name).await?; ensure_db_parent_dir(db_name, &db_path)?; + let open_lock = open_lock::open_lock_file(&db_path, true)? + .ok_or_else(|| anyhow::anyhow!("failed to create {db_name} database file"))?; + Box::pin(Self::open_locked(db_path, encryption_key, open_lock)).await + } + + pub async fn open_existing_at(db_path: impl AsRef) -> Result> { + let db_path = db_path.as_ref().to_path_buf(); + let Some(open_lock) = open_lock::open_lock_file(&db_path, false)? else { + return Ok(None); + }; + let encryption_key = resolve_encryption_key(&db_path, M::db_name()).await?; + Box::pin(Self::open_locked(db_path, encryption_key, open_lock)) + .await + .map(Some) + } + async fn open_locked( + db_path: PathBuf, + encryption_key: String, + open_lock: fs::File, + ) -> Result { + let db_name = M::db_name(); let retry_policy = resolve_connection_open_retry_policy::(); let busy_timeout = resolve_busy_timeout::(); let operation_name = format!("open encrypted {db_name} database connection"); @@ -1371,6 +1397,7 @@ impl EncryptedTursoDb { &operation_name, CONNECTION_OPEN_RETRY_HINT, |_| async { + open_lock::lock_shared(&open_lock, db_name, &db_path)?; let path_str = db_path.to_str().ok_or_else(|| { anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) })?; @@ -1403,6 +1430,7 @@ impl EncryptedTursoDb { let mut db = Self { core: TursoConnectionCore::new(conn), + _open_lock: open_lock, }; db.core diff --git a/cli/src/services/db/open_lock.rs b/cli/src/services/db/open_lock.rs new file mode 100644 index 000000000..0b8ae2df3 --- /dev/null +++ b/cli/src/services/db/open_lock.rs @@ -0,0 +1,76 @@ +use std::fs::{File, OpenOptions}; +use std::io; +use std::path::Path; + +use anyhow::{anyhow, Result}; + +#[cfg(target_os = "linux")] +const SHARED_LOCK_COMMAND: libc::c_int = libc::F_OFD_SETLK; +#[cfg(all(unix, not(target_os = "linux")))] +const SHARED_LOCK_COMMAND: libc::c_int = libc::F_SETLK; + +pub(super) fn open_lock_file(db_path: &Path, create: bool) -> Result> { + match OpenOptions::new() + .read(true) + .write(create) + .create(create) + .truncate(false) + .open(db_path) + { + Ok(file) => Ok(Some(file)), + Err(error) if !create && error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(anyhow!( + "failed to open database file {} for its compatibility lock: {error}", + db_path.display() + )), + } +} + +pub(super) fn lock_shared(file: &File, db_name: &str, db_path: &Path) -> Result<()> { + match try_lock_shared(file) { + Ok(()) => Ok(()), + Err(error) if is_lock_conflict(&error) => Err(anyhow!( + "{db_name} database at {} is open in an older SCE process that does not support shared access. Try: wait for older SCE processes to exit, upgrade every installed 'sce' binary, and rerun the command", + db_path.display() + )), + Err(error) => Err(anyhow!( + "failed to lock {db_name} database at {}: {error}", + db_path.display() + )), + } +} + +#[cfg(unix)] +fn try_lock_shared(file: &File) -> io::Result<()> { + use std::os::fd::AsRawFd; + + let mut lock: libc::flock = unsafe { std::mem::zeroed() }; + lock.l_type = lock_field(libc::F_RDLCK)?; + lock.l_whence = lock_field(libc::SEEK_SET)?; + if unsafe { libc::fcntl(file.as_raw_fd(), SHARED_LOCK_COMMAND, &raw const lock) } == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +#[cfg(unix)] +fn lock_field(value: impl TryInto) -> io::Result { + value + .try_into() + .map_err(|_| io::Error::from(io::ErrorKind::InvalidInput)) +} + +#[cfg(not(unix))] +fn try_lock_shared(_file: &File) -> io::Result<()> { + Ok(()) +} + +#[cfg(unix)] +fn is_lock_conflict(error: &io::Error) -> bool { + matches!(error.raw_os_error(), Some(libc::EAGAIN | libc::EACCES)) +} + +#[cfg(not(unix))] +fn is_lock_conflict(_error: &io::Error) -> bool { + false +} diff --git a/cli/src/services/hooks/claude_model_state.rs b/cli/src/services/hooks/claude_model_state.rs index 21cafe1d2..42f2730df 100644 --- a/cli/src/services/hooks/claude_model_state.rs +++ b/cli/src/services/hooks/claude_model_state.rs @@ -29,6 +29,9 @@ pub(super) async fn run_claude_model_state_subcommand< repository_root: &Path, logger: Option<&L>, ) -> String { + crate::services::managed_telemetry::trace_export::record_claude_hook_command( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::ClaudeModelState, + ); let stdin_payload = match read_hook_stdin() { Ok(payload) => payload, Err(error) => { @@ -36,10 +39,7 @@ pub(super) async fn run_claude_model_state_subcommand< return String::new(); } }; - crate::services::managed_telemetry::trace_export::record_claude_hook_payload( - crate::services::managed_telemetry::trace_export::ClaudeHookCommand::ClaudeModelState, - &stdin_payload, - ); + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); let session_id = fail_open_session_id(&stdin_payload); let observed_at_ms = match current_unix_time_ms() { diff --git a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs index cfb97469b..97f0a9bbb 100644 --- a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs +++ b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs @@ -47,11 +47,11 @@ pub(crate) async fn run_claude_mutation_scope_subcommand< >( logger: Option<&L>, ) -> Result { - let stdin_payload = hooks::read_hook_stdin()?; - crate::services::managed_telemetry::trace_export::record_claude_hook_payload( + crate::services::managed_telemetry::trace_export::record_claude_hook_command( crate::services::managed_telemetry::trace_export::ClaudeHookCommand::ClaudeMutationScope, - &stdin_payload, ); + let stdin_payload = hooks::read_hook_stdin()?; + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); run_claude_mutation_scope_from_payload(&stdin_payload, logger).await } @@ -542,7 +542,7 @@ mod tests { capture_requests_until, loopback_hook_telemetry, trace_spans, DecodedSpan, }; use crate::services::managed_telemetry::trace_export::{ - record_claude_hook_payload, ClaudeHookCommand, SHUTDOWN_TIMEOUT, + record_claude_hook_command, record_claude_hook_payload, ClaudeHookCommand, SHUTDOWN_TIMEOUT, }; use crate::services::mutation_trace::runtime::{ abandon_scope, advise_after_completed_boundary, coordinate, @@ -585,7 +585,8 @@ mod tests { for payload in HOOK_SEQUENCE { let output = telemetry .with_default_subscriber(&mut || async { - record_claude_hook_payload(ClaudeHookCommand::ClaudeMutationScope, payload); + record_claude_hook_command(ClaudeHookCommand::ClaudeMutationScope); + record_claude_hook_payload(payload); run_claude_mutation_scope_from_payload_with_resolver::( payload, None, @@ -702,7 +703,8 @@ mod tests { let telemetry = loopback_hook_telemetry(address, "bao-token-value"); let output = telemetry .with_default_subscriber(&mut || async { - record_claude_hook_payload(ClaudeHookCommand::ClaudeMutationScope, payload); + record_claude_hook_command(ClaudeHookCommand::ClaudeMutationScope); + record_claude_hook_payload(payload); run_claude_mutation_scope_from_payload_with_resolver::( payload, None, diff --git a/cli/src/services/hooks/conversation_trace.rs b/cli/src/services/hooks/conversation_trace.rs index 88c358cac..8237bee58 100644 --- a/cli/src/services/hooks/conversation_trace.rs +++ b/cli/src/services/hooks/conversation_trace.rs @@ -70,14 +70,14 @@ pub(crate) async fn run_conversation_trace_subcommand< repository_root: &Path, logger: Option<&L>, ) -> String { + crate::services::managed_telemetry::trace_export::record_claude_hook_command( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::ConversationTrace, + ); let stdin_payload = match read_hook_stdin() { Ok(payload) => payload, Err(error) => return log_conversation_trace_fail_open(&error, logger, None), }; - crate::services::managed_telemetry::trace_export::record_claude_hook_payload( - crate::services::managed_telemetry::trace_export::ClaudeHookCommand::ConversationTrace, - &stdin_payload, - ); + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); let session_id = conversation_trace_fail_open_session_id(&stdin_payload); match run_conversation_trace_subcommand_from_payload( diff --git a/cli/src/services/hooks/diff_trace.rs b/cli/src/services/hooks/diff_trace.rs index 33780be5f..05d918909 100644 --- a/cli/src/services/hooks/diff_trace.rs +++ b/cli/src/services/hooks/diff_trace.rs @@ -67,14 +67,14 @@ pub(crate) async fn run_diff_trace_subcommand, ) -> String { + crate::services::managed_telemetry::trace_export::record_claude_hook_command( + crate::services::managed_telemetry::trace_export::ClaudeHookCommand::DiffTrace, + ); let stdin_payload = match read_hook_stdin() { Ok(payload) => payload, Err(error) => return log_diff_trace_fail_open(&error, logger, None), }; - crate::services::managed_telemetry::trace_export::record_claude_hook_payload( - crate::services::managed_telemetry::trace_export::ClaudeHookCommand::DiffTrace, - &stdin_payload, - ); + crate::services::managed_telemetry::trace_export::record_claude_hook_payload(&stdin_payload); let session_id = diff_trace_fail_open_session_id(&stdin_payload); match run_diff_trace_subcommand_from_payload(repository_root, &stdin_payload, logger).await { diff --git a/cli/src/services/managed_telemetry/trace_export.rs b/cli/src/services/managed_telemetry/trace_export.rs index 491531fe5..aabab4cf7 100644 --- a/cli/src/services/managed_telemetry/trace_export.rs +++ b/cli/src/services/managed_telemetry/trace_export.rs @@ -391,12 +391,15 @@ pub fn record_command_name(name: &str) { Span::current().record(COMMAND_NAME_FIELD, name); } -pub fn record_claude_hook_payload(command: ClaudeHookCommand, stdin_payload: &str) { +pub fn record_claude_hook_command(command: ClaudeHookCommand) { + Span::current().record(HOOK_COMMAND_FIELD, command.as_str()); +} + +pub fn record_claude_hook_payload(stdin_payload: &str) { let span = Span::current(); if span.is_disabled() { return; } - span.record(HOOK_COMMAND_FIELD, command.as_str()); let Ok(serde_json::Value::Object(payload)) = serde_json::from_str(stdin_payload) else { return; }; @@ -824,7 +827,8 @@ pub(crate) mod tests { let result = telemetry .with_default_subscriber(&mut || async { record_command_name("hooks"); - record_claude_hook_payload(ClaudeHookCommand::DiffTrace, payload); + record_claude_hook_command(ClaudeHookCommand::DiffTrace); + record_claude_hook_payload(payload); tokio::task::yield_now().await; Ok(String::new()) }) diff --git a/cli/src/services/telemetry_credential_storage.rs b/cli/src/services/telemetry_credential_storage.rs index a42e7537e..6453bc4ea 100644 --- a/cli/src/services/telemetry_credential_storage.rs +++ b/cli/src/services/telemetry_credential_storage.rs @@ -6,8 +6,7 @@ use base64::Engine; use serde::Deserialize; use crate::services::db::{DbSpec, EncryptedTursoDb}; -use crate::services::default_paths::auth_db_path; -use crate::services::token_storage::{get_auth_db, TokenStorageError}; +use crate::services::token_storage::{get_auth_db, get_existing_auth_db, TokenStorageError}; const TELEMETRY_CREDENTIAL_ROW_ID: i64 = 1; const WORKOS_SESSION_ROW_ID: i64 = 1; @@ -87,27 +86,19 @@ pub async fn save_telemetry_credential( } pub async fn load_telemetry_credential() -> Result, TokenStorageError> { - if !auth_db_exists()? { + let Some(db) = get_existing_auth_db().await? else { return Ok(None); - } - load_with(get_auth_db().await?, current_unix_timestamp_seconds()?).await -} - -fn auth_db_exists() -> Result { - auth_db_path() - .map(|path| path.exists()) - .map_err(|error| TokenStorageError::PathResolution(error.to_string())) + }; + load_with(db, current_unix_timestamp_seconds()?).await } pub async fn load_repository_telemetry_credential( repository_id: String, ) -> Result, TokenStorageError> { - load_for_repository_with( - get_auth_db().await?, - current_unix_timestamp_seconds()?, - &repository_id, - ) - .await + let Some(db) = get_existing_auth_db().await? else { + return Ok(None); + }; + load_for_repository_with(db, current_unix_timestamp_seconds()?, &repository_id).await } pub async fn bind_repository_telemetry( diff --git a/cli/src/services/token_storage.rs b/cli/src/services/token_storage.rs index ccdb7977b..f42d80ed4 100644 --- a/cli/src/services/token_storage.rs +++ b/cli/src/services/token_storage.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use crate::services::auth::TokenResponse; use crate::services::auth_db::AuthDb; +use crate::services::default_paths::auth_db_path; /// Constant row ID for the single token row in `auth_credentials`. const DEFAULT_TOKEN_ROW_ID: i64 = 1; @@ -25,6 +26,21 @@ pub(crate) async fn get_auth_db() -> Result<&'static AuthDb, TokenStorageError> } } +pub(crate) async fn get_existing_auth_db() -> Result, TokenStorageError> { + if AUTH_DB.get().is_none() { + let db_path = + auth_db_path().map_err(|error| TokenStorageError::PathResolution(error.to_string()))?; + let Some(db) = AuthDb::open_existing_at(db_path) + .await + .map_err(|error| TokenStorageError::Database(error.to_string()))? + else { + return Ok(None); + }; + let _ = AUTH_DB.set(Ok(db)); + } + get_auth_db().await.map(Some) +} + #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct StoredTokens { pub access_token: String, diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md index 35dbcb27e..e250b8c1d 100644 --- a/context/plans/claude-hook-trace-propagation.md +++ b/context/plans/claude-hook-trace-propagation.md @@ -225,11 +225,44 @@ Persist this field in every plan; this is durable plan state, not chat state: - 32 reader processes × 3 rounds: 32/32 each round (about 60–190 ms per round, debug build). - 32 concurrent hook processes: every `(sce.claude.tool_use_id, sce.hook.command)` pair exported exactly once to a loopback collector. A wrong-key child resolves Noop with reason `database_error` and unchanged hook output. -- Mixed versions: unsafe for concurrent writers (silent lost update); sequential use and legacy reads are safe. See [Mixed-version rollout risk](../sce/auth-db.md#mixed-version-rollout-risk). +- Mixed versions: unsafe for concurrent writers (silent lost update); sequential use and legacy reads are safe. Superseded by the compatibility lock below; see [Mixed-version compatibility lock](../sce/auth-db.md#mixed-version-compatibility-lock). - Live: headless session `48a276ea-ce4a-4488-8244-649f20b87a42`, trace `132a62ceed757d0d8431011311ae76af`, 13 SCE hook invocations, zero local telemetry fallbacks. Dash0 delivery is **unverified**. ### Residual risks - Dash0 confirmation of the 13 spans, their `sce.hook.command` values, and `tool_use_id` equality is still needed. -- Mixed old/new `sce` writers can lose updates; rollout guidance is in `auth-db.md`. +- Mixed old/new `sce` writers can lose updates; rollout guidance is in `auth-db.md`. Resolved on Linux by the follow-up below. - Turso 0.8.1 has a multiprocess-WAL race when many processes concurrently create or migrate a DB under heavy load. It was not observed on read-only hook paths. + +## Follow-up validation: mixed-version lock, existing-only lookups, hook command identity (PR #330) + +**Date:** 2026-10-19 + +### Change + +- P1: every encrypted auth DB open takes a shared compatibility lock on `auth.db` (`cli/src/services/db/open_lock.rs`): an OFD `F_RDLCK` on Linux, a classic `F_SETLK` `F_RDLCK` on other Unix. It is held for the life of the `EncryptedTursoDb`. It conflicts with the classic whole-file `F_WRLCK` that legacy Turso 0.8.1/0.7.0 takes in pre-multiprocess SCE binaries, so old and new processes can no longer hold the DB at the same time. New processes still share it, and writes still go through multiprocess WAL. Details: [Mixed-version compatibility lock](../sce/auth-db.md#mixed-version-compatibility-lock). +- P2: `load_repository_telemetry_credential` (the `sce auth telemetry-headers` lookup) now uses the same existing-only path as `load_telemetry_credential`: `token_storage::get_existing_auth_db` → `EncryptedTursoDb::open_existing_at`, which opens the file without `O_CREAT` before resolving any key. +- P3: hook entry points call `record_claude_hook_command` before reading stdin. `record_claude_hook_payload` now parses the payload only. + +### Commands run + +- `nix flake check` -> exit 0, all checks passed (x86_64-linux; Darwin systems omitted by the check) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::auth_db::multiprocess_tests` -> 16 passed +- Targeted filters on the test binary: `telemetry_credential` 31, `trace_export` 14, `managed_telemetry` 26, `claude_mutation_scope` 7, `bash_policy` 36, `auth_command` 13, `services::db` 4, all passed +- Multiprocess suite stress, 6 parallel copies × 8 rounds -> 48 of 48 passed (one contended round took about 29 s per copy) +- Mutation checks: with the lock disabled, 3 tests fail, and the mixed-writer test loses 24 of 72 committed rows. With the repository lookup back on `get_auth_db`, the repository missing-DB test and the 32-helper test fail because `auth.db*` files are created. With the command recorded after the stdin read, the hook-entry test fails on the unreadable-stdin `diff-trace` span. +- Real binaries: the pre-fix `sce` (`nix build git+file://…?rev=9d4f5482…#default`) against this branch's debug build in a sandboxed `HOME`/`XDG_*`, with an env-derived key. + +### Results + +- Legacy holder (`/proc/locks`: `POSIX ADVISORY WRITE`): new `sce auth whoami` and `sce auth logout` exit 4 with `Authentication storage is unavailable`, and the log records the shared-access rejection. New holder (`OFDLCK ADVISORY READ`): a second new process succeeds, and legacy `whoami`/`logout` exit 4 (log: `File is locked by another process`). After all new processes exit, legacy `whoami` and `logout` exit 0. +- No silently lost committed write in any tested mixed-version scenario on Linux. +- Missing `auth.db`: both lookups and 32 concurrent `telemetry-headers` runs create no `auth.db`, `-wal`, or `-tshm` file and resolve no key, and the helpers fail closed. +- `sce.hook.command` is present on every `sce.hook` span for valid, empty, malformed, and unreadable stdin across all five commands. Fail-open hooks keep `sce.outcome=success`, and malformed `policy-bash` input records `failure`. No stdin content or token reaches the collector. +- Dash0: **unverified**. This session had no Dash0 access, and no live Claude run was made. + +### Residual risks + +- macOS uses a process-scoped lock and has not been built or run on a Mac. Windows has no compatibility lock. +- Mixed old/new processes still cannot run at the same time: one side fails explicitly until the other exits. Every `sce` binary used by the account must be upgraded. +- The upstream Turso multiprocess cold-open race for many concurrent creators or migrators is unchanged. diff --git a/context/sce/auth-db.md b/context/sce/auth-db.md index 5542c548e..e8560465d 100644 --- a/context/sce/auth-db.md +++ b/context/sce/auth-db.md @@ -52,19 +52,45 @@ Storage operations live in `cli/src/services/telemetry_credential_storage.rs`: ` - Many short-lived `sce` processes read `auth.db` at once: every admitted Claude hook resolves the telemetry credential at startup. `EncryptedTursoDb::new()` therefore opens with Turso multiprocess WAL, alongside encryption, so concurrent readers share the file. Multiprocess WAL creates `auth.db-wal` and the `auth.db-tshm` coordination file next to the database. - A current schema is opened read-only: no `CREATE TABLE IF NOT EXISTS __sce_migrations` or other DDL runs when every embedded migration ID is already recorded. A missing or outdated DB is migrated in one `BEGIN IMMEDIATE` transaction, with a retry only for genuine `Busy`. A partially migrated schema fails with the failing migration ID rather than being treated as current. Details: [shared-turso-db.md](shared-turso-db.md). -- `load_telemetry_credential` (the managed-telemetry lookup used by every command and admitted hook) is an existing-DB read path: when `auth.db` does not exist it returns no credential without creating the file or an OS-keyring key. Only `sce auth`, `sce setup`, and other writers create the DB. This also keeps concurrent hooks on a never-logged-in machine from racing to create it. +- Telemetry credential lookups are existing-DB read paths. Both `load_telemetry_credential` (the managed-telemetry lookup used by every command and admitted hook) and `load_repository_telemetry_credential` (the `sce auth telemetry-headers` lookup that Claude Code's `otelHeadersHelper` runs) go through `token_storage::get_existing_auth_db`. That helper reuses the process `AuthDb` singleton when it is already open. Otherwise it calls `EncryptedTursoDb::open_existing_at`, which opens the database file without `O_CREAT` before resolving the encryption key. An absent file returns no credential, with no DB file, `-wal`/`-tshm` file, encryption key, or migration created. The existence check is the `open()` itself, not a separate `Path::exists()`. Path-resolution errors stay `TokenStorageError::PathResolution`. Only `sce auth`, `sce setup`, and other writers create the DB. This also keeps concurrent hooks and header-helper calls on a never-logged-in machine from racing to create it. - The auth DB busy timeout is 1000 ms. Hook reads stay bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s), which aborts the lookup regardless of busy waits or retries. -- Regression coverage (`cli/src/services/auth_db/multiprocess_tests.rs`, child processes of the test binary with sandboxed `HOME`/`XDG_*` and an env-provided `SCE_AUTH_DB_ENCRYPTION_KEY`; never the real DB or keyring): a reader succeeds while another process holds the DB open; 32 reader processes × 3 rounds all resolve the credential; a current-schema reader succeeds while another process holds `BEGIN IMMEDIATE`, which proves it performs no migration write; a telemetry lookup on a missing DB returns none and creates nothing; 4 processes concurrently migrating an existing schemaless DB record each migration once and pass `PRAGMA integrity_check`; an outdated schema upgrades and a partial one fails clearly and rolls back; 32 concurrent hook processes export correlated `sce.hook` spans. +- Regression coverage (`cli/src/services/auth_db/multiprocess_tests.rs`, child processes of the test binary with sandboxed `HOME`/`XDG_*` and an env-provided `SCE_AUTH_DB_ENCRYPTION_KEY`; never the real DB or keyring): a reader succeeds while another process holds the DB open; 32 reader processes × 3 rounds all resolve the credential; a current-schema reader succeeds while another process holds `BEGIN IMMEDIATE`, which proves it performs no migration write; both telemetry lookups on a missing DB return none and create nothing (the repository lookup runs with an empty `SCE_AUTH_DB_ENCRYPTION_KEY`, which fails any key resolution, so it also proves no key is resolved or created); 32 concurrent `sce auth telemetry-headers` runs (through `app::run`, inside a Git repository) on a missing DB all fail closed with `Telemetry credential is unavailable.` and leave no `auth.db*` file; the helper prints the bearer only for the bound repository; 4 processes concurrently migrating an existing schemaless DB record each migration once and pass `PRAGMA integrity_check`; an outdated schema upgrades and a partial one fails clearly and rolls back; 32 concurrent hook processes export correlated `sce.hook` spans. The mixed-version tests are listed under [Mixed-version compatibility lock](#mixed-version-compatibility-lock). ### Upstream Turso multiprocess race Under heavy load, Turso 0.8.1 multiprocess WAL has an internal race on concurrent cold opens that involve writes. Stress runs (6 parallel copies of the suite, with 32 processes creating or migrating the same DB) occasionally failed: `short read on page 3`, a `turso_assert` panic `reader slot released by non-owner` in `shared_wal_coordination.rs`, or, in the Nix sandbox, a writer that stayed `database is locked` through all 10 migration attempts (about 49 s). The read-only paths never failed across about 3,800 cold-opening reader and hook processes, and that is all hooks do on a current schema. The migration regression therefore uses 4 concurrent migrators, which was 0 failures in 48 parallel stress runs. Concurrent creation and migration by many writers is left as a documented upstream limitation. It affects first-time initialization only, and the Agent Trace DB already runs in the same Turso mode. -### Mixed-version rollout risk +### Mixed-version compatibility lock -Binaries from before this change open `auth.db` in legacy (exclusive-lock) WAL mode. A probe on the pinned Turso 0.8.1 with the same encrypted builder showed that Turso's mixed-mode guards do not fire on this path. With a legacy holder, a multiprocess process opens and writes, and with a multiprocess holder, a legacy process opens and writes; neither gets a `LockingError`. When both modes hold the DB and both write, the last committer's stale WAL view wins and the other process's committed rows are silently lost, while `PRAGMA integrity_check` still reports `ok`. A legacy holder running only the no-op `CREATE TABLE IF NOT EXISTS` lost nothing, and sequential use is safe: after all multiprocess processes exit, the legacy `sce` binary opens, decrypts, reads, and writes the DB, and integrity stays `ok`. +SCE binaries from before multiprocess WAL (every release through `v0.4.0-pre-alpha-v11`, on Turso 0.8.1 or 0.7.0) open `auth.db` in legacy WAL mode. In that mode, Turso's `UnixFile::lock_file` takes a non-blocking classic POSIX record lock, `fcntl(F_SETLK, F_WRLCK)` over the whole file (`l_start = 0`, `l_len = 0`). It holds that lock while the database is open. Turso's own mixed-mode guards (`reject_live_*` in `turso_core::Database::resolve_default_storage`) never run for `turso::Builder`. `turso_sdk_kit` opens the file itself and calls `Database::open_async`, which skips them. Multiprocess mode opens the DB file with `NoLock`, so nothing stopped the two modes from overlapping. Without a guard, when an old and a new process both held the DB and both wrote, one process's committed rows were silently lost while `PRAGMA integrity_check` still reported `ok`. The regression test reproduced this: with the guard disabled, 24 of 72 committed rows were lost. -Hooks never write `auth.db`. Writers are `sce auth login`/`logout`, `sce setup` repository enrollment, telemetry-credential issuance, and the `sce sync` token refresh. So the hazard needs an old-version and a new-version `sce` process alive at the same time with at least one of them writing. Rollout: replace the binary while no `sce auth`, `sce setup`, or `sce sync` command is running, and do not keep two `sce` versions on `PATH` used by the same account. Short-lived old hook processes that are mid-flight during an upgrade only read. Closing the hazard in code would need a cross-mode lock guard (for example a shared `fcntl` lock held by new processes, which conflicts with the legacy exclusive lock). That needs a direct `rustix`/`libc` dependency and is not part of this change. +Every `EncryptedTursoDb` open now takes a shared compatibility lock (`cli/src/services/db/open_lock.rs`) on the `auth.db` file. It is acquired inside the connection-open retry loop, before Turso opens the file, and held for the life of the `EncryptedTursoDb` value: + +- Linux: `fcntl(F_OFD_SETLK, F_RDLCK)`, an open-file-description lock on the whole file, on a descriptor the `EncryptedTursoDb` owns. Linux reports conflicts between OFD locks and classic POSIX record locks. So the new read lock and the legacy write lock exclude each other in both directions, while any number of new processes share the read lock. The lock belongs to SCE's own descriptor, so Turso opening or closing its descriptors for `auth.db` cannot release it. It is released when the `EncryptedTursoDb` is dropped or the process exits. +- macOS and other Unix: `fcntl(F_SETLK, F_RDLCK)`, a classic process-scoped record lock, because those platforms have no public OFD lock. It conflicts with the legacy lock in the same way. POSIX releases every record lock a process holds on a file when that process closes any descriptor for the file. So this lock holds only while the process does not drop a second `EncryptedTursoDb` for `auth.db` while another one stays open. Production paths keep one `AuthDb` per process, the `token_storage` singleton. Lifecycle setup opens its own instance only while `auth.db` does not exist yet, and every open re-acquires the lock. **Not executed on macOS**: the macOS path has been reviewed against the `libc` definitions but has not been built or tested on a Mac. +- Windows: no compatibility lock. Mixed versions on Windows remain unprotected. + +Resulting coexistence on Linux, verified with separate processes, isolated encrypted databases, a legacy child that opens with the exact pre-fix builder configuration on the pinned Turso 0.8.1, and the real pre-fix `sce` binary: + +| First holder | Second process | Result | +| --- | --- | --- | +| legacy | new read (telemetry lookup) or write (`AuthDb::new`) | Rejected after the connection-open retries (under 1 s): `auth DB database at is open in an older SCE process that does not support shared access`. Commands surface the stable `Authentication storage is unavailable` error, and hooks fall back to Noop with reason `database_error`. Hook behavior is otherwise unchanged. | +| new | legacy read or write | Rejected immediately by Turso: `Failed locking file ''. File is locked by another process` | +| new | new read or write | Shared. Writes stay serialized by Turso multiprocess WAL and the 1000 ms busy timeout. | +| legacy and new, started together | concurrent writes | Whichever opens first proceeds. The other is rejected or opens after the first exits. No committed row is lost. | +| none (all new processes exited) | legacy | Opens, decrypts, reads, and writes normally | + +`/proc/locks` shows the legacy holder as `POSIX ADVISORY WRITE … 0 EOF` and a new holder as `OFDLCK ADVISORY READ -1 … 0 EOF`. Released binaries need no change. Unsafe combinations fail explicitly instead of losing data. + +Mixed-version tests (`multiprocess_tests.rs`), all verifying persisted `telemetry_repository_bindings` rows and the `telemetry_credentials`/`auth_credentials` rows plus `integrity_check` afterwards: + +- A legacy holder makes a new telemetry lookup and a new writer fail explicitly. The legacy holder's own later write persists. +- A new holder makes a legacy open fail with the Turso lock error. +- 3 rounds of 1 legacy writer and 2 new writers start together. Every row any writer reported committed is persisted. +- 4 concurrent new writers all commit all 32 rows. +- After every new process exits, a legacy process reopens the DB, reads the credential, and writes. + +Remaining limits: a mixed old/new deployment is still not usable at the same time, because one side fails while the other holds the DB. Upgrade by replacing every `sce` binary used by the account. Old hook processes still in flight during the switch make new processes fail (and hooks fall back to Noop) until they exit; they no longer lose data. Turso reopens `auth.db` by path after the lock is taken, so a file that something outside SCE deletes in that window would be recreated empty. SCE never deletes `auth.db`. ## Lifecycle integration diff --git a/context/sce/claude-hook-trace-propagation.md b/context/sce/claude-hook-trace-propagation.md index c77876e90..8c95f99aa 100644 --- a/context/sce/claude-hook-trace-propagation.md +++ b/context/sce/claude-hook-trace-propagation.md @@ -182,7 +182,7 @@ A later live run expected 15 SCE hook spans and Dash0 received 8. Claude Code ru - Secondary cause: with multiprocess WAL alone, every open still ran `run_migrations()`, and `CREATE TABLE IF NOT EXISTS __sce_migrations` takes the write lock even on a current schema, which caused transient `Busy("database is locked")`. - Reproduction at 32 concurrent processes (failures out of 256): current implementation 180; multiprocess WAL only 32; multiprocess WAL plus a 1000 ms busy timeout 0; multiprocess WAL plus skipping migrations when current 0; both remedies 0. - Fix: multiprocess WAL on the encrypted builder, a credential lookup that never creates a missing `auth.db`, a migration-free open for a current schema, serialized and Busy-retried migrations for a missing or outdated schema, and a 1000 ms auth DB busy timeout ([auth-db.md](auth-db.md), [shared-turso-db.md](shared-turso-db.md)). Every Noop fallback is now logged locally as `sce.telemetry.fallback` with its reason. -- Correlation: `sce.hook` now carries `sce.hook.command` (`diff-trace`, `conversation-trace`, `claude-mutation-scope`, `claude-model-state`, `policy-bash`), so spans from different SCE commands on the same Claude event are distinguishable. `tool_use_id` handling is unchanged. +- Correlation: `sce.hook` now carries `sce.hook.command` (`diff-trace`, `conversation-trace`, `claude-mutation-scope`, `claude-model-state`, `policy-bash`), so spans from different SCE commands on the same Claude event are distinguishable. Each entry point records it before reading stdin, so a hook whose stdin is empty, unreadable, or malformed, or that fails afterwards, still identifies itself. `tool_use_id` handling is unchanged. - Regression evidence (`cli/src/services/auth_db/multiprocess_tests.rs`): 32 hook child processes resolve the credential through the production `load_telemetry_credential` path. They export to a loopback collector, and every `(sce.claude.tool_use_id, sce.hook.command)` pair arrives exactly once. A child with a wrong encryption key resolves Noop with reason `database_error`, and its hook output is unchanged. The suite passed 48 of 48 runs with 6 copies in parallel. Removing the multiprocess flag fails the open-holder, 32-reader, held-write, concurrent-init, and hook-export tests. Restoring unconditional migrations fails the held-write test. - Live run on 2026-10-19, 11:08:50Z–11:09:03Z: headless Claude Code `2.1.301` (`claude -p --model haiku --allowedTools Write,Read`), session `48a276ea-ce4a-4488-8244-649f20b87a42`, `num_turns=3`, with a debug build of this fix first on `PATH` through a scratchpad shim. The shim logged only subcommand words. The stored credential was present and was not printed. The session wrote and read a gitignored `context/tmp/` file, which was then deleted. The shim recorded 13 SCE hook invocations: `claude-mutation-scope` ×7, `conversation-trace` ×4, `diff-trace` ×1, `claude-model-state` ×1. `policy bash` did not fire because its matcher is `Bash`, and `diff-trace` fired only for `Write`. Claude also ran `sce auth telemetry-headers` once. A scratchpad probe hook recorded tool hooks for `toolu_015xdt65shPcBzT4UNbbKib5` (Write) and `toolu_01VMf442BtXsW2XMvnd76YZJ` (Read), all with `TRACEPARENT` trace `132a62ceed757d0d8431011311ae76af` and parent `6a4ae45ade8a6191`. The repository log directory recorded zero `sce.telemetry.fallback` records, so none of the 13 hook processes fell back to Noop. The installed pre-fix `sce` still read the credential afterwards. - Dash0 status for this run: **unverified**. This session had no Dash0 access, and collector responses were not used as evidence. Expected in the `sce-crocoder` dataset, in trace `132a62ceed757d0d8431011311ae76af`: one `sce.hook` per tool-hook invocation, parented to `claude_code.interaction` `6a4ae45ade8a6191`, with `sce.claude.tool_use_id` equal to the matching `claude_code.tool`, and same-event hooks distinguished by `sce.hook.command` (for example, Write `PostToolUse` → `diff-trace`, `conversation-trace`, `claude-mutation-scope`). Session-boundary hooks should appear as independent `sce.hook` roots. diff --git a/context/sce/cli-observability-contract.md b/context/sce/cli-observability-contract.md index 1d2391d96..be511e4e7 100644 --- a/context/sce/cli-observability-contract.md +++ b/context/sce/cli-observability-contract.md @@ -99,7 +99,7 @@ Runtime observability consumes the shared resolved observability config from `cl - Admission (`managed_span_kind` in `cli/src/app.rs`, decided from argv before credential lookup): Claude-invoked SCE hooks resolve the `sce.hook` span kind, every other hook and the hidden `sce auth telemetry-headers` helper use `NoopTelemetry` without a credential lookup, and all remaining commands use the `sce.command` span kind. `sce hooks claude-mutation-scope` and `sce hooks claude-model-state` are admitted from argv alone. The shared `sce hooks diff-trace`, `sce hooks conversation-trace`, and `sce policy bash` are admitted only when both `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are present in the environment (checked for presence only; values never read); `CLAUDE_PROJECT_DIR` is set for Claude command hooks but not for the Claude Bash tool (evidence: [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md)). Other hooks (commit hooks, Codex, OpenCode, Pi) stay Noop. Admitted hooks use the same `CREDENTIAL_LOOKUP_DEADLINE`, `SHUTDOWN_TIMEOUT`, and runtime grace bounds as commands. Decision record: [2026-10-17-claude-hook-managed-trace-admission.md](../decisions/2026-10-17-claude-hook-managed-trace-admission.md). - Export is OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev/v1/traces`. A pinned HTTP client sets `Authorization: Bearer ` (token only, never the accessor), disables redirects, rebuilds the request to the fixed endpoint, and bounds requests to 3s. Ambient `OTEL_EXPORTER_OTLP_*` values cannot redirect it. - Each command emits one `sce.command` span with `sce.command.name`, `sce.outcome` (`success`/`failure`), and `sce.error.class` (the `SCE-ERR-*` code on failure); resource attributes are `service.name=sce` and `service.version`. Raw arguments, paths, and error chains are not exported. -- An admitted Claude hook process emits exactly one `sce.hook` span instead (no `sce.command` span), with `sce.agent=claude`, `sce.hook.command`, `sce.outcome`, `sce.error.class`, and, when present as non-empty strings in the Claude hook stdin payload, `sce.hook.event` (`hook_event_name`), `sce.hook.tool` (`tool_name`), `sce.claude.session_id`, and `sce.claude.tool_use_id`. `sce.hook.command` is the stable SCE entry point (`ClaudeHookCommand`: `diff-trace`, `conversation-trace`, `claude-mutation-scope`, `claude-model-state`, `policy-bash`), so the several SCE hooks that one Claude event runs produce distinguishable spans; it is recorded even when the payload is not JSON. `trace_export::record_claude_hook_payload(command, stdin)` is the only reader: each Claude hook entry point passes its command and stdin to it once, it returns immediately when the current span is disabled (Noop), and it copies only those four fields. Session and tool-use IDs are opaque correlation identifiers: span attributes only, never metric attributes, never used to derive a trace ID. Tool input and response, prompts, `cwd`, transcript paths, file contents, and the raw payload are never exported. Parent context: for the `sce.hook` kind only, `ManagedTraceTelemetry::new` reads `TRACEPARENT` (and `TRACESTATE` when present) from the process environment, extracts it with `TraceContextPropagator`, and keeps it only when the remote span context is valid; `root_span` then attaches it with `set_parent`, so `sce.hook` shares Claude Code's trace ID, its parent is the upstream span (Claude Code 2.1.301 supplies `claude_code.interaction`, making `sce.hook` a sibling of `claude_code.tool`), and its own span ID is freshly generated. Child spans created inside the instrumented hook future inherit that trace across `.await` points and worker migration. No global propagator is registered and `sce.command` never reads the variables. A missing, malformed, or all-zero traceparent (the normal case when Claude tracing is off) yields an independent SCE root. Claude `SessionStart` and `SessionEnd` hooks run outside any interaction span, so their `sce.hook` is always a root. Live status: Dash0 (dataset `sce-crocoder`) shows `sce doctor`, Claude session-boundary `sce.hook` spans, and parented tool-hook `sce.hook` spans with operation children under `claude_code.interaction`. Concurrent hook processes previously lost spans because the auth DB credential lookup failed with `LockingError` and fell back to Noop; that is fixed, and the fix is verified by multiprocess regression tests but not yet re-verified live in Dash0. The evidence and the opt-in smoke procedure are in [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md). Telemetry failures never change hook stdout, permission decisions, persisted state, or exit codes. +- An admitted Claude hook process emits exactly one `sce.hook` span instead (no `sce.command` span), with `sce.agent=claude`, `sce.hook.command`, `sce.outcome`, `sce.error.class`, and, when present as non-empty strings in the Claude hook stdin payload, `sce.hook.event` (`hook_event_name`), `sce.hook.tool` (`tool_name`), `sce.claude.session_id`, and `sce.claude.tool_use_id`. `sce.hook.command` is the stable SCE entry point (`ClaudeHookCommand`: `diff-trace`, `conversation-trace`, `claude-mutation-scope`, `claude-model-state`, `policy-bash`), so the several SCE hooks that one Claude event runs produce distinguishable spans; each Claude hook entry point records it with `trace_export::record_claude_hook_command` as its first step, before reading stdin, so the span carries it even when stdin is empty, cannot be read, or is not JSON, and when the hook then fails. `trace_export::record_claude_hook_payload(stdin)` is the only payload reader: each entry point passes its stdin to it once after a successful read, it returns immediately when the current span is disabled (Noop), and it copies only those four fields. Session and tool-use IDs are opaque correlation identifiers: span attributes only, never metric attributes, never used to derive a trace ID. Tool input and response, prompts, `cwd`, transcript paths, file contents, and the raw payload are never exported. Parent context: for the `sce.hook` kind only, `ManagedTraceTelemetry::new` reads `TRACEPARENT` (and `TRACESTATE` when present) from the process environment, extracts it with `TraceContextPropagator`, and keeps it only when the remote span context is valid; `root_span` then attaches it with `set_parent`, so `sce.hook` shares Claude Code's trace ID, its parent is the upstream span (Claude Code 2.1.301 supplies `claude_code.interaction`, making `sce.hook` a sibling of `claude_code.tool`), and its own span ID is freshly generated. Child spans created inside the instrumented hook future inherit that trace across `.await` points and worker migration. No global propagator is registered and `sce.command` never reads the variables. A missing, malformed, or all-zero traceparent (the normal case when Claude tracing is off) yields an independent SCE root. Claude `SessionStart` and `SessionEnd` hooks run outside any interaction span, so their `sce.hook` is always a root. Live status: Dash0 (dataset `sce-crocoder`) shows `sce doctor`, Claude session-boundary `sce.hook` spans, and parented tool-hook `sce.hook` spans with operation children under `claude_code.interaction`. Concurrent hook processes previously lost spans because the auth DB credential lookup failed with `LockingError` and fell back to Noop; that is fixed, and the fix is verified by multiprocess regression tests but not yet re-verified live in Dash0. The evidence and the opt-in smoke procedure are in [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md). Telemetry failures never change hook stdout, permission decisions, persisted state, or exit codes. - Operation child spans: `trace_export::in_child_span(ManagedChildSpan, future)` wraps an existing operation in one of three named spans, each carrying only `sce.outcome` (`success`/`failure` from the operation's `Result`): `sce.mutation_scope` around every Claude adapter seam call (one coordinate or abandon boundary), `sce.git_snapshot` around the coordinator's tree capture and pin, and `sce.persistence` around the mutation-store phase of a coordinate boundary (initialize, scope/provenance registration, CAS commit loop) and of an abandon. The hierarchy is `sce.hook → sce.mutation_scope → {sce.git_snapshot, sce.persistence}`. A child span exists only when its operation runs: a read-only tool, an untracked background shell, or a close with no matching attempt exports `sce.hook` alone. Snapshot-failure taint persistence and conversation/diff-trace Agent Trace DB writes have no span. The span filter depends on the span kind: a `sce.command` dispatch exports only `sce.command` (ordinary commands that reach the coordinator, such as ref maintenance, still export one span), and a `sce.hook` dispatch exports only `sce.hook` and the three child names. Without a managed hook dispatch the child spans are disabled, and they never change ordering, locking, or errors. - The exporter uses a bounded batch queue on a dedicated thread. Final stdout/stderr rendering and classified-error logging run inside the command span (`run_command_lifecycle_with_context`), so `sce.outcome` reflects the rendered result (a stdout write failure is `failure`) and the final `sce.error.*` log is exported. Trace and log providers shut down concurrently under one shared `SHUTDOWN_TIMEOUT` (3s) deadline after rendering; failures never change command results or exit codes. - Process lifetime: `app::run_process` owns the Tokio runtime (at least two workers) and ends it with `shutdown_timeout(RUNTIME_SHUTDOWN_GRACE)` (500ms), so a stalled keychain, database, or exporter worker cannot keep the process alive. The async deadlines bound when the command proceeds; the runtime grace is the hard process-level bound. Abandoned workers are not cancelled, only no longer awaited. diff --git a/context/sce/shared-turso-db.md b/context/sce/shared-turso-db.md index ca5b6136e..b664a4142 100644 --- a/context/sce/shared-turso-db.md +++ b/context/sce/shared-turso-db.md @@ -29,7 +29,7 @@ - `count_write_contention(body) -> (T, WriteContentionCounts)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports the write-contention `attempts`, `outer_retries`, and `exhaustions` it caused on the current thread. Test instrumentation only; not production telemetry and not shared across threads or processes. - `record_write_contention_timeline(body) -> (T, Vec<(Instant, WriteContentionTimelineEvent)>)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and returns the `Instant`-stamped attempt start/end, backoff requested and backoff slept events of the write-contention retry on the current thread. Test-only measurement instrumentation used by the lock-contention suite; never logged in production. - `count_read_statements(body) -> (T, usize)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports how many `TursoDb` read statements (`query_map`) it issued on the current thread, each counted once in its synchronous prelude before the retry wrapper. A deterministic seam for tests that must prove an operation reads from a single DB snapshot — one statement — rather than several independent `SELECT`s a concurrent commit could tear across (first used by `MutationTraceStore::load_tree_roots` / `load_all_tree_roots`). Not compiled into production builds. -- `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same static shape (connection and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, opens with `experimental_multiprocess_wal(true)` like `TursoDb` (without it Turso takes an exclusive, non-blocking lock on the whole file for the process lifetime, so a concurrent `sce` process fails with `LockingError`), installs the auth DB busy timeout, and wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`. After connect it migrates only when needed: a read-only readiness check (`__sce_migrations` exists and records every embedded ID) lets a current schema open with no write at all; otherwise all pending migrations run inside one `BEGIN IMMEDIATE` transaction, which serializes concurrent initializers (a waiter re-reads the metadata under the lock and applies nothing) and rolls back the whole attempt on any migration failure. Only `Busy`/`BusySnapshot` on begin or commit is retried, under `MIGRATION_RETRY_POLICY` (10 attempts, 25..200ms backoff, each attempt bounded by the 1000 ms busy timeout); a deterministic migration failure (for example a column added by a partially applied `004` whose ID was never recorded) fails after one attempt with the migration ID in the error chain. After migrating, the readiness check runs again and a schema that is still behind fails the open. +- `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same static shape (connection and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, opens with `experimental_multiprocess_wal(true)` like `TursoDb` (without it Turso takes an exclusive, non-blocking lock on the whole file for the process lifetime, so a concurrent `sce` process fails with `LockingError`), installs the auth DB busy timeout, and wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`. Inside that retry loop, before Turso opens the file, it takes the shared mixed-version compatibility lock from `db/open_lock.rs` (an OFD `F_RDLCK` on Linux, classic `F_SETLK` `F_RDLCK` elsewhere on Unix) on a descriptor the `EncryptedTursoDb` owns for its lifetime. That lock excludes pre-multiprocess SCE binaries, which hold a classic `F_WRLCK` on the whole file ([auth-db.md](auth-db.md#mixed-version-compatibility-lock)). `EncryptedTursoDb::open_existing_at(path)` is the existing-only variant: it opens the file without `O_CREAT` and returns `None` for an absent file before any encryption-key resolution, file creation, or migration. After connect it migrates only when needed: a read-only readiness check (`__sce_migrations` exists and records every embedded ID) lets a current schema open with no write at all; otherwise all pending migrations run inside one `BEGIN IMMEDIATE` transaction, which serializes concurrent initializers (a waiter re-reads the metadata under the lock and applies nothing) and rolls back the whole attempt on any migration failure. Only `Busy`/`BusySnapshot` on begin or commit is retried, under `MIGRATION_RETRY_POLICY` (10 attempts, 25..200ms backoff, each attempt bounded by the 1000 ms busy timeout); a deterministic migration failure (for example a column added by a partially applied `004` whose ID was never recorded) fails after one attempt with the migration ID in the error chain. After migrating, the readiness check runs again and a schema that is still behind fails the open. - `EncryptedTursoDb` exposes async `execute()` and `query_map()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. - Both public adapters keep concrete Turso connections and marker types; there is no shared runtime-owning core or synchronous executor bridge. - Shared lifecycle helpers: diff --git a/context/sce/telemetry-credential.md b/context/sce/telemetry-credential.md index c60c8cbc6..93a0885e4 100644 --- a/context/sce/telemetry-credential.md +++ b/context/sce/telemetry-credential.md @@ -29,7 +29,7 @@ flowchart LR ## Runtime use -- `managed_telemetry::RuntimeTelemetry::resolve` reads the stored credential at CLI startup (never issuing one); a valid unexpired `token` is used only as the `Authorization: Bearer` credential for managed OTLP trace and log export; the `accessor` is never transmitted. Absent, expired, malformed, or unavailable rows select `NoopTelemetry`. Every Noop fallback after a lookup is classified (`credential_unavailable`, `credential_expired`, `lookup_timeout`, `database_error`, `path_resolution_error`, `exporter_init_failed`) and logged locally as `sce.telemetry.fallback`; failures log at `warn` (forced, so they reach the log file at the default `error` level, never stderr while `log_to_file=true`), with the redacted `TokenStorageError` text for database and path errors, and missing or expired credentials log at `debug`. Concurrent lookups from many hook processes are supported by the auth DB's multiprocess WAL and migration-free current-schema open ([auth-db.md](auth-db.md)). Claude-invoked SCE hooks are admitted to this lookup and export one `sce.hook` span. Every other hook and this hidden header helper skip the lookup and stay Noop. See [cli-observability-contract.md](cli-observability-contract.md). +- `managed_telemetry::RuntimeTelemetry::resolve` reads the stored credential at CLI startup (never issuing one); a valid unexpired `token` is used only as the `Authorization: Bearer` credential for managed OTLP trace and log export; the `accessor` is never transmitted. Absent, expired, malformed, or unavailable rows select `NoopTelemetry`. Every Noop fallback after a lookup is classified (`credential_unavailable`, `credential_expired`, `lookup_timeout`, `database_error`, `path_resolution_error`, `exporter_init_failed`) and logged locally as `sce.telemetry.fallback`; failures log at `warn` (forced, so they reach the log file at the default `error` level, never stderr while `log_to_file=true`), with the redacted `TokenStorageError` text for database and path errors, and missing or expired credentials log at `debug`. Concurrent lookups from many hook processes are supported by the auth DB's multiprocess WAL and migration-free current-schema open ([auth-db.md](auth-db.md)). Both this lookup and the header helper's repository lookup are existing-DB reads: a missing `auth.db` yields no credential and creates no DB file or encryption key. Claude-invoked SCE hooks are admitted to this lookup and export one `sce.hook` span. Every other hook and this hidden header helper skip the lookup and stay Noop. See [cli-observability-contract.md](cli-observability-contract.md). ## Hidden header helper @@ -37,7 +37,7 @@ flowchart LR - Launch context (verified against Claude Code 2.1.284 with a real `claude -p` run): Claude runs the helper with no arguments, with its working directory set to the directory Claude was launched from (possibly a subdirectory), and without `CLAUDE_PROJECT_DIR`. Claude's documentation states that it does not pass `OTEL_*` variables to subprocesses. The helper therefore uses only the working directory, ignores `CLAUDE_PROJECT_DIR` (a value inherited from a parent session could name another project), and never requires an `OTEL_*` variable to be present. - Authorization: the helper walks up to the enclosing Git root, resolves the repository ID with the same precedence as Agent Trace storage, and prints headers only when that repository is bound to the identity of a valid credential that the current WorkOS session owns (see Repository bindings). An unbound repository, a repository bound to another user or workspace, a directory outside Git, or a repository without a resolvable identity gets no token. `.claude/settings.json`, SCE hook markers, and remote names are never treated as proof of ownership. - Destination tripwire: if `OTEL_EXPORTER_OTLP_ENDPOINT`, a per-signal `OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_ENDPOINT`, or `BETA_TRACING_ENDPOINT` is visible to the helper with a non-canonical value, the helper refuses. This catches visible misconfiguration only. It is not endpoint pinning (see Security boundaries). -- Repository resolution and the credential lookup together are bounded by `CREDENTIAL_LOOKUP_DEADLINE`. Resolution spawns one `git config --get` process, which is needed to compute the repository ID; there is no network call and no issuance. Every failure exits nonzero with the fixed stderr diagnostic `Telemetry credential is unavailable.` and empty stdout. The helper never initializes managed exporter telemetry. +- Repository resolution and the credential lookup together are bounded by `CREDENTIAL_LOOKUP_DEADLINE`. Resolution spawns one `git config --get` process, which is needed to compute the repository ID; there is no network call and no issuance. Every failure exits nonzero with the fixed stderr diagnostic `Telemetry credential is unavailable.` and empty stdout. That includes a missing `auth.db`, which the helper never creates or migrates, so concurrent helper calls on a never-logged-in machine cannot race to initialize it. The helper never initializes managed exporter telemetry. - It is omitted from `sce --help`, `sce auth --help`, and shell completions. ## Security boundaries From 1f44caf97542c429e068095aaa77062eb1384388 Mon Sep 17 00:00:00 2001 From: davidabram Date: Mon, 19 Oct 2026 15:31:14 +0200 Subject: [PATCH 10/17] runtime: Keep hooks and setup from prompting for auth DB credentials Claude hooks, managed telemetry startup, and `sce setup` could open or create the encrypted auth DB and trigger OS credential-store prompts (such as the macOS Keychain dialog) in unauthenticated or non-interactive contexts. Credential access is now explicit: `CredentialAccess::Interactive` is limited to `sce auth login`, `logout`, and `whoami`, while hooks, telemetry startup, `sce auth telemetry-headers`, and `sce sync` use `NonInteractive`, which only reads existing keys and, on macOS, disables Keychain user interaction. Inaccessible keys surface as a typed `CredentialAccessUnavailable` error. Credential-store access is serialized per process, and only a missing keyring entry may generate a new key. Add a non-secret telemetry readiness hint (`telemetry-ready.json`) that `sce auth login` writes after persisting a telemetry credential and that logout clears, so managed telemetry returns Noop (`not_configured`) without opening `auth.db` when no hint exists. `sce setup` now only creates the auth DB parent directory, no longer enrolls repositories, and is excluded from managed telemetry. Non-login paths open only an existing auth DB, and token storage caches only successful opens. Update regression coverage and the auth DB, telemetry credential, and observability documentation. Co-authored-by: SCE --- cli/Cargo.lock | 1 + cli/Cargo.toml | 1 + cli/src/app.rs | 4 + .../agent_trace_sync/control_plane.rs | 12 +- cli/src/services/auth_command/mod.rs | 187 +++++++++++++++--- cli/src/services/auth_db/lifecycle.rs | 11 +- .../services/auth_db/multiprocess_tests.rs | 52 ++++- cli/src/services/db/encryption_key.rs | 129 +++++++----- cli/src/services/db/mod.rs | 22 ++- cli/src/services/default_paths.rs | 15 +- cli/src/services/managed_telemetry.rs | 89 ++++++++- cli/src/services/mod.rs | 1 + cli/src/services/setup/command.rs | 100 ++-------- cli/src/services/sync/sync.rs | 5 + .../services/telemetry_credential_storage.rs | 29 +-- cli/src/services/telemetry_readiness.rs | 147 ++++++++++++++ cli/src/services/token_storage.rs | 86 +++++--- context/sce/auth-db.md | 16 +- context/sce/cli-observability-contract.md | 2 +- context/sce/telemetry-credential.md | 14 +- 20 files changed, 679 insertions(+), 244 deletions(-) create mode 100644 cli/src/services/telemetry_readiness.rs diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 2179dc7f2..ed2f23e9f 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -3833,6 +3833,7 @@ dependencies = [ "quint-connect", "rand 0.8.6", "reqwest", + "security-framework", "serde", "serde_json", "sha2 0.11.0", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 2cd7f2d45..6e5ce785d 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -66,6 +66,7 @@ zbus-secret-service-keyring-store = { version = "1.0.0", features = ["rt-tokio-c [target.'cfg(target_os = "macos")'.dependencies] apple-native-keyring-store = { version = "1", features = ["keychain"] } +security-framework = "3" [target.'cfg(target_os = "windows")'.dependencies] windows-native-keyring-store = "1" diff --git a/cli/src/app.rs b/cli/src/app.rs index 36996e35a..0d70cbae7 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -315,6 +315,7 @@ fn managed_span_kind(args: &[String], claude_hook_caller: bool) -> Option false, + (Some(first), _) if first == services::setup::NAME => false, (Some(first), Some("bash")) if first == services::bash_policy::NAME => claude_hook_caller, (Some(first), Some("telemetry-headers")) if first == services::auth_command::NAME => false, _ => return Some(ManagedSpanKind::Command), @@ -980,6 +981,7 @@ pub(crate) mod tests { let result = runtime.block_on(services::auth_command::telemetry_headers_with( Some(repo.path().to_path_buf()), false, + true, |_: &std::path::Path| Some("repository-a".to_string()), |_: String| stalled_keychain_lookup(), )); @@ -1131,6 +1133,8 @@ pub(crate) mod tests { assert_eq!(kind(&["sce", "hooks", "pi-mutation-scope"], caller), None); assert_eq!(kind(&["sce", "hooks"], caller), None); assert_eq!(kind(&["sce", "auth", "telemetry-headers"], caller), None); + assert_eq!(kind(&["sce", "setup"], caller), None); + assert_eq!(kind(&["sce", "setup", "--claude"], caller), None); assert_eq!(kind(&["sce", "auth", "login"], caller), command); assert_eq!(kind(&["sce", "config", "show"], caller), command); assert_eq!(kind(&["sce", "doctor"], caller), command); diff --git a/cli/src/services/agent_trace_sync/control_plane.rs b/cli/src/services/agent_trace_sync/control_plane.rs index 693e9ad93..491cff514 100644 --- a/cli/src/services/agent_trace_sync/control_plane.rs +++ b/cli/src/services/agent_trace_sync/control_plane.rs @@ -11,6 +11,7 @@ use crate::services::agent_trace_export::{ self, AgentTraceAgentTraceExportRow, AgentTraceMessageExportRow, AgentTracePartExportRow, }; use crate::services::auth::{self, AuthError, TokenResponse}; +use crate::services::db::encryption_key::CredentialAccess; use crate::services::resilience::{run_with_retry, RetryPolicy}; use crate::services::telemetry_credential_storage::{self, TelemetryCredential, TelemetryIdentity}; use crate::services::token_storage::{self, StoredTokens, TokenStorageError}; @@ -197,15 +198,17 @@ pub trait CredentialStore: Send + Sync + 'static { ) -> impl Future> + Send; } -pub struct SystemCredentialStore; +pub struct SystemCredentialStore { + access: CredentialAccess, +} impl CredentialStore for SystemCredentialStore { async fn load(&self) -> Result, ControlPlaneError> { - Ok(token_storage::load_tokens().await?) + Ok(token_storage::load_tokens(self.access).await?) } async fn save(&self, token: &TokenResponse) -> Result { - Ok(token_storage::save_tokens(token).await?) + Ok(token_storage::save_tokens(token, self.access).await?) } } @@ -300,13 +303,14 @@ impl AuthenticatedControlPlaneClient { base_url: impl Into, workos_api_base_url: impl Into, workos_client_id: impl Into, + access: CredentialAccess, ) -> Self { Self::with_credential_store( http, base_url, workos_api_base_url, workos_client_id, - SystemCredentialStore, + SystemCredentialStore { access }, ) } } diff --git a/cli/src/services/auth_command/mod.rs b/cli/src/services/auth_command/mod.rs index bd36da304..e84446462 100644 --- a/cli/src/services/auth_command/mod.rs +++ b/cli/src/services/auth_command/mod.rs @@ -12,6 +12,7 @@ use crate::services::agent_trace_sync::control_plane::{ }; use crate::services::auth::{self, AuthError, DeviceAuthFlowResult}; use crate::services::config; +use crate::services::db::encryption_key::CredentialAccess; use crate::services::error::{CliError, UserError}; use crate::services::managed_telemetry::claude_helper::{ repository_id_for, visible_destination_conflict, @@ -20,6 +21,7 @@ use crate::services::managed_telemetry::{run_with_deadline, CREDENTIAL_LOOKUP_DE use crate::services::output_format::OutputFormat; use crate::services::style::{label, prompt_label, prompt_value, value}; use crate::services::telemetry_credential_storage; +use crate::services::telemetry_readiness; use crate::services::token_storage::{self, StoredTokens, TokenStorageError}; pub const NAME: &str = "auth"; @@ -106,6 +108,7 @@ async fn run_telemetry_headers() -> Result { telemetry_headers_with( std::env::current_dir().ok(), visible_destination_conflict(|key| std::env::var(key).ok()), + telemetry_readiness::is_ready(current_unix_seconds()), repository_id_for, telemetry_credential_storage::load_repository_telemetry_credential, ) @@ -115,6 +118,7 @@ async fn run_telemetry_headers() -> Result { pub(crate) async fn telemetry_headers_with( working_dir: Option, destination_conflict: bool, + ready: bool, resolve_repository: R, lookup: L, ) -> Result @@ -129,7 +133,7 @@ where > + Send + 'static, { - let Some(working_dir) = working_dir.filter(|_| !destination_conflict) else { + let Some(working_dir) = working_dir.filter(|_| ready && !destination_conflict) else { return Err(telemetry_headers_unavailable()); }; let credential = run_with_deadline(CREDENTIAL_LOOKUP_DEADLINE, async move { @@ -171,8 +175,9 @@ pub async fn run_login(format: AuthFormat) -> Result { auth_config.control_plane_base_url.value.unwrap_or_default(), auth::WORKOS_DEFAULT_BASE_URL, client_id.clone(), + CredentialAccess::Interactive, ); - let stored_tokens = token_storage::load_tokens() + let stored_tokens = token_storage::load_tokens(CredentialAccess::Interactive) .await .map_err(auth_storage_error)?; @@ -199,21 +204,25 @@ pub async fn run_login(format: AuthFormat) -> Result { pub async fn run_logout(format: AuthFormat) -> Result { run_logout_with( format, + telemetry_readiness::clear, token_storage::delete_tokens(), telemetry_credential_storage::delete_telemetry_credential(), ) .await } -async fn run_logout_with( +async fn run_logout_with( format: AuthFormat, + clear_readiness: C, delete_workos: W, delete_telemetry: T, ) -> Result where + C: FnOnce() -> Result<()>, W: Future>, T: Future>, { + clear_readiness().map_err(unexpected_auth_command_error)?; let workos_result = delete_workos.await; let telemetry_result = delete_telemetry.await; let workos_deleted = workos_result.map_err(auth_storage_error)?; @@ -223,7 +232,7 @@ where } pub async fn run_whoami(format: AuthFormat) -> Result { - if token_storage::load_tokens() + if token_storage::load_tokens(CredentialAccess::Interactive) .await .map_err(auth_storage_error)? .is_none() @@ -241,6 +250,7 @@ pub async fn run_whoami(format: AuthFormat) -> Result { auth_config.control_plane_base_url.value.unwrap_or_default(), auth::WORKOS_DEFAULT_BASE_URL, auth_config.workos_client_id.value.unwrap_or_default(), + CredentialAccess::Interactive, ); let profile = client .me() @@ -263,7 +273,7 @@ async fn maybe_renew_stored_credentials( ) .await { - Ok(token) => token_storage::save_tokens(&token) + Ok(token) => token_storage::save_tokens(&token, CredentialAccess::Interactive) .await .map_err(auth_storage_error) .map(Some), @@ -299,21 +309,25 @@ where async fn save_new_session_tokens(token: &auth::TokenResponse) -> Result { replace_session_with( + telemetry_readiness::clear, telemetry_credential_storage::delete_telemetry_credential(), - || token_storage::save_tokens(token), + || token_storage::save_tokens(token, CredentialAccess::Interactive), ) .await } -async fn replace_session_with( +async fn replace_session_with( + clear_readiness: C, invalidate_telemetry: D, save_session: S, ) -> Result where + C: FnOnce() -> Result<()>, D: Future>, S: FnOnce() -> SF, SF: Future>, { + clear_readiness().map_err(unexpected_auth_command_error)?; invalidate_telemetry.await.map_err(auth_storage_error)?; save_session().await.map_err(auth_storage_error) } @@ -321,13 +335,40 @@ where async fn provision_telemetry_credential( client: &AuthenticatedControlPlaneClient, ) -> TelemetryProvisioningResult { - match client.issue_and_save_telemetry_credential().await { - Ok(_) => { - enroll_current_repository_if_unbound().await; - TelemetryProvisioningResult::Issued - } - Err(_) => TelemetryProvisioningResult::Unavailable, + let provisioned = client.issue_and_save_telemetry_credential().await; + let result = record_telemetry_readiness( + provisioned.as_ref().ok(), + telemetry_readiness::mark_ready, + telemetry_readiness::clear, + ); + if result == TelemetryProvisioningResult::Issued { + enroll_current_repository_if_unbound().await; + } + result +} + +fn record_telemetry_readiness( + provisioned: Option<&telemetry_credential_storage::TelemetryCredential>, + mark_ready: M, + clear: C, +) -> TelemetryProvisioningResult +where + M: FnOnce(u64) -> Result<()>, + C: FnOnce() -> Result<()>, +{ + let recorded = provisioned + .is_some_and(|credential| mark_ready(credential.expires_at_unix_seconds).is_ok()); + if recorded { + return TelemetryProvisioningResult::Issued; } + let _ = clear(); + TelemetryProvisioningResult::Unavailable +} + +fn current_unix_seconds() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) } async fn enroll_current_repository_if_unbound() { @@ -704,6 +745,10 @@ mod tests { async fn new_session_invalidates_previous_telemetry_credential_before_saving() { let order = std::sync::Mutex::new(Vec::new()); let saved = replace_session_with( + || { + order.lock().unwrap().push("clear"); + Ok(()) + }, async { order.lock().unwrap().push("invalidate"); Ok(true) @@ -716,13 +761,17 @@ mod tests { .await .expect("replacement should succeed"); assert_eq!(saved, stored_tokens()); - assert_eq!(*order.lock().unwrap(), ["invalidate", "save"]); + assert_eq!(*order.lock().unwrap(), ["clear", "invalidate", "save"]); let save_called = std::sync::atomic::AtomicBool::new(false); - let failed = replace_session_with(async { Err(storage_failure()) }, || async { - save_called.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(stored_tokens()) - }) + let failed = replace_session_with( + || Ok(()), + async { Err(storage_failure()) }, + || async { + save_called.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(stored_tokens()) + }, + ) .await; assert!(failed.is_err()); assert!(!save_called.load(std::sync::atomic::Ordering::SeqCst)); @@ -739,6 +788,17 @@ mod tests { }; let working_dir = Some(PathBuf::from("/workspace/repo")); + let error = telemetry_headers_with( + working_dir.clone(), + false, + false, + |_: &Path| Some("repo-a".to_string()), + lookup(std::sync::Arc::clone(&lookup_called)), + ) + .await + .expect_err("helper must fail closed without telemetry readiness"); + assert_eq!(error.to_string(), TELEMETRY_HEADERS_UNAVAILABLE); + for (dir, conflict, repository) in [ (working_dir.clone(), true, Some("repo-a")), (None, false, Some("repo-a")), @@ -747,6 +807,7 @@ mod tests { let error = telemetry_headers_with( dir, conflict, + true, move |_: &Path| repository.map(str::to_string), lookup(std::sync::Arc::clone(&lookup_called)), ) @@ -759,6 +820,7 @@ mod tests { let headers = telemetry_headers_with( working_dir, false, + true, |dir: &Path| (dir == Path::new("/workspace/repo")).then(|| "repo-a".to_string()), |repository_id: String| async move { assert_eq!(repository_id, "repo-a"); @@ -924,9 +986,11 @@ mod tests { #[tokio::test] async fn telemetry_credential_logout_reports_removal_when_only_telemetry_deleted() { - let output = run_logout_with(AuthFormat::Json, async { Ok(false) }, async { Ok(true) }) - .await - .expect("logout should succeed"); + let output = run_logout_with(AuthFormat::Json, || Ok(()), async { Ok(false) }, async { + Ok(true) + }) + .await + .expect("logout should succeed"); assert!(output.contains("\"credentials_removed\": true")); } @@ -935,6 +999,7 @@ mod tests { let workos_called = AtomicBool::new(false); let result = run_logout_with( AuthFormat::Json, + || Ok(()), async { workos_called.store(true, Ordering::SeqCst); Ok(true) @@ -949,12 +1014,84 @@ mod tests { #[tokio::test] async fn telemetry_credential_logout_deletes_telemetry_on_workos_failure() { let telemetry_called = AtomicBool::new(false); - let result = run_logout_with(AuthFormat::Json, async { Err(storage_failure()) }, async { - telemetry_called.store(true, Ordering::SeqCst); - Ok(true) - }) + let result = run_logout_with( + AuthFormat::Json, + || Ok(()), + async { Err(storage_failure()) }, + async { + telemetry_called.store(true, Ordering::SeqCst); + Ok(true) + }, + ) .await; assert!(result.is_err()); assert!(telemetry_called.load(Ordering::SeqCst)); } + + #[tokio::test] + async fn logout_clears_telemetry_readiness_even_when_credential_deletion_fails() { + let cleared = AtomicBool::new(false); + let result = run_logout_with( + AuthFormat::Json, + || { + cleared.store(true, Ordering::SeqCst); + Ok(()) + }, + async { Err(storage_failure()) }, + async { Err(storage_failure()) }, + ) + .await; + assert!(result.is_err()); + assert!(cleared.load(Ordering::SeqCst)); + } + + #[test] + fn telemetry_readiness_is_recorded_only_after_credential_persistence() { + let credential = telemetry_credential("tok-ready"); + let marked = std::cell::Cell::new(None); + let cleared = std::cell::Cell::new(false); + + let issued = record_telemetry_readiness( + Some(&credential), + |expires_at| { + marked.set(Some(expires_at)); + Ok(()) + }, + || { + cleared.set(true); + Ok(()) + }, + ); + assert_eq!(issued, TelemetryProvisioningResult::Issued); + assert_eq!(marked.get(), Some(credential.expires_at_unix_seconds)); + assert!(!cleared.get()); + + marked.set(None); + let failed = record_telemetry_readiness( + None, + |expires_at| { + marked.set(Some(expires_at)); + Ok(()) + }, + || { + cleared.set(true); + Ok(()) + }, + ); + assert_eq!(failed, TelemetryProvisioningResult::Unavailable); + assert_eq!(marked.get(), None); + assert!(cleared.get()); + + cleared.set(false); + let unrecorded = record_telemetry_readiness( + Some(&credential), + |_| Err(anyhow!("read-only state directory")), + || { + cleared.set(true); + Ok(()) + }, + ); + assert_eq!(unrecorded, TelemetryProvisioningResult::Unavailable); + assert!(cleared.get()); + } } diff --git a/cli/src/services/auth_db/lifecycle.rs b/cli/src/services/auth_db/lifecycle.rs index 79a5e3160..61826dd25 100644 --- a/cli/src/services/auth_db/lifecycle.rs +++ b/cli/src/services/auth_db/lifecycle.rs @@ -8,7 +8,7 @@ use crate::services::lifecycle::{ HealthProblemKind, HealthSeverity, ServiceLifecycle, SetupOutcome, }; -use super::{AuthDb, AuthDbSpec}; +use super::AuthDbSpec; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct AuthDbLifecycle; @@ -49,13 +49,8 @@ impl ServiceLifecycle for AuthDbLifecycle { } async fn setup(&self, _ctx: &C) -> Result { - if auth_db_path()?.exists() { - return Ok(SetupOutcome::default()); - } - - AuthDb::new() - .await - .context("Auth DB lifecycle setup failed while initializing auth DB")?; + bootstrap_auth_db_parent() + .context("Auth DB lifecycle setup failed while preparing the auth DB directory")?; Ok(SetupOutcome::default()) } } diff --git a/cli/src/services/auth_db/multiprocess_tests.rs b/cli/src/services/auth_db/multiprocess_tests.rs index 81f72d03a..c9aacfba2 100644 --- a/cli/src/services/auth_db/multiprocess_tests.rs +++ b/cli/src/services/auth_db/multiprocess_tests.rs @@ -354,6 +354,39 @@ fn current_schema_reader_performs_no_migration_write_under_held_write_lock() { sandbox.run("verify-schema"); } +struct NoRepositoryContext; + +impl crate::app::HasRepoRoot for NoRepositoryContext { + fn repo_root(&self) -> Option<&Path> { + None + } +} + +fn run_setup_lifecycle_child(sandbox: &Sandbox, role: &str) { + wait_success( + role, + sandbox + .command(role) + .env(AUTH_DB_ENCRYPTION_KEY_ENV, "") + .spawn() + .expect("spawn setup child"), + ); +} + +#[test] +fn setup_lifecycle_on_fresh_state_creates_no_auth_db_and_resolves_no_key() { + let _fan_out = serialize_fan_out(); + run_setup_lifecycle_child(&Sandbox::new(), "setup-fresh"); +} + +#[test] +fn setup_lifecycle_leaves_existing_auth_db_unchanged_and_resolves_no_key() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + run_setup_lifecycle_child(&sandbox, "setup-existing"); +} + #[test] fn telemetry_lookup_never_creates_a_missing_auth_db() { let _fan_out = serialize_fan_out(); @@ -811,6 +844,18 @@ async fn auth_db_child_process() { ); assert!(!auth_db_path().expect("auth DB path").exists()); } + "setup-fresh" | "setup-existing" => { + use crate::services::lifecycle::ServiceLifecycle; + let path = auth_db_path().expect("auth DB path"); + let before = std::fs::read(&path).ok(); + assert_eq!(before.is_some(), role == "setup-existing"); + super::lifecycle::AuthDbLifecycle + .setup(&NoRepositoryContext) + .await + .expect("setup needs no auth DB key"); + assert!(path.parent().expect("auth DB parent").is_dir()); + assert_eq!(std::fs::read(&path).ok(), before); + } "create-schemaless" => { let db = EncryptedTursoDb::::new() .await @@ -1000,6 +1045,7 @@ async fn seed() { }) .await .expect("telemetry credential seeds"); + crate::services::telemetry_readiness::mark_ready(FAR_FUTURE).expect("readiness seeds"); } async fn run_hook_child() { @@ -1045,7 +1091,11 @@ fn writer_rows() -> Vec { async fn open_legacy_auth_db() -> Result { let path = auth_db_path()?; - let hexkey = get_or_create_encryption_key(&path, AuthDbSpec::db_name())?; + let hexkey = get_or_create_encryption_key( + &path, + AuthDbSpec::db_name(), + crate::services::db::encryption_key::CredentialAccess::Interactive, + )?; let db = turso::Builder::new_local(path.to_str().expect("UTF-8 auth DB path")) .experimental_encryption(true) .with_encryption(turso::EncryptionOpts { diff --git a/cli/src/services/db/encryption_key.rs b/cli/src/services/db/encryption_key.rs index ef9cba9e1..e8a0fbd68 100644 --- a/cli/src/services/db/encryption_key.rs +++ b/cli/src/services/db/encryption_key.rs @@ -1,19 +1,8 @@ -//! Encryption key management backed by an env-secret override or the OS credential store. -//! -//! Provides a single entry point to get-or-create a 64-character hex -//! encryption key sourced from `SCE_AUTH_DB_ENCRYPTION_KEY` when set, or -//! stored in the platform-native credential store (macOS Keychain, -//! Linux Secret Service via zbus, Windows Credential Store) otherwise. -//! -//! On first use for a given database name (when the database file does -//! not yet exist), a random 32-byte key is generated, hex-encoded, -//! persisted in the credential store, and returned. Subsequent calls -//! read the key from the credential store. - +use std::fmt; use std::path::Path; use std::sync::{ atomic::{AtomicBool, Ordering}, - OnceLock, + Mutex, OnceLock, PoisonError, }; use std::thread; @@ -21,9 +10,34 @@ use anyhow::{Context, Result}; use keyring_core::Entry; use sha2::{Digest, Sha256}; -/// Environment variable that overrides keyring-backed auth DB encryption. pub const AUTH_DB_ENCRYPTION_KEY_ENV: &str = "SCE_AUTH_DB_ENCRYPTION_KEY"; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CredentialAccess { + Interactive, + NonInteractive, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CredentialAccessUnavailable { + db_name: String, +} + +impl fmt::Display for CredentialAccessUnavailable { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "encryption key for '{}' is not accessible without user interaction. \ + Try: run 'sce auth login' to authorize credential store access.", + self.db_name + ) + } +} + +impl std::error::Error for CredentialAccessUnavailable {} + +static CREDENTIAL_STORE_ACCESS: Mutex<()> = Mutex::new(()); + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum CredentialStorePlatform { #[cfg(target_os = "linux")] @@ -110,34 +124,42 @@ fn register_default_store() -> Result<()> { Ok(()) } -/// Retrieve or create the encryption key for the named database. -/// -/// The key is stored in the platform-native credential store under the -/// service name `"sce"` with the database name as the user/account -/// identifier. -/// -/// # Arguments -/// * `db_path` — Canonical path to the database file. Used to decide -/// whether this is a first-use (generate new key) or a subsequent -/// access (read existing key). -/// * `db_name` — Logical database name used as the credential store -/// username (e.g. `"auth_db"`, `"agent_trace_db"`). -/// -/// # Returns -/// A 64-character lowercase hex string that is the encryption key. -/// -/// # Errors -/// - Returns an error if the credential store cannot be initialised on -/// the current platform. -/// - Returns an error if the database file exists but the keyring entry -/// is missing (e.g. keyring was cleared on Linux). -/// - Returns an error if key generation or credential store I/O fails. -pub fn get_or_create_encryption_key(db_path: &Path, db_name: &str) -> Result { - get_or_create_encryption_key_with_keyring_resolver( - db_path, - db_name, - get_or_create_keyring_encryption_key, - ) +pub fn get_or_create_encryption_key( + db_path: &Path, + db_name: &str, + access: CredentialAccess, +) -> Result { + match access { + CredentialAccess::Interactive => get_or_create_encryption_key_with_keyring_resolver( + db_path, + db_name, + get_or_create_keyring_encryption_key, + ), + CredentialAccess::NonInteractive => { + get_or_create_encryption_key_with_keyring_resolver(db_path, db_name, |_, db_name| { + read_keyring_encryption_key_without_interaction(db_name) + }) + } + } +} + +fn read_keyring_encryption_key_without_interaction(db_name: &str) -> Result { + let unavailable = || { + anyhow::Error::new(CredentialAccessUnavailable { + db_name: db_name.to_owned(), + }) + }; + ensure_default_store().map_err(|_| unavailable())?; + let _serialized = CREDENTIAL_STORE_ACCESS + .lock() + .unwrap_or_else(PoisonError::into_inner); + #[cfg(target_os = "macos")] + let _interaction_disabled = + security_framework::os::macos::keychain::SecKeychain::disable_user_interaction() + .map_err(|_| unavailable())?; + Entry::new("sce", db_name) + .and_then(|entry| entry.get_password()) + .map_err(|_| unavailable()) } fn get_or_create_encryption_key_with_keyring_resolver( @@ -166,13 +188,23 @@ fn get_or_create_keyring_encryption_key(db_path: &Path, db_name: &str) -> Result ) })?; - // Try to retrieve an existing password from the credential store. - if let Ok(password) = entry.get_password() { - return Ok(password); + let _serialized = CREDENTIAL_STORE_ACCESS + .lock() + .unwrap_or_else(PoisonError::into_inner); + + match entry.get_password() { + Ok(password) => return Ok(password), + Err(keyring_core::Error::NoEntry) => {} + Err(error) => { + return Err(anyhow::Error::new(error)).with_context(|| { + format!( + "failed to read encryption key for '{db_name}' from credential store. {}", + credential_store_unavailable_message(current_credential_store_platform()) + ) + }); + } } - // No existing key was found. If the database file does not exist, - // this is a first-use scenario: generate and store a new key. if !db_path.exists() { let hex_key = generate_key(); @@ -186,9 +218,6 @@ fn get_or_create_keyring_encryption_key(db_path: &Path, db_name: &str) -> Result return Ok(hex_key); } - // The database file exists but the keyring entry is missing. - // This can happen when credentials were removed from the OS keyring. - // Provide clear remediation. anyhow::bail!( "encryption key for '{db_name}' not found in credential store.\n\ The database file exists at '{}' but no matching credential was found \ diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index 9f48718f4..e05150cba 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -876,11 +876,15 @@ fn apply_busy_timeout( .map_err(|e| anyhow::anyhow!("failed to set {db_name} database busy timeout: {e}")) } -async fn resolve_encryption_key(db_path: &Path, db_name: &str) -> Result { +async fn resolve_encryption_key( + db_path: &Path, + db_name: &str, + access: encryption_key::CredentialAccess, +) -> Result { let key_path = db_path.to_path_buf(); let key_name = db_name.to_owned(); tokio::task::spawn_blocking(move || { - encryption_key::get_or_create_encryption_key(&key_path, &key_name) + encryption_key::get_or_create_encryption_key(&key_path, &key_name, access) }) .await .context("encryption-key resolution worker failed")? @@ -1364,19 +1368,27 @@ impl EncryptedTursoDb { pub async fn new() -> Result { let db_name = M::db_name(); let db_path = M::db_path().with_context(|| format!("failed to resolve {db_name} path"))?; - let encryption_key = resolve_encryption_key(&db_path, db_name).await?; + let encryption_key = resolve_encryption_key( + &db_path, + db_name, + encryption_key::CredentialAccess::Interactive, + ) + .await?; ensure_db_parent_dir(db_name, &db_path)?; let open_lock = open_lock::open_lock_file(&db_path, true)? .ok_or_else(|| anyhow::anyhow!("failed to create {db_name} database file"))?; Box::pin(Self::open_locked(db_path, encryption_key, open_lock)).await } - pub async fn open_existing_at(db_path: impl AsRef) -> Result> { + pub async fn open_existing_at( + db_path: impl AsRef, + access: encryption_key::CredentialAccess, + ) -> Result> { let db_path = db_path.as_ref().to_path_buf(); let Some(open_lock) = open_lock::open_lock_file(&db_path, false)? else { return Ok(None); }; - let encryption_key = resolve_encryption_key(&db_path, M::db_name()).await?; + let encryption_key = resolve_encryption_key(&db_path, M::db_name(), access).await?; Box::pin(Self::open_locked(db_path, encryption_key, open_lock)) .await .map(Some) diff --git a/cli/src/services/default_paths.rs b/cli/src/services/default_paths.rs index 86ec44208..5f9a3eae3 100644 --- a/cli/src/services/default_paths.rs +++ b/cli/src/services/default_paths.rs @@ -250,21 +250,14 @@ pub fn local_db_path() -> anyhow::Result { Ok(sce_state_dir()?.join("local.db")) } -/// Returns the canonical path to the encrypted auth Turso database file. -/// -/// The path is `/sce/auth.db`, where `state_root` comes -/// from the shared default-path catalog (`XDG_STATE_HOME` or platform -/// equivalent). pub fn auth_db_path() -> anyhow::Result { Ok(sce_state_dir()?.join("auth.db")) } -/// Returns the canonical repository-scoped Agent Trace Turso database file path. -/// -/// The path is `/sce/repos//agent-trace.db`, where -/// `state_root` comes from the shared default-path catalog (`XDG_STATE_HOME` or -/// platform equivalent) and `repository_id` is the stable repository identity -/// hash from `services::repository_identity`. +pub fn telemetry_readiness_path() -> anyhow::Result { + Ok(sce_state_dir()?.join("telemetry-ready.json")) +} + pub fn agent_trace_db_path_for_repository(repository_id: &str) -> anyhow::Result { let state_root = resolve_sce_default_locations()? .roots() diff --git a/cli/src/services/managed_telemetry.rs b/cli/src/services/managed_telemetry.rs index 25f0a9a9d..839639413 100644 --- a/cli/src/services/managed_telemetry.rs +++ b/cli/src/services/managed_telemetry.rs @@ -13,6 +13,7 @@ use trace_export::{ManagedSpanKind, ManagedTraceTelemetry}; use crate::services::telemetry_credential_storage::{ load_telemetry_credential, TelemetryCredential, }; +use crate::services::telemetry_readiness; use crate::services::token_storage::TokenStorageError; pub const MANAGED_OTLP_ENDPOINT: &str = "https://otel.crocoderlab.dev"; @@ -115,6 +116,8 @@ where #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum TelemetryFallbackReason { + NotConfigured, + CredentialInaccessible, CredentialUnavailable, CredentialExpired, LookupTimeout, @@ -126,6 +129,8 @@ pub(crate) enum TelemetryFallbackReason { impl TelemetryFallbackReason { pub(crate) const fn as_str(self) -> &'static str { match self { + Self::NotConfigured => "not_configured", + Self::CredentialInaccessible => "credential_inaccessible", Self::CredentialUnavailable => "credential_unavailable", Self::CredentialExpired => "credential_expired", Self::LookupTimeout => "lookup_timeout", @@ -136,7 +141,13 @@ impl TelemetryFallbackReason { } const fn is_failure(self) -> bool { - !matches!(self, Self::CredentialUnavailable | Self::CredentialExpired) + !matches!( + self, + Self::NotConfigured + | Self::CredentialInaccessible + | Self::CredentialUnavailable + | Self::CredentialExpired + ) } } @@ -156,6 +167,9 @@ impl TelemetryFallback { fn from_storage_error(error: &TokenStorageError) -> Self { let reason = match error { + TokenStorageError::CredentialAccessUnavailable => { + return Self::new(TelemetryFallbackReason::CredentialInaccessible); + } TokenStorageError::Database(_) => TelemetryFallbackReason::DatabaseError, TokenStorageError::PathResolution(_) => TelemetryFallbackReason::PathResolutionError, }; @@ -222,14 +236,39 @@ impl RuntimeTelemetry { let Some(span_kind) = span_kind else { return Self::Noop(NoopTelemetry); }; - let (telemetry, fallback) = - Self::resolve_with(load_telemetry_credential(), span_kind).await; + let (telemetry, fallback) = Self::resolve_when_ready( + telemetry_readiness::is_ready(now_unix_seconds()), + load_telemetry_credential(), + span_kind, + ) + .await; if let Some(fallback) = fallback { fallback.log(logger); } telemetry } + pub(crate) async fn resolve_when_ready( + ready: bool, + lookup: Fut, + span_kind: ManagedSpanKind, + ) -> (Self, Option) + where + Fut: Future, TokenStorageError>> + + Send + + 'static, + { + if !ready { + return ( + Self::Noop(NoopTelemetry), + Some(TelemetryFallback::new( + TelemetryFallbackReason::NotConfigured, + )), + ); + } + Self::resolve_with(lookup, span_kind).await + } + pub(crate) async fn resolve_with( lookup: Fut, span_kind: ManagedSpanKind, @@ -351,6 +390,50 @@ mod tests { } } + #[tokio::test] + async fn claude_hook_without_readiness_never_looks_up_credentials() { + let looked_up = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = looked_up.clone(); + + let (telemetry, fallback) = RuntimeTelemetry::resolve_when_ready( + false, + async move { + flag.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(None) + }, + ManagedSpanKind::ClaudeHook, + ) + .await; + + assert!(matches!(telemetry, RuntimeTelemetry::Noop(_))); + let fallback = fallback.expect("disabled telemetry is diagnosed"); + assert_eq!(fallback.reason, TelemetryFallbackReason::NotConfigured); + assert!(!fallback.reason.is_failure()); + assert!(!looked_up.load(std::sync::atomic::Ordering::SeqCst)); + } + + #[tokio::test] + async fn claude_hook_with_inaccessible_credential_store_skips_quietly() { + let (telemetry, fallback) = RuntimeTelemetry::resolve_when_ready( + true, + async { Err(TokenStorageError::CredentialAccessUnavailable) }, + ManagedSpanKind::ClaudeHook, + ) + .await; + + assert!(matches!(telemetry, RuntimeTelemetry::Noop(_))); + let fallback = fallback.expect("inaccessible credentials are diagnosed"); + assert_eq!( + fallback.reason, + TelemetryFallbackReason::CredentialInaccessible + ); + assert!(!fallback.reason.is_failure()); + let output = telemetry + .with_default_subscriber(&mut || async { Ok("hook-output".to_string()) }) + .await; + assert_eq!(output.expect("hook result unaffected"), "hook-output"); + } + #[tokio::test] async fn claude_hook_credential_failure_or_expiry_resolves_noop_with_reason() { let (failed, fallback) = RuntimeTelemetry::resolve_with( diff --git a/cli/src/services/mod.rs b/cli/src/services/mod.rs index 2612c646b..2f484e8e0 100644 --- a/cli/src/services/mod.rs +++ b/cli/src/services/mod.rs @@ -37,5 +37,6 @@ pub mod structured_patch; pub mod style; pub mod sync; pub mod telemetry_credential_storage; +pub mod telemetry_readiness; pub mod token_storage; pub mod version; diff --git a/cli/src/services/setup/command.rs b/cli/src/services/setup/command.rs index ca2c6ebd7..f05f5a9fd 100644 --- a/cli/src/services/setup/command.rs +++ b/cli/src/services/setup/command.rs @@ -1,5 +1,3 @@ -use std::future::Future; - use anyhow::Context; use crate::app::ContextWithRepoRoot; @@ -7,12 +5,6 @@ use crate::services::error::{CliError, UserError}; use crate::services::lifecycle::{ lifecycle_providers, RequiredHookInstallStatus, RequiredHooksInstallOutcome, }; -use crate::services::managed_telemetry::claude_helper::repository_id_for; -use crate::services::managed_telemetry::{run_with_deadline, CREDENTIAL_LOOKUP_DEADLINE}; -use crate::services::telemetry_credential_storage::{ - bind_repository_telemetry, RepositoryBindingMode, -}; -use crate::services::token_storage::TokenStorageError; use crate::services::{config, setup}; pub struct SetupCommand { @@ -108,7 +100,7 @@ impl SetupCommand { sections.push(setup_message); if claude_telemetry_selected { - sections.push(claude_telemetry_report(&repository_root).await); + sections.push(claude_telemetry_report()); } } @@ -124,31 +116,17 @@ fn dispatch_targets_claude(dispatch: Option<&setup::SetupDispatch>) -> bool { ) } -async fn claude_telemetry_report(repository_root: &std::path::Path) -> String { - let repository_id = repository_id_for(repository_root); +fn claude_telemetry_report() -> String { render_claude_telemetry_section( setup::claude_telemetry::resolve_claude_settings_path().as_deref(), &setup::claude_telemetry::managed_claude_settings_path(), - async move { - match repository_id { - Some(repository_id) => { - bind_repository_telemetry(repository_id, RepositoryBindingMode::Enroll).await - } - None => Ok(false), - } - }, ) - .await } -async fn render_claude_telemetry_section( +fn render_claude_telemetry_section( settings_path: Option<&std::path::Path>, managed_settings_path: &std::path::Path, - repository_enrollment: Fut, -) -> String -where - Fut: Future> + Send + 'static, -{ +) -> String { use setup::claude_telemetry::{ format_claude_telemetry_report, install_claude_telemetry, ClaudeTelemetryInstallOutcome, }; @@ -158,11 +136,7 @@ where .unwrap_or(ClaudeTelemetryInstallOutcome::SkippedInvalid), None => ClaudeTelemetryInstallOutcome::SkippedInvalid, }; - let authentication_pending = installed == ClaudeTelemetryInstallOutcome::Configured - && !run_with_deadline(CREDENTIAL_LOOKUP_DEADLINE, repository_enrollment) - .await - .and_then(Result::ok) - .unwrap_or(false); + let authentication_pending = installed == ClaudeTelemetryInstallOutcome::Configured; format_claude_telemetry_report(&installed, authentication_pending) } @@ -275,80 +249,36 @@ mod tests { fn run_claude_telemetry_stage(dispatch: Option<&SetupDispatch>, root: &Path) -> Option { dispatch_targets_claude(dispatch).then(|| { - block_on_current_thread(render_claude_telemetry_section( + render_claude_telemetry_section( Some(&root.join("settings.json")), &root.join("managed.json"), - std::future::ready(Ok(false)), - )) + ) }) } - fn block_on_current_thread(future: F) -> F::Output { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap() - .block_on(future) - } - - #[tokio::test] - async fn setup_claude_telemetry_skips_repository_enrollment_when_install_is_skipped() { + #[test] + fn setup_claude_telemetry_reports_skip_without_repository_enrollment() { let dir = tempfile::tempdir().unwrap(); let settings = dir.path().join("settings.json"); fs::write(&settings, r#"{"otelHeadersHelper":"my-helper"}"#).unwrap(); - let looked_up = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let flag = looked_up.clone(); - let report = render_claude_telemetry_section( - Some(&settings), - &dir.path().join("managed.json"), - async move { - flag.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(false) - }, - ) - .await; + let report = + render_claude_telemetry_section(Some(&settings), &dir.path().join("managed.json")); assert!(report.contains("skipped")); - assert!(!looked_up.load(std::sync::atomic::Ordering::SeqCst)); + assert!(!report.contains("authentication pending")); } - #[tokio::test] - async fn setup_claude_telemetry_reports_pending_when_repository_enrollment_stalls() { + #[test] + fn setup_claude_telemetry_defers_repository_enrollment_to_auth_login() { let dir = tempfile::tempdir().unwrap(); - let started = std::time::Instant::now(); let report = render_claude_telemetry_section( Some(&dir.path().join("settings.json")), &dir.path().join("managed.json"), - std::future::pending(), - ) - .await; + ); assert!(report.contains("authentication pending")); - assert!(started.elapsed() < std::time::Duration::from_secs(5)); - } - - #[tokio::test] - async fn setup_claude_telemetry_reports_ready_only_when_repository_is_enrolled() { - for (enrolled, pending) in [(Ok(true), false), (Ok(false), true), (Err(storage()), true)] { - let dir = tempfile::tempdir().unwrap(); - let report = render_claude_telemetry_section( - Some(&dir.path().join("settings.json")), - &dir.path().join("managed.json"), - std::future::ready(enrolled), - ) - .await; - assert_eq!( - report.contains("authentication pending"), - pending, - "{report}" - ); - } - } - - fn storage() -> TokenStorageError { - TokenStorageError::Database("unavailable".to_string()) } fn assert_installed(root: &Path) { diff --git a/cli/src/services/sync/sync.rs b/cli/src/services/sync/sync.rs index dd41bee3b..104455c3a 100644 --- a/cli/src/services/sync/sync.rs +++ b/cli/src/services/sync/sync.rs @@ -25,6 +25,7 @@ use crate::services::agent_trace_sync::{ }; use crate::services::auth; use crate::services::config; +use crate::services::db::encryption_key::CredentialAccess; use crate::services::sync::progress::ProgressReporter; #[derive(Clone, Debug, PartialEq, Eq)] @@ -189,6 +190,7 @@ where auth_config.control_plane_base_url.value.unwrap_or_default(), auth::WORKOS_DEFAULT_BASE_URL, auth_config.workos_client_id.value.unwrap_or_default(), + CredentialAccess::NonInteractive, ); let result = run_sync_against_without_progress( @@ -547,6 +549,7 @@ mod tests { }; use crate::services::agent_trace_sync::StreamSyncError; use crate::services::auth::TokenResponse; + use crate::services::db::encryption_key::CredentialAccess; use crate::services::sync::progress::NoopProgressReporter; use crate::services::token_storage::StoredTokens; @@ -627,6 +630,7 @@ mod tests { "http://127.0.0.1:1", "http://127.0.0.1:1", "client", + CredentialAccess::NonInteractive, ); let mut reporter = NoopProgressReporter; let progress = Rc::new(RefCell::new(&mut reporter)); @@ -704,6 +708,7 @@ mod tests { "http://127.0.0.1:1", "http://127.0.0.1:1", "client", + CredentialAccess::NonInteractive, ) } diff --git a/cli/src/services/telemetry_credential_storage.rs b/cli/src/services/telemetry_credential_storage.rs index 6453bc4ea..557cdd04d 100644 --- a/cli/src/services/telemetry_credential_storage.rs +++ b/cli/src/services/telemetry_credential_storage.rs @@ -5,8 +5,11 @@ use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine; use serde::Deserialize; +use crate::services::db::encryption_key::CredentialAccess; use crate::services::db::{DbSpec, EncryptedTursoDb}; -use crate::services::token_storage::{get_auth_db, get_existing_auth_db, TokenStorageError}; +use crate::services::token_storage::{ + open_existing_auth_db, open_or_create_auth_db, TokenStorageError, +}; const TELEMETRY_CREDENTIAL_ROW_ID: i64 = 1; const WORKOS_SESSION_ROW_ID: i64 = 1; @@ -75,6 +78,7 @@ impl TelemetryCredential { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RepositoryBindingMode { + #[cfg_attr(not(test), expect(dead_code))] Enroll, EnrollIfUnbound, } @@ -82,11 +86,11 @@ pub enum RepositoryBindingMode { pub async fn save_telemetry_credential( credential: &TelemetryCredential, ) -> Result<(), TokenStorageError> { - save_with(get_auth_db().await?, credential).await + save_with(open_or_create_auth_db().await?, credential).await } pub async fn load_telemetry_credential() -> Result, TokenStorageError> { - let Some(db) = get_existing_auth_db().await? else { + let Some(db) = open_existing_auth_db(CredentialAccess::NonInteractive).await? else { return Ok(None); }; load_with(db, current_unix_timestamp_seconds()?).await @@ -95,7 +99,7 @@ pub async fn load_telemetry_credential() -> Result, pub async fn load_repository_telemetry_credential( repository_id: String, ) -> Result, TokenStorageError> { - let Some(db) = get_existing_auth_db().await? else { + let Some(db) = open_existing_auth_db(CredentialAccess::NonInteractive).await? else { return Ok(None); }; load_for_repository_with(db, current_unix_timestamp_seconds()?, &repository_id).await @@ -105,17 +109,17 @@ pub async fn bind_repository_telemetry( repository_id: String, mode: RepositoryBindingMode, ) -> Result { - bind_repository_with( - get_auth_db().await?, - current_unix_timestamp_seconds()?, - &repository_id, - mode, - ) - .await + let Some(db) = open_existing_auth_db(CredentialAccess::Interactive).await? else { + return Ok(false); + }; + bind_repository_with(db, current_unix_timestamp_seconds()?, &repository_id, mode).await } pub async fn delete_telemetry_credential() -> Result { - delete_with(get_auth_db().await?).await + let Some(db) = open_existing_auth_db(CredentialAccess::Interactive).await? else { + return Ok(false); + }; + delete_with(db).await } fn database_error(message: &str) -> TokenStorageError { @@ -913,6 +917,7 @@ mod tests { telemetry_headers_with( Some(working_dir), visible_destination_conflict(claude_helper_env), + true, repository_id_for, |repository_id: String| async move { let db = TestDb::new() diff --git a/cli/src/services/telemetry_readiness.rs b/cli/src/services/telemetry_readiness.rs new file mode 100644 index 000000000..a20f71c71 --- /dev/null +++ b/cli/src/services/telemetry_readiness.rs @@ -0,0 +1,147 @@ +use std::fs; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; + +use crate::services::default_paths::{auth_db_path, telemetry_readiness_path}; + +const TEMP_SUFFIX: &str = "tmp"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct TelemetryReadiness { + expires_at_unix_seconds: u64, +} + +pub fn mark_ready(expires_at_unix_seconds: u64) -> Result<()> { + mark_ready_at(&telemetry_readiness_path()?, expires_at_unix_seconds) +} + +pub fn clear() -> Result<()> { + clear_at(&telemetry_readiness_path()?) +} + +pub fn is_ready(now_unix_seconds: u64) -> bool { + let (Ok(readiness_path), Ok(auth_db_path)) = (telemetry_readiness_path(), auth_db_path()) + else { + return false; + }; + is_ready_at(&readiness_path, &auth_db_path, now_unix_seconds) +} + +pub(crate) fn mark_ready_at(path: &Path, expires_at_unix_seconds: u64) -> Result<()> { + let parent = path.parent().with_context(|| { + format!( + "telemetry readiness path '{}' has no parent", + path.display() + ) + })?; + fs::create_dir_all(parent) + .with_context(|| format!("failed to create '{}'", parent.display()))?; + let bytes = serde_json::to_vec(&TelemetryReadiness { + expires_at_unix_seconds, + }) + .context("failed to serialize telemetry readiness")?; + let temp_path = temp_path_for(path); + fs::write(&temp_path, bytes) + .with_context(|| format!("failed to write '{}'", temp_path.display()))?; + fs::rename(&temp_path, path).with_context(|| { + let _ = fs::remove_file(&temp_path); + format!("failed to replace '{}'", path.display()) + }) +} + +pub(crate) fn clear_at(path: &Path) -> Result<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| format!("failed to remove '{}'", path.display())), + } +} + +pub(crate) fn is_ready_at(path: &Path, auth_db_path: &Path, now_unix_seconds: u64) -> bool { + auth_db_path.is_file() + && fs::read(path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .is_some_and(|readiness| readiness.expires_at_unix_seconds > now_unix_seconds) +} + +fn temp_path_for(path: &Path) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(format!(".{}.{TEMP_SUFFIX}", std::process::id())); + PathBuf::from(name) +} + +#[cfg(test)] +mod tests { + use super::*; + + const NOW: u64 = 1_000; + + struct Paths { + _dir: tempfile::TempDir, + readiness: PathBuf, + auth_db: PathBuf, + } + + fn paths_with_auth_db() -> Paths { + let dir = tempfile::tempdir().unwrap(); + let readiness = dir.path().join("sce").join("telemetry-ready.json"); + let auth_db = dir.path().join("sce").join("auth.db"); + fs::create_dir_all(auth_db.parent().unwrap()).unwrap(); + fs::write(&auth_db, b"encrypted").unwrap(); + Paths { + _dir: dir, + readiness, + auth_db, + } + } + + #[test] + fn absent_readiness_is_not_ready() { + let paths = paths_with_auth_db(); + assert!(!is_ready_at(&paths.readiness, &paths.auth_db, NOW)); + } + + #[test] + fn marked_readiness_is_ready_until_expiry_and_holds_no_secret() { + let paths = paths_with_auth_db(); + mark_ready_at(&paths.readiness, NOW + 1).unwrap(); + + assert!(is_ready_at(&paths.readiness, &paths.auth_db, NOW)); + assert!(!is_ready_at(&paths.readiness, &paths.auth_db, NOW + 1)); + assert_eq!( + fs::read_to_string(&paths.readiness).unwrap(), + format!(r#"{{"expires_at_unix_seconds":{}}}"#, NOW + 1) + ); + } + + #[test] + fn readiness_without_auth_db_is_stale() { + let paths = paths_with_auth_db(); + mark_ready_at(&paths.readiness, NOW + 1).unwrap(); + fs::remove_file(&paths.auth_db).unwrap(); + + assert!(!is_ready_at(&paths.readiness, &paths.auth_db, NOW)); + } + + #[test] + fn clearing_readiness_disables_it_and_is_idempotent() { + let paths = paths_with_auth_db(); + mark_ready_at(&paths.readiness, NOW + 1).unwrap(); + + clear_at(&paths.readiness).unwrap(); + clear_at(&paths.readiness).unwrap(); + + assert!(!is_ready_at(&paths.readiness, &paths.auth_db, NOW)); + } + + #[test] + fn corrupt_readiness_is_not_ready() { + let paths = paths_with_auth_db(); + fs::write(&paths.readiness, b"not json").unwrap(); + assert!(!is_ready_at(&paths.readiness, &paths.auth_db, NOW)); + } +} diff --git a/cli/src/services/token_storage.rs b/cli/src/services/token_storage.rs index f42d80ed4..c4c9ff8ca 100644 --- a/cli/src/services/token_storage.rs +++ b/cli/src/services/token_storage.rs @@ -6,39 +6,46 @@ use serde::{Deserialize, Serialize}; use crate::services::auth::TokenResponse; use crate::services::auth_db::AuthDb; +use crate::services::db::encryption_key::{CredentialAccess, CredentialAccessUnavailable}; use crate::services::default_paths::auth_db_path; -/// Constant row ID for the single token row in `auth_credentials`. const DEFAULT_TOKEN_ROW_ID: i64 = 1; -/// Lazy singleton for the encrypted auth database. -/// -/// Stores `Result` so initialization failures are preserved across calls. -static AUTH_DB: OnceCell> = OnceCell::const_new(); - -pub(crate) async fn get_auth_db() -> Result<&'static AuthDb, TokenStorageError> { - let result = AUTH_DB - .get_or_init(async || AuthDb::new().await.map_err(|e| e.to_string())) - .await; - match result { - Ok(db) => Ok(db), - Err(msg) => Err(TokenStorageError::Database(msg.clone())), +static AUTH_DB: OnceCell = OnceCell::const_new(); + +fn open_error(error: &anyhow::Error) -> TokenStorageError { + if error + .downcast_ref::() + .is_some() + { + TokenStorageError::CredentialAccessUnavailable + } else { + TokenStorageError::Database(error.to_string()) } } -pub(crate) async fn get_existing_auth_db() -> Result, TokenStorageError> { - if AUTH_DB.get().is_none() { - let db_path = - auth_db_path().map_err(|error| TokenStorageError::PathResolution(error.to_string()))?; - let Some(db) = AuthDb::open_existing_at(db_path) - .await - .map_err(|error| TokenStorageError::Database(error.to_string()))? - else { - return Ok(None); - }; - let _ = AUTH_DB.set(Ok(db)); +pub(crate) async fn open_or_create_auth_db() -> Result<&'static AuthDb, TokenStorageError> { + AUTH_DB + .get_or_try_init(async || AuthDb::new().await.map_err(|error| open_error(&error))) + .await +} + +pub(crate) async fn open_existing_auth_db( + access: CredentialAccess, +) -> Result, TokenStorageError> { + if let Some(db) = AUTH_DB.get() { + return Ok(Some(db)); } - get_auth_db().await.map(Some) + let db_path = + auth_db_path().map_err(|error| TokenStorageError::PathResolution(error.to_string()))?; + let Some(db) = AuthDb::open_existing_at(db_path, access) + .await + .map_err(|error| open_error(&error))? + else { + return Ok(None); + }; + let _ = AUTH_DB.set(db); + Ok(AUTH_DB.get()) } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -69,6 +76,7 @@ impl StoredTokens { pub enum TokenStorageError { PathResolution(String), Database(String), + CredentialAccessUnavailable, } impl fmt::Display for TokenStorageError { @@ -82,14 +90,26 @@ impl fmt::Display for TokenStorageError { f, "Token storage database error: {reason}. Try: ensure the OS credential store is available and the auth database is accessible." ), + Self::CredentialAccessUnavailable => write!( + f, + "Token storage requires credential store authorization. Try: run 'sce auth login' to authorize access." + ), } } } impl std::error::Error for TokenStorageError {} -pub async fn save_tokens(token: &TokenResponse) -> Result { - let db = get_auth_db().await?; +pub async fn save_tokens( + token: &TokenResponse, + access: CredentialAccess, +) -> Result { + let db = match access { + CredentialAccess::Interactive => open_or_create_auth_db().await?, + CredentialAccess::NonInteractive => open_existing_auth_db(access) + .await? + .ok_or(TokenStorageError::CredentialAccessUnavailable)?, + }; let stored = StoredTokens::from_token_response(token)?; let expires_in = i64::try_from(stored.expires_in).map_err(|error| { @@ -125,8 +145,12 @@ pub async fn save_tokens(token: &TokenResponse) -> Result Result, TokenStorageError> { - let db = get_auth_db().await?; +pub async fn load_tokens( + access: CredentialAccess, +) -> Result, TokenStorageError> { + let Some(db) = open_existing_auth_db(access).await? else { + return Ok(None); + }; let sql = "SELECT access_token, token_type, expires_in, refresh_token, scope, \ stored_at_unix_seconds FROM auth_credentials WHERE id = ?1"; @@ -166,7 +190,9 @@ pub async fn load_tokens() -> Result, TokenStorageError> { } pub async fn delete_tokens() -> Result { - let db = get_auth_db().await?; + let Some(db) = open_existing_auth_db(CredentialAccess::Interactive).await? else { + return Ok(false); + }; let affected = db .execute( diff --git a/context/sce/auth-db.md b/context/sce/auth-db.md index e8560465d..e42071713 100644 --- a/context/sce/auth-db.md +++ b/context/sce/auth-db.md @@ -67,7 +67,7 @@ SCE binaries from before multiprocess WAL (every release through `v0.4.0-pre-alp Every `EncryptedTursoDb` open now takes a shared compatibility lock (`cli/src/services/db/open_lock.rs`) on the `auth.db` file. It is acquired inside the connection-open retry loop, before Turso opens the file, and held for the life of the `EncryptedTursoDb` value: - Linux: `fcntl(F_OFD_SETLK, F_RDLCK)`, an open-file-description lock on the whole file, on a descriptor the `EncryptedTursoDb` owns. Linux reports conflicts between OFD locks and classic POSIX record locks. So the new read lock and the legacy write lock exclude each other in both directions, while any number of new processes share the read lock. The lock belongs to SCE's own descriptor, so Turso opening or closing its descriptors for `auth.db` cannot release it. It is released when the `EncryptedTursoDb` is dropped or the process exits. -- macOS and other Unix: `fcntl(F_SETLK, F_RDLCK)`, a classic process-scoped record lock, because those platforms have no public OFD lock. It conflicts with the legacy lock in the same way. POSIX releases every record lock a process holds on a file when that process closes any descriptor for the file. So this lock holds only while the process does not drop a second `EncryptedTursoDb` for `auth.db` while another one stays open. Production paths keep one `AuthDb` per process, the `token_storage` singleton. Lifecycle setup opens its own instance only while `auth.db` does not exist yet, and every open re-acquires the lock. **Not executed on macOS**: the macOS path has been reviewed against the `libc` definitions but has not been built or tested on a Mac. +- macOS and other Unix: `fcntl(F_SETLK, F_RDLCK)`, a classic process-scoped record lock, because those platforms have no public OFD lock. It conflicts with the legacy lock in the same way. POSIX releases every record lock a process holds on a file when that process closes any descriptor for the file. So this lock holds only while the process does not drop a second `EncryptedTursoDb` for `auth.db` while another one stays open. Production paths keep one `AuthDb` per process, the `token_storage` singleton, and every open re-acquires the lock. Lifecycle setup never opens the database. **Not executed on macOS**: the macOS path has been reviewed against the `libc` definitions but has not been built or tested on a Mac. - Windows: no compatibility lock. Mixed versions on Windows remain unprotected. Resulting coexistence on Linux, verified with separate processes, isolated encrypted databases, a legacy child that opens with the exact pre-fix builder configuration on the pinned Turso 0.8.1, and the real pre-fix `sce` binary: @@ -97,18 +97,24 @@ Remaining limits: a mixed old/new deployment is still not usable at the same tim `AuthDbLifecycle` is registered in `cli/src/services/auth_db/lifecycle.rs` following the existing DB lifecycle pattern: - `diagnose` collects auth DB path health problems. - `fix` bootstraps missing auth DB parent directory. -- `setup` calls `AuthDb::new()` to initialize the encrypted database. +- `setup` only creates the auth DB parent directory. It never creates, opens, or migrates `auth.db` and never resolves an encryption key, so `sce setup` works offline, needs no login, and leaves an existing `auth.db` byte-for-byte unchanged. A missing `auth.db` is the normal unauthenticated state; only `sce auth login` creates it. - `LifecycleProviderId::AuthDb` is the provider identifier. - The lifecycle provider is registered in deterministic order: config → local_db → auth_db → agent_trace_db → hooks. ## Token storage integration -- `cli/src/services/token_storage.rs` uses `AuthDb` through async `save_tokens`, `load_tokens`, and `delete_tokens` operations and a `tokio::sync::OnceCell>`. Completed initialization success or failure is cached; cancellation before initialization completes leaves the cell uninitialized. +- `cli/src/services/token_storage.rs` uses `AuthDb` through async `save_tokens`, `load_tokens`, and `delete_tokens` operations and a `tokio::sync::OnceCell`. Only a successful open is cached, so a failed non-interactive open never blocks a later interactive one in the same process. +- Open paths: `open_or_create_auth_db()` (`AuthDb::new()`, interactive key access) is used only to persist a new session or telemetry credential during `sce auth login`. Every other path uses `open_existing_auth_db(access)` (`EncryptedTursoDb::open_existing_at(path, access)`), which returns `None` for a missing `auth.db` and never creates the file or a key. `load_tokens(access)` and `save_tokens(token, access)` take the caller's `CredentialAccess`; `delete_tokens` is interactive and returns `false` for a missing DB. - Auth commands and the generic async control-plane credential store directly await token persistence on the application runtime. Database operations do not cross a `spawn_blocking` boundary; the OS keyring adapter retains its platform-specific synchronous behavior where required. - `token_file_path()` returns the auth DB path from `auth_db_path()` instead of a JSON file path. -- `TokenStorageError` exposes `PathResolution` and `Database` variants; former `Io`, `Serialization`, `CorruptedTokenFile`, and `Permission` variants have been removed. +- `TokenStorageError` exposes `PathResolution`, `Database`, and `CredentialAccessUnavailable` variants (the last one is mapped from the typed `encryption_key::CredentialAccessUnavailable` error and carries no secret); former `Io`, `Serialization`, `CorruptedTokenFile`, and `Permission` variants have been removed. - No JSON file I/O remains in `token_storage.rs`. - The `auth_credentials` row uses constant integer ID `1` for single-row token storage. -- Encryption is required: `encryption_key::get_or_create_encryption_key()` first honors `SCE_AUTH_DB_ENCRYPTION_KEY` by deriving a deterministic 64-character hex Turso key from the non-empty env-secret text, then falls back to OS credential-store keyring get-or-create behavior when the env var is absent. No plaintext auth DB fallback exists. Missing/unavailable credential-store failures surface as `TokenStorageError::Database` and include platform-specific remediation for Linux system keyring/Secret Service, macOS Keychain, Windows Credential Store, or unsupported platforms, always mentioning `SCE_AUTH_DB_ENCRYPTION_KEY` for headless/CI use without printing the secret value. +- Encryption is required: `encryption_key::get_or_create_encryption_key()` first honors `SCE_AUTH_DB_ENCRYPTION_KEY` by deriving a deterministic 64-character hex Turso key from the non-empty env-secret text, then falls back to the OS credential store when the env var is absent. No plaintext auth DB fallback exists. +- Credential access is explicit (`encryption_key::CredentialAccess`): + - `Interactive`: used only by `sce auth login`, `logout`, and `whoami`. It may show the macOS Keychain authorization dialog. It generates and stores a new key only when the keyring reports `NoEntry` and `auth.db` does not exist. Any other read error is returned and never overwrites the stored key. + - `NonInteractive`: used by Claude hooks, managed telemetry startup, `sce auth telemetry-headers`, and `sce sync` (including post-commit auto-sync). It only reads, never creates. On macOS it wraps the read in `SecKeychain::disable_user_interaction()` (`SecKeychainSetUserInteractionAllowed(false)`, the UI switch for the legacy file-based keychain API that `apple-native-keyring-store`'s `keychain` feature uses through `SecKeychainFindGenericPassword`). An item the binary is not authorized for, or a locked keychain, fails with `errSecInteractionNotAllowed` instead of prompting. Every failure becomes `CredentialAccessUnavailable`. `kSecUseAuthenticationUIFail` and `LAContext.interactionNotAllowed` do not apply, because they belong to the `SecItem`/data-protection keychain and LocalAuthentication-protected items, which this adapter does not use. + - The interaction switch is process-wide, so all credential-store reads and writes in one process are serialized by a mutex. Interaction is turned back on before the next caller runs. + - Linux Secret Service and Windows have no equivalent switch in the adapter. `NonInteractive` there only guarantees no key creation. Missing/unavailable credential-store failures surface as `TokenStorageError::Database` and include platform-specific remediation for Linux system keyring/Secret Service, macOS Keychain, Windows Credential Store, or unsupported platforms, always mentioning `SCE_AUTH_DB_ENCRYPTION_KEY` for headless/CI use without printing the secret value. See also: [shared-turso-db.md](shared-turso-db.md), [../cli/default-path-catalog.md](../cli/default-path-catalog.md), [../context-map.md](../context-map.md) diff --git a/context/sce/cli-observability-contract.md b/context/sce/cli-observability-contract.md index be511e4e7..89d95865d 100644 --- a/context/sce/cli-observability-contract.md +++ b/context/sce/cli-observability-contract.md @@ -95,7 +95,7 @@ Runtime observability consumes the shared resolved observability config from `cl ## Managed trace export -- Defined in `cli/src/services/managed_telemetry.rs` and `cli/src/services/managed_telemetry/trace_export.rs` (plus `log_export.rs`). After startup, `RuntimeTelemetry::resolve` loads the stored telemetry credential from the encrypted auth DB in a spawned task bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s); absent, expired, malformed, unavailable, ownerless (no stored WorkOS session), or timed-out credentials select `NoopTelemetry`. Startup never issues or refreshes a credential. The fallback is not silent: `RuntimeTelemetry::resolve` logs one `sce.telemetry.fallback` record with `reason` = `credential_unavailable`, `credential_expired`, `lookup_timeout`, `database_error`, `path_resolution_error`, or `exporter_init_failed`, at forced `warn` for failures and `debug` otherwise. Database and path errors add the redacted storage error text as `error`; no credential, bearer, key, or payload is logged, and the record never reaches stdout or, with `log_to_file=true`, stderr. +- Defined in `cli/src/services/managed_telemetry.rs` and `cli/src/services/managed_telemetry/trace_export.rs` (plus `log_export.rs`). After startup, `RuntimeTelemetry::resolve` returns Noop (`not_configured`) without opening the auth DB unless the non-secret telemetry readiness hint is present ([telemetry-credential.md](telemetry-credential.md)). Otherwise it loads the stored telemetry credential from the encrypted auth DB, with non-interactive credential-store access, in a spawned task bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s); absent, expired, malformed, unavailable, ownerless (no stored WorkOS session), or timed-out credentials select `NoopTelemetry`. Startup never issues or refreshes a credential. The fallback is not silent: `RuntimeTelemetry::resolve` logs one `sce.telemetry.fallback` record with `reason` = `not_configured`, `credential_inaccessible`, `credential_unavailable`, `credential_expired`, `lookup_timeout`, `database_error`, `path_resolution_error`, or `exporter_init_failed`, at forced `warn` for failures and `debug` otherwise. Database and path errors add the redacted storage error text as `error`; no credential, bearer, key, or payload is logged, and the record never reaches stdout or, with `log_to_file=true`, stderr. - Admission (`managed_span_kind` in `cli/src/app.rs`, decided from argv before credential lookup): Claude-invoked SCE hooks resolve the `sce.hook` span kind, every other hook and the hidden `sce auth telemetry-headers` helper use `NoopTelemetry` without a credential lookup, and all remaining commands use the `sce.command` span kind. `sce hooks claude-mutation-scope` and `sce hooks claude-model-state` are admitted from argv alone. The shared `sce hooks diff-trace`, `sce hooks conversation-trace`, and `sce policy bash` are admitted only when both `CLAUDECODE` and `CLAUDE_PROJECT_DIR` are present in the environment (checked for presence only; values never read); `CLAUDE_PROJECT_DIR` is set for Claude command hooks but not for the Claude Bash tool (evidence: [claude-hook-trace-propagation.md](claude-hook-trace-propagation.md)). Other hooks (commit hooks, Codex, OpenCode, Pi) stay Noop. Admitted hooks use the same `CREDENTIAL_LOOKUP_DEADLINE`, `SHUTDOWN_TIMEOUT`, and runtime grace bounds as commands. Decision record: [2026-10-17-claude-hook-managed-trace-admission.md](../decisions/2026-10-17-claude-hook-managed-trace-admission.md). - Export is OTLP/HTTP protobuf to the hardcoded `https://otel.crocoderlab.dev/v1/traces`. A pinned HTTP client sets `Authorization: Bearer ` (token only, never the accessor), disables redirects, rebuilds the request to the fixed endpoint, and bounds requests to 3s. Ambient `OTEL_EXPORTER_OTLP_*` values cannot redirect it. - Each command emits one `sce.command` span with `sce.command.name`, `sce.outcome` (`success`/`failure`), and `sce.error.class` (the `SCE-ERR-*` code on failure); resource attributes are `service.name=sce` and `service.version`. Raw arguments, paths, and error chains are not exported. diff --git a/context/sce/telemetry-credential.md b/context/sce/telemetry-credential.md index 93a0885e4..2f3e54e6d 100644 --- a/context/sce/telemetry-credential.md +++ b/context/sce/telemetry-credential.md @@ -22,14 +22,19 @@ flowchart LR - Lookup never mints or refreshes; an expired, empty-field, negative-expiry, or identity-less (pre-`004` legacy) row is returned as `None`, so legacy rows require one `sce auth login`. - Ownership: every lookup decodes the stored WorkOS session's access-token claims and returns the credential only when `(sub, workspace_id)` equals the identity stored with it. Switching user or workspace therefore makes an older credential unusable even if it was never deleted. A new device-authorization login also deletes the stored telemetry credential before saving the new WorkOS session (`replace_session_with`), so a failed re-issuance leaves no credential; if that delete fails, the new session is not saved. Renewal of the same session keeps the credential. -- Repository bindings: `telemetry_repository_bindings` maps a repository ID (the canonical `repository_identity` hash, from `agent_trace.repository_id` or the configured remote) to the `(user_id, workspace_id)` it was enrolled under. Enrollment requires a valid, session-owned credential and records that credential's identity. `sce setup` with the Claude target enrolls the repository explicitly (`Enroll`, rebinding it to the current identity). A successful `sce auth login` enrolls the working-directory repository only if it is unbound (`EnrollIfUnbound`), so a login never silently moves a repository to another workspace. Bindings survive logout and rotation; they authorize nothing without a matching credential. +- Repository bindings: `telemetry_repository_bindings` maps a repository ID (the canonical `repository_identity` hash, from `agent_trace.repository_id` or the configured remote) to the `(user_id, workspace_id)` it was enrolled under. Enrollment requires a valid, session-owned credential and records that credential's identity. `sce setup` no longer enrolls repositories, because it must not open the encrypted auth DB or touch the OS credential store. The explicit rebinding mode (`Enroll`) currently has no production caller. A successful `sce auth login` enrolls the working-directory repository only if it is unbound (`EnrollIfUnbound`), so a login never silently moves a repository to another workspace. Bindings survive logout and rotation; they authorize nothing without a matching credential. +- Telemetry readiness hint: `cli/src/services/telemetry_readiness.rs` keeps a non-secret file, `/sce/telemetry-ready.json` (`{"expires_at_unix_seconds": N}`), next to `auth.db`. + - `sce auth login` writes it only after the credential has been issued and persisted. If provisioning, persistence, or writing the hint fails, the hint is cleared and the result is `unavailable`. + - Logout and a new device-authorization session clear it before anything else. + - It counts as ready only while the file parses, is unexpired, and `auth.db` exists. A missing, corrupt, expired, or orphaned hint means telemetry is disabled. + - It is a fast-path hint, not authorization. The credential is still validated from the auth DB. Its purpose is that unauthenticated hooks never open `auth.db` or touch the Keychain, including existing installs that have `auth.db` but no hint, until the next `sce auth login`. - Concurrent saves are single-statement atomic replacements of one fixed row; the last complete writer wins. - `sce auth logout` deletes the WorkOS credential and the telemetry credential, attempting both even when one fails and surfacing the first storage error. `credentials_removed` is true when either record was deleted; output fields are otherwise unchanged. - Telemetry save/delete never touches the WorkOS `auth_credentials` row. ## Runtime use -- `managed_telemetry::RuntimeTelemetry::resolve` reads the stored credential at CLI startup (never issuing one); a valid unexpired `token` is used only as the `Authorization: Bearer` credential for managed OTLP trace and log export; the `accessor` is never transmitted. Absent, expired, malformed, or unavailable rows select `NoopTelemetry`. Every Noop fallback after a lookup is classified (`credential_unavailable`, `credential_expired`, `lookup_timeout`, `database_error`, `path_resolution_error`, `exporter_init_failed`) and logged locally as `sce.telemetry.fallback`; failures log at `warn` (forced, so they reach the log file at the default `error` level, never stderr while `log_to_file=true`), with the redacted `TokenStorageError` text for database and path errors, and missing or expired credentials log at `debug`. Concurrent lookups from many hook processes are supported by the auth DB's multiprocess WAL and migration-free current-schema open ([auth-db.md](auth-db.md)). Both this lookup and the header helper's repository lookup are existing-DB reads: a missing `auth.db` yields no credential and creates no DB file or encryption key. Claude-invoked SCE hooks are admitted to this lookup and export one `sce.hook` span. Every other hook and this hidden header helper skip the lookup and stay Noop. See [cli-observability-contract.md](cli-observability-contract.md). +- `managed_telemetry::RuntimeTelemetry::resolve` first checks the readiness hint. Without it, the result is Noop with reason `not_configured` (logged at `debug`), and the auth DB is not opened. With the hint, it reads the stored credential through the non-interactive credential-store path at CLI startup (never issuing one). If the store refuses silent access, the reason is `credential_inaccessible` (logged at `debug`, so a denied Keychain does not flood the log). `sce setup` is excluded from managed telemetry so that it never reaches the credential store. a valid unexpired `token` is used only as the `Authorization: Bearer` credential for managed OTLP trace and log export; the `accessor` is never transmitted. Absent, expired, malformed, or unavailable rows select `NoopTelemetry`. Every Noop fallback is classified (`not_configured`, `credential_inaccessible`, `credential_unavailable`, `credential_expired`, `lookup_timeout`, `database_error`, `path_resolution_error`, `exporter_init_failed`) and logged locally as `sce.telemetry.fallback`; failures log at `warn` (forced, so they reach the log file at the default `error` level, never stderr while `log_to_file=true`), with the redacted `TokenStorageError` text for database and path errors, and a missing hint, an inaccessible credential store, or missing or expired credentials log at `debug`. Concurrent lookups from many hook processes are supported by the auth DB's multiprocess WAL and migration-free current-schema open ([auth-db.md](auth-db.md)). Both this lookup and the header helper's repository lookup are existing-DB reads: a missing `auth.db` yields no credential and creates no DB file or encryption key. Claude-invoked SCE hooks are admitted to this lookup and export one `sce.hook` span. Every other hook and this hidden header helper skip the lookup and stay Noop. See [cli-observability-contract.md](cli-observability-contract.md). ## Hidden header helper @@ -37,6 +42,7 @@ flowchart LR - Launch context (verified against Claude Code 2.1.284 with a real `claude -p` run): Claude runs the helper with no arguments, with its working directory set to the directory Claude was launched from (possibly a subdirectory), and without `CLAUDE_PROJECT_DIR`. Claude's documentation states that it does not pass `OTEL_*` variables to subprocesses. The helper therefore uses only the working directory, ignores `CLAUDE_PROJECT_DIR` (a value inherited from a parent session could name another project), and never requires an `OTEL_*` variable to be present. - Authorization: the helper walks up to the enclosing Git root, resolves the repository ID with the same precedence as Agent Trace storage, and prints headers only when that repository is bound to the identity of a valid credential that the current WorkOS session owns (see Repository bindings). An unbound repository, a repository bound to another user or workspace, a directory outside Git, or a repository without a resolvable identity gets no token. `.claude/settings.json`, SCE hook markers, and remote names are never treated as proof of ownership. - Destination tripwire: if `OTEL_EXPORTER_OTLP_ENDPOINT`, a per-signal `OTEL_EXPORTER_OTLP_{TRACES,METRICS,LOGS}_ENDPOINT`, or `BETA_TRACING_ENDPOINT` is visible to the helper with a non-canonical value, the helper refuses. This catches visible misconfiguration only. It is not endpoint pinning (see Security boundaries). +- Without the readiness hint the helper fails closed before resolving the repository or opening the auth DB. With it, the credential lookup uses the non-interactive credential-store path and never prompts. - Repository resolution and the credential lookup together are bounded by `CREDENTIAL_LOOKUP_DEADLINE`. Resolution spawns one `git config --get` process, which is needed to compute the repository ID; there is no network call and no issuance. Every failure exits nonzero with the fixed stderr diagnostic `Telemetry credential is unavailable.` and empty stdout. That includes a missing `auth.db`, which the helper never creates or migrates, so concurrent helper calls on a never-logged-in machine cannot race to initialize it. The helper never initializes managed exporter telemetry. - It is omitted from `sce --help`, `sce auth --help`, and shell completions. @@ -57,9 +63,9 @@ Claude Code's own OpenTelemetry export is separate from the SCE-CLI managed expo - Written keys: `otelHeadersHelper` set to `sce auth telemetry-headers`, plus an `env` block with `CLAUDE_CODE_ENABLE_TELEMETRY=1`, `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA=1` (required by Claude Code for span tracing), `OTEL_{TRACES,METRICS,LOGS}_EXPORTER=otlp`, `OTEL_EXPORTER_OTLP_PROTOCOL` / `OTEL_EXPORTER_OTLP_ENDPOINT` from `MANAGED_OTLP_PROTOCOL` / `MANAGED_OTLP_ENDPOINT`, `CLAUDE_CODE_OTEL_HEADERS_HELPER_DEBOUNCE_MS=300000`, and `OTEL_METRICS_INCLUDE_REPOSITORY=true` (unconditional because the Claude Code version cannot be detected during setup; unknown variables are ignored). - Authentication: no token is ever written to settings, generated assets, or setup output, and `OTEL_EXPORTER_OTLP_HEADERS` is never set. Claude runs the hidden helper (see Hidden header helper) and uses its `Authorization` header. - Rotation: Claude re-runs the helper at most every five minutes (debounce 300000 ms), so a credential replaced by a later `sce auth login` is picked up without rewriting settings. `sce auth logout` deletes the stored credential, after which the helper fails and Claude export stops authenticating; local deletion does not revoke the OpenBao token. -- Repository enrollment: after a successful install, setup enrolls the current repository under the stored credential's identity, bounded by `CREDENTIAL_LOOKUP_DEADLINE`. When no valid session-owned credential exists, setup succeeds and reports `authentication pending for this repository`; running `sce auth login` inside the repository enrolls it and starts export. +- Repository enrollment: setup never enrolls, because it does not open the auth DB. After a successful install it always reports `authentication pending for this repository`. Running `sce auth login` inside the repository enrolls it, if unbound, and starts export. - Failure handling: the helper's failure is nonzero with empty stdout and the fixed diagnostic `Telemetry credential is unavailable.`, and never affects other Claude behavior beyond missing telemetry auth. - Merge safety: repeated setup is byte-idempotent; unrelated settings, env, permissions, and hooks are preserved; writes are atomic via temp-file rename; an existing file's mode is kept and a new file uses `0600`; malformed JSON is never overwritten. -- Conflicts and limits: a user-owned `otelHeadersHelper`, an explicit opt-out (`CLAUDE_CODE_ENABLE_TELEMETRY`, `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA`, or `ENABLE_ENHANCED_TELEMETRY_BETA` not `1`, or any `OTEL_*_EXPORTER=none`), `OTEL_METRICS_INCLUDE_REPOSITORY` other than `true`, a different helper refresh interval, or a differing generic/per-signal OTLP endpoint, headers, protocol, or exporter in user settings or in the Linux/macOS managed-settings file (`/etc/claude-code/managed-settings.json`, `/Library/Application Support/ClaudeCode/managed-settings.json`), causes the telemetry install to be skipped and reported with the specific reason while repository setup still succeeds. Setup enrolls the repository only after a successful install. Environment variables present in a future Claude launch cannot be detected by setup and can override user settings. +- Conflicts and limits: a user-owned `otelHeadersHelper`, an explicit opt-out (`CLAUDE_CODE_ENABLE_TELEMETRY`, `CLAUDE_CODE_ENHANCED_TELEMETRY_BETA`, or `ENABLE_ENHANCED_TELEMETRY_BETA` not `1`, or any `OTEL_*_EXPORTER=none`), `OTEL_METRICS_INCLUDE_REPOSITORY` other than `true`, a different helper refresh interval, or a differing generic/per-signal OTLP endpoint, headers, protocol, or exporter in user settings or in the Linux/macOS managed-settings file (`/etc/claude-code/managed-settings.json`, `/Library/Application Support/ClaudeCode/managed-settings.json`), causes the telemetry install to be skipped and reported with the specific reason while repository setup still succeeds. Environment variables present in a future Claude launch cannot be detected by setup and can override user settings. See also: [auth-db.md](auth-db.md), [cli-observability-contract.md](cli-observability-contract.md), [../context-map.md](../context-map.md) From daaa92585fddcdcf5bc96100c6b8c58bdb6d8f61 Mon Sep 17 00:00:00 2001 From: davidabram Date: Mon, 19 Oct 2026 16:32:56 +0200 Subject: [PATCH 11/17] runtime: Use flock for the macOS auth DB compatibility lock On macOS the shared compatibility lock used `fcntl(F_SETLK, F_RDLCK)`, a process-scoped record lock that POSIX releases whenever the process closes any descriptor for `auth.db`. The guard could silently drop while the `EncryptedTursoDb` was still open, letting a legacy writer through. macOS now takes `flock(LOCK_SH | LOCK_NB)` on the SCE-owned descriptor. The lock belongs to the open file description, so it lasts until the `EncryptedTursoDb` is dropped or the process exits, and it still conflicts with the legacy whole-file `F_WRLCK` in both directions. Linux and other Unix locking is unchanged; `EWOULDBLOCK` is now also classified as a lock conflict. Add four macOS-only multiprocess tests covering mutual exclusion with the legacy lock, survival across an unrelated descriptor close, and concurrent new holders. Record macOS validation evidence and residual risks in the auth DB documentation and the claude-hook-trace-propagation plan. Co-authored-by: SCE --- .../services/auth_db/multiprocess_tests.rs | 122 ++++++++++++++++++ cli/src/services/db/open_lock.rs | 21 ++- .../plans/claude-hook-trace-propagation.md | 26 +++- context/sce/auth-db.md | 12 +- 4 files changed, 174 insertions(+), 7 deletions(-) diff --git a/cli/src/services/auth_db/multiprocess_tests.rs b/cli/src/services/auth_db/multiprocess_tests.rs index c9aacfba2..9e62d02e2 100644 --- a/cli/src/services/auth_db/multiprocess_tests.rs +++ b/cli/src/services/auth_db/multiprocess_tests.rs @@ -523,6 +523,83 @@ fn concurrent_new_writers_all_commit_and_legacy_reopens_afterwards() { assert_eq!(sandbox.rows(), expected); } +#[cfg(target_os = "macos")] +#[test] +fn macos_new_shared_lock_rejects_legacy_exclusive_lock_until_released() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let holder = sandbox.hold("hold-open"); + sandbox.run("legacy-lock-expect-rejected"); + sandbox.run("legacy-expect-rejected"); + holder.release(); + + sandbox.run("legacy-lock-acquire"); + sandbox.run("legacy-roundtrip"); +} + +#[cfg(target_os = "macos")] +#[test] +fn macos_legacy_exclusive_lock_rejects_new_shared_lock_until_released() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let legacy = sandbox.hold("legacy-lock-hold"); + sandbox.run("expect-rejected-read"); + sandbox.run("expect-rejected-write"); + legacy.release(); + + assert_eq!(read_token(&sandbox.run("read")), Some(TOKEN)); +} + +#[cfg(target_os = "macos")] +#[test] +fn macos_new_shared_lock_survives_unrelated_descriptor_close() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let holder = sandbox.hold("hold-open-close-other"); + sandbox.run("legacy-lock-expect-rejected"); + sandbox.run("legacy-expect-rejected"); + holder.release(); + + sandbox.run("legacy-roundtrip"); +} + +#[cfg(target_os = "macos")] +#[test] +fn macos_new_processes_share_the_lock_concurrently() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let holders = [ + sandbox.hold("hold-open"), + sandbox.hold("hold-open-close-other"), + ]; + let writers = (0..NEW_WRITERS) + .map(|index| { + ( + "new-writer".to_string(), + sandbox.writer("new-writer", &format!("shared-{index}")), + ) + }) + .collect(); + let (committed, rejected) = committed_rows(writers); + assert_eq!(rejected, 0); + assert_eq!(committed.len(), NEW_WRITERS * WRITER_ROWS); + sandbox.run("legacy-lock-expect-rejected"); + for holder in holders { + holder.release(); + } + + assert_eq!(sandbox.rows(), committed); + sandbox.run("legacy-roundtrip"); +} + #[test] fn repository_telemetry_lookup_never_creates_a_missing_auth_db() { let _fan_out = serialize_fan_out(); @@ -904,6 +981,29 @@ async fn run_compatibility_child(role: &str) { ); } "legacy-roundtrip" => legacy_roundtrip().await, + #[cfg(target_os = "macos")] + "hold-open-close-other" => { + let _db = AuthDb::new().await.expect("holder opens auth DB"); + let path = auth_db_path().expect("auth DB path"); + drop(std::fs::File::open(&path).expect("unrelated auth DB descriptor")); + wait_for_release(); + } + #[cfg(target_os = "macos")] + "legacy-lock-hold" => { + let file = auth_db_file(); + legacy_exclusive_lock(&file).expect("legacy lock acquires"); + wait_for_release(); + } + #[cfg(target_os = "macos")] + "legacy-lock-expect-rejected" => { + let error = legacy_exclusive_lock(&auth_db_file()) + .expect_err("legacy lock must be rejected while a new process holds the DB"); + assert_eq!(error.raw_os_error(), Some(libc::EAGAIN), "{error}"); + } + #[cfg(target_os = "macos")] + "legacy-lock-acquire" => { + legacy_exclusive_lock(&auth_db_file()).expect("legacy lock acquires"); + } "legacy-writer" => match open_legacy_auth_db().await { Ok(conn) => { for row in writer_rows() { @@ -1107,6 +1207,28 @@ async fn open_legacy_auth_db() -> Result { Ok(db.connect()?) } +#[cfg(target_os = "macos")] +fn auth_db_file() -> std::fs::File { + std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(auth_db_path().expect("auth DB path")) + .expect("auth DB file") +} + +#[cfg(target_os = "macos")] +fn legacy_exclusive_lock(file: &std::fs::File) -> std::io::Result<()> { + use std::os::fd::AsRawFd; + + let mut lock: libc::flock = unsafe { std::mem::zeroed() }; + lock.l_type = libc::c_short::try_from(libc::F_WRLCK).expect("lock type"); + lock.l_whence = libc::c_short::try_from(libc::SEEK_SET).expect("lock whence"); + if unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETLK, &raw const lock) } == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + async fn insert_binding(conn: &turso::Connection, row: &str) -> Result<()> { conn.execute(INSERT_BINDING_SQL, (row, USER_ID, WORKSPACE_ID)) .await?; diff --git a/cli/src/services/db/open_lock.rs b/cli/src/services/db/open_lock.rs index 0b8ae2df3..4e9c8a50b 100644 --- a/cli/src/services/db/open_lock.rs +++ b/cli/src/services/db/open_lock.rs @@ -6,7 +6,7 @@ use anyhow::{anyhow, Result}; #[cfg(target_os = "linux")] const SHARED_LOCK_COMMAND: libc::c_int = libc::F_OFD_SETLK; -#[cfg(all(unix, not(target_os = "linux")))] +#[cfg(all(unix, not(any(target_os = "linux", target_os = "macos"))))] const SHARED_LOCK_COMMAND: libc::c_int = libc::F_SETLK; pub(super) fn open_lock_file(db_path: &Path, create: bool) -> Result> { @@ -40,7 +40,17 @@ pub(super) fn lock_shared(file: &File, db_name: &str, db_path: &Path) -> Result< } } -#[cfg(unix)] +#[cfg(target_os = "macos")] +fn try_lock_shared(file: &File) -> io::Result<()> { + use std::os::fd::AsRawFd; + + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH | libc::LOCK_NB) } == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +#[cfg(all(unix, not(target_os = "macos")))] fn try_lock_shared(file: &File) -> io::Result<()> { use std::os::fd::AsRawFd; @@ -53,7 +63,7 @@ fn try_lock_shared(file: &File) -> io::Result<()> { Ok(()) } -#[cfg(unix)] +#[cfg(all(unix, not(target_os = "macos")))] fn lock_field(value: impl TryInto) -> io::Result { value .try_into() @@ -67,7 +77,10 @@ fn try_lock_shared(_file: &File) -> io::Result<()> { #[cfg(unix)] fn is_lock_conflict(error: &io::Error) -> bool { - matches!(error.raw_os_error(), Some(libc::EAGAIN | libc::EACCES)) + matches!( + error.raw_os_error(), + Some(code) if code == libc::EWOULDBLOCK || code == libc::EAGAIN || code == libc::EACCES + ) } #[cfg(not(unix))] diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md index e250b8c1d..830e9fe57 100644 --- a/context/plans/claude-hook-trace-propagation.md +++ b/context/plans/claude-hook-trace-propagation.md @@ -263,6 +263,30 @@ Persist this field in every plan; this is durable plan state, not chat state: ### Residual risks -- macOS uses a process-scoped lock and has not been built or run on a Mac. Windows has no compatibility lock. +- macOS: resolved by the follow-up below. Other non-Linux Unix still uses a process-scoped `F_SETLK` lock and is untested. Windows has no compatibility lock. - Mixed old/new processes still cannot run at the same time: one side fails explicitly until the other exits. Every `sce` binary used by the account must be upgraded. - The upstream Turso multiprocess cold-open race for many concurrent creators or migrators is unchanged. + +## Follow-up validation: macOS compatibility lock lifetime (PR #330) + +**Date:** 2026-10-19 + +### Change + +- `cli/src/services/db/open_lock.rs` on macOS now takes `flock(LOCK_SH | LOCK_NB)` on the SCE-owned `auth.db` descriptor instead of `fcntl(F_SETLK, F_RDLCK)`. The old process-scoped record lock was released whenever the process closed any descriptor for `auth.db`, even though `EncryptedTursoDb` still owned `_open_lock`. A `flock` lock belongs to the open file description, so it lasts until the `EncryptedTursoDb` is dropped or the process exits. Linux (`F_OFD_SETLK` `F_RDLCK`) and other Unix (`F_SETLK` `F_RDLCK`) are unchanged. `EWOULDBLOCK` is now listed with `EAGAIN`/`EACCES` as a lock conflict (they are the same value on Linux and macOS). Retry policy, diagnostics, and the rule that Turso is never opened after a failed lock are unchanged. No separate lock file, helper process, or dependency. +- Four `#[cfg(target_os = "macos")]` tests in `cli/src/services/auth_db/multiprocess_tests.rs`, described in [Mixed-version compatibility lock](../sce/auth-db.md#mixed-version-compatibility-lock). + +### Commands run (macOS 15.4.1, Darwin 24.4.0, arm64) + +- Before the fix: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::auth_db::multiprocess_tests` -> 18 passed. The existing suite did not detect the lifetime bug. +- Raw lock experiment with separate Python processes (`flock` vs `fcntl(F_SETLK)`): shared `flock` and legacy `F_WRLCK` reject each other in both directions with `EAGAIN`; two shared `flock`s coexist; a held `flock` survives an unrelated open/close in its process; an `F_SETLK` `F_RDLCK` holder that opens and closes another descriptor lets a legacy `F_WRLCK` through. +- After the fix: same command -> 22 passed, including `thirty_two_reader_processes_all_resolve_the_credential`, `concurrent_hook_processes_export_correlated_hook_spans`, `concurrent_legacy_and_new_writers_never_lose_committed_rows`, and the 4 new macOS tests. +- Mutation check: with `open_lock.rs` reverted to the old `F_SETLK` version, `macos_new_shared_lock_survives_unrelated_descriptor_close` fails (`legacy lock must be rejected while a new process holds the DB`). The other three macOS tests pass, as expected for the old lock. +- 3 parallel copies of the multiprocess suite -> 3 of 3 passed. +- `nix flake check` -> exit 0 on aarch64-darwin (`cli-clippy`, `cli-tests`, `cli-fmt`). Linux systems were omitted by the check. + +### Residual risks + +- Linux was not re-run in this session. Its lock code is unchanged; only the conflict check now also names `EWOULDBLOCK` (== `EAGAIN` on Linux). Run `nix flake check` on x86_64-linux before merge. +- Tested on macOS 15.4.1 only. Older macOS versions were not tested. +- macOS was not tested against the real pre-fix `sce` binary. The legacy side used the pre-fix Turso builder configuration and raw `F_SETLK` `F_WRLCK` children. diff --git a/context/sce/auth-db.md b/context/sce/auth-db.md index e42071713..1853925eb 100644 --- a/context/sce/auth-db.md +++ b/context/sce/auth-db.md @@ -67,10 +67,11 @@ SCE binaries from before multiprocess WAL (every release through `v0.4.0-pre-alp Every `EncryptedTursoDb` open now takes a shared compatibility lock (`cli/src/services/db/open_lock.rs`) on the `auth.db` file. It is acquired inside the connection-open retry loop, before Turso opens the file, and held for the life of the `EncryptedTursoDb` value: - Linux: `fcntl(F_OFD_SETLK, F_RDLCK)`, an open-file-description lock on the whole file, on a descriptor the `EncryptedTursoDb` owns. Linux reports conflicts between OFD locks and classic POSIX record locks. So the new read lock and the legacy write lock exclude each other in both directions, while any number of new processes share the read lock. The lock belongs to SCE's own descriptor, so Turso opening or closing its descriptors for `auth.db` cannot release it. It is released when the `EncryptedTursoDb` is dropped or the process exits. -- macOS and other Unix: `fcntl(F_SETLK, F_RDLCK)`, a classic process-scoped record lock, because those platforms have no public OFD lock. It conflicts with the legacy lock in the same way. POSIX releases every record lock a process holds on a file when that process closes any descriptor for the file. So this lock holds only while the process does not drop a second `EncryptedTursoDb` for `auth.db` while another one stays open. Production paths keep one `AuthDb` per process, the `token_storage` singleton, and every open re-acquires the lock. Lifecycle setup never opens the database. **Not executed on macOS**: the macOS path has been reviewed against the `libc` definitions but has not been built or tested on a Mac. +- macOS: `flock(LOCK_SH | LOCK_NB)` on the same SCE-owned `auth.db` descriptor. macOS has no public OFD lock, and a classic `F_SETLK` record lock is process-scoped: closing *any* descriptor for `auth.db` in the process released it, so the earlier `F_SETLK` `F_RDLCK` implementation could silently drop the guard while the `EncryptedTursoDb` was still open. A `flock` lock belongs to the open file description, so unrelated opens and closes of `auth.db` in the same process leave it in place. On macOS, `flock` and `fcntl` locks share one lock table and conflict across owners: the shared `flock` and the legacy whole-file `F_SETLK` `F_WRLCK` exclude each other in both directions (`EAGAIN`/`EWOULDBLOCK`), while any number of new processes share the `flock`. Turso's multiprocess open uses `NoLock` on the DB file and skips its legacy-WAL probe on the `turso::Builder` path, so it never requests a conflicting lock in the same process. Verified on macOS 15.4.1 (Darwin 24.4.0, arm64) with separate processes; see the macOS tests below. +- Other Unix (not Linux or macOS): `fcntl(F_SETLK, F_RDLCK)`, a classic process-scoped record lock. It conflicts with the legacy lock in the same way, but POSIX releases it when the process closes any descriptor for `auth.db`. Not built or tested on those platforms. - Windows: no compatibility lock. Mixed versions on Windows remain unprotected. -Resulting coexistence on Linux, verified with separate processes, isolated encrypted databases, a legacy child that opens with the exact pre-fix builder configuration on the pinned Turso 0.8.1, and the real pre-fix `sce` binary: +Resulting coexistence on Linux (and on macOS through the test suite, without the real pre-fix binary), verified with separate processes, isolated encrypted databases, a legacy child that opens with the exact pre-fix builder configuration on the pinned Turso 0.8.1, and the real pre-fix `sce` binary: | First holder | Second process | Result | | --- | --- | --- | @@ -90,6 +91,13 @@ Mixed-version tests (`multiprocess_tests.rs`), all verifying persisted `telemetr - 4 concurrent new writers all commit all 32 rows. - After every new process exits, a legacy process reopens the DB, reads the credential, and writes. +macOS-only tests (`#[cfg(target_os = "macos")]`, separate processes, raw `fcntl(F_SETLK, F_WRLCK)` children that take the exact legacy lock as well as legacy Turso opens): + +- `macos_new_shared_lock_rejects_legacy_exclusive_lock_until_released`: a new holder makes the raw legacy lock fail with `EAGAIN` and the legacy Turso open fail; after the holder exits, both succeed. +- `macos_legacy_exclusive_lock_rejects_new_shared_lock_until_released`: a raw legacy lock holder makes a new lookup and a new writer fail with the shared-access error; after it exits, a new reader resolves the credential. +- `macos_new_shared_lock_survives_unrelated_descriptor_close`: the new holder opens and closes a second `auth.db` descriptor after `AuthDb::new`; legacy locks and opens are still rejected. Against the old `F_SETLK` `F_RDLCK` implementation this test fails because the legacy lock is granted. +- `macos_new_processes_share_the_lock_concurrently`: two new holders (one of them after an unrelated close) and 4 new writers run together, every write commits, and the legacy lock is still rejected; after they exit, a legacy process reopens and writes. + Remaining limits: a mixed old/new deployment is still not usable at the same time, because one side fails while the other holds the DB. Upgrade by replacing every `sce` binary used by the account. Old hook processes still in flight during the switch make new processes fail (and hooks fall back to Noop) until they exit; they no longer lose data. Turso reopens `auth.db` by path after the lock is taken, so a file that something outside SCE deletes in that window would be recreated empty. SCE never deletes `auth.db`. ## Lifecycle integration From de2aedf2de9d6aad4e9c128957d6843aca1a2c05 Mon Sep 17 00:00:00 2001 From: David Abram Date: Mon, 19 Oct 2026 16:48:46 +0200 Subject: [PATCH 12/17] auth-db: Coordinate concurrent writer lifetimes Prevent compatibility regressions from masking insert failures when Turso WAL connection startup and teardown overlap concurrent writes. Keep all writers open through coordinated inserts, report row-specific errors, and document the lifecycle-race isolation while preserving the legacy reopen checks. Co-authored-by: SCE --- .../services/auth_db/multiprocess_tests.rs | 132 ++++++++++++------ context/sce/auth-db.md | 6 +- 2 files changed, 90 insertions(+), 48 deletions(-) diff --git a/cli/src/services/auth_db/multiprocess_tests.rs b/cli/src/services/auth_db/multiprocess_tests.rs index 9e62d02e2..4af2481ac 100644 --- a/cli/src/services/auth_db/multiprocess_tests.rs +++ b/cli/src/services/auth_db/multiprocess_tests.rs @@ -194,35 +194,21 @@ impl Sandbox { } fn hold(&self, role: &str) -> HeldChild { - let mut child = self - .command(role) - .stdin(Stdio::piped()) - .spawn() - .expect("spawn holder"); + Self::hold_command(role, self.command(role)) + } + + fn hold_command(role: &str, mut command: Command) -> HeldChild { + let mut child = command.stdin(Stdio::piped()).spawn().expect("spawn holder"); let stdin = child.stdin.take().expect("holder stdin"); - let mut stdout = BufReader::new(child.stdout.take().expect("holder stdout")); - let mut line = String::new(); - loop { - line.clear(); - let read = stdout.read_line(&mut line).expect("holder stdout line"); - if read == 0 { - let _ = child.kill(); - let output = child.wait_with_output().expect("holder output"); - panic!( - "{role} holder exited before ready: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - if line.trim_end().ends_with(READY) { - break; - } - } - HeldChild { + let stdout = BufReader::new(child.stdout.take().expect("holder stdout")); + let mut held = HeldChild { role: role.to_string(), child, stdin, stdout, - } + }; + held.wait_ready(); + held } fn seed(&self) { @@ -238,8 +224,42 @@ struct HeldChild { } impl HeldChild { - fn release(mut self) { + fn signal(&mut self) { writeln!(self.stdin).expect("release holder"); + } + + fn wait_ready(&mut self) -> String { + let mut captured = String::new(); + loop { + let mut line = String::new(); + let read = self + .stdout + .read_line(&mut line) + .expect("holder stdout line"); + if read == 0 { + let _ = self.child.kill(); + self.child.wait().expect("holder status"); + let mut stderr = String::new(); + self.child + .stderr + .take() + .expect("holder stderr") + .read_to_string(&mut stderr) + .expect("holder stderr output"); + panic!( + "{} holder exited before ready:\n{captured}\n{stderr}", + self.role + ); + } + if line.trim_end().ends_with(READY) { + return captured; + } + captured.push_str(&line); + } + } + + fn release(mut self) { + self.signal(); drop(self.stdin); let mut rest = String::new(); self.stdout @@ -292,6 +312,29 @@ fn committed_rows(children: Vec<(String, Child)>) -> (BTreeSet, usize) { (committed, rejected) } +fn coordinated_writer_rows(sandbox: &Sandbox) -> BTreeSet { + // Open every connection before writes start, and keep every connection + // alive until all writes finish so WAL startup/teardown cannot race them. + let mut writers = (0..NEW_WRITERS) + .map(|index| { + let mut command = sandbox.command("coordinated-writer"); + command.env(WRITER_ENV, format!("new-{index}")); + Sandbox::hold_command("coordinated-writer", command) + }) + .collect::>(); + for writer in &mut writers { + writer.signal(); + } + let mut committed = BTreeSet::new(); + for writer in &mut writers { + committed.extend(prefixed_lines(&writer.wait_ready(), COMMITTED_PREFIX)); + } + for writer in writers { + writer.release(); + } + committed +} + fn read_token(stdout: &str) -> Option<&str> { stdout .lines() @@ -503,17 +546,7 @@ fn concurrent_new_writers_all_commit_and_legacy_reopens_afterwards() { let _fan_out = serialize_fan_out(); let sandbox = Sandbox::new(); sandbox.seed(); - - let writers = (0..NEW_WRITERS) - .map(|index| { - ( - "new-writer".to_string(), - sandbox.writer("new-writer", &format!("new-{index}")), - ) - }) - .collect(); - let (committed, rejected) = committed_rows(writers); - assert_eq!(rejected, 0); + let committed = coordinated_writer_rows(&sandbox); assert_eq!(committed.len(), NEW_WRITERS * WRITER_ROWS); assert_eq!(sandbox.rows(), committed); @@ -580,16 +613,7 @@ fn macos_new_processes_share_the_lock_concurrently() { sandbox.hold("hold-open"), sandbox.hold("hold-open-close-other"), ]; - let writers = (0..NEW_WRITERS) - .map(|index| { - ( - "new-writer".to_string(), - sandbox.writer("new-writer", &format!("shared-{index}")), - ) - }) - .collect(); - let (committed, rejected) = committed_rows(writers); - assert_eq!(rejected, 0); + let committed = coordinated_writer_rows(&sandbox); assert_eq!(committed.len(), NEW_WRITERS * WRITER_ROWS); sandbox.run("legacy-lock-expect-rejected"); for holder in holders { @@ -956,6 +980,7 @@ async fn auth_db_child_process() { } "upgrade" => upgrade_outdated_schema().await, "partial" => reject_partial_schema().await, + "coordinated-writer" => run_coordinated_writer_child().await, "hook" => run_hook_child().await, "hook-entry" => run_hook_entry_child().await, other => run_compatibility_child(other).await, @@ -1123,6 +1148,21 @@ async fn run_new_version_child(role: &str) { } } +async fn run_coordinated_writer_child() { + let db = AuthDb::new() + .await + .expect("coordinated writer opens auth DB"); + wait_for_release(); + for row in writer_rows() { + db.execute(INSERT_BINDING_SQL, (row.as_str(), USER_ID, WORKSPACE_ID)) + .await + .unwrap_or_else(|error| panic!("writer failed at {row}: {error:#}")); + println!("{COMMITTED_PREFIX}{row}"); + tokio::time::sleep(WRITER_PAUSE).await; + } + wait_for_release(); +} + async fn seed() { let db = AuthDb::new().await.expect("seed opens auth DB"); db.execute( diff --git a/context/sce/auth-db.md b/context/sce/auth-db.md index 1853925eb..3cc136307 100644 --- a/context/sce/auth-db.md +++ b/context/sce/auth-db.md @@ -58,7 +58,9 @@ Storage operations live in `cli/src/services/telemetry_credential_storage.rs`: ` ### Upstream Turso multiprocess race -Under heavy load, Turso 0.8.1 multiprocess WAL has an internal race on concurrent cold opens that involve writes. Stress runs (6 parallel copies of the suite, with 32 processes creating or migrating the same DB) occasionally failed: `short read on page 3`, a `turso_assert` panic `reader slot released by non-owner` in `shared_wal_coordination.rs`, or, in the Nix sandbox, a writer that stayed `database is locked` through all 10 migration attempts (about 49 s). The read-only paths never failed across about 3,800 cold-opening reader and hook processes, and that is all hooks do on a current schema. The migration regression therefore uses 4 concurrent migrators, which was 0 failures in 48 parallel stress runs. Concurrent creation and migration by many writers is left as a documented upstream limitation. It affects first-time initialization only, and the Agent Trace DB already runs in the same Turso mode. +Under heavy load, Turso 0.8.1 multiprocess WAL has an internal race on concurrent cold opens that involve writes. Stress runs (6 parallel copies of the suite, with 32 processes creating or migrating the same DB) occasionally failed: `short read on page 3`, a `turso_assert` panic `reader slot released by non-owner` in `shared_wal_coordination.rs`, or, in the Nix sandbox, a writer that stayed `database is locked` through all 10 migration attempts (about 49 s). The read-only paths never failed across about 3,800 cold-opening reader and hook processes, and that is all hooks do on a current schema. The migration regression therefore uses 4 concurrent migrators, which was 0 failures in 48 parallel stress runs. Concurrent creation and migration by many writers is left as a documented upstream limitation. That migration limitation affects first-time initialization, and the Agent Trace DB already runs in the same Turso mode. + +The four-writer compatibility regression also reproduced exhausted `database is locked` retries against an already-seeded, current-schema DB when connection startup/teardown overlapped inserts. Its children previously swallowed insert failures, leaving only a short committed-row count in the parent. The regression now opens every writer before releasing concurrent inserts and keeps every connection alive until all writers finish; every insert must succeed, all 32 rows must persist, and a legacy process must reopen afterwards. This passed 80 stress repetitions across four parallel test runs. The macOS shared-lock writer regression uses the same coordination. This isolates the compatibility-lock guarantee from the Turso lifecycle race; production open/write/close behavior and retry limits are unchanged. ### Mixed-version compatibility lock @@ -88,7 +90,7 @@ Mixed-version tests (`multiprocess_tests.rs`), all verifying persisted `telemetr - A legacy holder makes a new telemetry lookup and a new writer fail explicitly. The legacy holder's own later write persists. - A new holder makes a legacy open fail with the Turso lock error. - 3 rounds of 1 legacy writer and 2 new writers start together. Every row any writer reported committed is persisted. -- 4 concurrent new writers all commit all 32 rows. +- 4 new writers with coordinated connection lifetimes insert concurrently and commit all 32 rows; any insert failure includes the row and database error. - After every new process exits, a legacy process reopens the DB, reads the credential, and writes. macOS-only tests (`#[cfg(target_os = "macos")]`, separate processes, raw `fcntl(F_SETLK, F_WRLCK)` children that take the exact legacy lock as well as legacy Turso opens): From dbdb4e8ed6879386c4e3c74fca7226dfb1d5f85f Mon Sep 17 00:00:00 2001 From: davidabram Date: Mon, 19 Oct 2026 19:34:31 +0200 Subject: [PATCH 13/17] doctor: Report managed telemetry health and auth recovery steps Expose read-only managed telemetry health in doctor text and JSON output, using readiness hints and the bounded non-interactive credential lookup. Keep unconfigured installs warning-free and report disabled telemetry as manual-only warnings without changing credentials or blocking hooks. Provide platform-specific auth storage recovery guidance and separate login failures to avoid circular instructions while preserving error identifiers and exit codes. Cover health classification, output, auth side effects, and real database states; correct the macOS multiprocess sandbox state path. Plan: claude-hook-trace-propagation (auth storage guidance and telemetry health follow-up). Co-authored-by: SCE --- .../app/doctor_reconciliation_cli_tests.rs | 99 +++++ cli/src/services/auth_command/mod.rs | 70 +++- .../services/auth_db/multiprocess_tests.rs | 70 +++- cli/src/services/doctor/managed_telemetry.rs | 345 ++++++++++++++++++ cli/src/services/doctor/mod.rs | 12 + cli/src/services/doctor/render.rs | 112 +++++- cli/src/services/doctor/types.rs | 6 + cli/src/services/error.rs | 283 +++++++++++++- cli/src/services/lifecycle.rs | 1 + cli/src/services/managed_telemetry.rs | 131 ++++++- cli/src/services/telemetry_readiness.rs | 67 +++- .../plans/claude-hook-trace-propagation.md | 33 ++ context/sce/auth-db.md | 13 +- context/sce/doctor-human-text-contract.md | 7 +- context/sce/telemetry-credential.md | 30 +- 15 files changed, 1240 insertions(+), 39 deletions(-) create mode 100644 cli/src/services/doctor/managed_telemetry.rs diff --git a/cli/src/app/doctor_reconciliation_cli_tests.rs b/cli/src/app/doctor_reconciliation_cli_tests.rs index ed1dfff00..8224a3bb6 100644 --- a/cli/src/app/doctor_reconciliation_cli_tests.rs +++ b/cli/src/app/doctor_reconciliation_cli_tests.rs @@ -5,6 +5,7 @@ use serde_json::Value; use super::run_with_dependency_check_and_streams; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::db::encryption_key::AUTH_DB_ENCRYPTION_KEY_ENV; use crate::services::default_paths::agent_trace_db_path_for_repository_at; use crate::services::repository_identity::resolve::resolve_repository_identity; @@ -15,6 +16,10 @@ const STATE_FILE: &str = "ref-maintenance.json"; const LOCK_FILE: &str = "mutation-cursor.lock"; const RECONCILIATION_DETAIL_PREFIX: &str = "Snapshot ref reconciliation"; +const AUTH_DB_TEST_SECRET: &str = "doctor-cli-test-secret"; +const UNREADABLE_AUTH_DB: &[u8] = b"not an encrypted auth database"; +const FAR_FUTURE_READINESS: &[u8] = br#"{"expires_at_unix_seconds":4102444800}"#; + const EXIT_SUCCESS: &str = "success"; const EXIT_FAILURE: &str = "failure"; @@ -138,6 +143,10 @@ impl Sandbox { } fn run(&self, args: &[&str]) -> CliRun { + self.run_with_env(args, &[]) + } + + fn run_with_env(&self, args: &[&str], envs: &[(&str, &str)]) -> CliRun { let out = self.dir.path().join(format!("out-{}", args.join("-"))); std::fs::create_dir_all(&out).expect("out dir"); let output = Command::new(std::env::current_exe().expect("current exe")) @@ -151,6 +160,8 @@ impl Sandbox { .env("XDG_CACHE_HOME", self.dir.path().join("cache")) .env("NO_COLOR", "1") .env_remove("SCE_CONFIG_FILE") + .env_remove(AUTH_DB_ENCRYPTION_KEY_ENV) + .envs(envs.iter().copied()) .output() .expect("child runs"); assert!( @@ -594,3 +605,91 @@ async fn doctor_fix_recovers_from_every_initial_maintenance_state_condition() { ); } } + +fn managed_telemetry_problems(report: &Value) -> Vec { + report["problems"] + .as_array() + .expect("problems array") + .iter() + .filter(|problem| { + problem["summary"] + .as_str() + .is_some_and(|summary| summary.contains("telemetry export is disabled")) + }) + .cloned() + .collect() +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_on_fresh_state_reports_unconfigured_telemetry_without_a_warning() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + let auth_dir = sandbox.state_root().join("sce"); + + let report = sandbox + .run_with_env( + &["doctor", "--format", "json"], + &[(AUTH_DB_ENCRYPTION_KEY_ENV, "")], + ) + .json(); + + assert_eq!(report["managed_telemetry"]["state"], "not_configured"); + assert!(managed_telemetry_problems(&report).is_empty()); + assert!(!auth_dir.join("auth.db").exists()); + assert!(!auth_dir.join("telemetry-ready.json").exists()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_reports_unreadable_telemetry_credentials_without_auth_side_effects() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + let auth_dir = sandbox.state_root().join("sce"); + std::fs::create_dir_all(&auth_dir).expect("auth dir"); + let auth_db = auth_dir.join("auth.db"); + let readiness = auth_dir.join("telemetry-ready.json"); + std::fs::write(&auth_db, UNREADABLE_AUTH_DB).expect("auth DB"); + std::fs::write(&readiness, FAR_FUTURE_READINESS).expect("readiness"); + let env = [(AUTH_DB_ENCRYPTION_KEY_ENV, AUTH_DB_TEST_SECRET)]; + + let fix = sandbox.run_with_env(&["doctor", "--fix", "--format", "json"], &env); + let report = fix.json(); + + assert_eq!( + report["managed_telemetry"], + serde_json::json!({ + "state": "disabled", + "reason": "database_error", + "kind": "managed_telemetry_database_error", + "severity": "warning", + "next_action": "manual_steps", + }), + "{}", + fix.stdout + ); + let problems = managed_telemetry_problems(&report); + assert_eq!(problems.len(), 1, "{}", fix.stdout); + assert_eq!(problems[0]["severity"], "warning"); + assert_eq!(problems[0]["fixability"], "manual_only"); + assert!(!problems[0]["remediation"]["text"] + .as_str() + .expect("remediation") + .contains("Always Allow")); + let telemetry_fixes = non_reconciliation_fix_results(&report, sandbox.dir.path()) + .into_iter() + .filter(|entry| { + entry["detail"] + .as_str() + .is_some_and(|detail| detail.contains("telemetry")) + }) + .collect::>(); + assert_eq!(telemetry_fixes.len(), 1, "{}", fix.stdout); + assert_eq!(telemetry_fixes[0]["outcome"], "manual"); + assert_eq!( + std::fs::read(&auth_db).expect("auth DB kept"), + UNREADABLE_AUTH_DB + ); + assert_eq!( + std::fs::read(&readiness).expect("readiness kept"), + FAR_FUTURE_READINESS + ); + assert!(!fix.stdout.contains(AUTH_DB_TEST_SECRET)); + assert!(!fix.stderr.contains(AUTH_DB_TEST_SECRET)); +} diff --git a/cli/src/services/auth_command/mod.rs b/cli/src/services/auth_command/mod.rs index e84446462..a6061c2d4 100644 --- a/cli/src/services/auth_command/mod.rs +++ b/cli/src/services/auth_command/mod.rs @@ -179,7 +179,7 @@ pub async fn run_login(format: AuthFormat) -> Result { ); let stored_tokens = token_storage::load_tokens(CredentialAccess::Interactive) .await - .map_err(auth_storage_error)?; + .map_err(login_storage_error)?; let client = &client; let control_plane = &control_plane; @@ -275,7 +275,7 @@ async fn maybe_renew_stored_credentials( { Ok(token) => token_storage::save_tokens(&token, CredentialAccess::Interactive) .await - .map_err(auth_storage_error) + .map_err(login_storage_error) .map(Some), Err(_) => Ok(None), } @@ -328,8 +328,8 @@ where SF: Future>, { clear_readiness().map_err(unexpected_auth_command_error)?; - invalidate_telemetry.await.map_err(auth_storage_error)?; - save_session().await.map_err(auth_storage_error) + invalidate_telemetry.await.map_err(login_storage_error)?; + save_session().await.map_err(login_storage_error) } async fn provision_telemetry_credential( @@ -484,7 +484,7 @@ fn write_login_prompt(authorization: &auth::DeviceAuthorizationResponse) -> Resu fn map_login_error(error: AuthError) -> CliError { let user_error = match &error { - AuthError::Io(_) | AuthError::Storage(_) => UserError::AuthStorageUnavailable, + AuthError::Io(_) | AuthError::Storage(_) => UserError::LoginAuthStorageUnavailable, _ => UserError::UnexpectedFailure, }; CliError::user_with_source(user_error, error) @@ -703,6 +703,10 @@ fn auth_storage_error(error: TokenStorageError) -> CliError { CliError::user_with_source(UserError::AuthStorageUnavailable, error) } +fn login_storage_error(error: TokenStorageError) -> CliError { + CliError::user_with_source(UserError::LoginAuthStorageUnavailable, error) +} + fn unexpected_auth_command_error(error: anyhow::Error) -> CliError { CliError::user_with_source(UserError::UnexpectedFailure, error) } @@ -773,8 +777,62 @@ mod tests { }, ) .await; - assert!(failed.is_err()); assert!(!save_called.load(std::sync::atomic::Ordering::SeqCst)); + assert_login_storage_guidance(&failed.expect_err("invalidation failure fails login")); + + let failed = replace_session_with( + || Ok(()), + async { Ok(true) }, + || async { Err(TokenStorageError::CredentialAccessUnavailable) }, + ) + .await; + assert_login_storage_guidance(&failed.expect_err("save failure fails login")); + } + + fn assert_login_storage_guidance(error: &CliError) { + let CliError::User { + error: user_error, .. + } = error + else { + panic!("expected a user-facing storage error"); + }; + assert_eq!(*user_error, UserError::LoginAuthStorageUnavailable); + assert_eq!(user_error.key(), "auth.storage_unavailable"); + assert_eq!(error.class().exit_code(), 4); + let message = error.to_string(); + assert!(message.contains("Running login again does not help until")); + assert!(!message.contains("Retry the original command.")); + } + + #[test] + fn login_device_flow_storage_failure_uses_noncircular_guidance() { + assert_login_storage_guidance(&map_login_error(AuthError::Storage( + TokenStorageError::CredentialAccessUnavailable, + ))); + let CliError::User { error, .. } = map_login_error(AuthError::MissingClientId) else { + panic!("expected a user-facing error"); + }; + assert_eq!(error, UserError::UnexpectedFailure); + } + + #[tokio::test] + async fn logout_storage_failure_keeps_command_storage_guidance() { + let error = run_logout_with( + AuthFormat::Json, + || Ok(()), + async { Err(TokenStorageError::CredentialAccessUnavailable) }, + async { Ok(false) }, + ) + .await + .expect_err("storage failure fails logout"); + let CliError::User { + error: user_error, .. + } = &error + else { + panic!("expected a user-facing storage error"); + }; + assert_eq!(*user_error, UserError::AuthStorageUnavailable); + assert!(error.to_string().contains("sce auth login")); } #[tokio::test] diff --git a/cli/src/services/auth_db/multiprocess_tests.rs b/cli/src/services/auth_db/multiprocess_tests.rs index 4af2481ac..236f13ef9 100644 --- a/cli/src/services/auth_db/multiprocess_tests.rs +++ b/cli/src/services/auth_db/multiprocess_tests.rs @@ -28,8 +28,8 @@ use crate::services::managed_telemetry::trace_export::{ record_claude_hook_command, record_claude_hook_payload, ClaudeHookCommand, ManagedSpanKind, }; use crate::services::managed_telemetry::{ - load_credential_within_deadline, select_mode, ManagedTelemetryMode, RuntimeTelemetry, - TelemetryFallbackReason, + inspect_health, load_credential_within_deadline, select_mode, ManagedTelemetryHealth, + ManagedTelemetryMode, RuntimeTelemetry, TelemetryFallbackReason, }; use crate::services::observability::traits::{NoopLogger, Telemetry}; use crate::services::telemetry_credential_storage::{ @@ -49,6 +49,7 @@ const USER_ID: &str = "user_mp"; const WORKSPACE_ID: &str = "workspace_mp"; const READY: &str = "SCE_AUTH_DB_CHILD_READY"; const TOKEN_PREFIX: &str = "SCE_AUTH_DB_CHILD_TOKEN="; +const HEALTH_PREFIX: &str = "SCE_AUTH_DB_CHILD_TELEMETRY_HEALTH="; const CONCURRENT_PROCESSES: usize = 32; const CONCURRENT_MIGRATORS: usize = 4; const READER_ROUNDS: usize = 3; @@ -180,8 +181,18 @@ impl Sandbox { prefixed_lines(&self.run("verify-rows"), ROW_PREFIX) } + fn sce_state_dir(&self) -> PathBuf { + let root = self.dir.path(); + let state_root = if cfg!(target_os = "macos") { + root.join("Library").join("Application Support") + } else { + root.join("state") + }; + state_root.join("sce") + } + fn auth_db_files(&self) -> Vec { - let state = self.dir.path().join("state").join("sce"); + let state = self.sce_state_dir(); AUTH_DB_FILES .iter() .filter(|name| state.join(name).exists()) @@ -436,6 +447,52 @@ fn telemetry_lookup_never_creates_a_missing_auth_db() { Sandbox::new().run("lookup-missing"); } +fn telemetry_health(sandbox: &Sandbox, secret: &str) -> String { + let stdout = wait_success( + "telemetry-health", + sandbox + .command("telemetry-health") + .env(AUTH_DB_ENCRYPTION_KEY_ENV, secret) + .spawn() + .expect("spawn health child"), + ); + let health = prefixed_lines(&stdout, HEALTH_PREFIX); + assert_eq!(health.len(), 1, "{stdout}"); + health.into_iter().next().expect("one health line") +} + +#[test] +fn doctor_telemetry_health_on_fresh_state_resolves_no_key_and_creates_no_auth_db() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + assert_eq!(telemetry_health(&sandbox, ""), "not_configured"); + assert!(sandbox.auth_db_files().is_empty()); +} + +#[test] +fn doctor_telemetry_health_classifies_real_auth_db_states_without_mutating_them() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + let state = sandbox.sce_state_dir(); + let auth_db = std::fs::read(state.join("auth.db")).expect("seeded auth DB"); + let readiness = state.join("telemetry-ready.json"); + + assert_eq!(telemetry_health(&sandbox, TEST_SECRET), "healthy"); + assert_eq!(telemetry_health(&sandbox, WRONG_SECRET), "database_error"); + assert_eq!(telemetry_health(&sandbox, ""), "database_error"); + assert_eq!( + std::fs::read(state.join("auth.db")).expect("auth DB"), + auth_db + ); + + std::fs::write(&readiness, br#"{"expires_at_unix_seconds":1}"#).expect("expire readiness"); + assert_eq!(telemetry_health(&sandbox, ""), "credential_expired"); + std::fs::remove_file(&readiness).expect("remove readiness"); + assert_eq!(telemetry_health(&sandbox, ""), "not_configured"); + assert_eq!(read_token(&sandbox.run("read")), Some(TOKEN)); +} + #[test] fn concurrent_migration_of_schemaless_db_applies_each_migration_once() { let _fan_out = serialize_fan_out(); @@ -936,6 +993,13 @@ async fn auth_db_child_process() { .expect("credential present"); println!("{TOKEN_PREFIX}{}", credential.token); } + "telemetry-health" => { + let label = match inspect_health().await { + ManagedTelemetryHealth::Healthy => "healthy", + ManagedTelemetryHealth::Disabled(reason) => reason.as_str(), + }; + println!("{HEALTH_PREFIX}{label}"); + } "lookup-missing" => { assert_eq!( load_telemetry_credential() diff --git a/cli/src/services/doctor/managed_telemetry.rs b/cli/src/services/doctor/managed_telemetry.rs new file mode 100644 index 000000000..cbe4958b7 --- /dev/null +++ b/cli/src/services/doctor/managed_telemetry.rs @@ -0,0 +1,345 @@ +use serde_json::{json, Value}; + +use crate::services::error::{CredentialStorePlatform, CURRENT_CREDENTIAL_STORE_PLATFORM}; +use crate::services::managed_telemetry::{ + ManagedTelemetryHealth, TelemetryFallbackReason, CREDENTIAL_LOOKUP_DEADLINE, +}; + +use super::types::{ + DoctorProblem, ProblemCategory, ProblemFixability, ProblemKind, ProblemSeverity, +}; + +const STATE_HEALTHY: &str = "healthy"; +const STATE_NOT_CONFIGURED: &str = "not_configured"; +const STATE_DISABLED: &str = "disabled"; +const SEVERITY_WARNING: &str = "warning"; +const NEXT_ACTION_AUTH_LOGIN: &str = "auth_login"; +const NEXT_ACTION_MANUAL_STEPS: &str = "manual_steps"; +pub(super) const HOOK_IMPACT: &str = + "Claude Code hooks continue working, but telemetry export is disabled."; +const MACOS_INACCESSIBLE_SUMMARY: &str = + "SCE cannot access its encryption key in macOS Keychain without user interaction."; +const MACOS_INACCESSIBLE_REMEDIATION: &str = "To fix: + 1. Run `sce auth login` in an interactive terminal. + 2. When macOS requests access to the SCE Keychain item, + select \"Always Allow\". + 3. Run `sce doctor` again to verify access."; +const RELOGIN_REMEDIATION: &str = "To fix: + 1. Run `sce auth login` in an interactive terminal to provision + a new telemetry credential. + 2. Run `sce doctor` again to verify access."; +const DATABASE_ERROR_REMEDIATION: &str = "To fix: + 1. Make sure no older `sce` executable is running; mixed versions + cannot share the auth DB. + 2. Run `sce doctor` again. + 3. If the problem persists, check the SCE log files for + `sce.telemetry.fallback` events with reason `database_error`."; +const PATH_RESOLUTION_REMEDIATION: &str = "To fix: + 1. Make sure HOME (and XDG_STATE_HOME, if set) point to a writable directory. + 2. Run `sce doctor` again."; +const EXPORTER_INIT_REMEDIATION: &str = "To fix: + 1. Run `sce doctor` again. + 2. If the problem persists, check the SCE log files for + `sce.telemetry.fallback` events with reason `exporter_init_failed`."; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct ManagedTelemetryDiagnostic { + pub(super) reason: &'static str, + pub(super) kind: &'static str, + pub(super) summary: String, + pub(super) remediation: String, + pub(super) next_action: &'static str, +} + +pub(super) fn diagnostic(health: ManagedTelemetryHealth) -> Option { + diagnostic_for_platform(health, CURRENT_CREDENTIAL_STORE_PLATFORM) +} + +fn diagnostic_for_platform( + health: ManagedTelemetryHealth, + platform: CredentialStorePlatform, +) -> Option { + let ManagedTelemetryHealth::Disabled(reason) = health else { + return None; + }; + let (kind, summary, remediation, next_action) = match reason { + TelemetryFallbackReason::NotConfigured => return None, + TelemetryFallbackReason::CredentialInaccessible => { + let (summary, remediation) = inaccessible_guidance(platform); + ( + "managed_telemetry_credential_inaccessible", + summary, + remediation, + NEXT_ACTION_AUTH_LOGIN, + ) + } + TelemetryFallbackReason::CredentialUnavailable => ( + "managed_telemetry_credential_unavailable", + "Managed telemetry was configured, but its credential is no longer available in the auth DB." + .to_string(), + RELOGIN_REMEDIATION.to_string(), + NEXT_ACTION_AUTH_LOGIN, + ), + TelemetryFallbackReason::CredentialExpired => ( + "managed_telemetry_credential_expired", + "The managed telemetry credential has expired.".to_string(), + RELOGIN_REMEDIATION.to_string(), + NEXT_ACTION_AUTH_LOGIN, + ), + TelemetryFallbackReason::LookupTimeout => ( + "managed_telemetry_lookup_timeout", + format!( + "The managed telemetry credential lookup did not finish within {} seconds.", + CREDENTIAL_LOOKUP_DEADLINE.as_secs() + ), + format!( + "To fix:\n 1. Run `sce doctor` again; another SCE process may have held the auth DB.\n 2. If the timeout persists, make sure {} is responsive and unlocked.", + platform.store_name() + ), + NEXT_ACTION_MANUAL_STEPS, + ), + TelemetryFallbackReason::DatabaseError => ( + "managed_telemetry_database_error", + "The auth DB could not be read to load the managed telemetry credential.".to_string(), + DATABASE_ERROR_REMEDIATION.to_string(), + NEXT_ACTION_MANUAL_STEPS, + ), + TelemetryFallbackReason::PathResolutionError => ( + "managed_telemetry_path_resolution_error", + "The SCE state directory could not be resolved, so the managed telemetry credential could not be located." + .to_string(), + PATH_RESOLUTION_REMEDIATION.to_string(), + NEXT_ACTION_MANUAL_STEPS, + ), + TelemetryFallbackReason::ExporterInitFailed => ( + "managed_telemetry_exporter_init_failed", + "The managed telemetry exporter could not be initialized.".to_string(), + EXPORTER_INIT_REMEDIATION.to_string(), + NEXT_ACTION_MANUAL_STEPS, + ), + }; + Some(ManagedTelemetryDiagnostic { + reason: reason.as_str(), + kind, + summary, + remediation, + next_action, + }) +} + +fn inaccessible_guidance(platform: CredentialStorePlatform) -> (String, String) { + if platform == CredentialStorePlatform::Macos { + return ( + MACOS_INACCESSIBLE_SUMMARY.to_string(), + MACOS_INACCESSIBLE_REMEDIATION.to_string(), + ); + } + let store = platform.store_name(); + ( + format!("SCE cannot access its encryption key in {store} without user interaction."), + format!( + "To fix:\n 1. Make sure {store} is available and unlocked.\n 2. Run `sce auth login` in an interactive terminal.\n 3. Run `sce doctor` again to verify access." + ), + ) +} + +pub(super) fn problem(health: ManagedTelemetryHealth) -> Option { + diagnostic(health).map(|diagnostic| DoctorProblem { + kind: ProblemKind::ManagedTelemetryUnavailable, + category: ProblemCategory::GlobalState, + severity: ProblemSeverity::Warning, + fixability: ProblemFixability::ManualOnly, + summary: format!("{} {HOOK_IMPACT}", diagnostic.summary), + remediation: diagnostic.remediation, + next_action: diagnostic.next_action, + scope: None, + mutation_scope_target: None, + }) +} + +pub(super) fn json(health: ManagedTelemetryHealth) -> Value { + match (health, diagnostic(health)) { + (_, Some(diagnostic)) => json!({ + "state": STATE_DISABLED, + "reason": diagnostic.reason, + "kind": diagnostic.kind, + "severity": SEVERITY_WARNING, + "next_action": diagnostic.next_action, + }), + (ManagedTelemetryHealth::Healthy, None) => json!({ + "state": STATE_HEALTHY, + "reason": null, + "kind": null, + "severity": null, + "next_action": null, + }), + (ManagedTelemetryHealth::Disabled(reason), None) => json!({ + "state": STATE_NOT_CONFIGURED, + "reason": reason.as_str(), + "kind": null, + "severity": null, + "next_action": null, + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ALL_REASONS: [TelemetryFallbackReason; 8] = [ + TelemetryFallbackReason::NotConfigured, + TelemetryFallbackReason::CredentialInaccessible, + TelemetryFallbackReason::CredentialUnavailable, + TelemetryFallbackReason::CredentialExpired, + TelemetryFallbackReason::LookupTimeout, + TelemetryFallbackReason::DatabaseError, + TelemetryFallbackReason::PathResolutionError, + TelemetryFallbackReason::ExporterInitFailed, + ]; + + fn disabled(reason: TelemetryFallbackReason) -> ManagedTelemetryHealth { + ManagedTelemetryHealth::Disabled(reason) + } + + #[test] + fn not_configured_and_healthy_telemetry_report_no_problem() { + assert_eq!(problem(ManagedTelemetryHealth::Healthy), None); + assert_eq!( + problem(disabled(TelemetryFallbackReason::NotConfigured)), + None + ); + assert_eq!( + json(ManagedTelemetryHealth::Healthy), + json!({ + "state": "healthy", + "reason": null, + "kind": null, + "severity": null, + "next_action": null, + }) + ); + assert_eq!( + json(disabled(TelemetryFallbackReason::NotConfigured)), + json!({ + "state": "not_configured", + "reason": "not_configured", + "kind": null, + "severity": null, + "next_action": null, + }) + ); + } + + #[test] + fn macos_inaccessible_credential_is_an_actionable_always_allow_warning() { + let diagnostic = diagnostic_for_platform( + disabled(TelemetryFallbackReason::CredentialInaccessible), + CredentialStorePlatform::Macos, + ) + .expect("inaccessible credentials are diagnosed"); + assert_eq!(diagnostic.reason, "credential_inaccessible"); + assert_eq!(diagnostic.kind, "managed_telemetry_credential_inaccessible"); + assert_eq!(diagnostic.next_action, "auth_login"); + assert_eq!( + diagnostic.summary, + "SCE cannot access its encryption key in macOS Keychain without user interaction." + ); + assert_eq!(diagnostic.remediation, MACOS_INACCESSIBLE_REMEDIATION); + assert!(diagnostic.remediation.contains("select \"Always Allow\".")); + } + + #[test] + fn always_allow_is_only_recommended_for_macos_keychain_authorization() { + for platform in [ + CredentialStorePlatform::Macos, + CredentialStorePlatform::Linux, + CredentialStorePlatform::Windows, + CredentialStorePlatform::Other, + ] { + for reason in ALL_REASONS { + let Some(diagnostic) = diagnostic_for_platform(disabled(reason), platform) else { + continue; + }; + let text = format!("{} {}", diagnostic.summary, diagnostic.remediation); + let expects_always_allow = platform == CredentialStorePlatform::Macos + && reason == TelemetryFallbackReason::CredentialInaccessible; + assert_eq!( + text.contains("Always Allow"), + expects_always_allow, + "{platform:?}/{reason:?}: {text}" + ); + if platform != CredentialStorePlatform::Macos { + assert!(!text.contains("macOS"), "{platform:?}/{reason:?}"); + } + } + } + } + + #[test] + fn every_disabled_reason_has_a_stable_kind_and_safe_remediation() { + let expected = [ + ( + TelemetryFallbackReason::CredentialInaccessible, + "managed_telemetry_credential_inaccessible", + "auth_login", + ), + ( + TelemetryFallbackReason::CredentialUnavailable, + "managed_telemetry_credential_unavailable", + "auth_login", + ), + ( + TelemetryFallbackReason::CredentialExpired, + "managed_telemetry_credential_expired", + "auth_login", + ), + ( + TelemetryFallbackReason::LookupTimeout, + "managed_telemetry_lookup_timeout", + "manual_steps", + ), + ( + TelemetryFallbackReason::DatabaseError, + "managed_telemetry_database_error", + "manual_steps", + ), + ( + TelemetryFallbackReason::PathResolutionError, + "managed_telemetry_path_resolution_error", + "manual_steps", + ), + ( + TelemetryFallbackReason::ExporterInitFailed, + "managed_telemetry_exporter_init_failed", + "manual_steps", + ), + ]; + for (reason, kind, next_action) in expected { + let health = disabled(reason); + let problem = problem(health).expect("disabled telemetry is a problem"); + assert_eq!(problem.kind, ProblemKind::ManagedTelemetryUnavailable); + assert_eq!(problem.severity, ProblemSeverity::Warning); + assert_eq!(problem.fixability, ProblemFixability::ManualOnly); + assert_eq!(problem.next_action, next_action); + assert!(problem.summary.ends_with(HOOK_IMPACT)); + let lowered = format!("{} {}", problem.summary, problem.remediation).to_lowercase(); + for forbidden in ["delete", "regenerat", "reset", "bearer", "token:"] { + assert!( + !lowered.contains(forbidden), + "{reason:?} mentions {forbidden}" + ); + } + assert_eq!( + json(health), + json!({ + "state": "disabled", + "reason": reason.as_str(), + "kind": kind, + "severity": "warning", + "next_action": next_action, + }) + ); + } + } +} diff --git a/cli/src/services/doctor/mod.rs b/cli/src/services/doctor/mod.rs index 8cc796382..74c5dbb28 100644 --- a/cli/src/services/doctor/mod.rs +++ b/cli/src/services/doctor/mod.rs @@ -11,6 +11,7 @@ use crate::services::lifecycle::{ lifecycle_providers, FixOutcome, HealthCategory, HealthFixability, HealthProblem, HealthProblemKind, HealthSeverity, LifecycleProvider, LifecycleProviderId, }; +use crate::services::managed_telemetry::ManagedTelemetryHealth; use crate::services::mutation_trace::runtime::{ inspect_reconciliation_recommendation, run_explicit_reconciliation, ReconciliationCounts, ReconciliationFix, ReconciliationRecommendation, @@ -20,6 +21,7 @@ use crate::services::setup; mod fixes; mod inspect; +mod managed_telemetry; mod render; pub(crate) mod types; @@ -60,10 +62,12 @@ struct DoctorDependencies<'a, G, S, C, P> { resolve_state_root: &'a S, resolve_global_config_path: &'a C, probe_codex_hook_policy: &'a P, + managed_telemetry: ManagedTelemetryHealth, } struct DoctorExecution { report: HookDoctorReport, + managed_telemetry: ManagedTelemetryHealth, fix_results: Vec, ref_reconciliation: Option, ref_reconciliation_fix: Option, @@ -122,6 +126,7 @@ async fn execute_doctor_with_context( Ok(resolve_sce_default_locations()?.global_config_file()) }, probe_codex_hook_policy: &codex_hook_policy::probe_default, + managed_telemetry: crate::services::managed_telemetry::inspect_health().await, }, mutation_scope_seam, reconcile_seam, @@ -150,12 +155,14 @@ async fn execute_doctor_with_lifecycle_providers( recommendation_seam: &impl Fn(&Path) -> Option, ) -> DoctorExecution { let policy_readiness = (dependencies.probe_codex_hook_policy)(); + let managed_telemetry = dependencies.managed_telemetry; let providers = lifecycle_providers(true); let initial_problems = diagnose_lifecycle_providers(context, &providers).await; let initial_doctor_problems = initial_problems .iter() .map(|problem| problem.problem.clone()) + .chain(managed_telemetry::problem(managed_telemetry)) .collect::>(); let initial_report = build_report_with_lifecycle_problems( request.mode, @@ -169,6 +176,7 @@ async fn execute_doctor_with_lifecycle_providers( if request.mode != DoctorMode::Fix { return DoctorExecution { report: initial_report, + managed_telemetry, fix_results: Vec::new(), ref_reconciliation: recommendation_seam(repository_root), ref_reconciliation_fix: None, @@ -186,6 +194,7 @@ async fn execute_doctor_with_lifecycle_providers( let final_doctor_problems = final_problems .into_iter() .map(|problem| problem.problem) + .chain(managed_telemetry::problem(managed_telemetry)) .collect::>(); let final_report = build_report_with_lifecycle_problems( request.mode, @@ -209,6 +218,7 @@ async fn execute_doctor_with_lifecycle_providers( DoctorExecution { report: final_report, + managed_telemetry, fix_results, ref_reconciliation: None, ref_reconciliation_fix: Some(reconciliation_fix), @@ -481,6 +491,7 @@ fn doctor_problem_kind(kind: HealthProblemKind) -> ProblemKind { } HealthProblemKind::MutationScopeHealthBlocked => ProblemKind::MutationScopeHealthBlocked, HealthProblemKind::MutationScopeHealthInvalid => ProblemKind::MutationScopeHealthInvalid, + HealthProblemKind::ManagedTelemetryUnavailable => ProblemKind::ManagedTelemetryUnavailable, } } @@ -560,6 +571,7 @@ fn health_problem_kind(kind: ProblemKind) -> HealthProblemKind { } ProblemKind::MutationScopeHealthBlocked => HealthProblemKind::MutationScopeHealthBlocked, ProblemKind::MutationScopeHealthInvalid => HealthProblemKind::MutationScopeHealthInvalid, + ProblemKind::ManagedTelemetryUnavailable => HealthProblemKind::ManagedTelemetryUnavailable, } } diff --git a/cli/src/services/doctor/render.rs b/cli/src/services/doctor/render.rs index e9555e168..25655df8d 100644 --- a/cli/src/services/doctor/render.rs +++ b/cli/src/services/doctor/render.rs @@ -12,7 +12,8 @@ use super::types::{ IntegrationGroupHealth, IntegrationGroupKey, IntegrationTarget, MutationScopeHealthRow, PostCommitAutoSyncState, ProblemKind, ProblemSeverity, Readiness, }; -use super::{DoctorExecution, DoctorFormat, DoctorMode, DoctorRequest, NAME}; +use super::{managed_telemetry, DoctorExecution, DoctorFormat, DoctorMode, DoctorRequest, NAME}; +use crate::services::managed_telemetry::ManagedTelemetryHealth; use crate::services::mutation_trace::runtime::{ ReconciliationCounts, ReconciliationFix, ReconciliationRecommendation, StateWarning, }; @@ -30,7 +31,7 @@ pub(super) fn render_report(request: DoctorRequest, execution: &DoctorExecution) fn format_execution(execution: &DoctorExecution) -> String { let report = &execution.report; - let base_report = format_report(report); + let base_report = format_report(report, execution.managed_telemetry); let mut lines = base_report .lines() .map(ToOwned::to_owned) @@ -103,11 +104,15 @@ fn format_age(age_ms: i64) -> String { } } -fn format_report(report: &HookDoctorReport) -> String { - format_report_with_color_policy(report, supports_color()) +fn format_report(report: &HookDoctorReport, managed_telemetry: ManagedTelemetryHealth) -> String { + format_report_with_color_policy(report, managed_telemetry, supports_color()) } -fn format_report_with_color_policy(report: &HookDoctorReport, color_enabled: bool) -> String { +fn format_report_with_color_policy( + report: &HookDoctorReport, + managed_telemetry: ManagedTelemetryHealth, + color_enabled: bool, +) -> String { let blocking_problem_count = report .problems .iter() @@ -125,7 +130,7 @@ fn format_report_with_color_policy(report: &HookDoctorReport, color_enabled: boo }); lines.push(format!("\n{}", heading("Environment"))); - for node in environment_nodes(report) { + for node in environment_nodes(report, managed_telemetry) { render_display_node(&mut lines, &node, color_enabled, 2, true); } @@ -248,7 +253,10 @@ fn format_human_text_row( ) } -fn environment_nodes(report: &HookDoctorReport) -> Vec { +fn environment_nodes( + report: &HookDoctorReport, + managed_telemetry: ManagedTelemetryHealth, +) -> Vec { vec![ top_level_node( "State", @@ -283,9 +291,32 @@ fn environment_nodes(report: &HookDoctorReport) -> Vec { ) }), ), + managed_telemetry_node(managed_telemetry), ] } +fn managed_telemetry_node(health: ManagedTelemetryHealth) -> DoctorDisplayNode { + match (health, managed_telemetry::diagnostic(health)) { + (_, Some(diagnostic)) => top_level_node( + "Managed telemetry", + DoctorDisplayStatus::Warn, + vec![DoctorDisplayDetail::ManagedTelemetry { + reason: diagnostic.reason, + summary: diagnostic.summary, + remediation: diagnostic.remediation, + }], + ), + (ManagedTelemetryHealth::Healthy, None) => { + top_level_node("Managed telemetry", DoctorDisplayStatus::Pass, Vec::new()) + } + (ManagedTelemetryHealth::Disabled(_), None) => top_level_node( + "Managed telemetry (not configured)", + DoctorDisplayStatus::Pass, + Vec::new(), + ), + } +} + fn top_level_node( label: &str, status: DoctorDisplayStatus, @@ -696,6 +727,16 @@ fn render_display_detail(lines: &mut Vec, detail: &DoctorDisplayDetail, lines.push(format!("{prefix}Remediation: {remediation}")); } } + DoctorDisplayDetail::ManagedTelemetry { + reason, + summary, + remediation, + } => { + lines.push(format!("{prefix}Reason: {reason}")); + lines.push(format!("{prefix}{summary}")); + lines.push(format!("{prefix}{}", managed_telemetry::HOOK_IMPACT)); + lines.extend(remediation.lines().map(|line| format!("{prefix}{line}"))); + } } } @@ -868,8 +909,6 @@ fn render_report_json(execution: &DoctorExecution) -> Result { }) .collect::>(); - let mutation_scope_health = mutation_scope_health_json(&report.mutation_scope_health); - let payload = json!({ "status": "ok", "command": NAME, @@ -917,7 +956,8 @@ fn render_report_json(execution: &DoctorExecution) -> Result { }, "config_paths": config_paths, "hooks": hooks, - "mutation_scope_health": mutation_scope_health, + "mutation_scope_health": mutation_scope_health_json(&report.mutation_scope_health), + "managed_telemetry": managed_telemetry::json(execution.managed_telemetry), "problems": report.problems.iter().map(|problem| json!({ "category": problem_category(problem.category), "severity": problem_severity(problem.severity), @@ -1228,3 +1268,55 @@ mod reconciliation_render_tests { } } } + +#[cfg(test)] +mod managed_telemetry_render_tests { + use super::{managed_telemetry_node, render_display_node}; + use crate::services::managed_telemetry::{ManagedTelemetryHealth, TelemetryFallbackReason}; + + fn rendered(health: ManagedTelemetryHealth) -> Vec { + let mut lines = Vec::new(); + render_display_node(&mut lines, &managed_telemetry_node(health), false, 2, true); + lines + } + + #[test] + fn healthy_and_unconfigured_telemetry_render_as_single_pass_rows() { + assert_eq!( + rendered(ManagedTelemetryHealth::Healthy), + [" [PASS] Managed telemetry"] + ); + assert_eq!( + rendered(ManagedTelemetryHealth::Disabled( + TelemetryFallbackReason::NotConfigured + )), + [" [PASS] Managed telemetry (not configured)"] + ); + } + + #[test] + fn inaccessible_credentials_render_reason_impact_and_login_steps() { + let lines = rendered(ManagedTelemetryHealth::Disabled( + TelemetryFallbackReason::CredentialInaccessible, + )); + assert_eq!(lines[0], " [WARN] Managed telemetry"); + assert_eq!(lines[1], " Reason: credential_inaccessible"); + assert!(lines[2].contains("without user interaction.")); + assert_eq!( + lines[3], + " Claude Code hooks continue working, but telemetry export is disabled." + ); + assert_eq!(lines[4], " To fix:"); + assert!(lines + .iter() + .any(|line| line.ends_with(". Run `sce auth login` in an interactive terminal."))); + assert_eq!( + lines.last().map(String::as_str), + Some(" 3. Run `sce doctor` again to verify access.") + ); + assert_eq!( + lines.iter().any(|line| line.contains("Always Allow")), + cfg!(target_os = "macos") + ); + } +} diff --git a/cli/src/services/doctor/types.rs b/cli/src/services/doctor/types.rs index f03489545..57ea0fa51 100644 --- a/cli/src/services/doctor/types.rs +++ b/cli/src/services/doctor/types.rs @@ -278,6 +278,11 @@ pub(super) enum DoctorDisplayDetail { detail: Option, remediation: Option, }, + ManagedTelemetry { + reason: &'static str, + summary: String, + remediation: String, + }, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -464,6 +469,7 @@ pub(crate) enum ProblemKind { MutationScopeHealthRecovering, MutationScopeHealthBlocked, MutationScopeHealthInvalid, + ManagedTelemetryUnavailable, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/cli/src/services/error.rs b/cli/src/services/error.rs index a99303543..b3b7004b9 100644 --- a/cli/src/services/error.rs +++ b/cli/src/services/error.rs @@ -57,6 +57,7 @@ pub enum UserError { NotGitRepository, NotGitRemote, AuthStorageUnavailable, + LoginAuthStorageUnavailable, UnexpectedFailure, } @@ -67,6 +68,7 @@ impl UserError { | Self::NotGitRepository | Self::NotGitRemote | Self::AuthStorageUnavailable + | Self::LoginAuthStorageUnavailable | Self::UnexpectedFailure => FailureClass::Runtime, } } @@ -76,7 +78,9 @@ impl UserError { Self::NotAuthenticated => "auth.not_authenticated", Self::NotGitRepository => "setup.not_git_repository", Self::NotGitRemote { .. } => "setup.not_git_remote", - Self::AuthStorageUnavailable => "auth.storage_unavailable", + Self::AuthStorageUnavailable | Self::LoginAuthStorageUnavailable => { + "auth.storage_unavailable" + } Self::UnexpectedFailure => "general.unexpected_failure", } } @@ -92,9 +96,14 @@ impl UserError { Self::NotGitRemote => { "The Git repository has no configured remote URL. Please run `git remote add `, then retry." } - Self::AuthStorageUnavailable => { - "Authentication storage is unavailable. Verify local credential storage is available, then retry the command." - } + Self::AuthStorageUnavailable => auth_storage_unavailable_message( + CURRENT_CREDENTIAL_STORE_PLATFORM, + AuthStorageOperation::Command, + ), + Self::LoginAuthStorageUnavailable => auth_storage_unavailable_message( + CURRENT_CREDENTIAL_STORE_PLATFORM, + AuthStorageOperation::Login, + ), Self::UnexpectedFailure => { "An unexpected error occurred. Check the log files for more details." } @@ -102,6 +111,163 @@ impl UserError { } } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CredentialStorePlatform { + Macos, + Linux, + Windows, + Other, +} + +impl CredentialStorePlatform { + pub(crate) const fn store_name(self) -> &'static str { + match self { + Self::Macos => "macOS Keychain", + Self::Linux => "the system keyring (Secret Service)", + Self::Windows => "Windows Credential Manager", + Self::Other => "the OS credential store", + } + } +} + +pub(crate) const CURRENT_CREDENTIAL_STORE_PLATFORM: CredentialStorePlatform = + if cfg!(target_os = "macos") { + CredentialStorePlatform::Macos + } else if cfg!(target_os = "linux") { + CredentialStorePlatform::Linux + } else if cfg!(target_os = "windows") { + CredentialStorePlatform::Windows + } else { + CredentialStorePlatform::Other + }; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AuthStorageOperation { + Command, + Login, +} + +const MACOS_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in macOS Keychain. + +To fix: + 1. Run `sce auth login` in an interactive terminal. + 2. When macOS asks for permission to access the SCE Keychain item, + choose \"Always Allow\". + 3. Retry the original command. + +If no prompt appears, verify that your login Keychain is unlocked +and retry using the same installed SCE executable."; + +const MACOS_LOGIN_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in macOS Keychain, so `sce auth login` +could not use its stored session. Running login again does not help until +Keychain access is allowed. + +To fix: + 1. Unlock your login Keychain (for example, in Keychain Access). + 2. Run `sce auth login` again in an interactive terminal. + 3. When macOS asks for permission to access the SCE Keychain item, + choose \"Always Allow\" to authorize this SCE executable. + +If no prompt appears, verify that your login Keychain is unlocked +and retry using the same installed SCE executable."; + +const LINUX_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in the system keyring (Secret Service). + +To fix: + 1. Make sure a Secret Service provider (such as GNOME Keyring or KWallet) + is running and its default collection is unlocked. + 2. Run `sce auth login` in an interactive terminal. + 3. Retry the original command."; + +const LINUX_LOGIN_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in the system keyring (Secret Service), +so `sce auth login` could not use its stored session. Running login again does +not help until the keyring is available. + +To fix: + 1. Make sure a Secret Service provider (such as GNOME Keyring or KWallet) + is running and its default collection is unlocked. + 2. Retry `sce auth login` in an interactive terminal."; + +const WINDOWS_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in Windows Credential Manager. + +To fix: + 1. Make sure you are signed in to your Windows user account and + Windows Credential Manager is available. + 2. Run `sce auth login` in an interactive terminal. + 3. Retry the original command."; + +const WINDOWS_LOGIN_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in Windows Credential Manager, +so `sce auth login` could not use its stored session. Running login again +does not help until Credential Manager is available. + +To fix: + 1. Make sure you are signed in to your Windows user account and + Windows Credential Manager is available. + 2. Retry `sce auth login` in an interactive terminal."; + +const OTHER_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in the OS credential store. + +To fix: + 1. Make sure a supported OS credential store is available and unlocked. + 2. Run `sce auth login` in an interactive terminal. + 3. Retry the original command."; + +const OTHER_LOGIN_AUTH_STORAGE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not access its encryption key in the OS credential store, +so `sce auth login` could not use its stored session. Running login again +does not help until the credential store is available. + +To fix: + 1. Make sure a supported OS credential store is available and unlocked. + 2. Retry `sce auth login` in an interactive terminal."; + +const fn auth_storage_unavailable_message( + platform: CredentialStorePlatform, + operation: AuthStorageOperation, +) -> &'static str { + match (platform, operation) { + (CredentialStorePlatform::Macos, AuthStorageOperation::Command) => { + MACOS_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Macos, AuthStorageOperation::Login) => { + MACOS_LOGIN_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Linux, AuthStorageOperation::Command) => { + LINUX_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Linux, AuthStorageOperation::Login) => { + LINUX_LOGIN_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Windows, AuthStorageOperation::Command) => { + WINDOWS_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Windows, AuthStorageOperation::Login) => { + WINDOWS_LOGIN_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Other, AuthStorageOperation::Command) => { + OTHER_AUTH_STORAGE_UNAVAILABLE + } + (CredentialStorePlatform::Other, AuthStorageOperation::Login) => { + OTHER_LOGIN_AUTH_STORAGE_UNAVAILABLE + } + } +} + impl std::fmt::Display for UserError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.message()) @@ -305,4 +471,113 @@ mod tests { assert_eq!(validation_error.class(), FailureClass::Validation); assert_eq!(validation_error.to_string(), "bad value"); } + + const ALL_PLATFORMS: [CredentialStorePlatform; 4] = [ + CredentialStorePlatform::Macos, + CredentialStorePlatform::Linux, + CredentialStorePlatform::Windows, + CredentialStorePlatform::Other, + ]; + const ALL_OPERATIONS: [AuthStorageOperation; 2] = + [AuthStorageOperation::Command, AuthStorageOperation::Login]; + + #[test] + fn auth_storage_unavailable_keeps_stable_key_class_and_exit_code() { + for user_error in [ + UserError::AuthStorageUnavailable, + UserError::LoginAuthStorageUnavailable, + ] { + let error = CliError::user_with_source( + user_error.clone(), + anyhow::anyhow!("keychain item secret-key-material denied"), + ); + assert_eq!(user_error.key(), "auth.storage_unavailable"); + assert_eq!(error.class(), FailureClass::Runtime); + assert_eq!(error.code(), "SCE-ERR-RUNTIME"); + assert_eq!(error.class().exit_code(), 4); + assert_eq!(error.to_string(), user_error.message()); + assert!(!error.to_string().contains("secret-key-material")); + } + } + + #[test] + fn macos_auth_storage_guidance_explains_login_and_always_allow() { + let message = auth_storage_unavailable_message( + CredentialStorePlatform::Macos, + AuthStorageOperation::Command, + ); + assert!(message.starts_with("Authentication storage is unavailable.\n")); + assert!(message.contains("macOS Keychain")); + assert!(message.contains("Run `sce auth login` in an interactive terminal.")); + assert!(message.contains("choose \"Always Allow\".")); + assert!(message.contains("Retry the original command.")); + assert!(message.contains("login Keychain is unlocked")); + } + + #[test] + fn macos_login_storage_guidance_is_not_circular() { + let message = auth_storage_unavailable_message( + CredentialStorePlatform::Macos, + AuthStorageOperation::Login, + ); + assert!(message.contains("Running login again does not help until")); + assert!(message.contains("Unlock your login Keychain")); + assert!(message.contains("\"Always Allow\" to authorize this SCE executable")); + assert!(!message.contains("Retry the original command.")); + } + + #[test] + fn non_macos_auth_storage_guidance_never_mentions_macos() { + for platform in [ + CredentialStorePlatform::Linux, + CredentialStorePlatform::Windows, + CredentialStorePlatform::Other, + ] { + for operation in ALL_OPERATIONS { + let message = auth_storage_unavailable_message(platform, operation); + assert!(message.starts_with("Authentication storage is unavailable.\n")); + assert!(message.contains("sce auth login"), "{platform:?}"); + assert!(!message.contains("macOS"), "{platform:?}"); + assert!(!message.contains("Keychain"), "{platform:?}"); + assert!(!message.contains("Always Allow"), "{platform:?}"); + } + } + } + + #[test] + fn auth_storage_guidance_never_recommends_destructive_repair() { + for platform in ALL_PLATFORMS { + for operation in ALL_OPERATIONS { + let message = auth_storage_unavailable_message(platform, operation).to_lowercase(); + for forbidden in ["auth.db", "delete", "regenerat", "reset", "permanent"] { + assert!( + !message.contains(forbidden), + "{platform:?}/{operation:?} mentions {forbidden:?}" + ); + } + } + } + } + + #[test] + fn current_platform_message_is_selected_for_display() { + assert_eq!( + UserError::AuthStorageUnavailable.message(), + auth_storage_unavailable_message( + CURRENT_CREDENTIAL_STORE_PLATFORM, + AuthStorageOperation::Command + ) + ); + assert_eq!( + UserError::LoginAuthStorageUnavailable.message(), + auth_storage_unavailable_message( + CURRENT_CREDENTIAL_STORE_PLATFORM, + AuthStorageOperation::Login + ) + ); + assert_eq!( + CURRENT_CREDENTIAL_STORE_PLATFORM == CredentialStorePlatform::Macos, + cfg!(target_os = "macos") + ); + } } diff --git a/cli/src/services/lifecycle.rs b/cli/src/services/lifecycle.rs index a343f1b2c..20490aca9 100644 --- a/cli/src/services/lifecycle.rs +++ b/cli/src/services/lifecycle.rs @@ -76,6 +76,7 @@ pub enum HealthProblemKind { MutationScopeHealthRecovering, MutationScopeHealthBlocked, MutationScopeHealthInvalid, + ManagedTelemetryUnavailable, } #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/cli/src/services/managed_telemetry.rs b/cli/src/services/managed_telemetry.rs index 839639413..df5dec860 100644 --- a/cli/src/services/managed_telemetry.rs +++ b/cli/src/services/managed_telemetry.rs @@ -13,7 +13,7 @@ use trace_export::{ManagedSpanKind, ManagedTraceTelemetry}; use crate::services::telemetry_credential_storage::{ load_telemetry_credential, TelemetryCredential, }; -use crate::services::telemetry_readiness; +use crate::services::telemetry_readiness::{self, ReadinessState}; use crate::services::token_storage::TokenStorageError; pub const MANAGED_OTLP_ENDPOINT: &str = "https://otel.crocoderlab.dev"; @@ -220,6 +220,47 @@ where } } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ManagedTelemetryHealth { + Healthy, + Disabled(TelemetryFallbackReason), +} + +pub(crate) async fn inspect_health() -> ManagedTelemetryHealth { + let now = now_unix_seconds(); + let Ok(readiness) = telemetry_readiness::state(now) else { + return ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::PathResolutionError); + }; + inspect_health_with(readiness, load_telemetry_credential(), now).await +} + +pub(crate) async fn inspect_health_with( + readiness: ReadinessState, + lookup: Fut, + now_unix_seconds: u64, +) -> ManagedTelemetryHealth +where + Fut: Future, TokenStorageError>> + Send + 'static, +{ + match readiness { + ReadinessState::NotConfigured => { + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::NotConfigured) + } + ReadinessState::Expired => { + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::CredentialExpired) + } + ReadinessState::Ready => match load_credential_within_deadline(lookup).await { + Err(fallback) => ManagedTelemetryHealth::Disabled(fallback.reason), + Ok(credential) => match select_mode(Some(credential), now_unix_seconds) { + ManagedTelemetryMode::Managed { .. } => ManagedTelemetryHealth::Healthy, + ManagedTelemetryMode::Noop => { + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::CredentialExpired) + } + }, + }, + } +} + fn now_unix_seconds() -> u64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -481,6 +522,94 @@ mod tests { ); } + fn tracked_lookup( + result: Result, TokenStorageError>, + ) -> ( + std::sync::Arc, + impl Future, TokenStorageError>> + Send + 'static, + ) { + let looked_up = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = looked_up.clone(); + (looked_up, async move { + flag.store(true, std::sync::atomic::Ordering::SeqCst); + result + }) + } + + #[tokio::test] + async fn health_without_readiness_or_with_expired_readiness_never_looks_up_credentials() { + for (readiness, reason) in [ + ( + ReadinessState::NotConfigured, + TelemetryFallbackReason::NotConfigured, + ), + ( + ReadinessState::Expired, + TelemetryFallbackReason::CredentialExpired, + ), + ] { + let (looked_up, lookup) = tracked_lookup(Ok(Some(credential("bao-token", NOW + 60)))); + let health = inspect_health_with(readiness, lookup, NOW).await; + assert_eq!(health, ManagedTelemetryHealth::Disabled(reason)); + assert!(!looked_up.load(std::sync::atomic::Ordering::SeqCst)); + } + } + + #[tokio::test] + async fn health_with_readiness_classifies_lookup_outcomes() { + let cases: Vec<( + Result, TokenStorageError>, + ManagedTelemetryHealth, + )> = vec![ + ( + Ok(Some(credential("bao-token", NOW + 60))), + ManagedTelemetryHealth::Healthy, + ), + ( + Err(TokenStorageError::CredentialAccessUnavailable), + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::CredentialInaccessible), + ), + ( + Ok(None), + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::CredentialUnavailable), + ), + ( + Ok(Some(credential("bao-token", NOW))), + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::CredentialExpired), + ), + ( + Err(TokenStorageError::Database("locked".to_string())), + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::DatabaseError), + ), + ( + Err(TokenStorageError::PathResolution("no home".to_string())), + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::PathResolutionError), + ), + ]; + for (result, expected) in cases { + let (looked_up, lookup) = tracked_lookup(result); + let health = inspect_health_with(ReadinessState::Ready, lookup, NOW).await; + assert_eq!(health, expected); + assert!(looked_up.load(std::sync::atomic::Ordering::SeqCst)); + } + } + + #[tokio::test] + async fn health_lookup_is_bounded_by_the_credential_lookup_deadline() { + let started = std::time::Instant::now(); + let health = inspect_health_with( + ReadinessState::Ready, + std::future::pending::, TokenStorageError>>(), + NOW, + ) + .await; + assert!(started.elapsed() < CREDENTIAL_LOOKUP_DEADLINE * 2); + assert_eq!( + health, + ManagedTelemetryHealth::Disabled(TelemetryFallbackReason::LookupTimeout) + ); + } + #[test] fn absent_expired_or_blank_credential_selects_noop() { assert_eq!(select_mode(None, NOW), ManagedTelemetryMode::Noop); diff --git a/cli/src/services/telemetry_readiness.rs b/cli/src/services/telemetry_readiness.rs index a20f71c71..606c7ccf2 100644 --- a/cli/src/services/telemetry_readiness.rs +++ b/cli/src/services/telemetry_readiness.rs @@ -9,6 +9,13 @@ use crate::services::default_paths::{auth_db_path, telemetry_readiness_path}; const TEMP_SUFFIX: &str = "tmp"; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReadinessState { + NotConfigured, + Expired, + Ready, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] struct TelemetryReadiness { expires_at_unix_seconds: u64, @@ -23,11 +30,15 @@ pub fn clear() -> Result<()> { } pub fn is_ready(now_unix_seconds: u64) -> bool { - let (Ok(readiness_path), Ok(auth_db_path)) = (telemetry_readiness_path(), auth_db_path()) - else { - return false; - }; - is_ready_at(&readiness_path, &auth_db_path, now_unix_seconds) + state(now_unix_seconds).is_ok_and(|state| state == ReadinessState::Ready) +} + +pub(crate) fn state(now_unix_seconds: u64) -> Result { + Ok(state_at( + &telemetry_readiness_path()?, + &auth_db_path()?, + now_unix_seconds, + )) } pub(crate) fn mark_ready_at(path: &Path, expires_at_unix_seconds: u64) -> Result<()> { @@ -60,12 +71,25 @@ pub(crate) fn clear_at(path: &Path) -> Result<()> { } } +#[cfg(test)] pub(crate) fn is_ready_at(path: &Path, auth_db_path: &Path, now_unix_seconds: u64) -> bool { - auth_db_path.is_file() - && fs::read(path) - .ok() - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) - .is_some_and(|readiness| readiness.expires_at_unix_seconds > now_unix_seconds) + state_at(path, auth_db_path, now_unix_seconds) == ReadinessState::Ready +} + +pub(crate) fn state_at(path: &Path, auth_db_path: &Path, now_unix_seconds: u64) -> ReadinessState { + if !auth_db_path.is_file() { + return ReadinessState::NotConfigured; + } + match fs::read(path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + { + None => ReadinessState::NotConfigured, + Some(readiness) if readiness.expires_at_unix_seconds > now_unix_seconds => { + ReadinessState::Ready + } + Some(_) => ReadinessState::Expired, + } } fn temp_path_for(path: &Path) -> PathBuf { @@ -118,6 +142,29 @@ mod tests { ); } + #[test] + fn readiness_state_distinguishes_unconfigured_expired_and_ready() { + let paths = paths_with_auth_db(); + assert_eq!( + state_at(&paths.readiness, &paths.auth_db, NOW), + ReadinessState::NotConfigured + ); + mark_ready_at(&paths.readiness, NOW + 1).unwrap(); + assert_eq!( + state_at(&paths.readiness, &paths.auth_db, NOW), + ReadinessState::Ready + ); + assert_eq!( + state_at(&paths.readiness, &paths.auth_db, NOW + 1), + ReadinessState::Expired + ); + fs::remove_file(&paths.auth_db).unwrap(); + assert_eq!( + state_at(&paths.readiness, &paths.auth_db, NOW + 1), + ReadinessState::NotConfigured + ); + } + #[test] fn readiness_without_auth_db_is_stale() { let paths = paths_with_auth_db(); diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md index 830e9fe57..efa9c8441 100644 --- a/context/plans/claude-hook-trace-propagation.md +++ b/context/plans/claude-hook-trace-propagation.md @@ -290,3 +290,36 @@ Persist this field in every plan; this is durable plan state, not chat state: - Linux was not re-run in this session. Its lock code is unchanged; only the conflict check now also names `EWOULDBLOCK` (== `EAGAIN` on Linux). Run `nix flake check` on x86_64-linux before merge. - Tested on macOS 15.4.1 only. Older macOS versions were not tested. - macOS was not tested against the real pre-fix `sce` binary. The legacy side used the pre-fix Turso builder configuration and raw `F_SETLK` `F_WRLCK` children. + +## Follow-up validation: auth storage guidance and doctor telemetry health (PR #330) + +**Date:** 2026-10-19 + +### Change + +- `UserError::AuthStorageUnavailable` now shows platform-specific recovery steps. On macOS these are: run `sce auth login` interactively, choose "Always Allow" in the Keychain dialog, then retry. A new `LoginAuthStorageUnavailable` variant covers storage failures inside `sce auth login`, so its text does not tell the user to rerun login before Keychain access is allowed. Both keep key `auth.storage_unavailable`, the runtime class, and exit code 4. Details: [User-facing storage guidance](../sce/auth-db.md#user-facing-storage-guidance). +- `sce doctor` has a read-only `Managed telemetry` row and a JSON `managed_telemetry` object. It reuses the readiness hint, the non-interactive existing-DB credential lookup, the lookup deadline, and `TelemetryFallbackReason`. Details: [Doctor health check](../sce/telemetry-credential.md#doctor-health-check). +- `telemetry_readiness::state` separates `NotConfigured`, `Expired`, and `Ready` without opening `auth.db`. `is_ready` behavior is unchanged. +- The multiprocess test sandbox now resolves the macOS state directory (`$HOME/Library/Application Support/sce`). Before, `auth_db_files()` checked `state/sce` on macOS, so its "no `auth.db*` files created" assertions passed without checking anything there. +- No changes to hook telemetry resolution, fallback logging levels, exporter lifecycle, setup, the compatibility lock, migrations, signing, or release workflows. + +### Commands run (macOS 15.4.1, Darwin 24.4.0, arm64) + +- `nix flake check` -> exit 0 on aarch64-darwin (`cli-tests`, `cli-clippy`, `cli-fmt`, and the other checks). The first run failed only in `concurrent_legacy_and_new_writers_never_lose_committed_rows`, on a `turso_core` `wal.rs` panic (`shared WAL frame ids must increase monotonically`). That is the upstream multiprocess-WAL race listed above. The test does not use the changed code, passed 6 of 6 isolated reruns, and passed on the full rerun. +- Targeted filters: `services::error` 15, `services::doctor` 11, `services::managed_telemetry` 29, `services::telemetry_readiness` 6, `services::auth_command` 17, `app::doctor_reconciliation_cli_tests` 9, `services::auth_db::multiprocess_tests` 24. All passed. +- Sandboxed debug binary (`HOME` in the scratchpad, `SCE_AUTH_DB_ENCRYPTION_KEY` set) with an unreadable `auth.db` and a ready hint: text shows `[WARN] Managed telemetry` / `Reason: database_error` and the step list. JSON has `managed_telemetry.kind = managed_telemetry_database_error`. With the hint removed: `[PASS] Managed telemetry (not configured)`. + +### Results + +- Fresh state: doctor reports `not_configured` with no warning and creates no `auth.db`, `telemetry-ready.json`, or key. The child runs with an empty `SCE_AUTH_DB_ENCRYPTION_KEY`, so any key resolution would have failed the check. +- Real seeded auth DB: `healthy` with the right key and `database_error` with a wrong or empty key. An expired hint gives `credential_expired` and a removed hint gives `not_configured`, both without a lookup. `auth.db` bytes stay unchanged and the credential still loads afterwards. +- `sce doctor --fix` with an unreadable `auth.db`: one `manual` fix result, and `auth.db` and the hint are byte-for-byte unchanged. Neither stdout nor stderr contains the secret. +- Unit tests cover the classification table, the lookup deadline (`lookup_timeout`), "Always Allow" only for macOS `credential_inaccessible`, no destructive or secret-bearing text, and noncircular login guidance. + +### Residual risks + +- Not run against the real macOS Keychain. No live `credential_inaccessible` doctor run or real "Always Allow" dialog was exercised in this session. Running it would have read the user's login Keychain. Its text is covered by unit tests, and the non-interactive read path is unchanged. +- "Always Allow" trust is tied to the executable's code signature. Unsigned or ad-hoc-signed upgrades can lose it, and the user then has to authorize again through `sce auth login`. This is out of scope without a Developer ID certificate. +- Interactive Keychain failures reach `AuthStorageUnavailable` as `TokenStorageError::Database`, so a DB failure during login also shows credential-store guidance. +- Doctor repeats the startup credential lookup (cached `AuthDb` on success), adding up to `CREDENTIAL_LOOKUP_DEADLINE` when the store hangs. Hook startup is unaffected. +- Linux was not re-run in this session. Run `nix flake check` on x86_64-linux before merge. diff --git a/context/sce/auth-db.md b/context/sce/auth-db.md index 3cc136307..58f83c732 100644 --- a/context/sce/auth-db.md +++ b/context/sce/auth-db.md @@ -123,8 +123,19 @@ Remaining limits: a mixed old/new deployment is still not usable at the same tim - Encryption is required: `encryption_key::get_or_create_encryption_key()` first honors `SCE_AUTH_DB_ENCRYPTION_KEY` by deriving a deterministic 64-character hex Turso key from the non-empty env-secret text, then falls back to the OS credential store when the env var is absent. No plaintext auth DB fallback exists. - Credential access is explicit (`encryption_key::CredentialAccess`): - `Interactive`: used only by `sce auth login`, `logout`, and `whoami`. It may show the macOS Keychain authorization dialog. It generates and stores a new key only when the keyring reports `NoEntry` and `auth.db` does not exist. Any other read error is returned and never overwrites the stored key. - - `NonInteractive`: used by Claude hooks, managed telemetry startup, `sce auth telemetry-headers`, and `sce sync` (including post-commit auto-sync). It only reads, never creates. On macOS it wraps the read in `SecKeychain::disable_user_interaction()` (`SecKeychainSetUserInteractionAllowed(false)`, the UI switch for the legacy file-based keychain API that `apple-native-keyring-store`'s `keychain` feature uses through `SecKeychainFindGenericPassword`). An item the binary is not authorized for, or a locked keychain, fails with `errSecInteractionNotAllowed` instead of prompting. Every failure becomes `CredentialAccessUnavailable`. `kSecUseAuthenticationUIFail` and `LAContext.interactionNotAllowed` do not apply, because they belong to the `SecItem`/data-protection keychain and LocalAuthentication-protected items, which this adapter does not use. + - `NonInteractive`: used by Claude hooks, managed telemetry startup, the `sce doctor` telemetry health check, `sce auth telemetry-headers`, and `sce sync` (including post-commit auto-sync). It only reads, never creates. On macOS it wraps the read in `SecKeychain::disable_user_interaction()` (`SecKeychainSetUserInteractionAllowed(false)`, the UI switch for the legacy file-based keychain API that `apple-native-keyring-store`'s `keychain` feature uses through `SecKeychainFindGenericPassword`). An item the binary is not authorized for, or a locked keychain, fails with `errSecInteractionNotAllowed` instead of prompting. Every failure becomes `CredentialAccessUnavailable`. `kSecUseAuthenticationUIFail` and `LAContext.interactionNotAllowed` do not apply, because they belong to the `SecItem`/data-protection keychain and LocalAuthentication-protected items, which this adapter does not use. - The interaction switch is process-wide, so all credential-store reads and writes in one process are serialized by a mutex. Interaction is turned back on before the next caller runs. - Linux Secret Service and Windows have no equivalent switch in the adapter. `NonInteractive` there only guarantees no key creation. Missing/unavailable credential-store failures surface as `TokenStorageError::Database` and include platform-specific remediation for Linux system keyring/Secret Service, macOS Keychain, Windows Credential Store, or unsupported platforms, always mentioning `SCE_AUTH_DB_ENCRYPTION_KEY` for headless/CI use without printing the secret value. +## User-facing storage guidance + +Storage failures from auth commands and `sce sync` render the `UserError` catalog entry for `auth.storage_unavailable` (runtime class, `SCE-ERR-RUNTIME`, exit code 4). The key, class, and exit code are shared by two variants in `cli/src/services/error.rs`: + +- `AuthStorageUnavailable`: logout, whoami, and sync. The text is platform-specific. On macOS it says SCE could not access its encryption key in macOS Keychain, then lists the steps: run `sce auth login` in an interactive terminal, choose "Always Allow" when macOS asks to access the SCE Keychain item, and retry. If no prompt appears, the user should check that the login Keychain is unlocked and use the same installed executable. Linux names Secret Service, Windows names Credential Manager, and other platforms name the OS credential store. None of them mention macOS or "Always Allow". +- `LoginAuthStorageUnavailable`: storage failures inside `sce auth login` (stored-session load, renewal save, session replacement, device-flow storage errors). It says that running login again does not help until access is allowed. On macOS the steps are: unlock the login Keychain, rerun `sce auth login`, and choose "Always Allow" to authorize this executable. + +The text never says that "Always Allow" survives a binary upgrade. Unsigned and ad-hoc-signed builds can lose Keychain trust when the executable changes. It never suggests deleting `auth.db`, regenerating the key, or resetting the Keychain. The `anyhow` source stays attached for logs. Only the fixed catalog text reaches stderr, after `redact_sensitive_text`. + +Limit: an interactive Keychain failure reaches this mapping as `TokenStorageError::Database`, the same variant as a real DB failure. So a corrupt or locked DB also shows the credential-store guidance. The precise cause is in the log file. + See also: [shared-turso-db.md](shared-turso-db.md), [../cli/default-path-catalog.md](../cli/default-path-catalog.md), [../context-map.md](../context-map.md) diff --git a/context/sce/doctor-human-text-contract.md b/context/sce/doctor-human-text-contract.md index e4116634e..1ce0250d0 100644 --- a/context/sce/doctor-human-text-contract.md +++ b/context/sce/doctor-human-text-contract.md @@ -12,8 +12,11 @@ Human text output renders these sections in this exact order: 2. `Repository` 3. `Integrations` -The `Environment` domain contains `State`, `Configuration`, and `Repository -identity`. The `Repository` domain contains `Git repository`, `Post-commit Agent +The `Environment` domain contains `State`, `Configuration`, `Repository +identity`, and `Managed telemetry` (read-only credential health; rows, JSON, and +remediation in [telemetry-credential.md](telemetry-credential.md#doctor-health-check)). +A warning row expands into `Reason: `, the summary, the hook-impact line, +and a `To fix:` step list. The `Repository` domain contains `Git repository`, `Post-commit Agent Trace auto-sync`, and `Git hooks`. When snapshot-ref reconciliation is recommended (or its maintenance state is diff --git a/context/sce/telemetry-credential.md b/context/sce/telemetry-credential.md index 2f3e54e6d..8226d690e 100644 --- a/context/sce/telemetry-credential.md +++ b/context/sce/telemetry-credential.md @@ -1,6 +1,6 @@ # Telemetry credential -The CLI holds one telemetry-only OpenBao credential, issued by the Control Plane and cached in the encrypted auth DB. `sce auth login` provisions it best-effort, and managed OTLP trace and log export reads it at startup as the bearer source. There is no user-facing provisioning command, refresh loop, or doctor check. +The CLI holds one telemetry-only OpenBao credential, issued by the Control Plane and cached in the encrypted auth DB. `sce auth login` provisions it best-effort, and managed OTLP trace and log export reads it at startup as the bearer source. There is no user-facing provisioning command or refresh loop. `sce doctor` reports read-only credential health ([Doctor health check](#doctor-health-check)). ## Components @@ -26,7 +26,7 @@ flowchart LR - Telemetry readiness hint: `cli/src/services/telemetry_readiness.rs` keeps a non-secret file, `/sce/telemetry-ready.json` (`{"expires_at_unix_seconds": N}`), next to `auth.db`. - `sce auth login` writes it only after the credential has been issued and persisted. If provisioning, persistence, or writing the hint fails, the hint is cleared and the result is `unavailable`. - Logout and a new device-authorization session clear it before anything else. - - It counts as ready only while the file parses, is unexpired, and `auth.db` exists. A missing, corrupt, expired, or orphaned hint means telemetry is disabled. + - It counts as ready only while the file parses, is unexpired, and `auth.db` exists. A missing, corrupt, expired, or orphaned hint means telemetry is disabled. `telemetry_readiness::state` reports the same check as `NotConfigured` (no `auth.db`, or a missing or corrupt hint), `Expired`, or `Ready`, without opening `auth.db`. `is_ready` is `state == Ready`. - It is a fast-path hint, not authorization. The credential is still validated from the auth DB. Its purpose is that unauthenticated hooks never open `auth.db` or touch the Keychain, including existing installs that have `auth.db` but no hint, until the next `sce auth login`. - Concurrent saves are single-statement atomic replacements of one fixed row; the last complete writer wins. - `sce auth logout` deletes the WorkOS credential and the telemetry credential, attempting both even when one fails and surfacing the first storage error. `credentials_removed` is true when either record was deleted; output fields are otherwise unchanged. @@ -36,6 +36,32 @@ flowchart LR - `managed_telemetry::RuntimeTelemetry::resolve` first checks the readiness hint. Without it, the result is Noop with reason `not_configured` (logged at `debug`), and the auth DB is not opened. With the hint, it reads the stored credential through the non-interactive credential-store path at CLI startup (never issuing one). If the store refuses silent access, the reason is `credential_inaccessible` (logged at `debug`, so a denied Keychain does not flood the log). `sce setup` is excluded from managed telemetry so that it never reaches the credential store. a valid unexpired `token` is used only as the `Authorization: Bearer` credential for managed OTLP trace and log export; the `accessor` is never transmitted. Absent, expired, malformed, or unavailable rows select `NoopTelemetry`. Every Noop fallback is classified (`not_configured`, `credential_inaccessible`, `credential_unavailable`, `credential_expired`, `lookup_timeout`, `database_error`, `path_resolution_error`, `exporter_init_failed`) and logged locally as `sce.telemetry.fallback`; failures log at `warn` (forced, so they reach the log file at the default `error` level, never stderr while `log_to_file=true`), with the redacted `TokenStorageError` text for database and path errors, and a missing hint, an inaccessible credential store, or missing or expired credentials log at `debug`. Concurrent lookups from many hook processes are supported by the auth DB's multiprocess WAL and migration-free current-schema open ([auth-db.md](auth-db.md)). Both this lookup and the header helper's repository lookup are existing-DB reads: a missing `auth.db` yields no credential and creates no DB file or encryption key. Claude-invoked SCE hooks are admitted to this lookup and export one `sce.hook` span. Every other hook and this hidden header helper skip the lookup and stay Noop. See [cli-observability-contract.md](cli-observability-contract.md). +## Doctor health check + +`sce doctor` reports managed telemetry as the last `Environment` row and as a top-level JSON `managed_telemetry` object. `managed_telemetry::inspect_health` computes it once per invocation, and it is read-only: + +- It never issues a credential, contacts the Control Plane, or starts an OTLP exporter. +- It reads the readiness hint first. `NotConfigured` and `Expired` hints return without opening `auth.db` or touching the credential store, so a fresh install never reaches the Keychain. +- With a `Ready` hint it runs the runtime lookup: `load_telemetry_credential` (existing DB only, `CredentialAccess::NonInteractive`) under `CREDENTIAL_LOOKUP_DEADLINE`. Errors are classified by the same `TelemetryFallbackReason` mapping the runtime uses. +- `--fix` adds no repair for it. The problem is `manual_only`, so fix mode reports one `manual` fix result. Doctor never runs `sce auth login`, changes Keychain ACLs, or recreates a key. + +| Health | Text row | JSON `state` / `reason` | Problem `kind` | `next_action` | +| --- | --- | --- | --- | --- | +| No hint or no `auth.db` | `[PASS] Managed telemetry (not configured)` | `not_configured` / `not_configured` | none | none | +| Credential valid | `[PASS] Managed telemetry` | `healthy` / `null` | none | none | +| Store refuses silent access | `[WARN]` | `disabled` / `credential_inaccessible` | `managed_telemetry_credential_inaccessible` | `auth_login` | +| Hint ready, no usable credential | `[WARN]` | `disabled` / `credential_unavailable` | `managed_telemetry_credential_unavailable` | `auth_login` | +| Hint or credential expired | `[WARN]` | `disabled` / `credential_expired` | `managed_telemetry_credential_expired` | `auth_login` | +| Lookup deadline exceeded | `[WARN]` | `disabled` / `lookup_timeout` | `managed_telemetry_lookup_timeout` | `manual_steps` | +| Auth DB open/read failure | `[WARN]` | `disabled` / `database_error` | `managed_telemetry_database_error` | `manual_steps` | +| State path unresolvable | `[WARN]` | `disabled` / `path_resolution_error` | `managed_telemetry_path_resolution_error` | `manual_steps` | + +- The JSON object has `state`, `reason`, `kind`, `severity` (`warning` or `null`), and `next_action`. A warning also appears in `problems` (category `global_state`, severity `warning`, fixability `manual_only`) with the same `next_action`. It never makes doctor `not_ready`. +- Every warning says that Claude Code hooks continue working and only telemetry export is disabled. +- Only macOS `credential_inaccessible` recommends choosing "Always Allow" in the Keychain dialog shown by `sce auth login`. Other platforms name their store (Secret Service, Windows Credential Manager) instead. Expiry, missing credentials, timeouts, and DB or path errors never mention it. +- Output uses fixed text only. It never includes `TokenStorageError` details, Keychain error codes, keys, WorkOS tokens, or OpenBao tokens. The detailed reason stays in the local `sce.telemetry.fallback` log event that the same doctor invocation's startup lookup writes. +- Code: `cli/src/services/doctor/managed_telemetry.rs` (classification text and JSON) and `cli/src/services/managed_telemetry.rs` (`inspect_health`, `inspect_health_with`). + ## Hidden header helper - `sce auth telemetry-headers` is a hidden (`hide = true`) subcommand intended for Claude Code's `otelHeadersHelper`. On every call it prints exactly one JSON object, `{"Authorization":"Bearer "}`, so a replaced credential is picked up on the next call. From 92ff51fc6548faecb6b4124bdc66058d2e3ed51e Mon Sep 17 00:00:00 2001 From: davidabram Date: Mon, 19 Oct 2026 21:19:46 +0200 Subject: [PATCH 14/17] auth-db: Classify auth storage failures with typed causes Auth storage failures from auth commands and `sce sync` always rendered the macOS Keychain guidance, even when the real cause was a locked or unreadable `auth.db`. The cause was lost because every downstream mapper collapsed into one variant and matched on rendered text. Introduce a typed `AuthStorageFailure` cause (`CredentialAccess`, `DatabaseLocked`, `Database`) and thread it from the sources to `UserError::AuthStorageUnavailable`/`LoginAuthStorageUnavailable`. Add typed markers so classification never depends on diagnostic text: `CredentialAccessDenied` for keyring authorization failures (tagged by `NoStorageAccess` or the macOS `OSStatus` codes), `db::DatabaseLocked` for the last encrypted-open attempt failing on the compatibility-lock conflict or a Turso `Busy`/`BusySnapshot`, and `open_lock::SharedLockConflict` with unchanged text. `ControlPlaneError::Storage` now carries the cause, and `is_storage_failure()` became `storage_failure() -> Option` across the control-plane, stream, and sync error types. Guidance now matches the cause: DB causes say the DB is locked or unreadable and never mention the Keychain, "Always Allow", or `sce auth login`; the macOS command guidance gained "or run `sce doctor` to verify telemetry access". Managed telemetry still reports `database_error` for a locked DB. The key `auth.storage_unavailable`, the runtime class, exit code 4, and the rendered compatibility-lock rejection text are unchanged. Plan: context/plans/claude-hook-trace-propagation.md Co-authored-by: SCE --- .../agent_trace_sync/control_plane.rs | 24 +- cli/src/services/agent_trace_sync/mod.rs | 7 +- cli/src/services/auth_command/mod.rs | 69 ++++-- .../services/auth_db/multiprocess_tests.rs | 6 +- cli/src/services/db/encryption_key.rs | 121 +++++++++- cli/src/services/db/mod.rs | 124 ++++++++--- cli/src/services/db/open_lock.rs | 31 ++- cli/src/services/error.rs | 208 +++++++++++++----- cli/src/services/managed_telemetry.rs | 4 +- cli/src/services/sync/command.rs | 4 +- cli/src/services/sync/sync.rs | 11 +- cli/src/services/token_storage.rs | 28 ++- .../plans/claude-hook-trace-propagation.md | 34 ++- context/sce/auth-db.md | 28 ++- context/sce/cli-error-code-taxonomy.md | 6 +- 15 files changed, 559 insertions(+), 146 deletions(-) diff --git a/cli/src/services/agent_trace_sync/control_plane.rs b/cli/src/services/agent_trace_sync/control_plane.rs index 491cff514..852368fa8 100644 --- a/cli/src/services/agent_trace_sync/control_plane.rs +++ b/cli/src/services/agent_trace_sync/control_plane.rs @@ -12,6 +12,7 @@ use crate::services::agent_trace_export::{ }; use crate::services::auth::{self, AuthError, TokenResponse}; use crate::services::db::encryption_key::CredentialAccess; +use crate::services::error::AuthStorageFailure; use crate::services::resilience::{run_with_retry, RetryPolicy}; use crate::services::telemetry_credential_storage::{self, TelemetryCredential, TelemetryIdentity}; use crate::services::token_storage::{self, StoredTokens, TokenStorageError}; @@ -108,7 +109,7 @@ pub enum ControlPlaneError { Conflict(String), ServerError(String), InvalidResponse(String), - Storage(String), + Storage(AuthStorageFailure, String), Protocol { status: StatusCode, message: String }, } @@ -137,7 +138,7 @@ impl fmt::Display for ControlPlaneError { Self::InvalidResponse(reason) => { write!(f, "Control-plane returned an unexpected response: {reason}") } - Self::Storage(reason) => write!(f, "Local credential storage error: {reason}"), + Self::Storage(_, reason) => write!(f, "Local credential storage error: {reason}"), Self::Protocol { status, message } => write!( f, "Control-plane rejected the request ({status}): {message}" @@ -156,14 +157,17 @@ impl ControlPlaneError { ) } - pub fn is_storage_failure(&self) -> bool { - matches!(self, Self::Storage(_)) + pub fn storage_failure(&self) -> Option { + match self { + Self::Storage(failure, _) => Some(*failure), + _ => None, + } } } impl From for ControlPlaneError { fn from(value: TokenStorageError) -> Self { - Self::Storage(value.to_string()) + Self::Storage(value.failure(), value.to_string()) } } @@ -172,7 +176,7 @@ impl From for ControlPlaneError { match value { AuthError::Unauthorized(reason) => Self::AuthenticationFailed(reason), AuthError::RequestFailed(error) => Self::Transport(error.to_string()), - AuthError::Storage(error) => Self::Storage(error.to_string()), + AuthError::Storage(error) => Self::Storage(error.failure(), error.to_string()), other => Self::AuthenticationFailed(other.to_string()), } } @@ -449,9 +453,9 @@ impl AuthenticatedControlPlaneClient { Fut: Future>, { let credential = self.issue_telemetry_credential().await?; - save(credential.clone()) - .await - .map_err(|_| ControlPlaneError::Storage(TELEMETRY_STORAGE_MESSAGE.to_string()))?; + save(credential.clone()).await.map_err(|error| { + ControlPlaneError::Storage(error.failure(), TELEMETRY_STORAGE_MESSAGE.to_string()) + })?; Ok(credential) } @@ -1263,7 +1267,7 @@ mod tests { &unauthorized, ControlPlaneError::AuthenticationFailed(_) | ControlPlaneError::Transport(_) - | ControlPlaneError::Storage(_) + | ControlPlaneError::Storage(..) ), "{unauthorized:?}" ); diff --git a/cli/src/services/agent_trace_sync/mod.rs b/cli/src/services/agent_trace_sync/mod.rs index cc9f2ed01..c10f607fd 100644 --- a/cli/src/services/agent_trace_sync/mod.rs +++ b/cli/src/services/agent_trace_sync/mod.rs @@ -5,6 +5,7 @@ use std::fmt; use crate::services::agent_trace_export::{ AgentTraceAgentTraceExportRow, AgentTraceMessageExportRow, AgentTracePartExportRow, }; +use crate::services::error::AuthStorageFailure; use control_plane::ControlPlaneError; pub const RECONCILIATION_MAX_ATTEMPTS: u32 = 5; @@ -93,10 +94,10 @@ impl StreamSyncError { /// or `Terminal` failure) means local credential storage is unavailable. /// `Read`, `InvalidResponse`, and `DidNotConverge` never carry a /// `ControlPlaneError` and are never storage failures. - pub fn is_storage_failure(&self) -> bool { + pub fn storage_failure(&self) -> Option { match self { - Self::Refresh(error) | Self::Terminal(error) => error.is_storage_failure(), - Self::Read(_) | Self::InvalidResponse(_) | Self::DidNotConverge => false, + Self::Refresh(error) | Self::Terminal(error) => error.storage_failure(), + Self::Read(_) | Self::InvalidResponse(_) | Self::DidNotConverge => None, } } } diff --git a/cli/src/services/auth_command/mod.rs b/cli/src/services/auth_command/mod.rs index a6061c2d4..e760caec0 100644 --- a/cli/src/services/auth_command/mod.rs +++ b/cli/src/services/auth_command/mod.rs @@ -13,7 +13,7 @@ use crate::services::agent_trace_sync::control_plane::{ use crate::services::auth::{self, AuthError, DeviceAuthFlowResult}; use crate::services::config; use crate::services::db::encryption_key::CredentialAccess; -use crate::services::error::{CliError, UserError}; +use crate::services::error::{AuthStorageFailure, CliError, UserError}; use crate::services::managed_telemetry::claude_helper::{ repository_id_for, visible_destination_conflict, }; @@ -484,7 +484,8 @@ fn write_login_prompt(authorization: &auth::DeviceAuthorizationResponse) -> Resu fn map_login_error(error: AuthError) -> CliError { let user_error = match &error { - AuthError::Io(_) | AuthError::Storage(_) => UserError::LoginAuthStorageUnavailable, + AuthError::Io(_) => UserError::LoginAuthStorageUnavailable(AuthStorageFailure::Database), + AuthError::Storage(storage) => UserError::LoginAuthStorageUnavailable(storage.failure()), _ => UserError::UnexpectedFailure, }; CliError::user_with_source(user_error, error) @@ -687,8 +688,8 @@ fn render_whoami_result(profile: &MeResponse, format: AuthFormat) -> Result CliError { let user_error = if error.is_authentication_failure() { UserError::NotAuthenticated - } else if error.is_storage_failure() { - UserError::AuthStorageUnavailable + } else if let Some(failure) = error.storage_failure() { + UserError::AuthStorageUnavailable(failure) } else { UserError::UnexpectedFailure }; @@ -700,11 +701,14 @@ fn map_whoami_control_plane_error(error: &ControlPlaneError) -> CliError { } fn auth_storage_error(error: TokenStorageError) -> CliError { - CliError::user_with_source(UserError::AuthStorageUnavailable, error) + CliError::user_with_source(UserError::AuthStorageUnavailable(error.failure()), error) } fn login_storage_error(error: TokenStorageError) -> CliError { - CliError::user_with_source(UserError::LoginAuthStorageUnavailable, error) + CliError::user_with_source( + UserError::LoginAuthStorageUnavailable(error.failure()), + error, + ) } fn unexpected_auth_command_error(error: anyhow::Error) -> CliError { @@ -778,7 +782,10 @@ mod tests { ) .await; assert!(!save_called.load(std::sync::atomic::Ordering::SeqCst)); - assert_login_storage_guidance(&failed.expect_err("invalidation failure fails login")); + assert_login_storage_guidance( + &failed.expect_err("invalidation failure fails login"), + AuthStorageFailure::Database, + ); let failed = replace_session_with( || Ok(()), @@ -786,29 +793,56 @@ mod tests { || async { Err(TokenStorageError::CredentialAccessUnavailable) }, ) .await; - assert_login_storage_guidance(&failed.expect_err("save failure fails login")); + assert_login_storage_guidance( + &failed.expect_err("save failure fails login"), + AuthStorageFailure::CredentialAccess, + ); } - fn assert_login_storage_guidance(error: &CliError) { + fn assert_login_storage_guidance(error: &CliError, failure: AuthStorageFailure) { let CliError::User { error: user_error, .. } = error else { panic!("expected a user-facing storage error"); }; - assert_eq!(*user_error, UserError::LoginAuthStorageUnavailable); + assert_eq!(*user_error, UserError::LoginAuthStorageUnavailable(failure)); assert_eq!(user_error.key(), "auth.storage_unavailable"); assert_eq!(error.class().exit_code(), 4); let message = error.to_string(); - assert!(message.contains("Running login again does not help until")); - assert!(!message.contains("Retry the original command.")); + assert!(!message.contains("Retry the original command")); + let keychain_guidance = message.contains("encryption key"); + assert_eq!( + keychain_guidance, + failure == AuthStorageFailure::CredentialAccess + ); + if keychain_guidance { + assert!(message.contains("Running login again does not help until")); + } } #[test] fn login_device_flow_storage_failure_uses_noncircular_guidance() { - assert_login_storage_guidance(&map_login_error(AuthError::Storage( - TokenStorageError::CredentialAccessUnavailable, - ))); + for (storage, failure) in [ + ( + TokenStorageError::CredentialAccessUnavailable, + AuthStorageFailure::CredentialAccess, + ), + ( + TokenStorageError::DatabaseLocked("busy".to_string()), + AuthStorageFailure::DatabaseLocked, + ), + ( + TokenStorageError::Database("corrupt".to_string()), + AuthStorageFailure::Database, + ), + ] { + assert_login_storage_guidance(&map_login_error(AuthError::Storage(storage)), failure); + } + assert_login_storage_guidance( + &map_login_error(AuthError::Io(std::io::Error::other("io"))), + AuthStorageFailure::Database, + ); let CliError::User { error, .. } = map_login_error(AuthError::MissingClientId) else { panic!("expected a user-facing error"); }; @@ -831,7 +865,10 @@ mod tests { else { panic!("expected a user-facing storage error"); }; - assert_eq!(*user_error, UserError::AuthStorageUnavailable); + assert_eq!( + *user_error, + UserError::AuthStorageUnavailable(AuthStorageFailure::CredentialAccess) + ); assert!(error.to_string().contains("sce auth login")); } diff --git a/cli/src/services/auth_db/multiprocess_tests.rs b/cli/src/services/auth_db/multiprocess_tests.rs index 236f13ef9..7ec7909a5 100644 --- a/cli/src/services/auth_db/multiprocess_tests.rs +++ b/cli/src/services/auth_db/multiprocess_tests.rs @@ -15,9 +15,9 @@ use crate::services::bash_policy::{ use crate::services::db::encryption_key::{ get_or_create_encryption_key, AUTH_DB_ENCRYPTION_KEY_ENV, }; -use crate::services::db::{DatabaseKind, DbSpec, EncryptedTursoDb}; +use crate::services::db::{DatabaseKind, DatabaseLocked, DbSpec, EncryptedTursoDb}; use crate::services::default_paths::auth_db_path; -use crate::services::error::CliError; +use crate::services::error::{AuthStorageFailure, CliError}; use crate::services::hooks::{run_hooks_subcommand, HookSubcommand}; use crate::services::managed_telemetry::claude_helper::repository_id_for; use crate::services::managed_telemetry::claude_helper::tests::git_repository_with_remote; @@ -1147,6 +1147,7 @@ async fn run_new_version_child(role: &str) { error.to_string().contains(SHARED_ACCESS_REJECTION), "{error}" ); + assert_eq!(error.failure(), AuthStorageFailure::DatabaseLocked); } "expect-rejected-write" => { let error = AuthDb::new() @@ -1157,6 +1158,7 @@ async fn run_new_version_child(role: &str) { format!("{error:#}").contains(SHARED_ACCESS_REJECTION), "{error:#}" ); + assert!(error.is::(), "{error:#}"); } "verify-rows" => print_verified_rows().await, "repository-lookup-missing" => { diff --git a/cli/src/services/db/encryption_key.rs b/cli/src/services/db/encryption_key.rs index e8a0fbd68..f79f0243c 100644 --- a/cli/src/services/db/encryption_key.rs +++ b/cli/src/services/db/encryption_key.rs @@ -36,6 +36,42 @@ impl fmt::Display for CredentialAccessUnavailable { impl std::error::Error for CredentialAccessUnavailable {} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CredentialAccessDenied { + db_name: String, +} + +impl fmt::Display for CredentialAccessDenied { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "credential store denied access to the encryption key for '{}'", + self.db_name + ) + } +} + +impl std::error::Error for CredentialAccessDenied {} + +#[cfg(target_os = "macos")] +const ERR_SEC_USER_CANCELED: i32 = -128; +#[cfg(target_os = "macos")] +const ERR_SEC_NO_ACCESS_FOR_ITEM: i32 = -25243; +#[cfg(target_os = "macos")] +const ERR_SEC_AUTH_FAILED: i32 = -25293; +#[cfg(target_os = "macos")] +const ERR_SEC_INTERACTION_NOT_ALLOWED: i32 = -25308; +#[cfg(target_os = "macos")] +const ERR_SEC_INTERACTION_REQUIRED: i32 = -25315; +#[cfg(target_os = "macos")] +const MACOS_AUTHORIZATION_FAILURE_CODES: [i32; 5] = [ + ERR_SEC_USER_CANCELED, + ERR_SEC_NO_ACCESS_FOR_ITEM, + ERR_SEC_AUTH_FAILED, + ERR_SEC_INTERACTION_NOT_ALLOWED, + ERR_SEC_INTERACTION_REQUIRED, +]; + static CREDENTIAL_STORE_ACCESS: Mutex<()> = Mutex::new(()); #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -196,7 +232,7 @@ fn get_or_create_keyring_encryption_key(db_path: &Path, db_name: &str) -> Result Ok(password) => return Ok(password), Err(keyring_core::Error::NoEntry) => {} Err(error) => { - return Err(anyhow::Error::new(error)).with_context(|| { + return Err(keyring_error(error, db_name)).with_context(|| { format!( "failed to read encryption key for '{db_name}' from credential store. {}", credential_store_unavailable_message(current_credential_store_platform()) @@ -208,12 +244,15 @@ fn get_or_create_keyring_encryption_key(db_path: &Path, db_name: &str) -> Result if !db_path.exists() { let hex_key = generate_key(); - entry.set_password(&hex_key).with_context(|| { - format!( - "failed to store encryption key for '{db_name}' in credential store. {}", - credential_store_unavailable_message(current_credential_store_platform()) - ) - })?; + entry + .set_password(&hex_key) + .map_err(|error| keyring_error(error, db_name)) + .with_context(|| { + format!( + "failed to store encryption key for '{db_name}' in credential store. {}", + credential_store_unavailable_message(current_credential_store_platform()) + ) + })?; return Ok(hex_key); } @@ -232,6 +271,29 @@ fn get_or_create_keyring_encryption_key(db_path: &Path, db_name: &str) -> Result ); } +fn keyring_error(error: keyring_core::Error, db_name: &str) -> anyhow::Error { + let denied = is_authorization_failure(&error); + let error = anyhow::Error::new(error); + if denied { + error.context(CredentialAccessDenied { + db_name: db_name.to_owned(), + }) + } else { + error + } +} + +fn is_authorization_failure(error: &keyring_core::Error) -> bool { + match error { + keyring_core::Error::NoStorageAccess(_) => true, + #[cfg(target_os = "macos")] + keyring_core::Error::PlatformFailure(platform) => platform + .downcast_ref::() + .is_some_and(|error| MACOS_AUTHORIZATION_FAILURE_CODES.contains(&error.code())), + _ => false, + } +} + fn current_credential_store_platform() -> CredentialStorePlatform { #[cfg(target_os = "linux")] { @@ -320,3 +382,48 @@ fn hex_encode(bytes: &[u8]) -> String { } hex } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn storage_access_failures_are_tagged_as_credential_access_denied() { + let error = keyring_error( + keyring_core::Error::NoStorageAccess(Box::new(std::io::Error::other("locked"))), + "auth_db", + ) + .context("failed to read encryption key"); + assert!(error.is::()); + assert_eq!(error.to_string(), "failed to read encryption key"); + + for untagged in [ + keyring_core::Error::NoEntry, + keyring_core::Error::PlatformFailure(Box::new(std::io::Error::other("other"))), + ] { + assert!(!keyring_error(untagged, "auth_db").is::()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_authorization_status_codes_are_tagged_by_code() { + let platform_failure = |code| { + keyring_core::Error::PlatformFailure(Box::new( + security_framework::base::Error::from_code(code), + )) + }; + for code in MACOS_AUTHORIZATION_FAILURE_CODES { + assert!( + keyring_error(platform_failure(code), "auth_db").is::(), + "{code}" + ); + } + for code in [-50, -25299, -34018] { + assert!( + !keyring_error(platform_failure(code), "auth_db").is::(), + "{code}" + ); + } + } +} diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index e05150cba..dc3a5f167 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -8,6 +8,7 @@ use std::{ fs, marker::PhantomData, path::{Path, PathBuf}, + sync::atomic::{AtomicBool, Ordering}, }; use anyhow::{Context, Result}; @@ -322,6 +323,42 @@ fn is_retryable_turso_error(error: &turso::Error) -> bool { matches!(error, turso::Error::Busy(_) | turso::Error::BusySnapshot(_)) } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct DatabaseLocked { + db_name: &'static str, +} + +impl std::fmt::Display for DatabaseLocked { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} database is locked by another process", self.db_name) + } +} + +impl std::error::Error for DatabaseLocked {} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct TursoBusy; + +impl std::fmt::Display for TursoBusy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "database is busy") + } +} + +impl std::error::Error for TursoBusy {} + +fn turso_open_error(error: &turso::Error, message: String) -> anyhow::Error { + if is_retryable_turso_error(error) { + anyhow::Error::new(TursoBusy).context(message) + } else { + anyhow::Error::msg(message) + } +} + +fn is_database_lock_conflict(error: &anyhow::Error) -> bool { + error.is::() || error.is::() +} + enum WriteAttemptFailure { Retryable(anyhow::Error), Deterministic(anyhow::Error), @@ -1404,41 +1441,30 @@ impl EncryptedTursoDb { let busy_timeout = resolve_busy_timeout::(); let operation_name = format!("open encrypted {db_name} database connection"); + let last_attempt_locked = AtomicBool::new(false); + let conn = run_with_retry_elapsed( retry_policy, &operation_name, CONNECTION_OPEN_RETRY_HINT, |_| async { - open_lock::lock_shared(&open_lock, db_name, &db_path)?; - let path_str = db_path.to_str().ok_or_else(|| { - anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) - })?; - - let encryption_opts = turso::EncryptionOpts { - hexkey: encryption_key.clone(), - cipher: ENCRYPTION_CIPHER_AEGIS256.to_string(), - }; - - let db = turso::Builder::new_local(path_str) - .experimental_multiprocess_wal(true) - .experimental_encryption(true) - .with_encryption(encryption_opts) - .build() - .await - .map_err(|e| { - anyhow::anyhow!( - "failed to open encrypted {db_name} database at {} with cipher {ENCRYPTION_CIPHER_AEGIS256}. Try: verify the credential store encryption key is valid and that local Turso encryption support is available: {e}", - db_path.display() - ) - })?; - - let conn = db.connect().map_err(|e| { - anyhow::anyhow!("failed to connect to encrypted {db_name} database: {e}") - })?; - apply_busy_timeout(&conn, db_name, busy_timeout)?; - Ok(conn) + let attempt = + Self::connect_locked(&db_path, &encryption_key, &open_lock, busy_timeout).await; + last_attempt_locked.store( + attempt.as_ref().is_err_and(is_database_lock_conflict), + Ordering::Relaxed, + ); + attempt }, - ).await?; + ) + .await + .map_err(|error| { + if last_attempt_locked.load(Ordering::Relaxed) { + anyhow::Error::new(DatabaseLocked { db_name }).context(format!("{error:#}")) + } else { + error + } + })?; let mut db = Self { core: TursoConnectionCore::new(conn), @@ -1453,6 +1479,46 @@ impl EncryptedTursoDb { Ok(db) } + async fn connect_locked( + db_path: &Path, + encryption_key: &str, + open_lock: &fs::File, + busy_timeout: std::time::Duration, + ) -> Result { + let db_name = M::db_name(); + open_lock::lock_shared(open_lock, db_name, db_path)?; + let path_str = db_path.to_str().ok_or_else(|| { + anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) + })?; + + let encryption_opts = turso::EncryptionOpts { + hexkey: encryption_key.to_owned(), + cipher: ENCRYPTION_CIPHER_AEGIS256.to_string(), + }; + + let db = turso::Builder::new_local(path_str) + .experimental_multiprocess_wal(true) + .experimental_encryption(true) + .with_encryption(encryption_opts) + .build() + .await + .map_err(|e| { + turso_open_error(&e, format!( + "failed to open encrypted {db_name} database at {} with cipher {ENCRYPTION_CIPHER_AEGIS256}. Try: verify the credential store encryption key is valid and that local Turso encryption support is available: {e}", + db_path.display() + )) + })?; + + let conn = db.connect().map_err(|e| { + turso_open_error( + &e, + format!("failed to connect to encrypted {db_name} database: {e}"), + ) + })?; + apply_busy_timeout(&conn, db_name, busy_timeout)?; + Ok(conn) + } + /// Execute a SQL statement that does not return rows. /// /// # Arguments diff --git a/cli/src/services/db/open_lock.rs b/cli/src/services/db/open_lock.rs index 4e9c8a50b..3ff596e70 100644 --- a/cli/src/services/db/open_lock.rs +++ b/cli/src/services/db/open_lock.rs @@ -1,6 +1,7 @@ +use std::fmt; use std::fs::{File, OpenOptions}; use std::io; -use std::path::Path; +use std::path::{Path, PathBuf}; use anyhow::{anyhow, Result}; @@ -26,13 +27,33 @@ pub(super) fn open_lock_file(db_path: &Path, create: bool) -> Result) -> fmt::Result { + write!( + f, + "{} database at {} is open in an older SCE process that does not support shared access. Try: wait for older SCE processes to exit, upgrade every installed 'sce' binary, and rerun the command", + self.db_name, + self.db_path.display() + ) + } +} + +impl std::error::Error for SharedLockConflict {} + pub(super) fn lock_shared(file: &File, db_name: &str, db_path: &Path) -> Result<()> { match try_lock_shared(file) { Ok(()) => Ok(()), - Err(error) if is_lock_conflict(&error) => Err(anyhow!( - "{db_name} database at {} is open in an older SCE process that does not support shared access. Try: wait for older SCE processes to exit, upgrade every installed 'sce' binary, and rerun the command", - db_path.display() - )), + Err(error) if is_lock_conflict(&error) => Err(SharedLockConflict { + db_name: db_name.to_owned(), + db_path: db_path.to_path_buf(), + } + .into()), Err(error) => Err(anyhow!( "failed to lock {db_name} database at {}: {error}", db_path.display() diff --git a/cli/src/services/error.rs b/cli/src/services/error.rs index b3b7004b9..786cf9a09 100644 --- a/cli/src/services/error.rs +++ b/cli/src/services/error.rs @@ -56,19 +56,28 @@ pub enum UserError { NotAuthenticated, NotGitRepository, NotGitRemote, - AuthStorageUnavailable, - LoginAuthStorageUnavailable, + AuthStorageUnavailable(AuthStorageFailure), + LoginAuthStorageUnavailable(AuthStorageFailure), UnexpectedFailure, } +/// Typed cause of an authentication storage failure, selected from typed +/// error sources so user guidance never depends on diagnostic text. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AuthStorageFailure { + CredentialAccess, + DatabaseLocked, + Database, +} + impl UserError { pub fn class(&self) -> FailureClass { match self { Self::NotAuthenticated | Self::NotGitRepository | Self::NotGitRemote - | Self::AuthStorageUnavailable - | Self::LoginAuthStorageUnavailable + | Self::AuthStorageUnavailable(_) + | Self::LoginAuthStorageUnavailable(_) | Self::UnexpectedFailure => FailureClass::Runtime, } } @@ -78,7 +87,7 @@ impl UserError { Self::NotAuthenticated => "auth.not_authenticated", Self::NotGitRepository => "setup.not_git_repository", Self::NotGitRemote { .. } => "setup.not_git_remote", - Self::AuthStorageUnavailable | Self::LoginAuthStorageUnavailable => { + Self::AuthStorageUnavailable(_) | Self::LoginAuthStorageUnavailable(_) => { "auth.storage_unavailable" } Self::UnexpectedFailure => "general.unexpected_failure", @@ -96,13 +105,15 @@ impl UserError { Self::NotGitRemote => { "The Git repository has no configured remote URL. Please run `git remote add `, then retry." } - Self::AuthStorageUnavailable => auth_storage_unavailable_message( + Self::AuthStorageUnavailable(failure) => auth_storage_unavailable_message( CURRENT_CREDENTIAL_STORE_PLATFORM, AuthStorageOperation::Command, + *failure, ), - Self::LoginAuthStorageUnavailable => auth_storage_unavailable_message( + Self::LoginAuthStorageUnavailable(failure) => auth_storage_unavailable_message( CURRENT_CREDENTIAL_STORE_PLATFORM, AuthStorageOperation::Login, + *failure, ), Self::UnexpectedFailure => { "An unexpected error occurred. Check the log files for more details." @@ -155,7 +166,8 @@ To fix: 1. Run `sce auth login` in an interactive terminal. 2. When macOS asks for permission to access the SCE Keychain item, choose \"Always Allow\". - 3. Retry the original command. + 3. Retry the original command, or run `sce doctor` to verify + telemetry access. If no prompt appears, verify that your login Keychain is unlocked and retry using the same installed SCE executable."; @@ -236,9 +248,41 @@ To fix: 1. Make sure a supported OS credential store is available and unlocked. 2. Retry `sce auth login` in an interactive terminal."; +const AUTH_DATABASE_LOCKED: &str = "Authentication storage is unavailable. + +The SCE authentication database is locked by another SCE process. + +To fix: + 1. Wait for other running `sce` processes, including Claude Code hooks, + to finish. + 2. Make sure every installed `sce` executable is the same version; + older versions cannot share the authentication database. + 3. Retry the command."; + +const AUTH_DATABASE_UNAVAILABLE: &str = "Authentication storage is unavailable. + +SCE could not open or read its local authentication database. + +To fix: + 1. Run `sce doctor` to check the SCE state directory. + 2. Retry the command. + 3. If the problem persists, check the SCE log files for details."; + const fn auth_storage_unavailable_message( platform: CredentialStorePlatform, operation: AuthStorageOperation, + failure: AuthStorageFailure, +) -> &'static str { + match failure { + AuthStorageFailure::CredentialAccess => credential_access_message(platform, operation), + AuthStorageFailure::DatabaseLocked => AUTH_DATABASE_LOCKED, + AuthStorageFailure::Database => AUTH_DATABASE_UNAVAILABLE, + } +} + +const fn credential_access_message( + platform: CredentialStorePlatform, + operation: AuthStorageOperation, ) -> &'static str { match (platform, operation) { (CredentialStorePlatform::Macos, AuthStorageOperation::Command) => { @@ -480,23 +524,30 @@ mod tests { ]; const ALL_OPERATIONS: [AuthStorageOperation; 2] = [AuthStorageOperation::Command, AuthStorageOperation::Login]; + const ALL_FAILURES: [AuthStorageFailure; 3] = [ + AuthStorageFailure::CredentialAccess, + AuthStorageFailure::DatabaseLocked, + AuthStorageFailure::Database, + ]; #[test] fn auth_storage_unavailable_keeps_stable_key_class_and_exit_code() { - for user_error in [ - UserError::AuthStorageUnavailable, - UserError::LoginAuthStorageUnavailable, - ] { - let error = CliError::user_with_source( - user_error.clone(), - anyhow::anyhow!("keychain item secret-key-material denied"), - ); - assert_eq!(user_error.key(), "auth.storage_unavailable"); - assert_eq!(error.class(), FailureClass::Runtime); - assert_eq!(error.code(), "SCE-ERR-RUNTIME"); - assert_eq!(error.class().exit_code(), 4); - assert_eq!(error.to_string(), user_error.message()); - assert!(!error.to_string().contains("secret-key-material")); + for failure in ALL_FAILURES { + for user_error in [ + UserError::AuthStorageUnavailable(failure), + UserError::LoginAuthStorageUnavailable(failure), + ] { + let error = CliError::user_with_source( + user_error.clone(), + anyhow::anyhow!("keychain item secret-key-material denied"), + ); + assert_eq!(user_error.key(), "auth.storage_unavailable"); + assert_eq!(error.class(), FailureClass::Runtime); + assert_eq!(error.code(), "SCE-ERR-RUNTIME"); + assert_eq!(error.class().exit_code(), 4); + assert_eq!(error.to_string(), user_error.message()); + assert!(!error.to_string().contains("secret-key-material")); + } } } @@ -505,12 +556,13 @@ mod tests { let message = auth_storage_unavailable_message( CredentialStorePlatform::Macos, AuthStorageOperation::Command, + AuthStorageFailure::CredentialAccess, ); assert!(message.starts_with("Authentication storage is unavailable.\n")); assert!(message.contains("macOS Keychain")); assert!(message.contains("Run `sce auth login` in an interactive terminal.")); assert!(message.contains("choose \"Always Allow\".")); - assert!(message.contains("Retry the original command.")); + assert!(message.contains("Retry the original command, or run `sce doctor`")); assert!(message.contains("login Keychain is unlocked")); } @@ -519,11 +571,50 @@ mod tests { let message = auth_storage_unavailable_message( CredentialStorePlatform::Macos, AuthStorageOperation::Login, + AuthStorageFailure::CredentialAccess, ); assert!(message.contains("Running login again does not help until")); assert!(message.contains("Unlock your login Keychain")); assert!(message.contains("\"Always Allow\" to authorize this SCE executable")); - assert!(!message.contains("Retry the original command.")); + assert!(!message.contains("Retry the original command")); + } + + #[test] + fn database_failures_never_blame_the_credential_store() { + for platform in ALL_PLATFORMS { + for operation in ALL_OPERATIONS { + let locked = auth_storage_unavailable_message( + platform, + operation, + AuthStorageFailure::DatabaseLocked, + ); + assert!(locked.contains("locked by another SCE process")); + let generic = auth_storage_unavailable_message( + platform, + operation, + AuthStorageFailure::Database, + ); + assert!( + generic.contains("could not open or read its local authentication database") + ); + for message in [locked, generic] { + assert!(message.starts_with("Authentication storage is unavailable.\n")); + assert!( + !message.contains("encryption key"), + "{platform:?}/{operation:?}" + ); + assert!(!message.contains("Keychain"), "{platform:?}/{operation:?}"); + assert!( + !message.contains("Always Allow"), + "{platform:?}/{operation:?}" + ); + assert!( + !message.contains("sce auth login"), + "{platform:?}/{operation:?}" + ); + } + } + } } #[test] @@ -534,12 +625,22 @@ mod tests { CredentialStorePlatform::Other, ] { for operation in ALL_OPERATIONS { - let message = auth_storage_unavailable_message(platform, operation); - assert!(message.starts_with("Authentication storage is unavailable.\n")); - assert!(message.contains("sce auth login"), "{platform:?}"); - assert!(!message.contains("macOS"), "{platform:?}"); - assert!(!message.contains("Keychain"), "{platform:?}"); - assert!(!message.contains("Always Allow"), "{platform:?}"); + for failure in ALL_FAILURES { + let message = auth_storage_unavailable_message(platform, operation, failure); + assert!(message.starts_with("Authentication storage is unavailable.\n")); + assert!(!message.contains("macOS"), "{platform:?}"); + assert!(!message.contains("Keychain"), "{platform:?}"); + assert!(!message.contains("Always Allow"), "{platform:?}"); + } + assert!( + auth_storage_unavailable_message( + platform, + operation, + AuthStorageFailure::CredentialAccess + ) + .contains("sce auth login"), + "{platform:?}" + ); } } } @@ -548,12 +649,15 @@ mod tests { fn auth_storage_guidance_never_recommends_destructive_repair() { for platform in ALL_PLATFORMS { for operation in ALL_OPERATIONS { - let message = auth_storage_unavailable_message(platform, operation).to_lowercase(); - for forbidden in ["auth.db", "delete", "regenerat", "reset", "permanent"] { - assert!( - !message.contains(forbidden), - "{platform:?}/{operation:?} mentions {forbidden:?}" - ); + for failure in ALL_FAILURES { + let message = auth_storage_unavailable_message(platform, operation, failure) + .to_lowercase(); + for forbidden in ["auth.db", "delete", "regenerat", "reset", "permanent"] { + assert!( + !message.contains(forbidden), + "{platform:?}/{operation:?}/{failure:?} mentions {forbidden:?}" + ); + } } } } @@ -561,20 +665,24 @@ mod tests { #[test] fn current_platform_message_is_selected_for_display() { - assert_eq!( - UserError::AuthStorageUnavailable.message(), - auth_storage_unavailable_message( - CURRENT_CREDENTIAL_STORE_PLATFORM, - AuthStorageOperation::Command - ) - ); - assert_eq!( - UserError::LoginAuthStorageUnavailable.message(), - auth_storage_unavailable_message( - CURRENT_CREDENTIAL_STORE_PLATFORM, - AuthStorageOperation::Login - ) - ); + for failure in ALL_FAILURES { + assert_eq!( + UserError::AuthStorageUnavailable(failure).message(), + auth_storage_unavailable_message( + CURRENT_CREDENTIAL_STORE_PLATFORM, + AuthStorageOperation::Command, + failure + ) + ); + assert_eq!( + UserError::LoginAuthStorageUnavailable(failure).message(), + auth_storage_unavailable_message( + CURRENT_CREDENTIAL_STORE_PLATFORM, + AuthStorageOperation::Login, + failure + ) + ); + } assert_eq!( CURRENT_CREDENTIAL_STORE_PLATFORM == CredentialStorePlatform::Macos, cfg!(target_os = "macos") diff --git a/cli/src/services/managed_telemetry.rs b/cli/src/services/managed_telemetry.rs index df5dec860..6b9a5ac16 100644 --- a/cli/src/services/managed_telemetry.rs +++ b/cli/src/services/managed_telemetry.rs @@ -170,7 +170,9 @@ impl TelemetryFallback { TokenStorageError::CredentialAccessUnavailable => { return Self::new(TelemetryFallbackReason::CredentialInaccessible); } - TokenStorageError::Database(_) => TelemetryFallbackReason::DatabaseError, + TokenStorageError::Database(_) | TokenStorageError::DatabaseLocked(_) => { + TelemetryFallbackReason::DatabaseError + } TokenStorageError::PathResolution(_) => TelemetryFallbackReason::PathResolutionError, }; Self { diff --git a/cli/src/services/sync/command.rs b/cli/src/services/sync/command.rs index 77adb11fb..c83165518 100644 --- a/cli/src/services/sync/command.rs +++ b/cli/src/services/sync/command.rs @@ -33,8 +33,8 @@ where #[allow(clippy::needless_pass_by_value)] fn classify_sync_error(err: TraceSyncError) -> CliError { - if err.is_storage_failure() { - CliError::user_with_source(UserError::AuthStorageUnavailable, err) + if let Some(failure) = err.storage_failure() { + CliError::user_with_source(UserError::AuthStorageUnavailable(failure), err) } else if err.is_authentication_failure() { CliError::user_with_source(UserError::NotAuthenticated, err) } else { diff --git a/cli/src/services/sync/sync.rs b/cli/src/services/sync/sync.rs index 104455c3a..a07ff6a39 100644 --- a/cli/src/services/sync/sync.rs +++ b/cli/src/services/sync/sync.rs @@ -26,6 +26,7 @@ use crate::services::agent_trace_sync::{ use crate::services::auth; use crate::services::config; use crate::services::db::encryption_key::CredentialAccess; +use crate::services::error::AuthStorageFailure; use crate::services::sync::progress::ProgressReporter; #[derive(Clone, Debug, PartialEq, Eq)] @@ -137,11 +138,11 @@ impl TraceSyncError { /// True when the failure came from local credential storage, whether it /// surfaced during the initial state request or a stream batch/refresh /// path. - pub fn is_storage_failure(&self) -> bool { + pub fn storage_failure(&self) -> Option { match self { - Self::ControlPlane(error) => error.is_storage_failure(), - Self::Stream { source, .. } => source.is_storage_failure(), - Self::Runtime(_) => false, + Self::ControlPlane(error) => error.storage_failure(), + Self::Stream { source, .. } => source.storage_failure(), + Self::Runtime(_) => None, } } } @@ -512,7 +513,7 @@ fn is_stream_terminal(error: &ControlPlaneError) -> bool { | ControlPlaneError::AuthenticationFailed(_) | ControlPlaneError::BadRequest(_) | ControlPlaneError::Forbidden(_) - | ControlPlaneError::Storage(_) + | ControlPlaneError::Storage(..) | ControlPlaneError::Protocol { .. } ) } diff --git a/cli/src/services/token_storage.rs b/cli/src/services/token_storage.rs index c4c9ff8ca..afedb8803 100644 --- a/cli/src/services/token_storage.rs +++ b/cli/src/services/token_storage.rs @@ -6,19 +6,22 @@ use serde::{Deserialize, Serialize}; use crate::services::auth::TokenResponse; use crate::services::auth_db::AuthDb; -use crate::services::db::encryption_key::{CredentialAccess, CredentialAccessUnavailable}; +use crate::services::db::encryption_key::{ + CredentialAccess, CredentialAccessDenied, CredentialAccessUnavailable, +}; +use crate::services::db::DatabaseLocked; use crate::services::default_paths::auth_db_path; +use crate::services::error::AuthStorageFailure; const DEFAULT_TOKEN_ROW_ID: i64 = 1; static AUTH_DB: OnceCell = OnceCell::const_new(); fn open_error(error: &anyhow::Error) -> TokenStorageError { - if error - .downcast_ref::() - .is_some() - { + if error.is::() || error.is::() { TokenStorageError::CredentialAccessUnavailable + } else if error.is::() { + TokenStorageError::DatabaseLocked(error.to_string()) } else { TokenStorageError::Database(error.to_string()) } @@ -76,9 +79,20 @@ impl StoredTokens { pub enum TokenStorageError { PathResolution(String), Database(String), + DatabaseLocked(String), CredentialAccessUnavailable, } +impl TokenStorageError { + pub fn failure(&self) -> AuthStorageFailure { + match self { + Self::CredentialAccessUnavailable => AuthStorageFailure::CredentialAccess, + Self::DatabaseLocked(_) => AuthStorageFailure::DatabaseLocked, + Self::PathResolution(_) | Self::Database(_) => AuthStorageFailure::Database, + } + } +} + impl fmt::Display for TokenStorageError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -90,6 +104,10 @@ impl fmt::Display for TokenStorageError { f, "Token storage database error: {reason}. Try: ensure the OS credential store is available and the auth database is accessible." ), + Self::DatabaseLocked(reason) => write!( + f, + "Token storage database is locked: {reason}. Try: wait for other SCE processes to exit and retry." + ), Self::CredentialAccessUnavailable => write!( f, "Token storage requires credential store authorization. Try: run 'sce auth login' to authorize access." diff --git a/context/plans/claude-hook-trace-propagation.md b/context/plans/claude-hook-trace-propagation.md index efa9c8441..9ba0f4ed4 100644 --- a/context/plans/claude-hook-trace-propagation.md +++ b/context/plans/claude-hook-trace-propagation.md @@ -320,6 +320,38 @@ Persist this field in every plan; this is durable plan state, not chat state: - Not run against the real macOS Keychain. No live `credential_inaccessible` doctor run or real "Always Allow" dialog was exercised in this session. Running it would have read the user's login Keychain. Its text is covered by unit tests, and the non-interactive read path is unchanged. - "Always Allow" trust is tied to the executable's code signature. Unsigned or ad-hoc-signed upgrades can lose it, and the user then has to authorize again through `sce auth login`. This is out of scope without a Developer ID certificate. -- Interactive Keychain failures reach `AuthStorageUnavailable` as `TokenStorageError::Database`, so a DB failure during login also shows credential-store guidance. +- Interactive Keychain failures reach `AuthStorageUnavailable` as `TokenStorageError::Database`, so a DB failure during login also shows credential-store guidance. Resolved in the follow-up below. - Doctor repeats the startup credential lookup (cached `AuthDb` on success), adding up to `CREDENTIAL_LOOKUP_DEADLINE` when the store hangs. Hook startup is unaffected. - Linux was not re-run in this session. Run `nix flake check` on x86_64-linux before merge. + +## Follow-up validation: typed auth storage failure classification (PR #330) + +**Date:** 2026-10-19 + +### Change + +- `UserError::AuthStorageUnavailable` and `LoginAuthStorageUnavailable` now carry a typed `AuthStorageFailure` cause: `CredentialAccess`, `DatabaseLocked`, or `Database`. Turso, lock, path, and I/O failures no longer show Keychain guidance. Key `auth.storage_unavailable`, the runtime class, and exit code 4 are unchanged. Details: [User-facing storage guidance](../sce/auth-db.md#user-facing-storage-guidance). +- Interactive keyring failures get the typed `CredentialAccessDenied` marker only for `NoStorageAccess`, or, on macOS, for the `OSStatus` codes `errSecUserCanceled`, `errSecNoAccessForItem`, `errSecAuthFailed`, `errSecInteractionNotAllowed`, and `errSecInteractionRequired`. +- The compatibility-lock conflict is a typed `SharedLockConflict` error with the same text. When the last encrypted-open retry fails on it, or on a Turso `Busy`/`BusySnapshot`, the error carries `db::DatabaseLocked`, which maps to `TokenStorageError::DatabaseLocked`. Managed telemetry still reports it as `database_error`. +- `ControlPlaneError::Storage` carries the cause, and `is_storage_failure()` became `storage_failure() -> Option` on the control-plane, stream, and sync error types. +- The macOS command guidance now ends with "Retry the original command, or run `sce doctor` to verify telemetry access." +- No changes to the non-interactive read path, `flock(LOCK_SH | LOCK_NB)`, lock lifetime, retry policy, Turso, setup, `doctor --fix`, signing, or release workflows. + +### Commands run (macOS 15.4.1, Darwin 24.4.0, arm64) + +- `nix flake check -L` -> exit 0 on aarch64-darwin (`cli-fmt`, `cli-clippy`, `cli-tests`: 671 passed, 0 failed). This includes all four `macos_*` compatibility-lock tests and the legacy-holder tests. +- Targeted filters: `services::error` 16, `services::auth_command` 17, `services::db::encryption_key` 2, `services::managed_telemetry` 29, `services::doctor` 11, `services::sync` 12, `services::agent_trace_sync` 14, `services::auth_db` 24. All passed. +- New or extended tests: per-cause message selection, with no Keychain text for DB causes on any platform or operation. Login guidance for all three causes plus `AuthError::Io`. Keyring tagging for `NoStorageAccess`, `NoEntry`, and generic platform failures, and for each macOS `OSStatus` code. The legacy-holder child roles now also assert `TokenStorageError::failure() == DatabaseLocked` and `error.is::()`. + +### Real macOS Keychain + +- Verified (read-only, user's login Keychain, existing `sce`/`auth DB` item left unchanged): a freshly built ad-hoc, linker-signed debug binary is a new code identity that is not in the item's ACL. `sce doctor --format json` from that binary exited 0 in about 2 s with empty stderr. It reported `managed_telemetry.reason = credential_inaccessible`, `kind = managed_telemetry_credential_inaccessible`, and the "Always Allow" remediation. It did not report `lookup_timeout`, so the lookup returned `errSecInteractionNotAllowed` without blocking on a dialog. +- Not verified: authorized silent access by the current build, locked-Keychain behavior, interactive authorization during `sce auth login`, persistence after "Always Allow", and real Claude Code hook runs against the Keychain. These need GUI interaction, locking the user's login Keychain, or changing its ACL or search list. The adapter only targets Keychain domains, not a disposable keychain file, so these could not be isolated. Hook fail-open and no-prompt behavior stays covered by the sandboxed multiprocess tests using `SCE_AUTH_DB_ENCRYPTION_KEY`. + +### Residual risks + +- "Always Allow" trust is tied to the executable's code signature. Unsigned or ad-hoc-signed upgrades can lose it. This is out of scope without a Developer ID certificate. +- Non-interactive failures, including a missing Keychain item while `auth.db` exists, still all map to `CredentialAccessUnavailable` (doctor `credential_inaccessible`), as before. +- `DatabaseLocked` is detected only while opening the encrypted DB. Busy errors from later queries after retries still classify as `Database`. +- Linux was not re-run in this session. + diff --git a/context/sce/auth-db.md b/context/sce/auth-db.md index 58f83c732..2a59211d6 100644 --- a/context/sce/auth-db.md +++ b/context/sce/auth-db.md @@ -117,25 +117,39 @@ Remaining limits: a mixed old/new deployment is still not usable at the same tim - Open paths: `open_or_create_auth_db()` (`AuthDb::new()`, interactive key access) is used only to persist a new session or telemetry credential during `sce auth login`. Every other path uses `open_existing_auth_db(access)` (`EncryptedTursoDb::open_existing_at(path, access)`), which returns `None` for a missing `auth.db` and never creates the file or a key. `load_tokens(access)` and `save_tokens(token, access)` take the caller's `CredentialAccess`; `delete_tokens` is interactive and returns `false` for a missing DB. - Auth commands and the generic async control-plane credential store directly await token persistence on the application runtime. Database operations do not cross a `spawn_blocking` boundary; the OS keyring adapter retains its platform-specific synchronous behavior where required. - `token_file_path()` returns the auth DB path from `auth_db_path()` instead of a JSON file path. -- `TokenStorageError` exposes `PathResolution`, `Database`, and `CredentialAccessUnavailable` variants (the last one is mapped from the typed `encryption_key::CredentialAccessUnavailable` error and carries no secret); former `Io`, `Serialization`, `CorruptedTokenFile`, and `Permission` variants have been removed. +- `TokenStorageError` exposes `PathResolution`, `Database`, `DatabaseLocked`, and `CredentialAccessUnavailable` variants. `CredentialAccessUnavailable` is mapped from the typed `encryption_key::CredentialAccessUnavailable` (non-interactive) or `encryption_key::CredentialAccessDenied` (interactive) errors and carries no secret. `DatabaseLocked` is mapped from the typed `db::DatabaseLocked` error. `TokenStorageError::failure()` returns the `AuthStorageFailure` cause used for user guidance. Former `Io`, `Serialization`, `CorruptedTokenFile`, and `Permission` variants have been removed. - No JSON file I/O remains in `token_storage.rs`. - The `auth_credentials` row uses constant integer ID `1` for single-row token storage. - Encryption is required: `encryption_key::get_or_create_encryption_key()` first honors `SCE_AUTH_DB_ENCRYPTION_KEY` by deriving a deterministic 64-character hex Turso key from the non-empty env-secret text, then falls back to the OS credential store when the env var is absent. No plaintext auth DB fallback exists. - Credential access is explicit (`encryption_key::CredentialAccess`): - - `Interactive`: used only by `sce auth login`, `logout`, and `whoami`. It may show the macOS Keychain authorization dialog. It generates and stores a new key only when the keyring reports `NoEntry` and `auth.db` does not exist. Any other read error is returned and never overwrites the stored key. + - `Interactive`: used only by `sce auth login`, `logout`, and `whoami`. It may show the macOS Keychain authorization dialog. It generates and stores a new key only when the keyring reports `NoEntry` and `auth.db` does not exist. Any other read error is returned and never overwrites the stored key. A read or write failure is tagged with the typed `CredentialAccessDenied` marker only when the keyring reports `NoStorageAccess`, or, on macOS, a `PlatformFailure` whose `OSStatus` is `errSecUserCanceled`, `errSecNoAccessForItem`, `errSecAuthFailed`, `errSecInteractionNotAllowed`, or `errSecInteractionRequired`. Classification uses these codes, never error text. - `NonInteractive`: used by Claude hooks, managed telemetry startup, the `sce doctor` telemetry health check, `sce auth telemetry-headers`, and `sce sync` (including post-commit auto-sync). It only reads, never creates. On macOS it wraps the read in `SecKeychain::disable_user_interaction()` (`SecKeychainSetUserInteractionAllowed(false)`, the UI switch for the legacy file-based keychain API that `apple-native-keyring-store`'s `keychain` feature uses through `SecKeychainFindGenericPassword`). An item the binary is not authorized for, or a locked keychain, fails with `errSecInteractionNotAllowed` instead of prompting. Every failure becomes `CredentialAccessUnavailable`. `kSecUseAuthenticationUIFail` and `LAContext.interactionNotAllowed` do not apply, because they belong to the `SecItem`/data-protection keychain and LocalAuthentication-protected items, which this adapter does not use. - The interaction switch is process-wide, so all credential-store reads and writes in one process are serialized by a mutex. Interaction is turned back on before the next caller runs. - Linux Secret Service and Windows have no equivalent switch in the adapter. `NonInteractive` there only guarantees no key creation. Missing/unavailable credential-store failures surface as `TokenStorageError::Database` and include platform-specific remediation for Linux system keyring/Secret Service, macOS Keychain, Windows Credential Store, or unsupported platforms, always mentioning `SCE_AUTH_DB_ENCRYPTION_KEY` for headless/CI use without printing the secret value. ## User-facing storage guidance -Storage failures from auth commands and `sce sync` render the `UserError` catalog entry for `auth.storage_unavailable` (runtime class, `SCE-ERR-RUNTIME`, exit code 4). The key, class, and exit code are shared by two variants in `cli/src/services/error.rs`: +Storage failures from auth commands and `sce sync` render the `UserError` catalog entry for `auth.storage_unavailable` (runtime class, `SCE-ERR-RUNTIME`, exit code 4). Two variants in `cli/src/services/error.rs` share that key, class, and exit code. Each carries a typed `AuthStorageFailure` cause: -- `AuthStorageUnavailable`: logout, whoami, and sync. The text is platform-specific. On macOS it says SCE could not access its encryption key in macOS Keychain, then lists the steps: run `sce auth login` in an interactive terminal, choose "Always Allow" when macOS asks to access the SCE Keychain item, and retry. If no prompt appears, the user should check that the login Keychain is unlocked and use the same installed executable. Linux names Secret Service, Windows names Credential Manager, and other platforms name the OS credential store. None of them mention macOS or "Always Allow". -- `LoginAuthStorageUnavailable`: storage failures inside `sce auth login` (stored-session load, renewal save, session replacement, device-flow storage errors). It says that running login again does not help until access is allowed. On macOS the steps are: unlock the login Keychain, rerun `sce auth login`, and choose "Always Allow" to authorize this executable. +- `AuthStorageUnavailable(cause)`: logout, whoami, and sync. +- `LoginAuthStorageUnavailable(cause)`: storage failures inside `sce auth login` (stored-session load, renewal save, session replacement, device-flow storage and I/O errors). -The text never says that "Always Allow" survives a binary upgrade. Unsigned and ad-hoc-signed builds can lose Keychain trust when the executable changes. It never suggests deleting `auth.db`, regenerating the key, or resetting the Keychain. The `anyhow` source stays attached for logs. Only the fixed catalog text reaches stderr, after `redact_sensitive_text`. +The cause comes from typed errors only: `TokenStorageError::failure()`, and `ControlPlaneError::Storage(cause, _)` through `storage_failure()` on the sync error types. It never comes from matching message text. -Limit: an interactive Keychain failure reaches this mapping as `TokenStorageError::Database`, the same variant as a real DB failure. So a corrupt or locked DB also shows the credential-store guidance. The precise cause is in the log file. +| Cause | Typed source | Guidance | +| --- | --- | --- | +| `CredentialAccess` | `CredentialAccessUnavailable` / `CredentialAccessDenied` | Platform-specific credential-store steps (below). | +| `DatabaseLocked` | `db::DatabaseLocked`: the last encrypted-open attempt failed on the compatibility-lock conflict or a Turso `Busy`/`BusySnapshot` | Another SCE process holds the auth database. Wait for other `sce` processes, including Claude Code hooks, make every installed `sce` the same version, then retry. | +| `Database` | Any other DB, path, or I/O failure | SCE could not open or read its local authentication database. Run `sce doctor`, retry, then check the log files. | + +`DatabaseLocked` and `Database` guidance is the same on every platform and for login. It never mentions the encryption key, Keychain, "Always Allow", or `sce auth login`, so it is never circular. + +`CredentialAccess` guidance is platform-specific: + +- Command, macOS: run `sce auth login` in an interactive terminal, choose "Always Allow" when macOS asks to access the SCE Keychain item, then retry the original command or run `sce doctor` to verify telemetry access. If no prompt appears, the user should check that the login Keychain is unlocked and use the same installed executable. +- Login, macOS: running login again does not help until access is allowed. Unlock the login Keychain, rerun `sce auth login`, and choose "Always Allow" to authorize this executable. +- Linux names Secret Service, Windows names Credential Manager, and other platforms name the OS credential store. None of them mention macOS or "Always Allow". + +The text never says that "Always Allow" survives a binary upgrade. Unsigned and ad-hoc-signed builds can lose Keychain trust when the executable changes. It never suggests deleting `auth.db`, regenerating the key, or resetting the Keychain. The `anyhow` source stays attached for logs. Only the fixed catalog text reaches stderr, after `redact_sensitive_text`. Typed markers are placed inside the `anyhow` chain, so rendered diagnostics, including the compatibility-lock rejection text, are unchanged. See also: [shared-turso-db.md](shared-turso-db.md), [../cli/default-path-catalog.md](../cli/default-path-catalog.md), [../context-map.md](../context-map.md) diff --git a/context/sce/cli-error-code-taxonomy.md b/context/sce/cli-error-code-taxonomy.md index 2ff3b4932..68382908b 100644 --- a/context/sce/cli-error-code-taxonomy.md +++ b/context/sce/cli-error-code-taxonomy.md @@ -33,10 +33,10 @@ It complements the numeric process exit-code classes documented in `context/sce/ ## Ownership - `FailureClass` in `cli/src/services/error.rs` owns class selection and stable code assignment (`FailureClass::code()`). -- `CliError::{User,Internal}` in `cli/src/services/error.rs` is the typed CLI-boundary error type; `CliError::code()`/`CliError::class()` delegate to the failure class. `CliError::User` carries a closed catalog `UserError` (`NotAuthenticated`, `NotGitRepository`, `NotGitRemote`, `AuthStorageUnavailable`, or `UnexpectedFailure`) for expected, deliberately-explained failures; `CliError::Internal` carries a live `anyhow::Error` source for every other failure, including non-classifiable setup preflight failures. `CliError::User` may also carry an optional preserved technical `source`, kept for observability only and never rendered to the terminal; the missing-remote source contains the configured remote name but no URL. -- `UserError` in `cli/src/services/error.rs` is the closed catalog of deliberately presented terminal failures. It has no arbitrary-message variant (no `Message(String)`/`Custom(...)` escape hatch): every entry returns a fixed reviewed sentence from `UserError::message()`, keyed for structured logging by `UserError::key()`. `NotAuthenticated` (`auth.not_authenticated`) covers Control Plane authentication failures from authenticated `sce auth whoami`; missing credentials from `sce auth logout`/`whoami` are successful state queries and do not enter the error catalog. `AuthStorageUnavailable` (`auth.storage_unavailable`) is used by `sce sync` and the `sce auth login`, `logout`, and `whoami` command boundary for token-storage plus `AuthError::Io`/`Storage` failures. Both auth-command mappings preserve technical sources for observability. `NotGitRepository` (`setup.not_git_repository`) is used by the setup command only when setup-owned repository-root resolution positively identifies a target as outside a Git repository; `NotGitRemote` (`setup.not_git_remote`) is used only when the configured named remote has no URL. Both preserve technical sources for observability. Other repository-root resolution, Git, and remote-lookup execution failures map to `UnexpectedFailure` (`general.unexpected_failure`). `UnexpectedFailure` is also used by `sce sync`, the config-command boundary for config execution failures, the version and doctor command boundaries for service execution failures, the auth-command boundary, and remaining setup execution failures; it renders one fixed, user-safe log-files guidance sentence and has no automatic `Try:` suffix or dynamic path input. +- `CliError::{User,Internal}` in `cli/src/services/error.rs` is the typed CLI-boundary error type; `CliError::code()`/`CliError::class()` delegate to the failure class. `CliError::User` carries a closed catalog `UserError` (`NotAuthenticated`, `NotGitRepository`, `NotGitRemote`, `AuthStorageUnavailable`, `LoginAuthStorageUnavailable`, or `UnexpectedFailure`) for expected, deliberately-explained failures; `CliError::Internal` carries a live `anyhow::Error` source for every other failure, including non-classifiable setup preflight failures. `CliError::User` may also carry an optional preserved technical `source`, kept for observability only and never rendered to the terminal; the missing-remote source contains the configured remote name but no URL. +- `UserError` in `cli/src/services/error.rs` is the closed catalog of deliberately presented terminal failures. It has no arbitrary-message variant (no `Message(String)`/`Custom(...)` escape hatch): every entry returns a fixed reviewed sentence from `UserError::message()`, keyed for structured logging by `UserError::key()`. `NotAuthenticated` (`auth.not_authenticated`) covers Control Plane authentication failures from authenticated `sce auth whoami`; missing credentials from `sce auth logout`/`whoami` are successful state queries and do not enter the error catalog. `AuthStorageUnavailable` and `LoginAuthStorageUnavailable` (both `auth.storage_unavailable`) are used by `sce sync` and the `sce auth login`, `logout`, and `whoami` command boundary for token-storage plus `AuthError::Io`/`Storage` failures; each carries a typed `AuthStorageFailure` cause (`CredentialAccess`, `DatabaseLocked`, `Database`) that selects the guidance text (see `context/sce/auth-db.md`). Both auth-command mappings preserve technical sources for observability. `NotGitRepository` (`setup.not_git_repository`) is used by the setup command only when setup-owned repository-root resolution positively identifies a target as outside a Git repository; `NotGitRemote` (`setup.not_git_remote`) is used only when the configured named remote has no URL. Both preserve technical sources for observability. Other repository-root resolution, Git, and remote-lookup execution failures map to `UnexpectedFailure` (`general.unexpected_failure`). `UnexpectedFailure` is also used by `sce sync`, the config-command boundary for config execution failures, the version and doctor command boundaries for service execution failures, the auth-command boundary, and remaining setup execution failures; it renders one fixed, user-safe log-files guidance sentence and has no automatic `Try:` suffix or dynamic path input. - Command and domain layers construct and return a `CliError`; they do not format terminal text, apply styling, or decide authentication/user-error semantics from string matching. Auth command orchestration classifies expected authentication and credential-storage failures into the existing `UserError` catalog by typed domain variants, never by string matching, and preserves the original technical chain as the optional user-error source. `app_support` is the sole owner of turning a `CliError` into the final stderr sentence. -- `UserError` in `cli/src/services/error.rs` is the closed catalog of deliberately presented terminal failures. It has no arbitrary-message variant (no `Message(String)`/`Custom(...)` escape hatch): every entry is a fixed, reviewed sentence returned by `UserError::message()`, keyed for structured logging by `UserError::key()`. `NotAuthenticated` (`auth.not_authenticated`) covers Control Plane authentication failures from authenticated `sce auth whoami`; missing credentials from `sce auth logout`/`whoami` are successful state queries and do not enter the error catalog. `AuthStorageUnavailable` (`auth.storage_unavailable`) is used by `sce sync` and the `sce auth login`, `logout`, and `whoami` command boundary for token-storage plus `AuthError::Io`/`Storage` failures. Both auth-command mappings preserve technical sources for observability. `NotGitRepository` (`setup.not_git_repository`) is used by the setup command only when setup-owned repository-root resolution positively identifies a target as outside a Git repository; it renders fixed Git-init/rerun guidance and preserves the technical source for observability. Other repository-root resolution failures, including nonexistent, inaccessible, process, and malformed-output failures, map to `UnexpectedFailure` (`general.unexpected_failure`). `UnexpectedFailure` is also used by `sce sync`, the config-command boundary for config execution failures, the version and doctor command boundaries for service execution failures, the auth-command boundary, and remaining setup execution failures; it renders one fixed, user-safe diagnostic sentence and has no automatic `Try:` suffix or dynamic path input. +- `UserError` in `cli/src/services/error.rs` is the closed catalog of deliberately presented terminal failures. It has no arbitrary-message variant (no `Message(String)`/`Custom(...)` escape hatch): every entry is a fixed, reviewed sentence returned by `UserError::message()`, keyed for structured logging by `UserError::key()`. `NotAuthenticated` (`auth.not_authenticated`) covers Control Plane authentication failures from authenticated `sce auth whoami`; missing credentials from `sce auth logout`/`whoami` are successful state queries and do not enter the error catalog. `AuthStorageUnavailable` and `LoginAuthStorageUnavailable` (both `auth.storage_unavailable`) are used by `sce sync` and the `sce auth login`, `logout`, and `whoami` command boundary for token-storage plus `AuthError::Io`/`Storage` failures; each carries a typed `AuthStorageFailure` cause (`CredentialAccess`, `DatabaseLocked`, `Database`) that selects the guidance text (see `context/sce/auth-db.md`). Both auth-command mappings preserve technical sources for observability. `NotGitRepository` (`setup.not_git_repository`) is used by the setup command only when setup-owned repository-root resolution positively identifies a target as outside a Git repository; it renders fixed Git-init/rerun guidance and preserves the technical source for observability. Other repository-root resolution failures, including nonexistent, inaccessible, process, and malformed-output failures, map to `UnexpectedFailure` (`general.unexpected_failure`). `UnexpectedFailure` is also used by `sce sync`, the config-command boundary for config execution failures, the version and doctor command boundaries for service execution failures, the auth-command boundary, and remaining setup execution failures; it renders one fixed, user-safe diagnostic sentence and has no automatic `Try:` suffix or dynamic path input. - `Logger::log_cli_error` in `cli/src/services/observability.rs` owns structured error logging with `sce.error.{code}` event IDs. - Config, version, doctor, and remaining setup execution boundaries map their unexpected failures to `UserError::UnexpectedFailure`; auth command mappings preserve their original technical sources for observability while terminal rendering remains user-safe. - `write_error_diagnostic` in `cli/src/services/app_support.rs` owns final stderr rendering: it redacts and writes the catalog variant's message without a wrapper or styling, while `CliError::Internal` retains code-bearing rendering and styles its rendered chain through `services::style::error_text_with_color_policy` under the stderr TTY/`NO_COLOR` policy (`services::style::supports_color_stderr()`), independent of stdout's TTY state. From 9d0caa8293df25d1990fa3d2f345872f180f7173 Mon Sep 17 00:00:00 2001 From: David Abram Date: Mon, 19 Oct 2026 21:41:13 +0200 Subject: [PATCH 15/17] auth: Normalize whitespace in login storage guidance assertions The Linux and Windows login guidance wraps "Running login again does not help until" across lines, so the exact-substring assertion in `assert_login_storage_guidance` failed on Linux CI. Collapse whitespace before asserting, and also assert that credential-store login guidance never tells the user to run `sce auth login` again. The messages, the `auth.storage_unavailable` key, and exit code 4 are unchanged. Co-authored-by: SCE --- cli/src/services/auth_command/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/src/services/auth_command/mod.rs b/cli/src/services/auth_command/mod.rs index e760caec0..89b196cc8 100644 --- a/cli/src/services/auth_command/mod.rs +++ b/cli/src/services/auth_command/mod.rs @@ -809,7 +809,11 @@ mod tests { assert_eq!(*user_error, UserError::LoginAuthStorageUnavailable(failure)); assert_eq!(user_error.key(), "auth.storage_unavailable"); assert_eq!(error.class().exit_code(), 4); - let message = error.to_string(); + let message = error + .to_string() + .split_whitespace() + .collect::>() + .join(" "); assert!(!message.contains("Retry the original command")); let keychain_guidance = message.contains("encryption key"); assert_eq!( @@ -818,6 +822,7 @@ mod tests { ); if keychain_guidance { assert!(message.contains("Running login again does not help until")); + assert!(!message.contains("Run `sce auth login` in an interactive terminal.")); } } From 464907a0cfd82292dfeddaface36784a74933640 Mon Sep 17 00:00:00 2001 From: David Abram Date: Mon, 19 Oct 2026 21:41:14 +0200 Subject: [PATCH 16/17] auth-db: Serialize Turso cold opens after legacy WAL writes A legacy-mode SCE process writes auth.db-wal without updating the multiprocess authority in auth.db-tshm. The next new-mode cold opens then take Turso 0.8.1's exclusive rebuild branch in `repair_or_reseed_authority_from_local_disk_scan`. That branch is guarded only by "no writer active" and a process-local publish mutex. When several processes open concurrently, they rebuild the shared frame index at the same time. The result is a `shared WAL frame index length changed` panic, which is the Linux CI failure, or a successful open that reads a stale snapshot, such as a missing credential or an outdated schema. Take an exclusive `File::lock` on a sidecar `auth.db-open.lock` around `Builder::build()`/`connect()` only, after the compatibility lock succeeds. Queries, writes, and migrations are not serialized, and the `F_OFD_SETLK` and macOS `flock` compatibility locks are unchanged. Add a regression that runs 8 concurrent cold opens after each of 20 legacy writes. It failed 20 of 20 runs without the lock. Co-authored-by: SCE --- .../services/auth_db/multiprocess_tests.rs | 29 +++++++++++++++++ cli/src/services/db/mod.rs | 1 + cli/src/services/db/open_lock.rs | 32 ++++++++++++++++++- context/sce/auth-db.md | 20 ++++++++++-- 4 files changed, 79 insertions(+), 3 deletions(-) diff --git a/cli/src/services/auth_db/multiprocess_tests.rs b/cli/src/services/auth_db/multiprocess_tests.rs index 7ec7909a5..00247e80c 100644 --- a/cli/src/services/auth_db/multiprocess_tests.rs +++ b/cli/src/services/auth_db/multiprocess_tests.rs @@ -73,6 +73,8 @@ const WRITER_ROWS: usize = 8; const WRITER_PAUSE: Duration = Duration::from_millis(20); const MIXED_WRITER_ROUNDS: usize = 3; const NEW_WRITERS: usize = 4; +const LEGACY_HANDOFF_ROUNDS: usize = 20; +const LEGACY_HANDOFF_READERS: usize = 8; const SHARED_ACCESS_REJECTION: &str = "does not support shared access"; const LEGACY_LOCK_REJECTION: &str = "File is locked by another process"; const LEGACY_CIPHER: &str = "aegis256"; @@ -598,6 +600,33 @@ fn concurrent_legacy_and_new_writers_never_lose_committed_rows() { sandbox.run("legacy-roundtrip"); } +#[test] +fn concurrent_cold_opens_after_legacy_write_all_see_committed_state() { + let _fan_out = serialize_fan_out(); + let sandbox = Sandbox::new(); + sandbox.seed(); + + let mut expected = BTreeSet::new(); + for round in 0..LEGACY_HANDOFF_ROUNDS { + let stdout = wait_success( + "legacy-writer", + sandbox.writer("legacy-writer", &format!("legacy-handoff-{round}")), + ); + let committed = prefixed_lines(&stdout, COMMITTED_PREFIX); + assert_eq!(committed.len(), WRITER_ROWS, "round {round}"); + expected.extend(committed); + let readers = (0..LEGACY_HANDOFF_READERS) + .map(|_| sandbox.command("read").spawn().expect("spawn reader")) + .collect::>(); + assert_eq!( + wait_readers(readers), + LEGACY_HANDOFF_READERS, + "round {round}" + ); + } + assert_eq!(sandbox.rows(), expected); +} + #[test] fn concurrent_new_writers_all_commit_and_legacy_reopens_afterwards() { let _fan_out = serialize_fan_out(); diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index dc3a5f167..67941f18c 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -1487,6 +1487,7 @@ impl EncryptedTursoDb { ) -> Result { let db_name = M::db_name(); open_lock::lock_shared(open_lock, db_name, db_path)?; + let _cold_open = open_lock::lock_cold_open(db_path, db_name).await?; let path_str = db_path.to_str().ok_or_else(|| { anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) })?; diff --git a/cli/src/services/db/open_lock.rs b/cli/src/services/db/open_lock.rs index 3ff596e70..f87c27cb8 100644 --- a/cli/src/services/db/open_lock.rs +++ b/cli/src/services/db/open_lock.rs @@ -3,7 +3,9 @@ use std::fs::{File, OpenOptions}; use std::io; use std::path::{Path, PathBuf}; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; + +const COLD_OPEN_LOCK_SUFFIX: &str = "-open.lock"; #[cfg(target_os = "linux")] const SHARED_LOCK_COMMAND: libc::c_int = libc::F_OFD_SETLK; @@ -61,6 +63,34 @@ pub(super) fn lock_shared(file: &File, db_name: &str, db_path: &Path) -> Result< } } +fn cold_open_lock_path(db_path: &Path) -> PathBuf { + let mut path = db_path.as_os_str().to_owned(); + path.push(COLD_OPEN_LOCK_SUFFIX); + PathBuf::from(path) +} + +pub(super) async fn lock_cold_open(db_path: &Path, db_name: &str) -> Result { + let lock_path = cold_open_lock_path(db_path); + let db_name = db_name.to_owned(); + tokio::task::spawn_blocking(move || { + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .and_then(|file| file.lock().map(|()| file)) + .map_err(|error| { + anyhow!( + "failed to serialize opening the {db_name} database with {}: {error}", + lock_path.display() + ) + }) + }) + .await + .context("database cold-open lock worker failed")? +} + #[cfg(target_os = "macos")] fn try_lock_shared(file: &File) -> io::Result<()> { use std::os::fd::AsRawFd; diff --git a/context/sce/auth-db.md b/context/sce/auth-db.md index 2a59211d6..e74776f7d 100644 --- a/context/sce/auth-db.md +++ b/context/sce/auth-db.md @@ -50,7 +50,7 @@ Storage operations live in `cli/src/services/telemetry_credential_storage.rs`: ` ## Concurrency and schema readiness -- Many short-lived `sce` processes read `auth.db` at once: every admitted Claude hook resolves the telemetry credential at startup. `EncryptedTursoDb::new()` therefore opens with Turso multiprocess WAL, alongside encryption, so concurrent readers share the file. Multiprocess WAL creates `auth.db-wal` and the `auth.db-tshm` coordination file next to the database. +- Many short-lived `sce` processes read `auth.db` at once: every admitted Claude hook resolves the telemetry credential at startup. `EncryptedTursoDb::new()` therefore opens with Turso multiprocess WAL, alongside encryption, so concurrent readers share the file. Multiprocess WAL creates `auth.db-wal` and the `auth.db-tshm` coordination file next to the database. SCE also creates `auth.db-open.lock` once `auth.db` exists, to serialize Turso cold opens (see [Serialized cold opens](#serialized-cold-opens-after-a-legacy-write)). - A current schema is opened read-only: no `CREATE TABLE IF NOT EXISTS __sce_migrations` or other DDL runs when every embedded migration ID is already recorded. A missing or outdated DB is migrated in one `BEGIN IMMEDIATE` transaction, with a retry only for genuine `Busy`. A partially migrated schema fails with the failing migration ID rather than being treated as current. Details: [shared-turso-db.md](shared-turso-db.md). - Telemetry credential lookups are existing-DB read paths. Both `load_telemetry_credential` (the managed-telemetry lookup used by every command and admitted hook) and `load_repository_telemetry_credential` (the `sce auth telemetry-headers` lookup that Claude Code's `otelHeadersHelper` runs) go through `token_storage::get_existing_auth_db`. That helper reuses the process `AuthDb` singleton when it is already open. Otherwise it calls `EncryptedTursoDb::open_existing_at`, which opens the database file without `O_CREAT` before resolving the encryption key. An absent file returns no credential, with no DB file, `-wal`/`-tshm` file, encryption key, or migration created. The existence check is the `open()` itself, not a separate `Path::exists()`. Path-resolution errors stay `TokenStorageError::PathResolution`. Only `sce auth`, `sce setup`, and other writers create the DB. This also keeps concurrent hooks and header-helper calls on a never-logged-in machine from racing to create it. - The auth DB busy timeout is 1000 ms. Hook reads stay bounded by `CREDENTIAL_LOOKUP_DEADLINE` (2s), which aborts the lookup regardless of busy waits or retries. @@ -58,10 +58,25 @@ Storage operations live in `cli/src/services/telemetry_credential_storage.rs`: ` ### Upstream Turso multiprocess race -Under heavy load, Turso 0.8.1 multiprocess WAL has an internal race on concurrent cold opens that involve writes. Stress runs (6 parallel copies of the suite, with 32 processes creating or migrating the same DB) occasionally failed: `short read on page 3`, a `turso_assert` panic `reader slot released by non-owner` in `shared_wal_coordination.rs`, or, in the Nix sandbox, a writer that stayed `database is locked` through all 10 migration attempts (about 49 s). The read-only paths never failed across about 3,800 cold-opening reader and hook processes, and that is all hooks do on a current schema. The migration regression therefore uses 4 concurrent migrators, which was 0 failures in 48 parallel stress runs. Concurrent creation and migration by many writers is left as a documented upstream limitation. That migration limitation affects first-time initialization, and the Agent Trace DB already runs in the same Turso mode. +Under heavy load, Turso 0.8.1 multiprocess WAL has an internal race on concurrent cold opens that involve writes. Stress runs (6 parallel copies of the suite, with 32 processes creating or migrating the same DB) occasionally failed: `short read on page 3`, a `turso_assert` panic `reader slot released by non-owner` in `shared_wal_coordination.rs`, or, in the Nix sandbox, a writer that stayed `database is locked` through all 10 migration attempts (about 49 s). The read-only paths never failed across about 3,800 cold-opening reader and hook processes, and that is all hooks do on a current schema. None of those runs followed a legacy-mode write; that case did fail, as described under [Serialized cold opens](#serialized-cold-opens-after-a-legacy-write). The migration regression therefore uses 4 concurrent migrators, which was 0 failures in 48 parallel stress runs. Concurrent creation and migration by many writers is left as a documented upstream limitation. That migration limitation affects first-time initialization, and the Agent Trace DB already runs in the same Turso mode. The four-writer compatibility regression also reproduced exhausted `database is locked` retries against an already-seeded, current-schema DB when connection startup/teardown overlapped inserts. Its children previously swallowed insert failures, leaving only a short committed-row count in the parent. The regression now opens every writer before releasing concurrent inserts and keeps every connection alive until all writers finish; every insert must succeed, all 32 rows must persist, and a legacy process must reopen afterwards. This passed 80 stress repetitions across four parallel test runs. The macOS shared-lock writer regression uses the same coordination. This isolates the compatibility-lock guarantee from the Turso lifecycle race; production open/write/close behavior and retry limits are unchanged. +### Serialized cold opens after a legacy write + +A legacy-mode process (any SCE release through `v0.4.0-pre-alpha-v11`) writes `auth.db-wal` without updating the multiprocess coordination authority in `auth.db-tshm`. The next new-mode open finds that its WAL disk scan disagrees with the persisted authority. Turso 0.8.1 then takes the rebuild branch of `WalFile::repair_or_reseed_authority_from_local_disk_scan` (`storage/wal.rs`, reached from `seed_or_sync_authority` during `turso::Builder::build`). That branch discards the durable shared frame index and republishes it from the local scan. It treats the open as exclusive, but its only guard is `!writer_or_checkpoint_lock_active()`. The publish mutex in `SharedWalCoordination::record_frame` is process-local. When several new processes cold-open at once, each runs the rebuild concurrently against the same shared index. This is an upstream Turso defect, not an SCE ordering bug or a test orchestration issue. The compatibility lock keeps legacy and new processes from holding the DB at the same time, but it cannot help here because the legacy process has already exited. + +Observed outcomes, all on read-only `load_telemetry_credential` children (the hook path) against a seeded, current-schema DB: + +- A `turso_assert` panic: `shared WAL frame index length changed while publishing an entry` (`wal.rs:1605`). This is the Linux CI failure in `concurrent_legacy_and_new_writers_never_lose_committed_rows`, where 2 new writers cold-opened together after a legacy writer from the previous round. +- A successful open that read a stale snapshot. Some processes found no telemetry credential, and others found an outdated schema (`failed to run auth DB migrations`). That is a silent wrong read, not only a crash. + +Probe (20 rounds: 1 legacy write, then 8 concurrent new readers): 7–14 failed opens out of 160 per run across 6 runs after a legacy write. In the control, with a new-mode write instead of the legacy write, 0 of 160 failed. With an authority that is current, Turso syncs from it and never rebuilds, so this needs a preceding legacy write (or another out-of-band WAL change). + +Mitigation (`open_lock::lock_cold_open`): every `EncryptedTursoDb` connection attempt takes an exclusive `std::fs::File::lock` (`flock` on Unix, `LockFileEx` on Windows) on the sidecar `auth.db-open.lock`. It is held only around `turso::Builder::build()` and `connect()`, after the compatibility lock succeeds, so a legacy holder is still rejected immediately. The wait runs on a blocking thread, so `CREDENTIAL_LOOKUP_DEADLINE` still bounds hook lookups. Queries, writes, and migrations are not serialized. Rationale: with the compatibility lock, a stale authority can exist only while no new process holds the DB. The first serialized opener therefore rebuilds alone, and later openers sync from the rebuilt authority. The sidecar is a separate file, so the Linux `F_OFD_SETLK` lock and the macOS `flock` lock on `auth.db` are unchanged. Cost: a burst of 32 simultaneous cold opens takes about 160 ms wall time instead of about 55 ms (roughly 3 ms per serialized open), well under the 2 s hook deadline. + +Residual risks: a process stuck inside `turso::Builder::build()` delays other cold opens until it exits; hook lookups still give up at their deadline, and the kernel releases the lock when the process dies. The Agent Trace DB (`TursoDb`, also multiprocess WAL) does not take this lock. The upstream defect remains in Turso 0.8.1, and SCE only avoids triggering it. + ### Mixed-version compatibility lock SCE binaries from before multiprocess WAL (every release through `v0.4.0-pre-alpha-v11`, on Turso 0.8.1 or 0.7.0) open `auth.db` in legacy WAL mode. In that mode, Turso's `UnixFile::lock_file` takes a non-blocking classic POSIX record lock, `fcntl(F_SETLK, F_WRLCK)` over the whole file (`l_start = 0`, `l_len = 0`). It holds that lock while the database is open. Turso's own mixed-mode guards (`reject_live_*` in `turso_core::Database::resolve_default_storage`) never run for `turso::Builder`. `turso_sdk_kit` opens the file itself and calls `Database::open_async`, which skips them. Multiprocess mode opens the DB file with `NoLock`, so nothing stopped the two modes from overlapping. Without a guard, when an old and a new process both held the DB and both wrote, one process's committed rows were silently lost while `PRAGMA integrity_check` still reported `ok`. The regression test reproduced this: with the guard disabled, 24 of 72 committed rows were lost. @@ -90,6 +105,7 @@ Mixed-version tests (`multiprocess_tests.rs`), all verifying persisted `telemetr - A legacy holder makes a new telemetry lookup and a new writer fail explicitly. The legacy holder's own later write persists. - A new holder makes a legacy open fail with the Turso lock error. - 3 rounds of 1 legacy writer and 2 new writers start together. Every row any writer reported committed is persisted. +- `concurrent_cold_opens_after_legacy_write_all_see_committed_state`: 20 rounds of one legacy writer (8 rows), then 8 concurrent new reader processes. Every reader must resolve the credential, and every legacy row must persist with `integrity_check` `ok`. Without the cold-open lock, this test failed 20 of 20 runs. - 4 new writers with coordinated connection lifetimes insert concurrently and commit all 32 rows; any insert failure includes the row and database error. - After every new process exits, a legacy process reopens the DB, reads the credential, and writes. From 78bada7d7bd71ee44eddde066ddfb260b28f3c47 Mon Sep 17 00:00:00 2001 From: David Abram Date: Tue, 20 Oct 2026 12:06:19 +0200 Subject: [PATCH 17/17] tests: Harden auth and telemetry integration coverage Strengthen staged coverage for credential lifecycle ordering, readiness state handling, managed telemetry routing, and concurrent auth database access. Isolate environment-sensitive exporter tests in fault-process children and assert decoded telemetry attributes, redirect behavior, and failure preservation instead of relying on serialized or substring-based checks. This makes the integration suite deterministic and verifies security-sensitive export and persistence behavior without changing runtime code. Co-authored-by: SCE --- cli/src/app.rs | 66 +- .../app/doctor_reconciliation_cli_tests.rs | 41 +- cli/src/services/auth_command/mod.rs | 179 +++-- .../services/auth_db/multiprocess_tests.rs | 621 ++---------------- cli/src/services/doctor/managed_telemetry.rs | 55 +- .../managed_telemetry/trace_export.rs | 215 +++--- cli/src/services/telemetry_readiness.rs | 83 ++- 7 files changed, 398 insertions(+), 862 deletions(-) diff --git a/cli/src/app.rs b/cli/src/app.rs index 0d70cbae7..2a863fc0d 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -1105,47 +1105,37 @@ pub(crate) mod tests { #[test] fn only_claude_hook_invocations_are_admitted_to_managed_hook_telemetry() { - let kind = |args: &[&str], claude_caller: bool| { + const HOOK: Option = Some(ManagedSpanKind::ClaudeHook); + const COMMAND: Option = Some(ManagedSpanKind::Command); + let cases: [(&[&str], bool, Option); 20] = [ + (&["sce", "hooks", "claude-mutation-scope"], false, HOOK), + (&["sce", "hooks", "claude-model-state"], false, HOOK), + (&["sce", "hooks", "diff-trace"], true, HOOK), + (&["sce", "hooks", "diff-trace"], false, None), + (&["sce", "hooks", "conversation-trace"], true, HOOK), + (&["sce", "hooks", "conversation-trace"], false, None), + (&["sce", "policy", "bash"], true, HOOK), + (&["sce", "policy", "bash"], false, None), + (&["sce", "hooks", "pre-commit"], true, None), + (&["sce", "--verbose", "hooks", "codex"], true, None), + (&["sce", "hooks", "codex-mutation-scope"], true, None), + (&["sce", "hooks", "opencode-mutation-scope"], true, None), + (&["sce", "hooks", "pi-mutation-scope"], true, None), + (&["sce", "hooks"], true, None), + (&["sce", "auth", "telemetry-headers"], true, None), + (&["sce", "setup"], true, None), + (&["sce", "setup", "--claude"], true, None), + (&["sce", "auth", "login"], true, COMMAND), + (&["sce", "config", "show"], false, COMMAND), + (&["sce", "doctor"], true, COMMAND), + ]; + for (args, claude_caller, expected) in cases { let args = args.iter().map(ToString::to_string).collect::>(); - managed_span_kind(&args, claude_caller) - }; - let claude_hook = Some(ManagedSpanKind::ClaudeHook); - let command = Some(ManagedSpanKind::Command); - for caller in [false, true] { - assert_eq!( - kind(&["sce", "hooks", "claude-mutation-scope"], caller), - claude_hook - ); - assert_eq!( - kind(&["sce", "hooks", "claude-model-state"], caller), - claude_hook - ); - assert_eq!(kind(&["sce", "hooks", "pre-commit"], caller), None); - assert_eq!(kind(&["sce", "--verbose", "hooks", "codex"], caller), None); - assert_eq!( - kind(&["sce", "hooks", "codex-mutation-scope"], caller), - None - ); assert_eq!( - kind(&["sce", "hooks", "opencode-mutation-scope"], caller), - None + managed_span_kind(&args, claude_caller), + expected, + "{args:?} claude_caller={claude_caller}" ); - assert_eq!(kind(&["sce", "hooks", "pi-mutation-scope"], caller), None); - assert_eq!(kind(&["sce", "hooks"], caller), None); - assert_eq!(kind(&["sce", "auth", "telemetry-headers"], caller), None); - assert_eq!(kind(&["sce", "setup"], caller), None); - assert_eq!(kind(&["sce", "setup", "--claude"], caller), None); - assert_eq!(kind(&["sce", "auth", "login"], caller), command); - assert_eq!(kind(&["sce", "config", "show"], caller), command); - assert_eq!(kind(&["sce", "doctor"], caller), command); - } - for shared in [ - &["sce", "hooks", "diff-trace"][..], - &["sce", "hooks", "conversation-trace"], - &["sce", "policy", "bash"], - ] { - assert_eq!(kind(shared, true), claude_hook); - assert_eq!(kind(shared, false), None); } } diff --git a/cli/src/app/doctor_reconciliation_cli_tests.rs b/cli/src/app/doctor_reconciliation_cli_tests.rs index 8224a3bb6..bc798c2a0 100644 --- a/cli/src/app/doctor_reconciliation_cli_tests.rs +++ b/cli/src/app/doctor_reconciliation_cli_tests.rs @@ -606,16 +606,12 @@ async fn doctor_fix_recovers_from_every_initial_maintenance_state_condition() { } } -fn managed_telemetry_problems(report: &Value) -> Vec { +fn global_state_warnings(report: &Value) -> Vec { report["problems"] .as_array() .expect("problems array") .iter() - .filter(|problem| { - problem["summary"] - .as_str() - .is_some_and(|summary| summary.contains("telemetry export is disabled")) - }) + .filter(|problem| problem["category"] == "global_state" && problem["severity"] == "warning") .cloned() .collect() } @@ -633,7 +629,8 @@ async fn doctor_on_fresh_state_reports_unconfigured_telemetry_without_a_warning( .json(); assert_eq!(report["managed_telemetry"]["state"], "not_configured"); - assert!(managed_telemetry_problems(&report).is_empty()); + let warnings = global_state_warnings(&report); + assert!(warnings.is_empty(), "{warnings:#?}"); assert!(!auth_dir.join("auth.db").exists()); assert!(!auth_dir.join("telemetry-ready.json").exists()); } @@ -664,24 +661,24 @@ async fn doctor_fix_reports_unreadable_telemetry_credentials_without_auth_side_e "{}", fix.stdout ); - let problems = managed_telemetry_problems(&report); - assert_eq!(problems.len(), 1, "{}", fix.stdout); - assert_eq!(problems[0]["severity"], "warning"); - assert_eq!(problems[0]["fixability"], "manual_only"); - assert!(!problems[0]["remediation"]["text"] - .as_str() - .expect("remediation") - .contains("Always Allow")); - let telemetry_fixes = non_reconciliation_fix_results(&report, sandbox.dir.path()) - .into_iter() + let warnings = global_state_warnings(&report); + assert_eq!(warnings.len(), 1, "{}", fix.stdout); + assert_eq!(warnings[0]["fixability"], "manual_only"); + assert_eq!(warnings[0]["remediation"]["next_action"], "manual_steps"); + let summary = warnings[0]["summary"].as_str().expect("summary"); + let fixes = report["fix_results"] + .as_array() + .expect("fix_results array") + .iter() .filter(|entry| { - entry["detail"] - .as_str() - .is_some_and(|detail| detail.contains("telemetry")) + entry["category"] == "global_state" + && entry["detail"] + .as_str() + .is_some_and(|detail| detail.starts_with(summary)) }) .collect::>(); - assert_eq!(telemetry_fixes.len(), 1, "{}", fix.stdout); - assert_eq!(telemetry_fixes[0]["outcome"], "manual"); + assert_eq!(fixes.len(), 1, "{}", fix.stdout); + assert_eq!(fixes[0]["outcome"], "manual"); assert_eq!( std::fs::read(&auth_db).expect("auth DB kept"), UNREADABLE_AUTH_DB diff --git a/cli/src/services/auth_command/mod.rs b/cli/src/services/auth_command/mod.rs index 89b196cc8..384fc9643 100644 --- a/cli/src/services/auth_command/mod.rs +++ b/cli/src/services/auth_command/mod.rs @@ -717,8 +717,6 @@ fn unexpected_auth_command_error(error: anyhow::Error) -> CliError { #[cfg(test)] mod tests { - use std::sync::atomic::{AtomicBool, Ordering}; - use super::*; fn storage_failure() -> TokenStorageError { @@ -971,55 +969,39 @@ mod tests { } #[tokio::test] - async fn valid_stored_credentials_are_provisioned_after_workos_persistence() { - let provision_calls = AtomicBool::new(false); - let tokens = stored_tokens(); - let output = run_login_with_stored_credentials( - AuthFormat::Json, - Some(tokens.clone()), - |stored| async move { - assert_eq!(stored, tokens); - Ok(Some(stored)) - }, - || async { - provision_calls.store(true, Ordering::SeqCst); - TelemetryProvisioningResult::Issued - }, - |_format| async { panic!("valid stored credentials must not use device login") }, - ) - .await - .expect("valid stored login should succeed"); - - assert!(provision_calls.load(Ordering::SeqCst)); - assert!(output.contains("\"renewed\": true")); - assert!(output.contains("\"status\": \"issued\"")); - } - - #[tokio::test] - async fn renewed_stored_credentials_are_provisioned_after_workos_persistence() { - let provision_calls = AtomicBool::new(false); + async fn stored_credentials_are_provisioned_only_after_renewal_completes() { let original = stored_tokens(); let renewed = StoredTokens { access_token: "renewed-access".to_string(), refresh_token: "renewed-refresh".to_string(), ..original.clone() }; - let output = run_login_with_stored_credentials( - AuthFormat::Json, - Some(original), - |_stored| async move { Ok(Some(renewed)) }, - || async { - provision_calls.store(true, Ordering::SeqCst); - TelemetryProvisioningResult::Issued - }, - |_format| async { panic!("renewed stored credentials must not use device login") }, - ) - .await - .expect("renewed stored login should succeed"); + for returned in [original.clone(), renewed] { + let events = std::sync::Mutex::new(Vec::new()); + let events = &events; + let expected_stored = original.clone(); + let output = run_login_with_stored_credentials( + AuthFormat::Json, + Some(original.clone()), + |stored| async move { + assert_eq!(stored, expected_stored); + tokio::task::yield_now().await; + events.lock().unwrap().push("renewal-complete"); + Ok(Some(returned)) + }, + || async move { + events.lock().unwrap().push("provision"); + TelemetryProvisioningResult::Issued + }, + |_format| async { panic!("stored credentials must not use device login") }, + ) + .await + .expect("stored login should succeed"); - assert!(provision_calls.load(Ordering::SeqCst)); - assert!(output.contains("\"renewed\": true")); - assert!(output.contains("\"status\": \"issued\"")); + assert_eq!(*events.lock().unwrap(), ["renewal-complete", "provision"]); + assert!(output.contains("\"renewed\": true")); + assert!(output.contains("\"status\": \"issued\"")); + } } #[tokio::test] @@ -1085,64 +1067,59 @@ mod tests { } #[tokio::test] - async fn telemetry_credential_logout_reports_removal_when_only_telemetry_deleted() { - let output = run_logout_with(AuthFormat::Json, || Ok(()), async { Ok(false) }, async { - Ok(true) - }) - .await - .expect("logout should succeed"); - assert!(output.contains("\"credentials_removed\": true")); - } - - #[tokio::test] - async fn telemetry_credential_logout_attempts_both_deletions_on_telemetry_failure() { - let workos_called = AtomicBool::new(false); - let result = run_logout_with( - AuthFormat::Json, - || Ok(()), - async { - workos_called.store(true, Ordering::SeqCst); - Ok(true) - }, - async { Err(storage_failure()) }, - ) - .await; - assert!(result.is_err()); - assert!(workos_called.load(Ordering::SeqCst)); - } - - #[tokio::test] - async fn telemetry_credential_logout_deletes_telemetry_on_workos_failure() { - let telemetry_called = AtomicBool::new(false); - let result = run_logout_with( - AuthFormat::Json, - || Ok(()), - async { Err(storage_failure()) }, - async { - telemetry_called.store(true, Ordering::SeqCst); - Ok(true) - }, - ) - .await; - assert!(result.is_err()); - assert!(telemetry_called.load(Ordering::SeqCst)); - } + async fn logout_clears_readiness_and_attempts_both_deletions_before_reporting() { + use AuthStorageFailure::{CredentialAccess, DatabaseLocked}; + + fn deletion(result: Result) -> Result { + result.map_err(|failure| match failure { + CredentialAccess => TokenStorageError::CredentialAccessUnavailable, + _ => TokenStorageError::DatabaseLocked("busy".to_string()), + }) + } - #[tokio::test] - async fn logout_clears_telemetry_readiness_even_when_credential_deletion_fails() { - let cleared = AtomicBool::new(false); - let result = run_logout_with( - AuthFormat::Json, - || { - cleared.store(true, Ordering::SeqCst); - Ok(()) - }, - async { Err(storage_failure()) }, - async { Err(storage_failure()) }, - ) - .await; - assert!(result.is_err()); - assert!(cleared.load(Ordering::SeqCst)); + for (workos, telemetry, expected) in [ + (Ok(false), Ok(true), Ok(())), + (Ok(true), Err(DatabaseLocked), Err(DatabaseLocked)), + (Err(CredentialAccess), Ok(true), Err(CredentialAccess)), + ( + Err(CredentialAccess), + Err(DatabaseLocked), + Err(CredentialAccess), + ), + ] { + let events = std::sync::Mutex::new(Vec::new()); + let result = run_logout_with( + AuthFormat::Json, + || { + events.lock().unwrap().push("clear-readiness"); + Ok(()) + }, + async { + events.lock().unwrap().push("delete-workos"); + deletion(workos) + }, + async { + events.lock().unwrap().push("delete-telemetry"); + deletion(telemetry) + }, + ) + .await; + + assert_eq!( + *events.lock().unwrap(), + ["clear-readiness", "delete-workos", "delete-telemetry"], + "{workos:?} {telemetry:?}" + ); + match (result, expected) { + (Ok(output), Ok(())) => assert!(output.contains("\"credentials_removed\": true")), + (Err(CliError::User { error, .. }), Err(failure)) => { + assert_eq!(error, UserError::AuthStorageUnavailable(failure)); + } + (result, expected) => { + panic!("{workos:?} {telemetry:?}: {result:?} != {expected:?}") + } + } + } } #[test] diff --git a/cli/src/services/auth_db/multiprocess_tests.rs b/cli/src/services/auth_db/multiprocess_tests.rs index 00247e80c..e35be5e2a 100644 --- a/cli/src/services/auth_db/multiprocess_tests.rs +++ b/cli/src/services/auth_db/multiprocess_tests.rs @@ -1,7 +1,8 @@ use std::collections::BTreeSet; use std::io::{BufRead, BufReader, Read, Write}; use std::path::{Path, PathBuf}; -use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; +use std::process::{Child, ChildStdin, Command, Stdio}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; use std::sync::{Mutex, MutexGuard}; use std::time::{Duration, Instant}; @@ -12,25 +13,18 @@ use crate::generated_migrations; use crate::services::bash_policy::{ run_bash_policy_request, BashPolicyRequest, PolicyInputMode, PolicyOutputMode, }; -use crate::services::db::encryption_key::{ - get_or_create_encryption_key, AUTH_DB_ENCRYPTION_KEY_ENV, -}; -use crate::services::db::{DatabaseKind, DatabaseLocked, DbSpec, EncryptedTursoDb}; +use crate::services::db::encryption_key::AUTH_DB_ENCRYPTION_KEY_ENV; +use crate::services::db::{DatabaseKind, DbSpec, EncryptedTursoDb}; use crate::services::default_paths::auth_db_path; -use crate::services::error::{AuthStorageFailure, CliError}; +use crate::services::error::CliError; use crate::services::hooks::{run_hooks_subcommand, HookSubcommand}; use crate::services::managed_telemetry::claude_helper::repository_id_for; use crate::services::managed_telemetry::claude_helper::tests::git_repository_with_remote; use crate::services::managed_telemetry::trace_export::tests::{ capture_requests_until, loopback_hook_telemetry, trace_spans, }; -use crate::services::managed_telemetry::trace_export::{ - record_claude_hook_command, record_claude_hook_payload, ClaudeHookCommand, ManagedSpanKind, -}; -use crate::services::managed_telemetry::{ - inspect_health, load_credential_within_deadline, select_mode, ManagedTelemetryHealth, - ManagedTelemetryMode, RuntimeTelemetry, TelemetryFallbackReason, -}; +use crate::services::managed_telemetry::trace_export::ClaudeHookCommand; +use crate::services::managed_telemetry::{inspect_health, ManagedTelemetryHealth}; use crate::services::observability::traits::{NoopLogger, Telemetry}; use crate::services::telemetry_credential_storage::{ bind_repository_telemetry, load_repository_telemetry_credential, load_telemetry_credential, @@ -52,7 +46,6 @@ const TOKEN_PREFIX: &str = "SCE_AUTH_DB_CHILD_TOKEN="; const HEALTH_PREFIX: &str = "SCE_AUTH_DB_CHILD_TELEMETRY_HEALTH="; const CONCURRENT_PROCESSES: usize = 32; const CONCURRENT_MIGRATORS: usize = 4; -const READER_ROUNDS: usize = 3; const CHILD_KILL_AFTER: Duration = Duration::from_mins(1); const CHILD_POLL_INTERVAL: Duration = Duration::from_millis(10); const FAR_FUTURE: u64 = 4_102_444_800; @@ -68,16 +61,9 @@ const BINDING_MIGRATION_ID: &str = "004_bind_telemetry_credentials"; const WRITER_ENV: &str = "SCE_AUTH_DB_TEST_WRITER"; const COMMITTED_PREFIX: &str = "SCE_AUTH_DB_CHILD_COMMITTED="; const ROW_PREFIX: &str = "SCE_AUTH_DB_CHILD_ROW="; -const REJECTED: &str = "SCE_AUTH_DB_CHILD_REJECTED"; const WRITER_ROWS: usize = 8; const WRITER_PAUSE: Duration = Duration::from_millis(20); -const MIXED_WRITER_ROUNDS: usize = 3; -const NEW_WRITERS: usize = 4; -const LEGACY_HANDOFF_ROUNDS: usize = 20; -const LEGACY_HANDOFF_READERS: usize = 8; -const SHARED_ACCESS_REJECTION: &str = "does not support shared access"; -const LEGACY_LOCK_REJECTION: &str = "File is locked by another process"; -const LEGACY_CIPHER: &str = "aegis256"; +const CONCURRENT_WRITERS: usize = 4; const INSERT_BINDING_SQL: &str = "INSERT INTO telemetry_repository_bindings \ (repository_id, user_id, workspace_id, bound_at_unix_seconds) VALUES (?1, ?2, ?3, 1)"; const AUTH_DB_FILES: [&str; 3] = ["auth.db", "auth.db-wal", "auth.db-tshm"]; @@ -172,13 +158,6 @@ impl Sandbox { command } - fn writer(&self, role: &str, writer: &str) -> Child { - self.command(role) - .env(WRITER_ENV, writer) - .spawn() - .expect("spawn writer") - } - fn rows(&self) -> BTreeSet { prefixed_lines(&self.run("verify-rows"), ROW_PREFIX) } @@ -213,12 +192,20 @@ impl Sandbox { fn hold_command(role: &str, mut command: Command) -> HeldChild { let mut child = command.stdin(Stdio::piped()).spawn().expect("spawn holder"); let stdin = child.stdin.take().expect("holder stdin"); - let stdout = BufReader::new(child.stdout.take().expect("holder stdout")); + let stdout = child.stdout.take().expect("holder stdout"); + let (sender, lines) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines().map_while(Result::ok) { + if sender.send(line).is_err() { + break; + } + } + }); let mut held = HeldChild { role: role.to_string(), child, stdin, - stdout, + lines, }; held.wait_ready(); held @@ -233,7 +220,7 @@ struct HeldChild { role: String, child: Child, stdin: ChildStdin, - stdout: BufReader, + lines: Receiver, } impl HeldChild { @@ -242,42 +229,41 @@ impl HeldChild { } fn wait_ready(&mut self) -> String { + let deadline = Instant::now() + CHILD_KILL_AFTER; let mut captured = String::new(); loop { - let mut line = String::new(); - let read = self - .stdout - .read_line(&mut line) - .expect("holder stdout line"); - if read == 0 { - let _ = self.child.kill(); - self.child.wait().expect("holder status"); - let mut stderr = String::new(); - self.child - .stderr - .take() - .expect("holder stderr") - .read_to_string(&mut stderr) - .expect("holder stderr output"); - panic!( - "{} holder exited before ready:\n{captured}\n{stderr}", - self.role - ); - } - if line.trim_end().ends_with(READY) { - return captured; + match self + .lines + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + { + Ok(line) if line.trim_end().ends_with(READY) => return captured, + Ok(line) => { + captured.push_str(&line); + captured.push('\n'); + } + Err(RecvTimeoutError::Timeout) => self.abort( + &format!("was not ready within {CHILD_KILL_AFTER:?}"), + &captured, + ), + Err(RecvTimeoutError::Disconnected) => self.abort("exited before ready", &captured), } - captured.push_str(&line); } } + fn abort(&mut self, reason: &str, stdout: &str) -> ! { + let _ = self.child.kill(); + let _ = self.child.wait(); + let mut stderr = String::new(); + if let Some(mut pipe) = self.child.stderr.take() { + let _ = pipe.read_to_string(&mut stderr); + } + panic!("{} holder {reason}:\n{stdout}\n{stderr}", self.role); + } + fn release(mut self) { self.signal(); - drop(self.stdin); - let mut rest = String::new(); - self.stdout - .read_to_string(&mut rest) - .expect("holder stdout"); + wait_bounded(&self.role, &mut self.child); + let rest = self.lines.iter().collect::>().join("\n"); let output = self.child.wait_with_output().expect("holder output"); assert!( output.status.success() && rest.contains("1 passed"), @@ -288,15 +274,20 @@ impl HeldChild { } } -fn wait_success(role: &str, mut child: Child) -> String { +fn wait_bounded(role: &str, child: &mut Child) { let started = Instant::now(); while child.try_wait().expect("child status").is_none() { if started.elapsed() > CHILD_KILL_AFTER { let _ = child.kill(); + let _ = child.wait(); panic!("{role} child did not exit within {CHILD_KILL_AFTER:?}"); } std::thread::sleep(CHILD_POLL_INTERVAL); } +} + +fn wait_success(role: &str, mut child: Child) -> String { + wait_bounded(role, &mut child); let output = child.wait_with_output().expect("child output"); let stdout = String::from_utf8_lossy(&output.stdout).to_string(); assert!( @@ -314,24 +305,13 @@ fn prefixed_lines(stdout: &str, prefix: &str) -> BTreeSet { .collect() } -fn committed_rows(children: Vec<(String, Child)>) -> (BTreeSet, usize) { - let mut committed = BTreeSet::new(); - let mut rejected = 0; - for (role, child) in children { - let stdout = wait_success(&role, child); - committed.extend(prefixed_lines(&stdout, COMMITTED_PREFIX)); - rejected += usize::from(stdout.contains(REJECTED)); - } - (committed, rejected) -} - fn coordinated_writer_rows(sandbox: &Sandbox) -> BTreeSet { // Open every connection before writes start, and keep every connection // alive until all writes finish so WAL startup/teardown cannot race them. - let mut writers = (0..NEW_WRITERS) + let mut writers = (0..CONCURRENT_WRITERS) .map(|index| { let mut command = sandbox.command("coordinated-writer"); - command.env(WRITER_ENV, format!("new-{index}")); + command.env(WRITER_ENV, format!("writer-{index}")); Sandbox::hold_command("coordinated-writer", command) }) .collect::>(); @@ -348,6 +328,12 @@ fn coordinated_writer_rows(sandbox: &Sandbox) -> BTreeSet { committed } +fn expected_writer_rows() -> BTreeSet { + (0..CONCURRENT_WRITERS) + .flat_map(|index| (0..WRITER_ROWS).map(move |row| format!("writer-{index}-{row:02}"))) + .collect() +} + fn read_token(stdout: &str) -> Option<&str> { stdout .lines() @@ -386,15 +372,7 @@ fn thirty_two_reader_processes_all_resolve_the_credential() { let sandbox = Sandbox::new(); sandbox.seed(); - for round in 0..READER_ROUNDS { - let started = Instant::now(); - let succeeded = wait_readers(spawn_readers(&sandbox)); - println!( - "round {round}: {succeeded}/{CONCURRENT_PROCESSES} readers in {:?}", - started.elapsed() - ); - assert_eq!(succeeded, CONCURRENT_PROCESSES, "round {round}"); - } + assert_eq!(wait_readers(spawn_readers(&sandbox)), CONCURRENT_PROCESSES); } #[test] @@ -518,196 +496,29 @@ fn outdated_schema_is_migrated_and_partial_schema_fails_clearly() { } #[test] -fn concurrent_hook_processes_export_correlated_hook_spans() { - let _fan_out = serialize_fan_out(); - let sandbox = Sandbox::new(); - sandbox.seed(); - tokio::runtime::Builder::new_multi_thread() - .worker_threads(4) - .enable_all() - .build() - .expect("test runtime") - .block_on(export_concurrent_hook_spans(&sandbox)); -} - -#[test] -fn legacy_holder_rejects_new_readers_and_writers_explicitly() { - let _fan_out = serialize_fan_out(); - let sandbox = Sandbox::new(); - sandbox.seed(); - - let legacy = sandbox.hold("legacy-hold"); - sandbox.run("expect-rejected-read"); - sandbox.run("expect-rejected-write"); - legacy.release(); - - assert_eq!(sandbox.rows(), BTreeSet::from(["legacy-held".to_string()])); - sandbox.run("legacy-roundtrip"); -} - -#[test] -fn new_holder_rejects_legacy_writer_explicitly() { - let _fan_out = serialize_fan_out(); - let sandbox = Sandbox::new(); - sandbox.seed(); - - let holder = sandbox.hold("hold-open"); - sandbox.run("legacy-expect-rejected"); - holder.release(); - - sandbox.run("legacy-roundtrip"); - assert_eq!( - sandbox.rows(), - BTreeSet::from(["legacy-roundtrip".to_string()]) - ); -} - -#[test] -fn concurrent_legacy_and_new_writers_never_lose_committed_rows() { - let _fan_out = serialize_fan_out(); - let sandbox = Sandbox::new(); - sandbox.seed(); - - let mut committed = BTreeSet::new(); - let mut rejected = 0; - for round in 0..MIXED_WRITER_ROUNDS { - let writers = vec![ - ( - "legacy-writer".to_string(), - sandbox.writer("legacy-writer", &format!("legacy-{round}")), - ), - ( - "new-writer".to_string(), - sandbox.writer("new-writer", &format!("new-{round}-a")), - ), - ( - "new-writer".to_string(), - sandbox.writer("new-writer", &format!("new-{round}-b")), - ), - ]; - let (round_committed, round_rejected) = committed_rows(writers); - committed.extend(round_committed); - rejected += round_rejected; - } - println!( - "mixed writers: {} committed rows, {rejected} rejected opens", - committed.len() - ); - - let persisted = sandbox.rows(); - let lost = committed.difference(&persisted).collect::>(); - assert!(lost.is_empty(), "committed rows lost: {lost:?}"); - sandbox.run("legacy-roundtrip"); -} - -#[test] -fn concurrent_cold_opens_after_legacy_write_all_see_committed_state() { - let _fan_out = serialize_fan_out(); - let sandbox = Sandbox::new(); - sandbox.seed(); - - let mut expected = BTreeSet::new(); - for round in 0..LEGACY_HANDOFF_ROUNDS { - let stdout = wait_success( - "legacy-writer", - sandbox.writer("legacy-writer", &format!("legacy-handoff-{round}")), - ); - let committed = prefixed_lines(&stdout, COMMITTED_PREFIX); - assert_eq!(committed.len(), WRITER_ROWS, "round {round}"); - expected.extend(committed); - let readers = (0..LEGACY_HANDOFF_READERS) - .map(|_| sandbox.command("read").spawn().expect("spawn reader")) - .collect::>(); - assert_eq!( - wait_readers(readers), - LEGACY_HANDOFF_READERS, - "round {round}" - ); - } - assert_eq!(sandbox.rows(), expected); -} - -#[test] -fn concurrent_new_writers_all_commit_and_legacy_reopens_afterwards() { - let _fan_out = serialize_fan_out(); - let sandbox = Sandbox::new(); - sandbox.seed(); - let committed = coordinated_writer_rows(&sandbox); - assert_eq!(committed.len(), NEW_WRITERS * WRITER_ROWS); - assert_eq!(sandbox.rows(), committed); - - sandbox.run("legacy-roundtrip"); - let mut expected = committed; - expected.insert("legacy-roundtrip".to_string()); - assert_eq!(sandbox.rows(), expected); -} - -#[cfg(target_os = "macos")] -#[test] -fn macos_new_shared_lock_rejects_legacy_exclusive_lock_until_released() { +fn concurrent_current_version_writers_persist_every_row() { let _fan_out = serialize_fan_out(); let sandbox = Sandbox::new(); sandbox.seed(); - let holder = sandbox.hold("hold-open"); - sandbox.run("legacy-lock-expect-rejected"); - sandbox.run("legacy-expect-rejected"); - holder.release(); - - sandbox.run("legacy-lock-acquire"); - sandbox.run("legacy-roundtrip"); + assert_eq!(coordinated_writer_rows(&sandbox), expected_writer_rows()); + assert_eq!(sandbox.rows(), expected_writer_rows()); } #[cfg(target_os = "macos")] #[test] -fn macos_legacy_exclusive_lock_rejects_new_shared_lock_until_released() { +fn macos_current_version_processes_share_the_lock_concurrently() { let _fan_out = serialize_fan_out(); let sandbox = Sandbox::new(); sandbox.seed(); - let legacy = sandbox.hold("legacy-lock-hold"); - sandbox.run("expect-rejected-read"); - sandbox.run("expect-rejected-write"); - legacy.release(); - + let holders = [sandbox.hold("hold-open"), sandbox.hold("hold-open")]; + assert_eq!(coordinated_writer_rows(&sandbox), expected_writer_rows()); assert_eq!(read_token(&sandbox.run("read")), Some(TOKEN)); -} - -#[cfg(target_os = "macos")] -#[test] -fn macos_new_shared_lock_survives_unrelated_descriptor_close() { - let _fan_out = serialize_fan_out(); - let sandbox = Sandbox::new(); - sandbox.seed(); - - let holder = sandbox.hold("hold-open-close-other"); - sandbox.run("legacy-lock-expect-rejected"); - sandbox.run("legacy-expect-rejected"); - holder.release(); - - sandbox.run("legacy-roundtrip"); -} - -#[cfg(target_os = "macos")] -#[test] -fn macos_new_processes_share_the_lock_concurrently() { - let _fan_out = serialize_fan_out(); - let sandbox = Sandbox::new(); - sandbox.seed(); - - let holders = [ - sandbox.hold("hold-open"), - sandbox.hold("hold-open-close-other"), - ]; - let committed = coordinated_writer_rows(&sandbox); - assert_eq!(committed.len(), NEW_WRITERS * WRITER_ROWS); - sandbox.run("legacy-lock-expect-rejected"); for holder in holders { holder.release(); } - - assert_eq!(sandbox.rows(), committed); - sandbox.run("legacy-roundtrip"); + assert_eq!(sandbox.rows(), expected_writer_rows()); } #[test] @@ -928,74 +739,6 @@ async fn run_hook_entry_child() { telemetry.shutdown().await; } -async fn export_concurrent_hook_spans(sandbox: &Sandbox) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind"); - let address = listener.local_addr().expect("addr"); - let (stop, stopped) = tokio::sync::oneshot::channel(); - let receiver = tokio::spawn(capture_requests_until(listener, stopped)); - - let hooks = (0..CONCURRENT_PROCESSES) - .map(|index| { - sandbox - .command("hook") - .env(COLLECTOR_ENV, address.to_string()) - .env(HOOK_INDEX_ENV, index.to_string()) - .spawn() - .expect("spawn hook") - }) - .collect::>(); - let mut failing = sandbox.command("hook-credential-failure"); - failing.env(AUTH_DB_ENCRYPTION_KEY_ENV, WRONG_SECRET); - let failing = failing.spawn().expect("spawn failing hook"); - tokio::task::spawn_blocking(move || { - for child in hooks { - wait_success("hook", child); - } - wait_success("hook-credential-failure", failing); - }) - .await - .expect("hook children join"); - stop.send(()).expect("stop collector"); - - let spans = trace_spans(&receiver.await.expect("collector join")); - let observed = spans - .iter() - .filter(|span| span.name == "sce.hook") - .map(|span| { - let attribute = |key: &str| { - span.attributes - .iter() - .find(|(name, _)| name == key) - .map(|(_, value)| value.clone()) - .unwrap_or_default() - }; - ( - attribute("sce.claude.tool_use_id"), - attribute("sce.hook.command"), - ) - }) - .collect::>(); - let expected = (0..CONCURRENT_PROCESSES) - .map(|index| { - ( - hook_tool_use_id(index), - hook_command(index).as_str().to_string(), - ) - }) - .collect::>(); - assert_eq!(observed, expected); -} - -fn hook_command(index: usize) -> ClaudeHookCommand { - HOOK_COMMANDS[index % HOOK_COMMANDS.len()] -} - -fn hook_tool_use_id(index: usize) -> String { - format!("toolu_mp_{index:02}") -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn auth_db_child_process() { let Ok(role) = std::env::var(ROLE_ENV) else { @@ -1073,123 +816,15 @@ async fn auth_db_child_process() { } "upgrade" => upgrade_outdated_schema().await, "partial" => reject_partial_schema().await, + "verify-rows" => print_verified_rows().await, "coordinated-writer" => run_coordinated_writer_child().await, - "hook" => run_hook_child().await, "hook-entry" => run_hook_entry_child().await, - other => run_compatibility_child(other).await, - } -} - -async fn run_compatibility_child(role: &str) { - match role { - "legacy-hold" => { - let conn = open_legacy_auth_db().await.expect("legacy holder opens"); - wait_for_release(); - insert_binding(&conn, "legacy-held") - .await - .expect("legacy holder writes after release"); - } - "legacy-expect-rejected" => { - let Err(error) = open_legacy_auth_db().await else { - panic!("legacy open must be rejected while a new process holds the DB"); - }; - assert!( - format!("{error:#}").contains(LEGACY_LOCK_REJECTION), - "{error:#}" - ); - } - "legacy-roundtrip" => legacy_roundtrip().await, - #[cfg(target_os = "macos")] - "hold-open-close-other" => { - let _db = AuthDb::new().await.expect("holder opens auth DB"); - let path = auth_db_path().expect("auth DB path"); - drop(std::fs::File::open(&path).expect("unrelated auth DB descriptor")); - wait_for_release(); - } - #[cfg(target_os = "macos")] - "legacy-lock-hold" => { - let file = auth_db_file(); - legacy_exclusive_lock(&file).expect("legacy lock acquires"); - wait_for_release(); - } - #[cfg(target_os = "macos")] - "legacy-lock-expect-rejected" => { - let error = legacy_exclusive_lock(&auth_db_file()) - .expect_err("legacy lock must be rejected while a new process holds the DB"); - assert_eq!(error.raw_os_error(), Some(libc::EAGAIN), "{error}"); - } - #[cfg(target_os = "macos")] - "legacy-lock-acquire" => { - legacy_exclusive_lock(&auth_db_file()).expect("legacy lock acquires"); - } - "legacy-writer" => match open_legacy_auth_db().await { - Ok(conn) => { - for row in writer_rows() { - if insert_binding(&conn, &row).await.is_err() { - break; - } - println!("{COMMITTED_PREFIX}{row}"); - tokio::time::sleep(WRITER_PAUSE).await; - } - } - Err(error) => { - assert!( - format!("{error:#}").contains(LEGACY_LOCK_REJECTION), - "{error:#}" - ); - println!("{REJECTED}"); - } - }, - other => run_new_version_child(other).await, + other => run_binding_child(other).await, } } -async fn run_new_version_child(role: &str) { +async fn run_binding_child(role: &str) { match role { - "new-writer" => match AuthDb::new().await { - Ok(db) => { - for row in writer_rows() { - if db - .execute(INSERT_BINDING_SQL, (row.as_str(), USER_ID, WORKSPACE_ID)) - .await - .is_err() - { - break; - } - println!("{COMMITTED_PREFIX}{row}"); - tokio::time::sleep(WRITER_PAUSE).await; - } - } - Err(error) => { - assert!( - format!("{error:#}").contains(SHARED_ACCESS_REJECTION), - "{error:#}" - ); - println!("{REJECTED}"); - } - }, - "expect-rejected-read" => { - let error = load_telemetry_credential() - .await - .expect_err("lookup must fail while a legacy process holds the DB"); - assert!( - error.to_string().contains(SHARED_ACCESS_REJECTION), - "{error}" - ); - assert_eq!(error.failure(), AuthStorageFailure::DatabaseLocked); - } - "expect-rejected-write" => { - let error = AuthDb::new() - .await - .err() - .expect("writer open must fail while a legacy process holds the DB"); - assert!( - format!("{error:#}").contains(SHARED_ACCESS_REJECTION), - "{error:#}" - ); - assert!(error.is::(), "{error:#}"); - } - "verify-rows" => print_verified_rows().await, "repository-lookup-missing" => { assert_eq!( load_repository_telemetry_credential("repo-missing".to_string()) @@ -1225,20 +860,6 @@ async fn run_new_version_child(role: &str) { assert_eq!(code, std::process::ExitCode::from(4)); } } - "hook-credential-failure" => { - let (telemetry, fallback) = RuntimeTelemetry::resolve_with( - load_telemetry_credential(), - ManagedSpanKind::ClaudeHook, - ) - .await; - assert!(matches!(telemetry, RuntimeTelemetry::Noop(_))); - let fallback = fallback.expect("credential failure is diagnosed"); - assert_eq!(fallback.reason, TelemetryFallbackReason::DatabaseError); - let output = telemetry - .with_default_subscriber(&mut || async { Ok("hook-output".to_string()) }) - .await; - assert_eq!(output.expect("hook result unaffected"), "hook-output"); - } other => panic!("unknown child role {other}"), } } @@ -1283,40 +904,6 @@ async fn seed() { crate::services::telemetry_readiness::mark_ready(FAR_FUTURE).expect("readiness seeds"); } -async fn run_hook_child() { - let index = std::env::var(HOOK_INDEX_ENV) - .expect("hook index") - .parse::() - .expect("numeric hook index"); - let address = std::env::var(COLLECTOR_ENV) - .expect("collector address") - .parse() - .expect("socket address"); - let credential = load_credential_within_deadline(load_telemetry_credential()) - .await - .expect("managed credential resolves"); - let ManagedTelemetryMode::Managed { .. } = select_mode(Some(credential.clone()), 0) else { - panic!("credential selects managed telemetry"); - }; - let telemetry = loopback_hook_telemetry(address, &credential.token); - let payload = serde_json::json!({ - "hook_event_name": "PostToolUse", - "tool_name": "Write", - "session_id": "session-mp", - "tool_use_id": hook_tool_use_id(index), - }) - .to_string(); - let output = telemetry - .with_default_subscriber(&mut || async { - record_claude_hook_command(hook_command(index)); - record_claude_hook_payload(&payload); - Ok(String::new()) - }) - .await; - assert_eq!(output.expect("hook result"), ""); - telemetry.shutdown().await; -} - fn writer_rows() -> Vec { let writer = std::env::var(WRITER_ENV).expect("writer id"); (0..WRITER_ROWS) @@ -1324,72 +911,6 @@ fn writer_rows() -> Vec { .collect() } -async fn open_legacy_auth_db() -> Result { - let path = auth_db_path()?; - let hexkey = get_or_create_encryption_key( - &path, - AuthDbSpec::db_name(), - crate::services::db::encryption_key::CredentialAccess::Interactive, - )?; - let db = turso::Builder::new_local(path.to_str().expect("UTF-8 auth DB path")) - .experimental_encryption(true) - .with_encryption(turso::EncryptionOpts { - hexkey, - cipher: LEGACY_CIPHER.to_string(), - }) - .build() - .await?; - Ok(db.connect()?) -} - -#[cfg(target_os = "macos")] -fn auth_db_file() -> std::fs::File { - std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(auth_db_path().expect("auth DB path")) - .expect("auth DB file") -} - -#[cfg(target_os = "macos")] -fn legacy_exclusive_lock(file: &std::fs::File) -> std::io::Result<()> { - use std::os::fd::AsRawFd; - - let mut lock: libc::flock = unsafe { std::mem::zeroed() }; - lock.l_type = libc::c_short::try_from(libc::F_WRLCK).expect("lock type"); - lock.l_whence = libc::c_short::try_from(libc::SEEK_SET).expect("lock whence"); - if unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETLK, &raw const lock) } == -1 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) -} - -async fn insert_binding(conn: &turso::Connection, row: &str) -> Result<()> { - conn.execute(INSERT_BINDING_SQL, (row, USER_ID, WORKSPACE_ID)) - .await?; - Ok(()) -} - -async fn legacy_roundtrip() { - let conn = open_legacy_auth_db() - .await - .expect("legacy binary reopens the auth DB"); - let mut rows = conn - .query("SELECT token FROM telemetry_credentials WHERE id = 1", ()) - .await - .expect("legacy reads telemetry credential"); - let row = rows - .next() - .await - .expect("legacy credential row fetch") - .expect("legacy sees credential row"); - assert_eq!(row.get::(0).expect("token"), TOKEN); - drop(rows); - insert_binding(&conn, "legacy-roundtrip") - .await - .expect("legacy writes after new processes exit"); -} - async fn print_verified_rows() { let db = AuthDb::new().await.expect("verifier opens auth DB"); assert_credential_rows_intact(&db).await; diff --git a/cli/src/services/doctor/managed_telemetry.rs b/cli/src/services/doctor/managed_telemetry.rs index cbe4958b7..825e67ee9 100644 --- a/cli/src/services/doctor/managed_telemetry.rs +++ b/cli/src/services/doctor/managed_telemetry.rs @@ -187,17 +187,6 @@ pub(super) fn json(health: ManagedTelemetryHealth) -> Value { mod tests { use super::*; - const ALL_REASONS: [TelemetryFallbackReason; 8] = [ - TelemetryFallbackReason::NotConfigured, - TelemetryFallbackReason::CredentialInaccessible, - TelemetryFallbackReason::CredentialUnavailable, - TelemetryFallbackReason::CredentialExpired, - TelemetryFallbackReason::LookupTimeout, - TelemetryFallbackReason::DatabaseError, - TelemetryFallbackReason::PathResolutionError, - TelemetryFallbackReason::ExporterInitFailed, - ]; - fn disabled(reason: TelemetryFallbackReason) -> ManagedTelemetryHealth { ManagedTelemetryHealth::Disabled(reason) } @@ -251,28 +240,30 @@ mod tests { #[test] fn always_allow_is_only_recommended_for_macos_keychain_authorization() { - for platform in [ - CredentialStorePlatform::Macos, - CredentialStorePlatform::Linux, - CredentialStorePlatform::Windows, - CredentialStorePlatform::Other, + use CredentialStorePlatform::{Linux, Macos, Other, Windows}; + use TelemetryFallbackReason::{CredentialInaccessible, LookupTimeout}; + + for (platform, reason, expects_always_allow) in [ + (Macos, CredentialInaccessible, true), + (Macos, LookupTimeout, false), + (Linux, CredentialInaccessible, false), + (Windows, CredentialInaccessible, false), + (Other, CredentialInaccessible, false), + (Linux, LookupTimeout, false), ] { - for reason in ALL_REASONS { - let Some(diagnostic) = diagnostic_for_platform(disabled(reason), platform) else { - continue; - }; - let text = format!("{} {}", diagnostic.summary, diagnostic.remediation); - let expects_always_allow = platform == CredentialStorePlatform::Macos - && reason == TelemetryFallbackReason::CredentialInaccessible; - assert_eq!( - text.contains("Always Allow"), - expects_always_allow, - "{platform:?}/{reason:?}: {text}" - ); - if platform != CredentialStorePlatform::Macos { - assert!(!text.contains("macOS"), "{platform:?}/{reason:?}"); - } - } + let diagnostic = diagnostic_for_platform(disabled(reason), platform) + .expect("disabled telemetry is diagnosed"); + let text = format!("{} {}", diagnostic.summary, diagnostic.remediation); + assert_eq!( + text.contains("Always Allow"), + expects_always_allow, + "{platform:?}/{reason:?}: {text}" + ); + assert_eq!( + text.contains("macOS"), + platform == Macos, + "{platform:?}/{reason:?}: {text}" + ); } } diff --git a/cli/src/services/managed_telemetry/trace_export.rs b/cli/src/services/managed_telemetry/trace_export.rs index aabab4cf7..00e104219 100644 --- a/cli/src/services/managed_telemetry/trace_export.rs +++ b/cli/src/services/managed_telemetry/trace_export.rs @@ -448,12 +448,20 @@ pub(crate) mod tests { pub(crate) async fn capture_requests_until( listener: TcpListener, + stop: tokio::sync::oneshot::Receiver<()>, + ) -> Vec { + capture_requests_with_status_until(listener, "HTTP/1.1 200 OK", stop).await + } + + async fn capture_requests_with_status_until( + listener: TcpListener, + status_line: &'static str, mut stop: tokio::sync::oneshot::Receiver<()>, ) -> Vec { let mut captured = Vec::new(); loop { tokio::select! { - request = capture_one_request(&listener, "HTTP/1.1 200 OK") => captured.push(request), + request = capture_one_request(&listener, status_line) => captured.push(request), _ = &mut stop => return captured, } } @@ -802,18 +810,21 @@ pub(crate) mod tests { ] { assert!(!log_body.contains(leaked), "leaked {leaked}"); } + let spans = decode_spans(&trace.body); + assert_eq!(spans.len(), 1, "{spans:?}"); + assert_eq!(spans[0].name, "sce.command"); + assert_eq!( + spans[0].sce_attributes(), + [("sce.command.name", "config"), ("sce.outcome", "success")] + .map(|(key, value)| (key.to_string(), value.to_string())) + ); + assert!(spans[0] + .resource_attributes + .contains(&("service.name".to_string(), "sce".to_string()))); let body = String::from_utf8_lossy(&trace.body).to_string(); - for expected in [ - "sce.command", - "sce.command.name", - "config", - "service.name", - "success", - ] { - assert!(body.contains(expected), "missing {expected}"); + for leaked in ["local-accessor-secret", "bao-token-value"] { + assert!(!body.contains(leaked), "leaked {leaked}"); } - assert!(!body.contains("local-accessor-secret")); - assert!(!body.contains("bao-token-value")); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -998,29 +1009,45 @@ pub(crate) mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn managed_trace_export_does_not_follow_redirects_or_alter_results() { + let destination = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind destination"); + let destination_address = destination.local_addr().expect("destination addr"); let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); let address = listener.local_addr().expect("addr"); - let receiver = tokio::spawn(capture_requests( - listener, - "HTTP/1.1 307 Temporary Redirect\r\nlocation: http://127.0.0.1:1/steal", - 1, + let redirect: &'static str = Box::leak( + format!( + "HTTP/1.1 307 Temporary Redirect\r\nlocation: http://{destination_address}/v1/traces" + ) + .into_boxed_str(), + ); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let receiver = tokio::spawn(capture_requests_with_status_until( + listener, redirect, stopped, )); let telemetry = loopback_telemetry(address, "bao-token-value"); + let expected = CliError::runtime(anyhow::anyhow!("boom")); - let result = telemetry + let error = telemetry .with_default_subscriber(&mut || async { Err(CliError::runtime(anyhow::anyhow!("boom"))) }) - .await; - assert!(result.is_err()); + .await + .expect_err("command error is preserved"); + assert_eq!(error.code(), expected.code()); + assert_eq!(error.to_string(), expected.to_string()); telemetry.shutdown().await; + stop.send(()).expect("stop receiver"); let requests = receiver.await.expect("receiver join"); - assert!(!requests.is_empty()); + assert!(!trace_spans(&requests).is_empty()); for request in &requests { let head = request.head.to_ascii_lowercase(); - assert!(head.starts_with("post /v1/traces ") || head.starts_with("post /v1/logs ")); + assert!(head.contains("authorization: bearer bao-token-value")); } + let redirected = + tokio::time::timeout(Duration::from_millis(300), destination.accept()).await; + assert!(redirected.is_err(), "redirect destination was contacted"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1039,53 +1066,77 @@ pub(crate) mod tests { assert!(started.elapsed() < Duration::from_secs(6)); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn ambient_otlp_environment_cannot_redirect_or_inject_headers() { - let alternate = TcpListener::bind("127.0.0.1:0").await.expect("bind alt"); - let alternate_address = alternate.local_addr().expect("alt addr"); - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let address = listener.local_addr().expect("addr"); - std::env::set_var( - "OTEL_EXPORTER_OTLP_ENDPOINT", - format!("http://{alternate_address}"), - ); - std::env::set_var( - "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", - format!("http://{alternate_address}/v1/traces"), + #[test] + fn ambient_otlp_environment_cannot_redirect_or_inject_headers() { + crate::app::tests::run_fault_child( + "services::managed_telemetry::trace_export::tests::isolated_ambient_otlp_environment", ); - std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", "x-ambient-leak=1"); - std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc"); - let receiver = tokio::spawn(capture_requests(listener, "HTTP/1.1 200 OK", 2)); - let telemetry = loopback_telemetry(address, "bao-token-value"); + } - let result = telemetry - .with_default_subscriber(&mut || async { - record_command_name("config"); - tracing::info!(target: "sce", event_id = "sce.test.event", event_message = "Starting command"); - Ok("payload".to_string()) - }) - .await; - assert_eq!(result.expect("command result"), "payload"); - telemetry.shutdown().await; - for name in [ - "OTEL_EXPORTER_OTLP_ENDPOINT", - "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", - "OTEL_EXPORTER_OTLP_HEADERS", - "OTEL_EXPORTER_OTLP_PROTOCOL", + #[test] + fn isolated_ambient_otlp_environment() { + if !crate::app::tests::is_fault_child() { + return; + } + let bind = || { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + listener.set_nonblocking(true).expect("nonblocking"); + listener + }; + let (alternate, collector) = (bind(), bind()); + let alternate_address = alternate.local_addr().expect("alt addr"); + let address = collector.local_addr().expect("addr"); + for (name, value) in [ + ( + "OTEL_EXPORTER_OTLP_ENDPOINT", + format!("http://{alternate_address}"), + ), + ( + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + format!("http://{alternate_address}/v1/traces"), + ), + ( + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + format!("http://{alternate_address}/v1/logs"), + ), + ( + "OTEL_EXPORTER_OTLP_HEADERS", + "x-ambient-leak=1,authorization=Bearer ambient-leak".to_string(), + ), + ("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc".to_string()), ] { - std::env::remove_var(name); + std::env::set_var(name, value); } + let runtime = crate::app::build_runtime().expect("runtime"); + runtime.block_on(async { + let alternate = TcpListener::from_std(alternate).expect("alt listener"); + let collector = TcpListener::from_std(collector).expect("listener"); + let receiver = tokio::spawn(capture_requests(collector, "HTTP/1.1 200 OK", 2)); + let telemetry = loopback_telemetry(address, "bao-token-value"); - let requests = receiver.await.expect("receiver join"); - assert_eq!(requests.len(), 2); - for request in &requests { - let head = request.head.to_ascii_lowercase(); - assert!(head.contains("authorization: bearer bao-token-value")); - assert!(!head.contains("x-ambient-leak")); - assert!(head.contains("application/x-protobuf")); - } - let unexpected = tokio::time::timeout(Duration::from_millis(300), alternate.accept()).await; - assert!(unexpected.is_err(), "alternate receiver was contacted"); + let result = telemetry + .with_default_subscriber(&mut || async { + record_command_name("config"); + tracing::info!(target: "sce", event_id = "sce.test.event", event_message = "Starting command"); + Ok("payload".to_string()) + }) + .await; + assert_eq!(result.expect("command result"), "payload"); + telemetry.shutdown().await; + + let requests = receiver.await.expect("receiver join"); + assert_eq!(requests.len(), 2); + for request in &requests { + let head = request.head.to_ascii_lowercase(); + assert!(head.contains("authorization: bearer bao-token-value")); + assert!(!head.contains("ambient-leak")); + assert!(head.contains("application/x-protobuf")); + } + let unexpected = + tokio::time::timeout(Duration::from_millis(300), alternate.accept()).await; + assert!(unexpected.is_err(), "alternate receiver was contacted"); + }); + crate::app::finish_runtime(runtime); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1116,7 +1167,8 @@ pub(crate) mod tests { async fn concurrent_commands_keep_results_and_export_isolated() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); let address = listener.local_addr().expect("addr"); - let receiver = tokio::spawn(capture_requests(listener, "HTTP/1.1 200 OK", 1)); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let receiver = tokio::spawn(capture_requests_until(listener, stopped)); let telemetry = std::sync::Arc::new(loopback_telemetry(address, "bao-token-value")); let first = { @@ -1150,21 +1202,30 @@ pub(crate) mod tests { .shutdown() .await; - let requests = receiver.await.expect("receiver join"); - let trace = requests + stop.send(()).expect("stop receiver"); + + let mut commands = trace_spans(&receiver.await.expect("receiver join")) .iter() - .find(|request| { - request - .head - .to_ascii_lowercase() - .starts_with("post /v1/traces ") - }) - .expect("trace request"); - let body = String::from_utf8_lossy(&trace.body).to_string(); - assert!(body.contains("first")); - assert!(body.contains("second")); - assert!(body.contains("success")); - assert!(body.contains("failure")); + .filter(|span| span.name == "sce.command") + .map(DecodedSpan::sce_attributes) + .collect::>(); + commands.sort(); + let failure_class = CliError::runtime(anyhow::anyhow!("boom")).code(); + let expected = [ + vec![("sce.command.name", "first"), ("sce.outcome", "success")], + vec![ + ("sce.command.name", "second"), + ("sce.error.class", failure_class), + ("sce.outcome", "failure"), + ], + ] + .map(|attributes| { + attributes + .into_iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect::>() + }); + assert_eq!(commands, expected); } #[test] diff --git a/cli/src/services/telemetry_readiness.rs b/cli/src/services/telemetry_readiness.rs index 606c7ccf2..00e9b2444 100644 --- a/cli/src/services/telemetry_readiness.rs +++ b/cli/src/services/telemetry_readiness.rs @@ -124,9 +124,47 @@ mod tests { } #[test] - fn absent_readiness_is_not_ready() { - let paths = paths_with_auth_db(); - assert!(!is_ready_at(&paths.readiness, &paths.auth_db, NOW)); + fn readiness_state_requires_valid_unexpired_marker_and_auth_db() { + for (case, marker, auth_db, expected) in [ + ("missing marker", None, true, ReadinessState::NotConfigured), + ( + "malformed marker", + Some(b"not json".to_vec()), + true, + ReadinessState::NotConfigured, + ), + ( + "valid marker", + Some(format!(r#"{{"expires_at_unix_seconds":{}}}"#, NOW + 1).into_bytes()), + true, + ReadinessState::Ready, + ), + ( + "expired marker", + Some(format!(r#"{{"expires_at_unix_seconds":{NOW}}}"#).into_bytes()), + true, + ReadinessState::Expired, + ), + ( + "missing auth DB", + Some(format!(r#"{{"expires_at_unix_seconds":{}}}"#, NOW + 1).into_bytes()), + false, + ReadinessState::NotConfigured, + ), + ] { + let paths = paths_with_auth_db(); + if let Some(marker) = marker { + fs::write(&paths.readiness, marker).unwrap(); + } + if !auth_db { + fs::remove_file(&paths.auth_db).unwrap(); + } + assert_eq!( + state_at(&paths.readiness, &paths.auth_db, NOW), + expected, + "{case}" + ); + } } #[test] @@ -142,38 +180,6 @@ mod tests { ); } - #[test] - fn readiness_state_distinguishes_unconfigured_expired_and_ready() { - let paths = paths_with_auth_db(); - assert_eq!( - state_at(&paths.readiness, &paths.auth_db, NOW), - ReadinessState::NotConfigured - ); - mark_ready_at(&paths.readiness, NOW + 1).unwrap(); - assert_eq!( - state_at(&paths.readiness, &paths.auth_db, NOW), - ReadinessState::Ready - ); - assert_eq!( - state_at(&paths.readiness, &paths.auth_db, NOW + 1), - ReadinessState::Expired - ); - fs::remove_file(&paths.auth_db).unwrap(); - assert_eq!( - state_at(&paths.readiness, &paths.auth_db, NOW + 1), - ReadinessState::NotConfigured - ); - } - - #[test] - fn readiness_without_auth_db_is_stale() { - let paths = paths_with_auth_db(); - mark_ready_at(&paths.readiness, NOW + 1).unwrap(); - fs::remove_file(&paths.auth_db).unwrap(); - - assert!(!is_ready_at(&paths.readiness, &paths.auth_db, NOW)); - } - #[test] fn clearing_readiness_disables_it_and_is_idempotent() { let paths = paths_with_auth_db(); @@ -184,11 +190,4 @@ mod tests { assert!(!is_ready_at(&paths.readiness, &paths.auth_db, NOW)); } - - #[test] - fn corrupt_readiness_is_not_ready() { - let paths = paths_with_auth_db(); - fs::write(&paths.readiness, b"not json").unwrap(); - assert!(!is_ready_at(&paths.readiness, &paths.auth_db, NOW)); - } }