diff --git a/docs/llm-task-artifacts.md b/docs/llm-task-artifacts.md index d0361c2..be0e4b0 100644 --- a/docs/llm-task-artifacts.md +++ b/docs/llm-task-artifacts.md @@ -433,3 +433,73 @@ no-run lifecycle mode. Classifier failures are non-blocking: the gate warns and continues with normal review. Successful and failed classifier task metadata still lives under the prospective run artifact root so provider-session resume and local artifact inspection use the same lifecycle shape. + +## Request-Cost Observations (Collection Only) + +Optional `request_cost_evidence` and `request_cost_checkpoint` fields supplement +schema 4 without changing output reuse or reviewer-tool coverage semantics. Only +the concrete direct OpenAI Responses adapter produces v1 request observations. +Unknown or malformed optional cost data is isolated in both metadata readers; +valid output and reviewer-tool evidence remain available. Schemas 1/2/3 retain +the existing rejection behavior. This change does not enable cost estimation. + +Each observed adapter attempt, including transient retries and structured-output +corrections, gets a distinct ordinal. Usage and observed response identity are +captured independently of accepted output, including no-text, incomplete, +non-2xx and cancellation paths. Missing/null values remain unknown. Requested +and observed model/service tier are separate. Invalid counters are diagnostic +only. No prompt, raw output, refusal/error body, URL, header or credential is +copied into these observations. + +The adapter-owned OpenAI client uses a fresh pinned standard HTTP transport, +disables redirects and clears the replay body. Caller-supplied clients remain +opaque. Only the actual canonical global Responses destination qualifies as +`official_global`; gateway/custom routes remain unverified. Entering `Client.Do` +means `possibly_dispatched`, including network errors; it does not prove a free +request. A pre-dispatch failure is recorded separately. + +The lifecycle holds the same fail-fast advisory task lock across cache checking, +checkpoint updates, invocation and metadata, including no-run mode. Reset uses +that lock too. The bounded sidecar lives at +`llm-tasks/.request-cost/.json`, outside the task directory removed +by fingerprint resets and inside the same root covered by retention/purge. +Before dispatch it atomically records a pending generation. After return it +writes the final checkpoint before ledger insertion or task output/metadata. +If the pending write fails, no adapter call starts. A failed final write leaves +the on-disk pending marker as unknown. Cache loads never create generations or +ledger rows. Reset reconciles metadata observations with the sidecar before +removing metadata, retaining a known prefix even if the sidecar was lost. + +Histories retain at most 64 attempts and 64 finalized generation descriptors, +16 fixed gap reasons and 256 KiB encoded checkpoints. A 4 KiB reserve keeps room +for pending/unknown state even at capacity. Prefixes, saturating omission counts +and gaps survive later success. The whole legacy metadata read is still +unbounded; only its isolated cost envelope and the sidecar reads are bounded. + +Task-history completeness remains false: this slice does not establish a fresh +artifact-root accounting origin. An absent directory, empty ledger, new session +ID or successful latest request cannot establish it. Interrupted, legacy, +opaque, conflicting, missing and overflowing observations remain unknown. +Primary and repair histories remain separate detached values. Direct OpenAI +costs are not passed to the old aggregate estimator; Codex cannot borrow direct +API pricing. Existing compatibility token counters and reported-cost semantics +are not redefined as sums of the new request history. + +Evidence and checkpoints have fixed-order, map-free compact JSON SHA-256 +projections with explicit nulls, normalized empty arrays, sorted gap enums and +separate versioned digest domains. They bind scope, ordered attempts, generation +fingerprints, pending state and omission counts. Prices are excluded. + +Two writes per generation do not preserve every pre-crash observation: a crash +during an invocation may lose that generation's already observed prefix. Its +pending marker makes the missing history explicit. The repository's atomic +writer uses rename without file/directory fsync; no power-loss durability or +provider-invoice completeness is claimed. Per-attempt durable checkpoints, +fresh-origin initialization, request-local pricing and pricing receipts require +separate implementation and verification before estimate activation. + +Cost-bearing metadata uses compact JSON so its durable nested-envelope size +matches the collector/reader admission budget; unrelated legacy metadata keeps +its existing formatting. Recovered metadata-only generations contribute a known +lower-bound count, with missing finalized descriptors recorded as omissions and +gaps. Their fingerprints are not reconstructed by guessing. diff --git a/internal/llm/adapter.go b/internal/llm/adapter.go index c0ec63b..801e5f7 100644 --- a/internal/llm/adapter.go +++ b/internal/llm/adapter.go @@ -135,6 +135,7 @@ type Response struct { Usage Usage DurationMS int64 ReviewerToolEvidence *ReviewerToolEvidence + RequestCostEvidence *RequestCostEvidence } // DiffToolStatus records the terminal state of the required reviewer diff tool. @@ -224,9 +225,25 @@ func (e *StructuredValidationError) Is(target error) bool { // conversation no longer exists is retried once as a fresh conversation // instead of failing; callers that store the session for later reuse must // persist the returned SessionID, which replaces a dangling stored one. -func RunStructuredWithSessionResume[T any](ctx context.Context, adapter Adapter, resumeSessionID string, req Request, decode Decoder[T]) (StructuredResult[T], error) { +func RunStructuredWithSessionResume[T any](ctx context.Context, adapter Adapter, resumeSessionID string, req Request, decode Decoder[T]) (result StructuredResult[T], resultErr error) { + collector := newRequestCostCollector(adapter) + defer func() { + result.Response = cloneResponse(result.Response) + result.Response.RequestCostEvidence = collector.snapshot() + // Validation error payload and returned snapshots must not share mutable + // response evidence with each other or with the adapter. + var copied []StructuredValidationAttempt + if result.ValidationAttempts != nil { + copied = make([]StructuredValidationAttempt, len(result.ValidationAttempts)) + } + for i, attempt := range result.ValidationAttempts { + attempt.Response = cloneResponse(attempt.Response) + copied[i] = attempt + } + result.ValidationAttempts = copied + }() var zero T - sessionID, response, err := runOnceWithSession(ctx, adapter, resumeSessionID, req) + sessionID, response, err := runOnceWithSession(ctx, adapter, resumeSessionID, req, collector, "initial") if err != nil { return StructuredResult[T]{Response: response, SessionID: sessionID}, err } @@ -255,7 +272,7 @@ func RunStructuredWithSessionResume[T any](ctx context.Context, adapter Adapter, if strings.TrimSpace(retryResumeSessionID) == "" && !retryReq.FreshValidationRetrySession { retryResumeSessionID = resumeSessionID } - retrySessionID, retryResponse, err := runOnceWithSession(ctx, adapter, retryResumeSessionID, retryReq) + retrySessionID, retryResponse, err := runOnceWithSession(ctx, adapter, retryResumeSessionID, retryReq, collector, "correction") if err != nil { return StructuredResult[T]{Response: retryResponse, SessionID: retrySessionID, ValidationAttempts: attempts}, err } @@ -306,10 +323,15 @@ func decodeStructuredAccepted[T any](decode Decoder[T], data []byte) (T, []byte, // prior context, neither of which the review depends on. Without the fallback a // dangling session ID fails the task outright, and a whole round of reviewers // can fail before doing any work while still reporting zero findings. -func runOnceWithSession(ctx context.Context, adapter Adapter, resumeSessionID string, req Request) (string, Response, error) { +func runOnceWithSession(ctx context.Context, adapter Adapter, resumeSessionID string, req Request, collector *requestCostCollector, phase string) (string, Response, error) { + var adapterAttempt uint32 sessionDropped := false for attempt := 0; ; attempt++ { + if adapterAttempt < ^uint32(0) { + adapterAttempt++ + } sid, resp, err := runOnceAttempt(ctx, adapter, resumeSessionID, req) + collector.observe(resp, phase, adapterAttempt) if err == nil { return sid, resp, nil } @@ -339,15 +361,19 @@ func runOnceAttempt(ctx context.Context, adapter Adapter, resumeSessionID string ) if req.ReviewerWorkspace != nil { if err := RequireReviewerWorkspace(adapter); err != nil { - return "", Response{}, err + return "", nonDispatchedCostResponse(adapter, req), err } } - if strings.TrimSpace(resumeSessionID) != "" && adapter.SupportsResume() { + resuming := strings.TrimSpace(resumeSessionID) != "" && adapter.SupportsResume() + if resuming { stream, err = adapter.Resume(ctx, resumeSessionID, req) } else { stream, err = adapter.Start(ctx, req) } if err != nil { + if stream == nil && !resuming { + return "", nonDispatchedCostResponse(adapter, req), err + } return "", Response{}, err } response, err := stream.Wait(ctx) @@ -356,7 +382,15 @@ func runOnceAttempt(ctx context.Context, adapter Adapter, resumeSessionID string func cloneResponse(response Response) Response { response.StructuredOutput = append([]byte(nil), response.StructuredOutput...) + response.Usage.TokensIn = costCopy(response.Usage.TokensIn) + response.Usage.TokensOut = costCopy(response.Usage.TokensOut) + response.Usage.CacheRead = costCopy(response.Usage.CacheRead) + response.Usage.CacheCreate = costCopy(response.Usage.CacheCreate) + response.Usage.CacheCreate5m = costCopy(response.Usage.CacheCreate5m) + response.Usage.CacheCreate1h = costCopy(response.Usage.CacheCreate1h) + response.Usage.CostUSD = costCopy(response.Usage.CostUSD) response.ReviewerToolEvidence = CloneReviewerToolEvidence(response.ReviewerToolEvidence) + response.RequestCostEvidence = CloneRequestCostEvidence(response.RequestCostEvidence) return response } diff --git a/internal/llm/request_cost_evidence.go b/internal/llm/request_cost_evidence.go new file mode 100644 index 0000000..2d0ebc9 --- /dev/null +++ b/internal/llm/request_cost_evidence.go @@ -0,0 +1,570 @@ +package llm + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "math" + "slices" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// Cost evidence is bounded client telemetry, never provider-reported cost. +const ( + MaxRequestCostAttempts = 64 + MaxRequestCostEvidenceBytes = 256 * 1024 + RequestCostReservedBytes = 4 * 1024 + MaxRequestCostGaps = 16 +) + +// CostEvidenceGap is a finite allow-listed reason for incomplete evidence. +type CostEvidenceGap string + +// Cost evidence gaps describe why observations are incomplete or untrusted. +const ( + CostGapMissing CostEvidenceGap = "missing_evidence" + CostGapInvalid CostEvidenceGap = "invalid_evidence" + CostGapUsage CostEvidenceGap = "missing_or_invalid_usage" + CostGapIdentity CostEvidenceGap = "missing_or_invalid_identity" + CostGapResponse CostEvidenceGap = "unsupported_response" + CostGapTransport CostEvidenceGap = "opaque_transport" + CostGapEndpoint CostEvidenceGap = "custom_or_unverified_endpoint" + CostGapDispatch CostEvidenceGap = "unresolved_dispatch" + CostGapOverflow CostEvidenceGap = "overflow" + CostGapLegacy CostEvidenceGap = "legacy_history_unknown" + CostGapInterrupted CostEvidenceGap = "interrupted_generation" + CostGapCheckpoint CostEvidenceGap = "checkpoint_unavailable" + CostGapConflict CostEvidenceGap = "conflicting_evidence" + CostGapDuplicate CostEvidenceGap = "duplicate_identity" + CostGapScope CostEvidenceGap = "scope_mismatch" + CostGapPending CostEvidenceGap = "pending_generation" +) + +// RequestCostProducer opts into collection. Start errors without a stream MUST +// mean no dispatch. Returning both a stream and an error is unsupported and +// remains missing/possibly-dispatched evidence; the direct API never does so. +// Only the concrete direct OpenAI adapter opts in in v1. +type RequestCostProducer interface{ RequestCostEvidenceSource() string } + +// RequestCostSource returns only an explicitly supported producer contract. +func RequestCostSource(adapter Adapter) string { + p, ok := adapter.(RequestCostProducer) + if ok && p.RequestCostEvidenceSource() == "openai_responses" { + return "openai_responses" + } + return "" +} + +// RequestCostEvidence preserves ordered observations, independently of output. +type RequestCostEvidence struct { + Version int `json:"version"` + Source string `json:"source"` + Scope string `json:"scope"` + ArtifactScope string `json:"artifact_scope"` + RunID string `json:"run_id"` + TaskID string `json:"task_id"` + AttemptsObserved uint64 `json:"attempts_observed"` + AttemptCountExact bool `json:"attempt_count_exact"` + Truncated bool `json:"truncated"` + DroppedAttempts uint64 `json:"dropped_attempts"` + Gaps []CostEvidenceGap `json:"gaps"` + Attempts []RequestCostAttempt `json:"attempts"` +} + +// RequestCostAttempt excludes bodies, URLs, headers and raw errors. +type RequestCostAttempt struct { + GenerationID string `json:"generation_id"` + Ordinal uint32 `json:"ordinal"` + ValidationPhase string `json:"validation_phase"` + AdapterAttempt uint32 `json:"adapter_attempt"` + RuntimeID string `json:"runtime_id"` + EndpointKind string `json:"endpoint_kind"` + TransportMode string `json:"transport_mode"` + Dispatch string `json:"dispatch"` + HTTPStatus *int `json:"http_status"` + BodyState string `json:"body_state"` + ProviderResponseID *string `json:"provider_response_id"` + ObservedModel *string `json:"observed_model"` + ObservedServiceTier *string `json:"observed_service_tier"` + ResponseStatus *string `json:"response_status"` + RequestedModel string `json:"requested_model"` + RequestedServiceTier *string `json:"requested_service_tier"` + Usage *OpenAIRequestUsage `json:"usage"` + Issues []CostEvidenceGap `json:"issues"` +} + +// OpenAIRequestUsage preserves absence and negative observations without pricing. +type OpenAIRequestUsage struct { + InputTokens *int64 `json:"input_tokens"` + OutputTokens *int64 `json:"output_tokens"` + CacheReadTokens *int64 `json:"cache_read_tokens"` + CacheWriteTokens *int64 `json:"cache_write_tokens"` + TotalTokens *int64 `json:"total_tokens"` +} + +func costCopy[T any](p *T) *T { + if p == nil { + return nil + } + v := *p + return &v +} + +// CloneRequestCostEvidence makes a detached copy, including all nested pointers. +func CloneRequestCostEvidence(e *RequestCostEvidence) *RequestCostEvidence { + if e == nil { + return nil + } + c := *e + c.Gaps = append([]CostEvidenceGap{}, e.Gaps...) + c.Attempts = make([]RequestCostAttempt, len(e.Attempts)) + for i, a := range e.Attempts { + a.HTTPStatus = costCopy(a.HTTPStatus) + a.ProviderResponseID = costCopy(a.ProviderResponseID) + a.ObservedModel = costCopy(a.ObservedModel) + a.ObservedServiceTier = costCopy(a.ObservedServiceTier) + a.ResponseStatus = costCopy(a.ResponseStatus) + a.RequestedServiceTier = costCopy(a.RequestedServiceTier) + a.Issues = append([]CostEvidenceGap{}, a.Issues...) + if a.Usage != nil { + u := *a.Usage + u.InputTokens = costCopy(u.InputTokens) + u.OutputTokens = costCopy(u.OutputTokens) + u.CacheReadTokens = costCopy(u.CacheReadTokens) + u.CacheWriteTokens = costCopy(u.CacheWriteTokens) + u.TotalTokens = costCopy(u.TotalTokens) + a.Usage = &u + } + c.Attempts[i] = a + } + return &c +} + +// SaturatingCostAdd prevents observation/omission counters wrapping to zero. +func SaturatingCostAdd(a, b uint64) uint64 { + if math.MaxUint64-a < b { + return math.MaxUint64 + } + return a + b +} + +// SafeCostString accepts bounded UTF-8 without controls, preserving exact bytes. +func SafeCostString(s string, limit int) bool { + if s == "" || len(s) > limit || !utf8.ValidString(s) { + return false + } + for _, r := range s { + if unicode.IsControl(r) { + return false + } + } + return true +} + +func validCostGap(g CostEvidenceGap) bool { + switch g { + case CostGapMissing, CostGapInvalid, CostGapUsage, CostGapIdentity, CostGapResponse, CostGapTransport, + CostGapEndpoint, CostGapDispatch, CostGapOverflow, CostGapLegacy, CostGapInterrupted, CostGapCheckpoint, + CostGapConflict, CostGapDuplicate, CostGapScope, CostGapPending: + return true + } + return false +} + +// AddCostGap deduplicates/sorts the finite enum set; arbitrary errors are excluded. +func AddCostGap(gaps []CostEvidenceGap, gap CostEvidenceGap) []CostEvidenceGap { + if !validCostGap(gap) { + gap = CostGapInvalid + } + if !slices.Contains(gaps, gap) { + gaps = append(gaps, gap) + } + slices.Sort(gaps) + return gaps +} + +// UnknownRequestCostEvidence constructs an explicit bounded unknown value. +func UnknownRequestCostEvidence(gap CostEvidenceGap) *RequestCostEvidence { + return &RequestCostEvidence{Version: 1, Source: "openai_responses", Scope: "invocation", Gaps: []CostEvidenceGap{gap}, Attempts: []RequestCostAttempt{}} +} + +// NormalizeRequestCostEvidence bounds and sanitizes a detached snapshot. It +// never repairs a gap or manufactures usage/identity. The retained prefix wins. +func NormalizeRequestCostEvidence(e *RequestCostEvidence) *RequestCostEvidence { + if e == nil { + return nil + } + c := *e + c.Gaps = []CostEvidenceGap{} + c.Attempts = []RequestCostAttempt{} + for _, g := range e.Gaps { + c.Gaps = AddCostGap(c.Gaps, g) + } + if c.Version != 1 || c.Source != "openai_responses" || (c.Scope != "invocation" && c.Scope != "task_artifact_history") { + return UnknownRequestCostEvidence(CostGapInvalid) + } + for _, p := range []*string{&c.ArtifactScope, &c.RunID, &c.TaskID} { + if *p != "" && !SafeCostString(*p, 256) { + *p = "" + c.Gaps = AddCostGap(c.Gaps, CostGapScope) + } + } + for i, raw := range e.Attempts { + if i >= MaxRequestCostAttempts { + c.DroppedAttempts = SaturatingCostAdd(c.DroppedAttempts, uint64(len(e.Attempts[i:]))) + c.Truncated = true + break + } + a := raw + a.Issues = []CostEvidenceGap{} + for _, g := range raw.Issues { + a.Issues = AddCostGap(a.Issues, g) + } + for _, p := range []*string{&a.GenerationID, &a.RequestedModel} { + if *p != "" && !SafeCostString(*p, 256) { + *p = "" + a.Issues = AddCostGap(a.Issues, CostGapIdentity) + } + } + for _, field := range []struct { + p **string + limit int + }{{&a.ProviderResponseID, 256}, {&a.ObservedModel, 256}, {&a.ObservedServiceTier, 64}, {&a.ResponseStatus, 64}, {&a.RequestedServiceTier, 64}} { + if *field.p != nil && !SafeCostString(**field.p, field.limit) { + *field.p = nil + a.Issues = AddCostGap(a.Issues, CostGapIdentity) + } + } + if a.RuntimeID != "openai-api-key" { + a.RuntimeID = "" + a.Issues = AddCostGap(a.Issues, CostGapInvalid) + } + if a.ValidationPhase != "" && a.ValidationPhase != "initial" && a.ValidationPhase != "correction" { + a.ValidationPhase = "" + a.Issues = AddCostGap(a.Issues, CostGapInvalid) + } + if a.EndpointKind != "official_global" { + a.EndpointKind = "custom_or_unverified" + a.Issues = AddCostGap(a.Issues, CostGapEndpoint) + } + if a.TransportMode != "single_send_v1" { + a.TransportMode = "opaque" + a.Issues = AddCostGap(a.Issues, CostGapTransport) + if a.Dispatch != "not_dispatched" { + c.AttemptCountExact = false + } + } + if a.Dispatch != "not_dispatched" { + a.Dispatch = "possibly_dispatched" + } + switch a.BodyState { + case "complete", "missing", "read_error", "over_limit", "malformed": + default: + a.BodyState = "missing" + a.Issues = AddCostGap(a.Issues, CostGapInvalid) + } + if a.HTTPStatus != nil && (*a.HTTPStatus < 100 || *a.HTTPStatus > 599) { + a.HTTPStatus = nil + a.Issues = AddCostGap(a.Issues, CostGapInvalid) + } + if a.Dispatch != "not_dispatched" { + validateObservedCostAttempt(&a) + } + c.Attempts = append(c.Attempts, a) + } + if c.Truncated || c.DroppedAttempts != 0 { + c.Truncated = true + c.Gaps = AddCostGap(c.Gaps, CostGapOverflow) + c.AttemptCountExact = false + } + if c.AttemptsObserved < uint64(len(c.Attempts)) { + c.AttemptsObserved = uint64(len(c.Attempts)) + c.Gaps = AddCostGap(c.Gaps, CostGapInvalid) + } + if c.AttemptsObserved != SaturatingCostAdd(uint64(len(c.Attempts)), c.DroppedAttempts) { + c.Gaps = AddCostGap(c.Gaps, CostGapMissing) + } + if len(c.Gaps) != 0 { + c.AttemptCountExact = false + } + for len(c.Attempts) > 0 { + b, _ := json.Marshal(c) + if len(b) <= MaxRequestCostEvidenceBytes-RequestCostReservedBytes { + break + } + c.Attempts = c.Attempts[:len(c.Attempts)-1] + c.Truncated = true + c.DroppedAttempts = SaturatingCostAdd(c.DroppedAttempts, 1) + c.Gaps = AddCostGap(c.Gaps, CostGapOverflow) + c.AttemptCountExact = false + } + return CloneRequestCostEvidence(&c) +} + +func validateObservedCostAttempt(a *RequestCostAttempt) { + if a.BodyState != "complete" || a.HTTPStatus == nil || *a.HTTPStatus < 200 || *a.HTTPStatus > 299 { + a.Issues = AddCostGap(a.Issues, CostGapDispatch) + } + if a.ProviderResponseID == nil || a.ObservedModel == nil || a.ObservedServiceTier == nil { + a.Issues = AddCostGap(a.Issues, CostGapIdentity) + } + if a.ResponseStatus == nil || (*a.ResponseStatus != "completed" && *a.ResponseStatus != "incomplete") { + a.Issues = AddCostGap(a.Issues, CostGapResponse) + } + if a.ObservedServiceTier != nil { + switch *a.ObservedServiceTier { + case "default", "standard", "priority", "fast", "flex", "scale": + default: + a.Issues = AddCostGap(a.Issues, CostGapIdentity) + } + } + u := a.Usage + if u == nil || u.InputTokens == nil || u.OutputTokens == nil || u.CacheReadTokens == nil || u.CacheWriteTokens == nil { + a.Issues = AddCostGap(a.Issues, CostGapUsage) + return + } + in, out, read, write := *u.InputTokens, *u.OutputTokens, *u.CacheReadTokens, *u.CacheWriteTokens + if in < 0 || out < 0 || read < 0 || write < 0 || read > in || write > in-read { + a.Issues = AddCostGap(a.Issues, CostGapUsage) + } + if u.TotalTokens != nil && (in < 0 || out < 0 || in > math.MaxInt64-out || *u.TotalTokens != in+out) { + a.Issues = AddCostGap(a.Issues, CostGapUsage) + } +} + +// RequestCostEvidenceBytes supplies fixed-order, map-free v1 digest bytes. +func RequestCostEvidenceBytes(e *RequestCostEvidence) ([]byte, error) { + return json.Marshal(NormalizeRequestCostEvidence(e)) +} + +// RequestCostEvidenceDigest binds the full scoped raw evidence, excluding prices. +func RequestCostEvidenceDigest(e *RequestCostEvidence) (string, error) { + b, err := RequestCostEvidenceBytes(e) + if err != nil { + return "", err + } + h := sha256.New() + _, _ = h.Write([]byte("cr-request-cost-evidence/v1\n")) + _, _ = h.Write(b) + return hex.EncodeToString(h.Sum(nil)), nil +} + +// DecodeRequestCostEvidence isolates malformed/unsupported optional telemetry. +func DecodeRequestCostEvidence(data []byte) *RequestCostEvidence { + if len(data) == 0 || string(data) == "null" { + return nil + } + if len(data) > MaxRequestCostEvidenceBytes { + return UnknownRequestCostEvidence(CostGapOverflow) + } + var e RequestCostEvidence + if !RequestCostJSONValid(data) || json.Unmarshal(data, &e) != nil { + return UnknownRequestCostEvidence(CostGapInvalid) + } + if len(e.Attempts) > MaxRequestCostAttempts || len(e.Gaps) > MaxRequestCostGaps { + return UnknownRequestCostEvidence(CostGapOverflow) + } + for _, a := range e.Attempts { + if len(a.Issues) > MaxRequestCostGaps { + return UnknownRequestCostEvidence(CostGapOverflow) + } + } + return NormalizeRequestCostEvidence(&e) +} + +type requestCostCollector struct { + evidence *RequestCostEvidence + ordinal uint64 +} + +func newRequestCostCollector(adapter Adapter) *requestCostCollector { + if RequestCostSource(adapter) == "" { + return nil + } + return &requestCostCollector{evidence: &RequestCostEvidence{Version: 1, Source: "openai_responses", Scope: "invocation", AttemptCountExact: true, Gaps: []CostEvidenceGap{}, Attempts: []RequestCostAttempt{}}} +} +func (c *requestCostCollector) snapshot() *RequestCostEvidence { + if c == nil { + return nil + } + return NormalizeRequestCostEvidence(c.evidence) +} +func (c *requestCostCollector) observe(resp Response, phase string, attempt uint32) { + if c == nil { + return + } + c.ordinal = SaturatingCostAdd(c.ordinal, 1) + c.evidence.AttemptsObserved = SaturatingCostAdd(c.evidence.AttemptsObserved, 1) + e := NormalizeRequestCostEvidence(resp.RequestCostEvidence) + a := RequestCostAttempt{RuntimeID: "openai-api-key", TransportMode: "opaque", EndpointKind: "custom_or_unverified", Dispatch: "possibly_dispatched", BodyState: "missing", Issues: []CostEvidenceGap{CostGapMissing}} + if e != nil && len(e.Attempts) == 1 && e.AttemptsObserved == 1 { + a = e.Attempts[0] + } else { + c.evidence.Gaps = AddCostGap(c.evidence.Gaps, CostGapMissing) + c.evidence.AttemptCountExact = false + } + if e != nil { + for _, g := range e.Gaps { + c.evidence.Gaps = AddCostGap(c.evidence.Gaps, g) + } + c.evidence.AttemptCountExact = c.evidence.AttemptCountExact && e.AttemptCountExact + } + a.ValidationPhase = phase + a.AdapterAttempt = attempt + if c.ordinal <= math.MaxUint32 { + a.Ordinal = uint32(c.ordinal) + } else { + c.evidence.Gaps = AddCostGap(c.evidence.Gaps, CostGapOverflow) + } + if len(c.evidence.Attempts) < MaxRequestCostAttempts && !c.evidence.Truncated { + c.evidence.Attempts = append(c.evidence.Attempts, a) + } else { + c.evidence.Truncated = true + c.evidence.DroppedAttempts = SaturatingCostAdd(c.evidence.DroppedAttempts, 1) + } + c.evidence = NormalizeRequestCostEvidence(c.evidence) +} + +func nonDispatchedCostResponse(adapter Adapter, req Request) Response { + if RequestCostSource(adapter) == "" { + return Response{} + } + attempt := RequestCostAttempt{RuntimeID: "openai-api-key", EndpointKind: "custom_or_unverified", TransportMode: "opaque", Dispatch: "not_dispatched", BodyState: "missing", RequestedModel: req.Model} + if req.Fast { + tier := "fast" + attempt.RequestedServiceTier = &tier + } + return Response{RequestCostEvidence: NormalizeRequestCostEvidence(&RequestCostEvidence{Version: 1, Source: "openai_responses", Scope: "invocation", AttemptsObserved: 1, AttemptCountExact: true, Attempts: []RequestCostAttempt{attempt}})} +} + +// RequestCostJSONValid rejects duplicate keys and excessive nesting before a +// bounded telemetry envelope is decoded. Unknown unique fields remain allowed. +func RequestCostJSONValid(data []byte) bool { + if !CostJSONUnicodeValid(data) { + return false + } + d := json.NewDecoder(bytes.NewReader(data)) + d.UseNumber() + if !costJSONValue(d, 0) { + return false + } + _, err := d.Token() + return err == io.EOF +} +func costJSONValue(d *json.Decoder, depth int) bool { + if depth > 64 { + return false + } + token, err := d.Token() + if err != nil { + return false + } + delim, ok := token.(json.Delim) + if !ok { + return true + } + switch delim { + case '{': + seen := map[string]bool{} + for d.More() { + t, err := d.Token() + if err != nil { + return false + } + key, ok := t.(string) + if !ok { + return false + } + folded := costKeyFold(key) + if seen[folded] { + return false + } + seen[folded] = true + if !costJSONValue(d, depth+1) { + return false + } + } + end, err := d.Token() + return err == nil && end == json.Delim('}') + case '[': + for d.More() { + if !costJSONValue(d, depth+1) { + return false + } + } + end, err := d.Token() + return err == nil && end == json.Delim(']') + default: + return false + } +} + +// CostJSONUnicodeValid prevents encoding/json from turning invalid UTF-8 or +// unpaired surrogate escapes into a different, apparently valid identity. It +// does not replace JSON syntax validation and does not unescape provider data. +func CostJSONUnicodeValid(data []byte) bool { + if !utf8.Valid(data) { + return false + } + inString := false + for i := 0; i < len(data); i++ { + if data[i] == '"' { + inString = !inString + continue + } + if !inString || data[i] != '\\' { + continue + } + i++ + if i >= len(data) { + return false + } + if data[i] != 'u' { + continue + } + if i+4 >= len(data) { + return false + } + value, err := strconv.ParseUint(string(data[i+1:i+5]), 16, 16) + if err != nil { + return false + } + i += 4 + if value >= 0xdc00 && value <= 0xdfff { + return false + } + if value >= 0xd800 && value <= 0xdbff { + if i+6 >= len(data) || data[i+1] != '\\' || data[i+2] != 'u' { + return false + } + low, err := strconv.ParseUint(string(data[i+3:i+7]), 16, 16) + if err != nil || low < 0xdc00 || low > 0xdfff { + return false + } + i += 6 + } + } + return true +} + +// encoding/json matches tagged fields with Unicode simple-fold semantics. +// Use the same equivalence classes so alternate casing cannot become a second +// spelling of one field and silently overwrite it during typed decoding. +func costKeyFold(key string) string { + var out strings.Builder + for _, r := range key { + least := r + for next := unicode.SimpleFold(r); next != r; next = unicode.SimpleFold(next) { + if next < least { + least = next + } + } + out.WriteRune(least) + } + return out.String() +} diff --git a/internal/llm/request_cost_evidence_test.go b/internal/llm/request_cost_evidence_test.go new file mode 100644 index 0000000..2074639 --- /dev/null +++ b/internal/llm/request_cost_evidence_test.go @@ -0,0 +1,361 @@ +package llm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "reflect" + "slices" + "strings" + "testing" +) + +type costFakeAdapter struct { + *FakeAdapter + waits []error +} + +func (a *costFakeAdapter) Queue(r FakeResult) { + a.waits = append(a.waits, r.WaitErr) + r.WaitErr = nil + a.FakeAdapter.Queue(r) +} +func (a *costFakeAdapter) Start(ctx context.Context, req Request) (Stream, error) { + stream, err := a.FakeAdapter.Start(ctx, req) + var waitErr error + if len(a.waits) > 0 { + waitErr = a.waits[0] + a.waits = a.waits[1:] + } + if err != nil { + return nil, err + } + return costEvidenceStream{Stream: stream, waitErr: waitErr}, nil +} + +type costEvidenceStream struct { + Stream + waitErr error +} + +func (s costEvidenceStream) Wait(ctx context.Context) (Response, error) { + response, err := s.Stream.Wait(ctx) + if err != nil { + return response, err + } + return response, s.waitErr +} +func (*costFakeAdapter) RequestCostEvidenceSource() string { return "openai_responses" } +func costPtr[T any](v T) *T { return &v } +func costResponse(id, text string) Response { + return Response{StructuredOutput: []byte(text), RequestCostEvidence: &RequestCostEvidence{ + Version: 1, Source: "openai_responses", Scope: "invocation", AttemptsObserved: 1, AttemptCountExact: true, + Attempts: []RequestCostAttempt{{RuntimeID: "openai-api-key", EndpointKind: "official_global", TransportMode: "single_send_v1", Dispatch: "possibly_dispatched", HTTPStatus: costPtr(200), BodyState: "complete", ProviderResponseID: costPtr(id), ObservedModel: costPtr("observed"), ObservedServiceTier: costPtr("default"), ResponseStatus: costPtr("completed"), RequestedModel: "requested", Usage: &OpenAIRequestUsage{InputTokens: costPtr(int64(10)), OutputTokens: costPtr(int64(3)), CacheReadTokens: costPtr(int64(2)), CacheWriteTokens: costPtr(int64(0)), TotalTokens: costPtr(int64(13))}}}, + }} +} +func costDecode(b []byte) (bool, error) { + if string(b) == `{"ok":true}` { + return true, nil + } + return false, errors.New("invalid synthetic output") +} + +func TestRequestCostCollectorValidationAndTerminalPaths(t *testing.T) { + for _, tc := range []struct { + name string + second FakeResult + callback bool + wantAttempts uint64 + wantErr bool + }{ + {"correction success", FakeResult{Response: costResponse("B", `{"ok":true}`)}, false, 2, false}, + {"two invalid", FakeResult{Response: costResponse("B", `invalid`)}, false, 2, true}, + {"correction start", FakeResult{StartErr: errors.New("start")}, false, 2, true}, + {"correction wait", FakeResult{Response: costResponse("B", ``), WaitErr: errors.New("wait")}, false, 2, true}, + {"correction callback", FakeResult{}, true, 1, true}, + } { + t.Run(tc.name, func(t *testing.T) { + a := &costFakeAdapter{FakeAdapter: &FakeAdapter{}} + first := costResponse("A", `invalid`) + a.Queue(FakeResult{Response: first}) + a.Queue(tc.second) + req := Request{Model: "requested", FreshValidationRetrySession: true} + if tc.callback { + req.OnValidationRetry = func(*Request) error { return errors.New("callback") } + } + got, err := RunStructuredWithSessionResume(context.Background(), a, "", req, costDecode) + if (err != nil) != tc.wantErr { + t.Fatalf("error = %v", err) + } + e := got.Response.RequestCostEvidence + if e == nil || uint64(len(e.Attempts)) != tc.wantAttempts || e.AttemptsObserved != tc.wantAttempts { + t.Fatalf("history = %#v", e) + } + if !tc.wantErr && string(got.AcceptedOutput) != `{"ok":true}` { + t.Fatal("accepted output replaced by initial attempt") + } + if *e.Attempts[0].ProviderResponseID != "A" || e.Attempts[0].Ordinal != 1 || e.Attempts[0].ValidationPhase != "initial" { + t.Fatal("initial receipt lost") + } + if tc.wantAttempts == 2 && (e.Attempts[1].Ordinal != 2 || e.Attempts[1].ValidationPhase != "correction" || e.Attempts[1].AdapterAttempt != 1) { + t.Fatal("phase ordering lost") + } + if tc.name == "correction start" && e.Attempts[1].Dispatch != "not_dispatched" { + t.Fatal("Start failure is not explicit non-dispatch") + } + *first.RequestCostEvidence.Attempts[0].Usage.InputTokens = 999 + *got.ValidationAttempts[0].Response.RequestCostEvidence.Attempts[0].Usage.InputTokens = 777 + if *e.Attempts[0].Usage.InputTokens != 10 { + t.Fatal("history aliases adapter or validation snapshot") + } + }) + } +} + +func TestRequestCostCollectorTransientPhasesAndExhaustion(t *testing.T) { + old := activeRetryPolicy + activeRetryPolicy = retryPolicy{MaxRetries: 2} + t.Cleanup(func() { activeRetryPolicy = old }) + for _, success := range []bool{false, true} { + a := &costFakeAdapter{FakeAdapter: &FakeAdapter{}} + for i := 0; i < 3; i++ { + waitErr := error(ErrTransient) + if success && i == 2 { + waitErr = nil + } + a.Queue(FakeResult{Response: costResponse(fmt.Sprint(i), `{"ok":true}`), WaitErr: waitErr}) + } + got, err := RunStructuredWithSessionResume(context.Background(), a, "", Request{Model: "requested"}, costDecode) + if (err == nil) != success || len(got.Response.RequestCostEvidence.Attempts) != 3 { + t.Fatalf("err=%v history=%#v", err, got.Response.RequestCostEvidence) + } + for i, a := range got.Response.RequestCostEvidence.Attempts { + if a.Ordinal != uint32(i+1) || a.AdapterAttempt != uint32(i+1) { + t.Fatal("attempt budget inferred or reset") + } + } + } +} + +func TestRequestCostCollectorMissingUnsupportedAndCancellation(t *testing.T) { + a := &costFakeAdapter{FakeAdapter: &FakeAdapter{}} + a.Queue(FakeResult{Response: Response{StructuredOutput: []byte(`{"ok":true}`)}}) + got, err := RunStructuredWithSessionResume(context.Background(), a, "", Request{Model: "requested"}, costDecode) + if err != nil || !slices.Contains(got.Response.RequestCostEvidence.Gaps, CostGapMissing) { + t.Fatal("missing expected receipt was omitted") + } + plain := &FakeAdapter{} + plain.Queue(FakeResult{Response: costResponse("ignored", `{"ok":true}`)}) + unsupported, _ := RunStructuredWithSessionResume(context.Background(), plain, "", Request{}, costDecode) + if unsupported.Response.RequestCostEvidence != nil { + t.Fatal("unsupported producer gained collection") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + canceled, _ := RunStructuredWithSessionResume(ctx, &costFakeAdapter{FakeAdapter: &FakeAdapter{}}, "", Request{Model: "requested"}, costDecode) + if len(canceled.Response.RequestCostEvidence.Attempts) != 1 || canceled.Response.RequestCostEvidence.Attempts[0].Dispatch != "not_dispatched" { + t.Fatal("canceled Start omitted") + } +} + +func TestRequestCostBoundsCounterSaturationAndDetachedToolTrace(t *testing.T) { + collector := newRequestCostCollector(&costFakeAdapter{FakeAdapter: &FakeAdapter{}}) + for i := 0; i < 65; i++ { + collector.observe(costResponse(fmt.Sprint(i), ""), "initial", uint32(i+1)) + } + e := collector.snapshot() + if len(e.Attempts) != 64 || e.AttemptsObserved != 65 || e.DroppedAttempts != 1 || !e.Truncated || e.AttemptCountExact { + t.Fatalf("bounds = %#v", e) + } + if *e.Attempts[0].ProviderResponseID != "0" || *e.Attempts[63].ProviderResponseID != "63" { + t.Fatal("retained last success instead of prefix") + } + if SaturatingCostAdd(math.MaxUint64, 1) != math.MaxUint64 { + t.Fatal("counter wrapped") + } + original := costResponse("clone", "") + original.ReviewerToolEvidence = &ReviewerToolEvidence{DiffStatus: DiffToolStatusSucceeded} + cloned := cloneResponse(original) + cloned.RequestCostEvidence.Attempts[0].Issues = append(cloned.RequestCostEvidence.Attempts[0].Issues, CostGapInvalid) + *cloned.RequestCostEvidence.Attempts[0].Usage.CacheWriteTokens = 9 + cloned.ReviewerToolEvidence.DiffStatus = DiffToolStatusFailed + if *original.RequestCostEvidence.Attempts[0].Usage.CacheWriteTokens != 0 || original.ReviewerToolEvidence.DiffStatus != DiffToolStatusSucceeded { + t.Fatal("response ownership regression") + } + data, _ := RequestCostEvidenceBytes(e) + if len(data) > MaxRequestCostEvidenceBytes-RequestCostReservedBytes { + t.Fatal("evidence exceeded budget") + } +} + +func TestRequestCostDecoderMalformedOptionalAndIdentityBounds(t *testing.T) { + for _, raw := range []string{`[]`, `{"version":2}`, `{"version":1,"version":1}`, `{"version":"bad"}`, strings.Repeat(" ", MaxRequestCostEvidenceBytes) + `{}`} { + e := DecodeRequestCostEvidence([]byte(raw)) + if e == nil || e.AttemptCountExact || len(e.Gaps) == 0 { + t.Fatalf("malformed envelope became complete: %s", raw[:min(len(raw), 40)]) + } + } + e := costResponse("safe", "").RequestCostEvidence + e.Attempts[0].ObservedModel = costPtr("malicious\nmodel") + e.Attempts[0].ProviderResponseID = costPtr(strings.Repeat("x", 257)) + n := NormalizeRequestCostEvidence(e) + if n.Attempts[0].ObservedModel != nil || n.Attempts[0].ProviderResponseID != nil { + t.Fatal("invalid identity retained") + } + if e.Attempts[0].ObservedModel == nil { + t.Fatal("normalization mutated input") + } +} + +func TestRequestCostDigestNullZeroAndOrder(t *testing.T) { + e := costResponse("A", "").RequestCostEvidence + e.Attempts[0].Usage.CacheWriteTokens = nil + first, _ := RequestCostEvidenceDigest(e) + e.Attempts[0].Usage.CacheWriteTokens = costPtr(int64(0)) + second, _ := RequestCostEvidenceDigest(e) + if first == second { + t.Fatal("null equals numeric zero") + } + e.Attempts = append(e.Attempts, costResponse("B", "").RequestCostEvidence.Attempts[0]) + e.AttemptsObserved = 2 + third, _ := RequestCostEvidenceDigest(e) + slices.Reverse(e.Attempts) + fourth, _ := RequestCostEvidenceDigest(e) + if third == fourth { + t.Fatal("chronological order absent from digest") + } + clone := CloneRequestCostEvidence(e) + if !reflect.DeepEqual(NormalizeRequestCostEvidence(clone), NormalizeRequestCostEvidence(e)) { + t.Fatal("clone changed values") + } + data, _ := RequestCostEvidenceBytes(e) + var decoded RequestCostEvidence + if json.Unmarshal(data, &decoded) != nil { + t.Fatal("canonical bytes are not JSON") + } +} + +type costBackoffContext struct { + context.Context + canceled bool + done chan struct{} +} + +func (c *costBackoffContext) Done() <-chan struct{} { + if !c.canceled { + c.canceled = true + close(c.done) + } + return c.done +} +func (c *costBackoffContext) Err() error { + if c.canceled { + return context.Canceled + } + return nil +} +func TestRequestCostCancellationInsideBackoffRetainsReceipt(t *testing.T) { + old := activeRetryPolicy + activeRetryPolicy = retryPolicy{MaxRetries: 3, Base: 1_000_000_000, Multiplier: 2} + t.Cleanup(func() { activeRetryPolicy = old }) + ctx := &costBackoffContext{Context: context.Background(), done: make(chan struct{})} + a := &costFakeAdapter{FakeAdapter: &FakeAdapter{}} + a.Queue(FakeResult{Response: costResponse("observed-before-backoff", ``), WaitErr: ErrTransient}) + got, err := RunStructuredWithSessionResume(ctx, a, "", Request{Model: "requested"}, costDecode) + if err == nil || !ctx.canceled || len(a.Requests()) != 1 || len(got.Response.RequestCostEvidence.Attempts) < 1 || *got.Response.RequestCostEvidence.Attempts[0].ProviderResponseID != "observed-before-backoff" { + t.Fatal("canceled backoff erased or retried receipt") + } +} +func TestRequestCostTransientBudgetsAcrossValidationPhases(t *testing.T) { + old := activeRetryPolicy + activeRetryPolicy = retryPolicy{MaxRetries: 1} + t.Cleanup(func() { activeRetryPolicy = old }) + a := &costFakeAdapter{FakeAdapter: &FakeAdapter{}} + for _, r := range []FakeResult{ + {Response: costResponse("1", ""), WaitErr: ErrTransient}, {Response: costResponse("2", "invalid")}, + {Response: costResponse("3", ""), WaitErr: ErrTransient}, {Response: costResponse("4", `{"ok":true}`)}, + } { + a.Queue(r) + } + got, err := RunStructuredWithSessionResume(context.Background(), a, "", Request{Model: "requested", FreshValidationRetrySession: true}, costDecode) + if err != nil || len(got.Response.RequestCostEvidence.Attempts) != 4 || len(got.ValidationAttempts) != 1 { + t.Fatalf("err=%v result=%#v", err, got) + } + for i, r := range got.Response.RequestCostEvidence.Attempts { + if r.Ordinal != uint32(i+1) || r.AdapterAttempt != uint32(i%2+1) { + t.Fatal("phase budget reset global ordinal") + } + } +} + +func TestRequestCostEvidenceCanonicalGolden(t *testing.T) { + want, err := os.ReadFile("testdata/request-cost-evidence-v1.golden.json") + if err != nil { + t.Fatal(err) + } + var e RequestCostEvidence + if json.Unmarshal(want, &e) != nil { + t.Fatal("golden fixture") + } + got, err := RequestCostEvidenceBytes(&e) + if err != nil || string(got) != string(want) { + t.Fatalf("canonical bytes changed: %s %v", got, err) + } + digest, err := RequestCostEvidenceDigest(&e) + if err != nil || digest != "40a0d89467879b363d192692bb0df2599c92fb476b4e1f7db8ad88368ad842d0" { + t.Fatalf("digest=%s err=%v", digest, err) + } +} + +func TestRequestCostRejectsUnicodeReplacementIdentities(t *testing.T) { + for _, raw := range [][]byte{[]byte(`"\ud800"`), []byte(`"\udc00"`), []byte(`"\ud800\u0041"`), {'"', 0xff, '"'}} { + if CostJSONUnicodeValid(raw) { + t.Fatalf("invalid Unicode accepted: %q", raw) + } + } + for _, raw := range []string{`"\ud83d\ude00"`, `"\\ud800"`, `"valid\u0041"`, `"literal �"`} { + if !CostJSONUnicodeValid([]byte(raw)) { + t.Fatalf("valid Unicode rejected: %s", raw) + } + } + if RequestCostJSONValid([]byte(`{"identity":"\ud800"}`)) { + t.Fatal("optional cost decoder could replace identity") + } +} + +func TestRequestCostBaseStreamReturnsDetachedSnapshots(t *testing.T) { + stream := NewBaseStream(func() {}) + source := costResponse("source", `{"ok":true}`) + source.Usage.TokensIn = costPtr(10) + source.ReviewerToolEvidence = &ReviewerToolEvidence{DiffStatus: DiffToolStatusSucceeded, Trace: &ReviewerToolTrace{Version: 1, Source: "pi_rpc", Calls: []ReviewerToolCallEvidence{{CallID: "one"}}}} + stream.Finish(source, nil) + *source.RequestCostEvidence.Attempts[0].Usage.InputTokens = 999 + *source.Usage.TokensIn = 999 + source.ReviewerToolEvidence.Trace.Calls[0].CallID = "mutated" + first, err := stream.Wait(context.Background()) + if err != nil { + t.Fatal(err) + } + *first.RequestCostEvidence.Attempts[0].Usage.InputTokens = 777 + *first.Usage.TokensIn = 777 + first.ReviewerToolEvidence.Trace.Calls[0].CallID = "again" + second, err := stream.Wait(context.Background()) + if err != nil { + t.Fatal(err) + } + if *second.RequestCostEvidence.Attempts[0].Usage.InputTokens != 10 || *second.Usage.TokensIn != 10 || second.ReviewerToolEvidence.Trace.Calls[0].CallID != "one" { + t.Fatal("Finish or Wait retained shared mutable response") + } + results := make(chan Response, 2) + for i := 0; i < 2; i++ { + go func() { r, _ := stream.Wait(context.Background()); results <- r }() + } + a, b := <-results, <-results + *a.RequestCostEvidence.Attempts[0].Usage.InputTokens = 666 + if *b.RequestCostEvidence.Attempts[0].Usage.InputTokens != 10 { + t.Fatal("concurrent Wait copies alias") + } +} diff --git a/internal/llm/subprocess.go b/internal/llm/subprocess.go index e4cb0b2..0780015 100644 --- a/internal/llm/subprocess.go +++ b/internal/llm/subprocess.go @@ -206,6 +206,7 @@ func (s *BaseStream) Wait(ctx context.Context) (Response, error) { s.mu.Lock() result := s.result + result.response = cloneResponse(result.response) if result.err == nil && ctx.Err() != nil { result.err = ctx.Err() } @@ -281,7 +282,7 @@ func (s *BaseStream) Cleanup() { // Finish publishes a stream result and wakes waiters. func (s *BaseStream) Finish(response Response, err error) { s.mu.Lock() - s.result = streamResult{response: response, err: err} + s.result = streamResult{response: cloneResponse(response), err: err} s.mu.Unlock() close(s.done) } diff --git a/internal/llm/testdata/request-cost-evidence-v1.golden.json b/internal/llm/testdata/request-cost-evidence-v1.golden.json new file mode 100644 index 0000000..953a6ae --- /dev/null +++ b/internal/llm/testdata/request-cost-evidence-v1.golden.json @@ -0,0 +1 @@ +{"version":1,"source":"openai_responses","scope":"invocation","artifact_scope":"","run_id":"","task_id":"","attempts_observed":1,"attempt_count_exact":true,"truncated":false,"dropped_attempts":0,"gaps":[],"attempts":[{"generation_id":"","ordinal":1,"validation_phase":"initial","adapter_attempt":1,"runtime_id":"openai-api-key","endpoint_kind":"official_global","transport_mode":"single_send_v1","dispatch":"not_dispatched","http_status":null,"body_state":"missing","provider_response_id":null,"observed_model":null,"observed_service_tier":null,"response_status":null,"requested_model":"requested\u003cmodel\u003e","requested_service_tier":null,"usage":null,"issues":[]}]} \ No newline at end of file diff --git a/internal/llmadapters/api.go b/internal/llmadapters/api.go index bee2917..41c564b 100644 --- a/internal/llmadapters/api.go +++ b/internal/llmadapters/api.go @@ -22,6 +22,8 @@ import ( // ErrAPIAdapterConfig reports invalid direct API adapter configuration. var ErrAPIAdapterConfig = errors.New("llm api: invalid configuration") +var errAPIResponseOverLimit = errors.New("llm api: response body exceeds limit") + const ( defaultAnthropicBaseURL = "https://api.anthropic.com/" defaultOpenAIBaseURL = "https://api.openai.com/" @@ -50,13 +52,14 @@ type APIOptions struct { // APIAdapter calls a direct provider HTTP API as an LLM adapter. type APIAdapter struct { - kind apiKind - apiKey string - httpClient *http.Client - baseURL *url.URL - maxTokens int - anthropicVersion string - fastModeModels []string + kind apiKind + apiKey string + httpClient *http.Client + ownedOpenAIClient *http.Client + baseURL *url.URL + maxTokens int + anthropicVersion string + fastModeModels []string } var _ llm.Adapter = (*APIAdapter)(nil) @@ -132,21 +135,28 @@ func newAPIAdapter(kind apiKind, opts APIOptions) (*APIAdapter, error) { return nil, err } httpClient := opts.HTTPClient + var ownedOpenAIClient *http.Client if httpClient == nil { - httpClient = &http.Client{Timeout: defaultAPIClientTimeout} + if kind == apiOpenAI { + ownedOpenAIClient = newSingleSendOpenAIClient() + httpClient = ownedOpenAIClient + } else { + httpClient = &http.Client{Timeout: defaultAPIClientTimeout} + } } version := strings.TrimSpace(opts.AnthropicVersion) if version == "" { version = defaultAnthropicVersion } return &APIAdapter{ - kind: kind, - apiKey: opts.APIKey, - httpClient: httpClient, - baseURL: baseURL, - maxTokens: maxTokens, - anthropicVersion: version, - fastModeModels: append([]string(nil), opts.FastModeModels...), + kind: kind, + apiKey: opts.APIKey, + httpClient: httpClient, + ownedOpenAIClient: ownedOpenAIClient, + baseURL: baseURL, + maxTokens: maxTokens, + anthropicVersion: version, + fastModeModels: append([]string(nil), opts.FastModeModels...), }, nil } @@ -229,18 +239,35 @@ func (s *apiStream) run(ctx context.Context, adapter *APIAdapter, req Request) { s.Finish(response, err) } -func (a *APIAdapter) execute(ctx context.Context, req Request) (string, Response, error) { +func (a *APIAdapter) execute(ctx context.Context, req Request) (sessionID string, response Response, resultErr error) { + cost := a.newCostAttempt(req) + defer func() { response.RequestCostEvidence = singleCostEvidence(cost) }() endpoint, requestBody, err := a.buildProviderRequest(req) if err != nil { return "", Response{}, err } + if cost != nil && officialResponsesEndpoint(endpoint) { + cost.EndpointKind = "official_global" + } httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(requestBody)) if err != nil { return "", Response{}, err } a.applyHeaders(httpReq, req) - + if cost != nil { + // Clearing GetBody blocks consumed-body transport/redirect replay for + // the owned client. Injected clients remain explicitly opaque. + httpReq.GetBody = nil + if err := ctx.Err(); err != nil { + return "", Response{}, err + } + cost.Dispatch = "possibly_dispatched" + } httpResp, err := a.httpClient.Do(httpReq) + if cost != nil && httpResp != nil { + status := httpResp.StatusCode + cost.HTTPStatus = &status + } if err != nil { if isTransientTransportError(err) { return "", Response{}, fmt.Errorf("%w: %w", ErrTransient, err) @@ -248,10 +275,24 @@ func (a *APIAdapter) execute(ctx context.Context, req Request) (string, Response return "", Response{}, err } defer httpResp.Body.Close() + if cost != nil { + status := httpResp.StatusCode + cost.HTTPStatus = &status + } responseBody, err := readAPIResponseBody(httpResp.Body) if err != nil { + if cost != nil { + cost.BodyState = "read_error" + if errors.Is(err, errAPIResponseOverLimit) { + cost.BodyState = "over_limit" + } + } return "", Response{}, err } + if cost != nil { + cost.BodyState = "complete" + observeCostEnvelope(cost, responseBody) + } if httpResp.StatusCode < 200 || httpResp.StatusCode > 299 { statusErr := fmt.Errorf("llm api %s: provider returned %s", a.kind, httpResp.Status) if classifyHTTPStatusTransient(httpResp.StatusCode) { @@ -449,7 +490,8 @@ type openAIUsage struct { } type openAIInputTokensDetails struct { - CachedTokens *int `json:"cached_tokens"` + CachedTokens *int `json:"cached_tokens"` + CacheWriteTokens *int `json:"cache_write_tokens"` } func parseOpenAIResponse(body []byte) (string, Response, error) { @@ -468,9 +510,6 @@ func parseOpenAIResponse(body []byte) (string, Response, error) { if text.Len() == 0 && payload.OutputText != "" { text.WriteString(payload.OutputText) } - if text.Len() == 0 { - return payload.ID, Response{}, errors.New("llm api openai_api: no text output") - } speed := "" switch strings.ToLower(strings.TrimSpace(payload.ServiceTier)) { case "fast", "priority": @@ -478,15 +517,22 @@ func parseOpenAIResponse(body []byte) (string, Response, error) { case "standard", "default": speed = "standard" } - return payload.ID, Response{ + // OpenAI's input total includes cache reads and writes. Keep that total + // intact rather than treating cache writes as an additional input bucket. + response := Response{ StructuredOutput: []byte(text.String()), Usage: Usage{ - TokensIn: payload.Usage.InputTokens, - TokensOut: payload.Usage.OutputTokens, - CacheRead: payload.Usage.InputTokensDetails.CachedTokens, - Speed: speed, + TokensIn: payload.Usage.InputTokens, + TokensOut: payload.Usage.OutputTokens, + CacheRead: payload.Usage.InputTokensDetails.CachedTokens, + CacheCreate: payload.Usage.InputTokensDetails.CacheWriteTokens, + Speed: speed, }, - }, nil + } + if text.Len() == 0 { + return payload.ID, response, errors.New("llm api openai_api: no text output") + } + return payload.ID, response, nil } func readAPIResponseBody(reader io.Reader) ([]byte, error) { @@ -495,7 +541,7 @@ func readAPIResponseBody(reader io.Reader) ([]byte, error) { return nil, err } if len(body) > apiResponseLogLimit { - return nil, errors.New("llm api: response body exceeds limit") + return nil, errAPIResponseOverLimit } return body, nil } diff --git a/internal/llmadapters/api_test.go b/internal/llmadapters/api_test.go index 27b0a13..968561a 100644 --- a/internal/llmadapters/api_test.go +++ b/internal/llmadapters/api_test.go @@ -16,6 +16,8 @@ import ( "github.com/open-cli-collective/codereview-cli/internal/config" "github.com/open-cli-collective/codereview-cli/internal/credentials" + "github.com/open-cli-collective/codereview-cli/internal/modelcatalog" + "github.com/open-cli-collective/codereview-cli/internal/pricing" ) func TestAnthropicAPIAdapterRequestAndResponse(t *testing.T) { @@ -234,6 +236,111 @@ func TestOpenAIAPIAdapterFastRequest(t *testing.T) { } } +func TestParseOpenAIResponseCacheWrites(t *testing.T) { + zero, five, seven, twelve, negative := 0, 5, 7, 12, -3 + for _, tt := range []struct { + name string + details string + want *int + wantRead *int + }{ + {name: "missing details"}, + {name: "null details", details: `,"input_tokens_details":null`}, + {name: "missing count", details: `,"input_tokens_details":{"cached_tokens":5}`, wantRead: &five}, + {name: "null count", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":null}`, wantRead: &five}, + {name: "zero count", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":0}`, want: &zero, wantRead: &five}, + {name: "positive count", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":7}`, want: &seven, wantRead: &five}, + {name: "negative count retained", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":-3}`, want: &negative, wantRead: &five}, + {name: "overlapping counts retained", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":12}`, want: &twelve, wantRead: &five}, + {name: "unknown field ignored", details: `,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":7,"future_detail":{"count":9}}`, want: &seven, wantRead: &five}, + } { + t.Run(tt.name, func(t *testing.T) { + body := fmt.Sprintf(`{"id":"resp-cache","service_tier":"default","output_text":"{}","usage":{"input_tokens":16,"output_tokens":3%s}}`, tt.details) + _, response, err := parseOpenAIResponse([]byte(body)) + if err != nil { + t.Fatalf("parseOpenAIResponse: %v", err) + } + got := response.Usage.CacheCreate + if (got == nil) != (tt.want == nil) || got != nil && *got != *tt.want { + t.Fatalf("CacheCreate = %v, want %v", got, tt.want) + } + read := response.Usage.CacheRead + if (read == nil) != (tt.wantRead == nil) || read != nil && *read != *tt.wantRead { + t.Fatalf("CacheRead = %v, want %v", read, tt.wantRead) + } + // OpenAI input is inclusive of cache reads and writes. Preserve the + // provider's total. Raw negative or overlapping counts are not + // proof of billable usage; a future estimator must validate them. + if response.Usage.TokensIn == nil || *response.Usage.TokensIn != 16 || + response.Usage.TokensOut == nil || *response.Usage.TokensOut != 3 { + t.Fatalf("Usage = %#v, want unchanged input/output totals", response.Usage) + } + if response.Usage.CacheCreate5m != nil || response.Usage.CacheCreate1h != nil || response.Usage.CostUSD != nil { + t.Fatalf("Usage = %#v, want unknown TTL buckets and cost", response.Usage) + } + if response.Usage.Speed != "standard" { + t.Fatalf("Speed = %q, want observed standard tier", response.Usage.Speed) + } + }) + } +} + +func TestParseOpenAIResponseRejectsMalformedCacheWriteCount(t *testing.T) { + for _, count := range []string{`"7"`, `1.5`, `true`, `{}`, `[]`, `9223372036854775808`} { + t.Run(count, func(t *testing.T) { + body := fmt.Sprintf(`{"output_text":"{}","usage":{"input_tokens_details":{"cache_write_tokens":%s}}}`, count) + if _, _, err := parseOpenAIResponse([]byte(body)); err == nil { + t.Fatal("expected malformed cache-write count to fail response decoding") + } + }) + } +} + +func TestParsedOpenAIUsageLeavesBundledEstimatesUnavailable(t *testing.T) { + catalog, err := modelcatalog.LoadBundled() + if err != nil { + t.Fatalf("LoadBundled: %v", err) + } + for _, count := range []string{`null`, `0`, `7`} { + t.Run(count, func(t *testing.T) { + body := fmt.Sprintf(`{"service_tier":"default","output_text":"{}","usage":{"input_tokens":16,"output_tokens":3,"input_tokens_details":{"cached_tokens":5,"cache_write_tokens":%s}}}`, count) + _, response, err := parseOpenAIResponse([]byte(body)) + if err != nil { + t.Fatalf("parseOpenAIResponse: %v", err) + } + usage := response.Usage + _, ok := pricing.EstimateUsageUSDFor(catalog, "gpt-6.1-sol", pricing.Usage{ + TokensIn: usage.TokensIn, + TokensOut: usage.TokensOut, + CacheRead: usage.CacheRead, + CacheCreateTotal: usage.CacheCreate, + CacheCreate5m: usage.CacheCreate5m, + CacheCreate1h: usage.CacheCreate1h, + Speed: usage.Speed, + }) + if ok { + t.Fatal("parsed cache-write telemetry must not activate bundled context-banded estimates") + } + }) + } +} + +func TestParseAnthropicResponseKeepsSeparateCacheBuckets(t *testing.T) { + _, response, err := parseAnthropicResponse([]byte(`{"content":[{"type":"text","text":"{}"}],"usage":{"input_tokens":16,"output_tokens":3,"cache_read_input_tokens":5,"cache_creation_input_tokens":7,"cache_creation":{"ephemeral_5m_input_tokens":3,"ephemeral_1h_input_tokens":4},"input_tokens_details":{"cached_tokens":100,"cache_write_tokens":200}}}`)) + if err != nil { + t.Fatalf("parseAnthropicResponse: %v", err) + } + assertIntPtr(t, "TokensIn", response.Usage.TokensIn, 16) + assertIntPtr(t, "TokensOut", response.Usage.TokensOut, 3) + assertIntPtr(t, "CacheRead", response.Usage.CacheRead, 5) + assertIntPtr(t, "CacheCreate", response.Usage.CacheCreate, 7) + assertIntPtr(t, "CacheCreate5m", response.Usage.CacheCreate5m, 3) + assertIntPtr(t, "CacheCreate1h", response.Usage.CacheCreate1h, 4) + if response.Usage.CostUSD != nil { + t.Fatal("cache usage must not become a reported cost") + } +} + func TestParseOpenAIResponseSpeed(t *testing.T) { for _, tt := range []struct { name string diff --git a/internal/llmadapters/request_cost_evidence.go b/internal/llmadapters/request_cost_evidence.go new file mode 100644 index 0000000..7bcf12a --- /dev/null +++ b/internal/llmadapters/request_cost_evidence.go @@ -0,0 +1,192 @@ +package llmadapters + +import ( + "bytes" + "encoding/json" + "net" + "net/http" + "net/url" + "time" + "unicode/utf8" + + "github.com/open-cli-collective/codereview-cli/internal/llm" +) + +// RequestCostEvidenceSource opts in by concrete adapter ownership, never by +// model prefix. Start's synchronous configuration errors occur before dispatch. +func (a *APIAdapter) RequestCostEvidenceSource() string { + if a.kind == apiOpenAI { + return "openai_responses" + } + return "" +} + +func newSingleSendOpenAIClient() *http.Client { + // Pin a fresh standard transport, not mutable http.DefaultTransport. With + // GetBody cleared and no idempotency header, consumed request bodies cannot + // be replayed. Explicit outer retries produce independent receipts. + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + ForceAttemptHTTP2: true, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: time.Second, + } + return &http.Client{Transport: transport, Timeout: defaultAPIClientTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} +} + +func officialResponsesEndpoint(endpoint string) bool { + u, err := url.Parse(endpoint) + return err == nil && u.Scheme == "https" && u.Host == "api.openai.com" && u.User == nil && u.Path == "/v1/responses" && u.RawPath == "" && u.RawQuery == "" && !u.ForceQuery && u.Fragment == "" +} + +func (a *APIAdapter) newCostAttempt(req Request) *llm.RequestCostAttempt { + if a.kind != apiOpenAI { + return nil + } + attempt := &llm.RequestCostAttempt{RuntimeID: "openai-api-key", RequestedModel: req.Model, EndpointKind: "custom_or_unverified", TransportMode: "opaque", Dispatch: "not_dispatched", BodyState: "missing", Issues: []llm.CostEvidenceGap{}} + if req.Fast { + tier := "fast" + attempt.RequestedServiceTier = &tier + } + if a.ownedOpenAIClient != nil && a.httpClient == a.ownedOpenAIClient { + attempt.TransportMode = "single_send_v1" + } + return attempt +} + +func singleCostEvidence(a *llm.RequestCostAttempt) *llm.RequestCostEvidence { + if a == nil { + return nil + } + return llm.NormalizeRequestCostEvidence(&llm.RequestCostEvidence{Version: 1, Source: "openai_responses", Scope: "invocation", AttemptsObserved: 1, AttemptCountExact: a.Dispatch == "not_dispatched" || a.TransportMode == "single_send_v1", Gaps: []llm.CostEvidenceGap{}, Attempts: []llm.RequestCostAttempt{*a}}) +} + +// costObject isolates known keys, detecting duplicates without last-key-wins. +// Full-document syntax must be validated by the caller first. Unknown keys are +// ignored and no arbitrary provider metadata enters the returned evidence. +func costObject(data []byte, known ...string) (map[string]json.RawMessage, map[string]bool, bool) { + allowed := map[string]bool{} + for _, key := range known { + allowed[key] = true + } + fields := map[string]json.RawMessage{} + duplicates := map[string]bool{} + d := json.NewDecoder(bytes.NewReader(data)) + token, err := d.Token() + if err != nil || token != json.Delim('{') { + return nil, nil, false + } + for d.More() { + token, err = d.Token() + if err != nil { + return nil, nil, false + } + key, ok := token.(string) + if !ok { + return nil, nil, false + } + var raw json.RawMessage + if d.Decode(&raw) != nil { + return nil, nil, false + } + if !allowed[key] { + continue + } + if _, exists := fields[key]; exists { + duplicates[key] = true + } + fields[key] = raw + } + _, err = d.Token() + return fields, duplicates, err == nil +} + +func observeCostEnvelope(a *llm.RequestCostAttempt, body []byte) { + if a == nil { + return + } + if !utf8.Valid(body) || !json.Valid(body) { + a.BodyState = "malformed" + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapInvalid) + return + } + fields, duplicates, ok := costObject(body, "id", "model", "service_tier", "status", "usage") + if !ok { + a.BodyState = "malformed" + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapInvalid) + return + } + for _, field := range []struct { + key string + target **string + limit int + }{{"id", &a.ProviderResponseID, 256}, {"model", &a.ObservedModel, 256}, {"service_tier", &a.ObservedServiceTier, 64}, {"status", &a.ResponseStatus, 64}} { + raw, exists := fields[field.key] + if duplicates[field.key] { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapDuplicate) + continue + } + if !exists || bytes.Equal(raw, []byte("null")) { + continue + } + var value string + if !llm.CostJSONUnicodeValid(raw) || json.Unmarshal(raw, &value) != nil || !llm.SafeCostString(value, field.limit) { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapIdentity) + continue + } + *field.target = &value + } + if duplicates["usage"] { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapDuplicate) + return + } + raw, exists := fields["usage"] + if !exists || bytes.Equal(raw, []byte("null")) { + return + } + usage, dups, ok := costObject(raw, "input_tokens", "output_tokens", "total_tokens", "input_tokens_details") + if !ok { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapUsage) + return + } + u := &llm.OpenAIRequestUsage{} + a.Usage = u + for _, field := range []struct { + key string + target **int64 + }{{"input_tokens", &u.InputTokens}, {"output_tokens", &u.OutputTokens}, {"total_tokens", &u.TotalTokens}} { + decodeCostCounter(a, usage, dups, field.key, field.target) + } + if dups["input_tokens_details"] { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapDuplicate) + return + } + details, exists := usage["input_tokens_details"] + if !exists || bytes.Equal(details, []byte("null")) { + return + } + detailFields, detailDups, ok := costObject(details, "cached_tokens", "cache_write_tokens") + if !ok { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapUsage) + return + } + decodeCostCounter(a, detailFields, detailDups, "cached_tokens", &u.CacheReadTokens) + decodeCostCounter(a, detailFields, detailDups, "cache_write_tokens", &u.CacheWriteTokens) +} + +func decodeCostCounter(a *llm.RequestCostAttempt, fields map[string]json.RawMessage, duplicates map[string]bool, key string, target **int64) { + if duplicates[key] { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapDuplicate) + return + } + raw, exists := fields[key] + if !exists || bytes.Equal(raw, []byte("null")) { + return + } + var value int64 + if json.Unmarshal(raw, &value) != nil { + a.Issues = llm.AddCostGap(a.Issues, llm.CostGapUsage) + return + } + *target = &value +} diff --git a/internal/llmadapters/request_cost_evidence_test.go b/internal/llmadapters/request_cost_evidence_test.go new file mode 100644 index 0000000..13bb75f --- /dev/null +++ b/internal/llmadapters/request_cost_evidence_test.go @@ -0,0 +1,360 @@ +package llmadapters + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/llm" +) + +const costWire = `{"id":"resp-synthetic","model":"observed-model","service_tier":"priority","status":"completed","usage":{"input_tokens":10,"output_tokens":3,"total_tokens":13,"input_tokens_details":{"cached_tokens":2,"cache_write_tokens":0}},"output_text":"{\"ok\":true}"}` + +type costRoundTrip func(*http.Request) (*http.Response, error) + +func (f costRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } +func costBodyAdapter(t *testing.T, status int, body io.ReadCloser) *APIAdapter { + t.Helper() + a, err := newAPIAdapter(apiOpenAI, APIOptions{APIKey: "synthetic-test-key", HTTPClient: &http.Client{Transport: costRoundTrip(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: status, Status: fmt.Sprint(status), Body: body, Header: make(http.Header)}, nil + })}}) + if err != nil { + t.Fatal(err) + } + return a +} +func oneCostAttempt(t *testing.T, r Response) llm.RequestCostAttempt { + t.Helper() + if r.RequestCostEvidence == nil || len(r.RequestCostEvidence.Attempts) != 1 { + t.Fatalf("missing bounded receipt: %#v", r.RequestCostEvidence) + } + return r.RequestCostEvidence.Attempts[0] +} + +func TestOpenAIRequestCostRetainsUsageAndObservedIdentity(t *testing.T) { + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(costWire))) + _, response, err := a.execute(context.Background(), Request{Model: "requested-model", Fast: true}) + if err != nil { + t.Fatal(err) + } + r := oneCostAttempt(t, response) + if r.RequestedModel != "requested-model" || *r.ObservedModel != "observed-model" || *r.RequestedServiceTier != "fast" || *r.ObservedServiceTier != "priority" || *r.Usage.CacheWriteTokens != 0 { + t.Fatalf("collapsed request/delivered values: %#v", r) + } + if r.TransportMode != "opaque" || response.RequestCostEvidence.AttemptCountExact { + t.Fatal("injected client gained ownership") + } + *r.Usage.InputTokens = 999 + if *response.RequestCostEvidence.Attempts[0].Usage.InputTokens != 999 { + t.Fatal("unexpected local pointer semantics") + } + clone := llm.CloneRequestCostEvidence(response.RequestCostEvidence) + *clone.Attempts[0].Usage.InputTokens = 1 + if *response.RequestCostEvidence.Attempts[0].Usage.InputTokens != 999 { + t.Fatal("detached copy aliases evidence") + } +} + +func TestOpenAIRequestCostInvalidCountersMissingNullAndDuplicateFields(t *testing.T) { + for _, value := range []string{`null`, `-1`, `10.5`, `"10"`, `true`, `[]`, `9223372036854775808`} { + body := strings.Replace(costWire, `"cache_write_tokens":0`, `"cache_write_tokens":`+value, 1) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, _ := a.execute(context.Background(), Request{Model: "request"}) + r := oneCostAttempt(t, response) + if r.Usage == nil || *r.Usage.InputTokens != 10 || !slices.Contains(r.Issues, llm.CostGapUsage) { + t.Fatalf("invalid counter erased safe usage or became eligible: %s %#v", value, r) + } + if value == `-1` && (r.Usage.CacheWriteTokens == nil || *r.Usage.CacheWriteTokens != -1) { + t.Fatal("negative observation not retained") + } + if value != `-1` && r.Usage.CacheWriteTokens != nil { + t.Fatal("invalid/null counter manufactured a value") + } + } + for _, body := range []string{ + strings.Replace(costWire, `"usage":{`, `"usage":null,"ignored_usage":{`, 1), + strings.Replace(costWire, `"input_tokens_details":{`, `"input_tokens_details":null,"ignored_details":{`, 1), + strings.Replace(costWire, `"cache_write_tokens":0`, `"cache_write_tokens":0,"cache_write_tokens":1`, 1), + strings.Replace(costWire, `"id":"resp-synthetic"`, `"id":"first","id":"second"`, 1), + strings.Replace(costWire, `"input_tokens":10`, `"input_tokens":10,"input_tokens":11`, 1), + strings.Replace(costWire, `"cached_tokens":2`, `"cached_tokens":11`, 1), + strings.Replace(costWire, `"total_tokens":13`, `"total_tokens":14`, 1), + } { + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, _ := a.execute(context.Background(), Request{Model: "request"}) + r := oneCostAttempt(t, response) + if !slices.Contains(r.Issues, llm.CostGapUsage) && !slices.Contains(r.Issues, llm.CostGapDuplicate) { + t.Fatalf("malformed evidence became eligible: %#v", r) + } + } +} + +func TestOpenAIRequestCostNoTextMalformedOutputAndErrorStatus(t *testing.T) { + for _, tc := range []struct { + name, body string + status int + bodyState string + }{ + {"completed refusal", strings.Replace(costWire, `"output_text":"{\"ok\":true}"`, `"output":[{"type":"message","content":[{"type":"refusal","refusal":"synthetic-secret-canary"}]}]`, 1), 200, "complete"}, + {"incomplete", strings.Replace(strings.Replace(costWire, `"status":"completed"`, `"status":"incomplete"`, 1), `"output_text":"{\"ok\":true}"`, `"output":[]`, 1), 200, "complete"}, + {"output shape", strings.Replace(costWire, `"output_text":"{\"ok\":true}"`, `"output":42`, 1), 200, "complete"}, + {"HTTP error", costWire, 429, "complete"}, + {"truncated", costWire[:len(costWire)-1], 200, "malformed"}, + } { + t.Run(tc.name, func(t *testing.T) { + a := costBodyAdapter(t, tc.status, io.NopCloser(strings.NewReader(tc.body))) + _, response, err := a.execute(context.Background(), Request{Model: "request"}) + if err == nil { + t.Fatal("task error converted to success") + } + r := oneCostAttempt(t, response) + if r.BodyState != tc.bodyState || *r.HTTPStatus != tc.status { + t.Fatalf("receipt=%#v", r) + } + if tc.bodyState == "complete" && (r.Usage == nil || *r.Usage.InputTokens != 10) { + t.Fatal("no-text/error discarded usage") + } + if tc.bodyState != "complete" && r.Usage != nil { + t.Fatal("malformed JSON prefix scraped") + } + encoded, _ := json.Marshal(response.RequestCostEvidence) + if strings.Contains(string(encoded), "synthetic-secret-canary") { + t.Fatal("refusal text leaked into cost evidence") + } + }) + } +} + +type failingCostBody struct{} + +func (failingCostBody) Read([]byte) (int, error) { return 0, io.ErrUnexpectedEOF } +func (failingCostBody) Close() error { return nil } +func TestOpenAIRequestCostTransportBodyLimitAndCancellation(t *testing.T) { + for _, tc := range []struct { + body io.ReadCloser + state string + }{{failingCostBody{}, "read_error"}, {io.NopCloser(strings.NewReader(strings.Repeat("x", apiResponseLogLimit+1))), "over_limit"}} { + a := costBodyAdapter(t, 200, tc.body) + _, r, err := a.execute(context.Background(), Request{Model: "request"}) + if err == nil { + t.Fatal("body error lost") + } + receipt := oneCostAttempt(t, r) + if receipt.BodyState != tc.state || receipt.Dispatch != "possibly_dispatched" || receipt.Usage != nil { + t.Fatalf("receipt=%#v", receipt) + } + } + var calls atomic.Int32 + a, err := newAPIAdapter(apiOpenAI, APIOptions{APIKey: "synthetic-test-key", HTTPClient: &http.Client{Transport: costRoundTrip(func(*http.Request) (*http.Response, error) { calls.Add(1); return nil, context.DeadlineExceeded })}}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, r, _ := a.execute(ctx, Request{Model: "request"}) + if calls.Load() != 0 || oneCostAttempt(t, r).Dispatch != "not_dispatched" { + t.Fatal("pre-Do cancel dispatched") + } + _, r, _ = a.execute(context.Background(), Request{Model: "request"}) + if calls.Load() != 1 || oneCostAttempt(t, r).Dispatch != "possibly_dispatched" { + t.Fatal("Do error incorrectly free") + } + ctx, cancel = context.WithCancel(context.Background()) + a.httpClient = &http.Client{Transport: costRoundTrip(func(*http.Request) (*http.Response, error) { + cancel() + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(costWire)), Header: make(http.Header)}, nil + })} + _, r, _ = a.execute(ctx, Request{Model: "request"}) + if oneCostAttempt(t, r).Usage == nil { + t.Fatal("complete response lost on cancellation") + } +} + +// This _test.go-only assembly seam redirects the adapter-owned standard TLS +// transport to a synthetic local server. No production HTTPClient trust toggle +// is exposed, and arbitrary injected clients remain opaque. +func ownedCostTestAdapter(t *testing.T, handler http.Handler) (*APIAdapter, *httptest.Server) { + t.Helper() + server := httptest.NewTLSServer(handler) + t.Cleanup(server.Close) + a, err := newAPIAdapter(apiOpenAI, APIOptions{APIKey: "synthetic-test-key"}) + if err != nil { + t.Fatal(err) + } + transport := a.ownedOpenAIClient.Transport.(*http.Transport) + transport.Proxy = nil // Route this synthetic owned-transport fixture locally even when HTTPS_PROXY is set. + pool := x509.NewCertPool() + pool.AddCert(server.Certificate()) + transport.TLSClientConfig = &tls.Config{RootCAs: pool, ServerName: server.Certificate().DNSNames[0], MinVersion: tls.VersionTLS12} + transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String()) + } + t.Cleanup(transport.CloseIdleConnections) + return a, server +} +func TestOpenAIOwnedClientStopsRedirectsAndHasNoReplayBody(t *testing.T) { + for _, status := range []int{301, 302, 303, 307, 308} { + var calls atomic.Int32 + a, _ := ownedCostTestAdapter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.Header().Set("Location", "https://api.openai.com/again") + w.WriteHeader(status) + })) + _, response, err := a.execute(context.Background(), Request{Model: "requested", Prompt: "synthetic"}) + if err == nil || calls.Load() != 1 { + t.Fatalf("redirect %d calls=%d err=%v", status, calls.Load(), err) + } + r := oneCostAttempt(t, response) + if r.TransportMode != "single_send_v1" || r.EndpointKind != "official_global" { + t.Fatal("owned transport classification missing") + } + } + a, _ := ownedCostTestAdapter(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + _, _ = io.WriteString(w, costWire) + })) + original := a.httpClient + // Inspect the outgoing GetBody through a test-only opaque wrapper. The + // wrapper is intentionally NOT eligible as a single-send producer. + a.httpClient = &http.Client{Transport: costRoundTrip(func(r *http.Request) (*http.Response, error) { + if r.GetBody != nil { + t.Error("replayable body") + } + return original.Transport.RoundTrip(r) + })} + _, response, err := a.execute(context.Background(), Request{Model: "requested"}) + if err != nil { + t.Fatal(err) + } + if oneCostAttempt(t, response).TransportMode != "opaque" { + t.Fatal("test wrapper inherited owned trust") + } +} + +func TestOpenAICostEnvelopeUnknownFieldsAndUnsafeValues(t *testing.T) { + for _, field := range []string{"id", "model", "service_tier", "status"} { + var body map[string]json.RawMessage + if json.Unmarshal([]byte(costWire), &body) != nil { + t.Fatal("fixture") + } + delete(body, field) + body["unknown_future"] = json.RawMessage(`{"any":[1,2,3]}`) + encoded, _ := json.Marshal(body) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(string(encoded)))) + _, response, _ := a.execute(context.Background(), Request{Model: "requested"}) + r := oneCostAttempt(t, response) + if len(r.Issues) == 0 || r.Usage == nil { + t.Fatal("unknown fields discarded known usage or missing identity inferred") + } + } + for _, unsafe := range []string{strings.Repeat("x", 257), "model\nsecret", "model\x00secret"} { + encoded, _ := json.Marshal(unsafe) + body := strings.Replace(costWire, `"observed-model"`, string(encoded), 1) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, _ := a.execute(context.Background(), Request{Model: "requested"}) + if oneCostAttempt(t, response).ObservedModel != nil { + t.Fatal("unsafe identity retained/truncated") + } + } + if officialResponsesEndpoint("https://api.openai.com/v1/responses?secret=x") || officialResponsesEndpoint("https://gateway.example/v1/responses") || officialResponsesEndpoint("http://api.openai.com/v1/responses") { + t.Fatal("unverified route qualified") + } +} + +func TestOpenAIRequestCostEachMissingOrNullCounterRemainsUnknown(t *testing.T) { + for _, field := range []string{"input_tokens", "output_tokens", "cached_tokens", "cache_write_tokens"} { + for _, missing := range []bool{false, true} { + var body map[string]any + if err := json.Unmarshal([]byte(costWire), &body); err != nil { + t.Fatal(err) + } + usage := body["usage"].(map[string]any) + target := usage + if field == "cached_tokens" || field == "cache_write_tokens" { + target = usage["input_tokens_details"].(map[string]any) + } + if missing { + delete(target, field) + } else { + target[field] = nil + } + encoded, _ := json.Marshal(body) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(string(encoded)))) + _, response, _ := a.execute(context.Background(), Request{Model: "request"}) + receipt := oneCostAttempt(t, response) + if receipt.Usage == nil || !slices.Contains(receipt.Issues, llm.CostGapUsage) { + t.Fatalf("%s absence manufactured zero", field) + } + } + } +} +func TestOpenAIRequestCostConstructionAndUnknownDeliveredTier(t *testing.T) { + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(costWire))) + _, response, err := a.execute(nil, Request{Model: "request"}) //nolint:staticcheck // SA1012: nil deliberately tests request-construction failure before dispatch. + if err == nil || oneCostAttempt(t, response).Dispatch != "not_dispatched" { + t.Fatal("construction failure dispatched") + } + body := strings.Replace(costWire, `"service_tier":"priority"`, `"service_tier":"future-tier"`, 1) + a = costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, err = a.execute(context.Background(), Request{Model: "request"}) + if err != nil { + t.Fatal(err) + } + receipt := oneCostAttempt(t, response) + if receipt.ObservedServiceTier == nil || *receipt.ObservedServiceTier != "future-tier" || !slices.Contains(receipt.Issues, llm.CostGapIdentity) { + t.Fatal("unknown delivered tier inferred or discarded") + } +} +func TestOpenAIOwnedExplicitRetryProducesIndependentReceipts(t *testing.T) { + var calls atomic.Int32 + a, _ := ownedCostTestAdapter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(429) + _, _ = io.WriteString(w, `{"error":{"message":"synthetic transient"}}`) + return + } + _, _ = io.WriteString(w, costWire) + })) + result, err := llm.RunStructuredWithSessionResume(context.Background(), a, "", Request{Model: "requested"}, func(b []byte) (bool, error) { + var payload struct { + OK bool `json:"ok"` + } + err := json.Unmarshal(b, &payload) + return payload.OK, err + }) + if err != nil || !result.Value || calls.Load() != 2 || len(result.Response.RequestCostEvidence.Attempts) != 2 { + t.Fatalf("explicit retry err=%v history=%#v", err, result.Response.RequestCostEvidence) + } + if !slices.Contains(result.Response.RequestCostEvidence.Attempts[0].Issues, llm.CostGapDispatch) { + t.Fatal("HTTP error erased by later success") + } +} + +func TestOpenAIRequestCostUnicodeIdentityIsNeverReplaced(t *testing.T) { + for _, raw := range []string{`"\ud800"`, `"\udc00"`, `"\ud800\u0041"`} { + body := strings.Replace(costWire, `"observed-model"`, raw, 1) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, _ := a.execute(context.Background(), Request{Model: "request"}) + receipt := oneCostAttempt(t, response) + if receipt.ObservedModel != nil || receipt.Usage == nil || !slices.Contains(receipt.Issues, llm.CostGapIdentity) { + t.Fatal("surrogate changed identity or erased unrelated usage") + } + } + body := strings.Replace(costWire, `"observed-model"`, "\"\xff\"", 1) + a := costBodyAdapter(t, 200, io.NopCloser(strings.NewReader(body))) + _, response, _ := a.execute(context.Background(), Request{Model: "request"}) + receipt := oneCostAttempt(t, response) + if receipt.BodyState != "malformed" || receipt.ObservedModel != nil || receipt.Usage != nil { + t.Fatal("invalid UTF-8 whole document prefix parsed") + } +} diff --git a/internal/llmlifecycle/lifecycle.go b/internal/llmlifecycle/lifecycle.go index ce351e9..54e0e68 100644 --- a/internal/llmlifecycle/lifecycle.go +++ b/internal/llmlifecycle/lifecycle.go @@ -202,31 +202,34 @@ func (d SessionDraft) ToLedger(runID string) ledger.Session { // Metadata is the durable descriptor for one structured LLM task. type Metadata struct { - SchemaVersion int `json:"schema_version"` - TaskID string `json:"task_id"` - Phase string `json:"phase"` - DependencyTaskIDs []string `json:"dependency_task_ids,omitempty"` - InputFingerprint string `json:"input_fingerprint"` - AgentID string `json:"agent_id,omitempty"` - Status Status `json:"status"` - SessionRowID string `json:"session_row_id,omitempty"` - ProviderSessionID string `json:"provider_session_id,omitempty"` - Adapter string `json:"adapter"` - Model string `json:"model"` - Effort string `json:"effort,omitempty"` - LogPath string `json:"log_path,omitempty"` - ValidatedOutputPath string `json:"validated_output_path,omitempty"` - Error string `json:"error,omitempty"` - TokensIn *int `json:"tokens_in,omitempty"` - TokensOut *int `json:"tokens_out,omitempty"` - CacheRead *int `json:"cache_read,omitempty"` - CacheCreate *int `json:"cache_create,omitempty"` - CacheCreate5m *int `json:"cache_create_5m,omitempty"` - CacheCreate1h *int `json:"cache_create_1h,omitempty"` - CostUSD *float64 `json:"cost_usd,omitempty"` - Speed string `json:"speed,omitempty"` - ReviewerToolEvidence *llm.ReviewerToolEvidence `json:"reviewer_tool_evidence,omitempty"` - Attempts []AttemptMetadata `json:"attempts,omitempty"` + requestCostScopeValid bool + SchemaVersion int `json:"schema_version"` + TaskID string `json:"task_id"` + Phase string `json:"phase"` + DependencyTaskIDs []string `json:"dependency_task_ids,omitempty"` + InputFingerprint string `json:"input_fingerprint"` + AgentID string `json:"agent_id,omitempty"` + Status Status `json:"status"` + SessionRowID string `json:"session_row_id,omitempty"` + ProviderSessionID string `json:"provider_session_id,omitempty"` + Adapter string `json:"adapter"` + Model string `json:"model"` + Effort string `json:"effort,omitempty"` + LogPath string `json:"log_path,omitempty"` + ValidatedOutputPath string `json:"validated_output_path,omitempty"` + Error string `json:"error,omitempty"` + TokensIn *int `json:"tokens_in,omitempty"` + TokensOut *int `json:"tokens_out,omitempty"` + CacheRead *int `json:"cache_read,omitempty"` + CacheCreate *int `json:"cache_create,omitempty"` + CacheCreate5m *int `json:"cache_create_5m,omitempty"` + CacheCreate1h *int `json:"cache_create_1h,omitempty"` + CostUSD *float64 `json:"cost_usd,omitempty"` + Speed string `json:"speed,omitempty"` + ReviewerToolEvidence *llm.ReviewerToolEvidence `json:"reviewer_tool_evidence,omitempty"` + RequestCostEvidence *llm.RequestCostEvidence `json:"request_cost_evidence,omitempty"` + RequestCostCheckpoint *RequestCostCheckpointBinding `json:"request_cost_checkpoint,omitempty"` + Attempts []AttemptMetadata `json:"attempts,omitempty"` } // AttemptMetadata records one invalid structured-output attempt. @@ -268,7 +271,23 @@ func (e *TaskError) Status() Status { } // RunStructured executes or loads one durable structured LLM task. -func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (Result[T], error) { +func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (result Result[T], resultErr error) { + if err := validateRequest(req, decode); err != nil { + return result, err + } + lock, err := taskCostLock(req.Paths, req.TaskID) + if err != nil { + return result, err + } + defer func() { + if releaseErr := lock.Release(); releaseErr != nil { + resultErr = errors.Join(resultErr, releaseErr) + } + }() + return runStructuredLocked(ctx, req, decode) +} + +func runStructuredLocked[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (Result[T], error) { var zero Result[T] if err := validateRequest(req, decode); err != nil { return zero, err @@ -276,7 +295,7 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T if strings.TrimSpace(req.RunID) == "" && !req.AllowNoRunCache { return zero, fmt.Errorf("llmlifecycle: structured task %q requires a persisted run", req.TaskID) } - if loaded, ok, err := LoadStructured(ctx, req, decode); err != nil || ok { + if loaded, ok, err := loadStructuredLocked(ctx, req, decode); err != nil || ok { return loaded, err } @@ -288,6 +307,15 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T resumeSessionID = taskSessionID } } + rowID := req.NewSessionRowID() + var checkpoint *costCheckpoint + if llm.RequestCostSource(req.Adapter) != "" { + var err error + checkpoint, err = beginCostGeneration(req, rowID) + if err != nil { + return zero, err + } + } progressEvent := NewProgressEvent(req, resumeSessionID) progressSpan := startProgress(req.Progress, progressEvent) @@ -302,7 +330,7 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T structured, runErr := llm.RunStructuredWithSessionResume(ctx, req.Adapter, resumeSessionID, request, decode) completed := now() draft := SessionDraft{ - RowID: req.NewSessionRowID(), + RowID: rowID, ProviderReportedSessionID: structured.SessionID, ProviderSessionID: structured.SessionID, Role: req.Role, @@ -325,9 +353,21 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T role = ledger.SessionRoleOrchestrator } draft.Role = role + if checkpoint != nil { + finalErr := finishCostGeneration(req.Paths, checkpoint, structured.Response.RequestCostEvidence) + if finalErr != nil { + costGap(checkpoint, llm.CostGapCheckpoint) + } + draft.Response.RequestCostEvidence = llm.CloneRequestCostEvidence(checkpoint.Evidence) + if finalErr != nil { + endProgress(progressSpan, finalErr, ProgressResult{Usage: draft.Response.Usage}) + return Result[T]{Draft: draft}, errors.Join(runErr, fmt.Errorf("llmlifecycle: finalize request-cost checkpoint: %w", finalErr)) + } + } hasRun := strings.TrimSpace(req.RunID) != "" meta := BaseMetadata(req, draft) + meta.RequestCostCheckpoint = checkpointBinding(checkpoint) if !hasRun { meta.SessionRowID = "" } @@ -336,14 +376,14 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T if req.Store == nil { err := fmt.Errorf("llmlifecycle: store is required") endProgress(progressSpan, err, progressResult(meta, structured, false, structured.Response.Usage)) - return zero, err + return Result[T]{Draft: draft, Session: session}, err } session = draft.ToLedger(req.RunID) if err := req.Store.InsertSession(ctx, session); err != nil { meta.Status = StatusFailedBlocking meta.ProviderSessionID = draft.ProviderSessionID endProgress(progressSpan, err, progressResult(meta, structured, false, structured.Response.Usage)) - return zero, err + return Result[T]{Draft: draft, Session: session}, err } } @@ -353,7 +393,7 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T meta.ProviderSessionID = draft.ProviderSessionID if err := WriteSuccess(req.Paths, &meta, structured.AcceptedOutput); err != nil { endProgress(progressSpan, err, progressResult(meta, structured, false, structured.Response.Usage)) - return zero, err + return Result[T]{Draft: draft, Session: session}, err } endProgress(progressSpan, nil, progressResult(meta, structured, false, structured.Response.Usage)) return Result[T]{Value: structured.Value, Draft: draft, Session: session}, nil @@ -372,7 +412,20 @@ func RunStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T } // LoadStructured loads a reusable task result without calling the provider. -func LoadStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (Result[T], bool, error) { +func LoadStructured[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (result Result[T], loaded bool, resultErr error) { + lock, err := taskCostLock(req.Paths, req.TaskID) + if err != nil { + return result, false, err + } + defer func() { + if releaseErr := lock.Release(); releaseErr != nil { + resultErr = errors.Join(resultErr, releaseErr) + } + }() + return loadStructuredLocked(ctx, req, decode) +} + +func loadStructuredLocked[T any](ctx context.Context, req Request, decode llm.Decoder[T]) (Result[T], bool, error) { var zero Result[T] meta, ok, err := ReadMetadata(req.Paths, req.TaskID) if err != nil || !ok { @@ -381,6 +434,9 @@ func LoadStructured[T any](ctx context.Context, req Request, decode llm.Decoder[ if err := ValidateMetadata(meta, req, adapterName(req.Adapter)); err != nil { return zero, true, err } + if llm.RequestCostSource(req.Adapter) != "" || meta.RequestCostEvidence != nil || meta.Adapter == "openai_api" { + overlayCostMetadata(req.Paths, req.TaskID, req.RunID, &meta) + } switch meta.Status { case StatusSucceeded: outputPath, err := req.Paths.ValidatedOutput(req.TaskID) @@ -417,6 +473,7 @@ func LoadStructured[T any](ctx context.Context, req Request, decode llm.Decoder[ draft := SessionDraftFromLedger(session) draft.Response.Usage.Speed = meta.Speed draft.Response.ReviewerToolEvidence = llm.CloneReviewerToolEvidence(meta.ReviewerToolEvidence) + draft.Response.RequestCostEvidence = llm.CloneRequestCostEvidence(meta.RequestCostEvidence) progress := progressResult(meta, llm.StructuredResult[T]{SessionID: meta.ProviderSessionID}, true, draft.Response.Usage) loadProgress(req.Progress, NewProgressEvent(req, ResumeSessionID(meta)), progress) return Result[T]{Value: value, Draft: draft, Session: session, Cached: true}, true, nil @@ -572,7 +629,7 @@ func ListMetadata(paths Paths) ([]Metadata, error) { return nil, fmt.Errorf("llmlifecycle: read task metadata %q: %w", entry.Name(), err) } var meta Metadata - if err := json.Unmarshal(data, &meta); err != nil { + if err := decodeMetadata(data, &meta); err != nil { return nil, fmt.Errorf("llmlifecycle: decode task metadata %q: %w", entry.Name(), err) } metadata = append(metadata, meta) @@ -593,14 +650,23 @@ func readMetadata(paths Paths, taskID, readError, decodeError string) (Metadata, return Metadata{}, false, fmt.Errorf(readError, taskID, err) } var meta Metadata - if err := json.Unmarshal(data, &meta); err != nil { + if err := decodeMetadata(data, &meta); err != nil { return Metadata{}, false, fmt.Errorf(decodeError, taskID, err) } return meta, true, nil } // ResetIfInputFingerprintChanged removes stale task artifacts when inputs change. -func ResetIfInputFingerprintChanged(paths Paths, taskID, fingerprint string) error { +func ResetIfInputFingerprintChanged(paths Paths, taskID, fingerprint string) (resultErr error) { + lock, err := taskCostLock(paths, taskID) + if err != nil { + return err + } + defer func() { + if releaseErr := lock.Release(); releaseErr != nil { + resultErr = errors.Join(resultErr, releaseErr) + } + }() meta, ok, err := readMetadata(paths, taskID, "pipeline: read LLM task %q metadata: %w", "pipeline: decode LLM task %q metadata: %w", @@ -611,6 +677,13 @@ func ResetIfInputFingerprintChanged(paths Paths, taskID, fingerprint string) err if strings.TrimSpace(meta.InputFingerprint) == strings.TrimSpace(fingerprint) { return nil } + checkpoint, err := reconcileCostCheckpoint(paths, taskID, "", &meta, true) + if err != nil { + return err + } + if err := writeCostCheckpoint(paths, checkpoint); err != nil { + return fmt.Errorf("llmlifecycle: preserve request-cost observations before reset: %w", err) + } taskDir, err := paths.TaskDir(taskID) if err != nil { return err @@ -664,6 +737,12 @@ func WriteMetadata(paths Paths, meta Metadata) error { return err } data, err := json.MarshalIndent(meta, "", " ") + if meta.RequestCostEvidence != nil { + // Admission and the bounded optional-envelope reader use compact JSON. + // Pretty-printing a near-cap cost prefix would inflate it past that + // bound and lose observations when the sidecar is unavailable. + data, err = json.Marshal(meta) + } if err != nil { return err } @@ -702,6 +781,7 @@ func BaseMetadata(req Request, draft SessionDraft) Metadata { CostUSD: draft.Response.Usage.CostUSD, Speed: draft.Response.Usage.Speed, ReviewerToolEvidence: llm.CloneReviewerToolEvidence(draft.Response.ReviewerToolEvidence), + RequestCostEvidence: llm.CloneRequestCostEvidence(draft.Response.RequestCostEvidence), } } @@ -832,6 +912,7 @@ func SessionDraftFromMetadata(meta Metadata) SessionDraft { Effort: meta.Effort, Response: llm.Response{ ReviewerToolEvidence: llm.CloneReviewerToolEvidence(meta.ReviewerToolEvidence), + RequestCostEvidence: llm.CloneRequestCostEvidence(meta.RequestCostEvidence), Usage: llm.Usage{ TokensIn: meta.TokensIn, TokensOut: meta.TokensOut, @@ -871,6 +952,7 @@ func loadOptionalTaskSession(ctx context.Context, store Store, runID string, met draft := SessionDraftFromLedger(session) draft.Response.Usage.Speed = meta.Speed draft.Response.ReviewerToolEvidence = llm.CloneReviewerToolEvidence(meta.ReviewerToolEvidence) + draft.Response.RequestCostEvidence = llm.CloneRequestCostEvidence(meta.RequestCostEvidence) return session, draft, nil } diff --git a/internal/llmlifecycle/request_cost_evidence.go b/internal/llmlifecycle/request_cost_evidence.go new file mode 100644 index 0000000..baec6aa --- /dev/null +++ b/internal/llmlifecycle/request_cost_evidence.go @@ -0,0 +1,665 @@ +package llmlifecycle + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/open-cli-collective/codereview-cli/internal/fsatomic" + "github.com/open-cli-collective/codereview-cli/internal/llm" + "github.com/open-cli-collective/codereview-cli/internal/runlock" +) + +const maxCostGenerations = 64 + +// RequestCostCheckpointBinding binds optional telemetry, never output reuse. +type RequestCostCheckpointBinding struct { + Version int `json:"version"` + ArtifactScope string `json:"artifact_scope"` + RunID string `json:"run_id"` + TaskID string `json:"task_id"` + Sequence uint64 `json:"sequence"` + Digest string `json:"digest"` +} + +type costGeneration struct { + GenerationID string `json:"generation_id"` + InputFingerprint string `json:"input_fingerprint"` + EvidenceDigest string `json:"evidence_digest"` +} + +type pendingCostGeneration struct { + GenerationID string `json:"generation_id"` + InputFingerprint string `json:"input_fingerprint"` +} + +// Map-free fixed-order v1 digest projection; only self-digest is excluded. +type costCheckpointProjection struct { + Version int `json:"version"` + ArtifactScope string `json:"artifact_scope"` + RunID string `json:"run_id"` + TaskID string `json:"task_id"` + Sequence uint64 `json:"sequence"` + FreshOrigin bool `json:"fresh_origin"` + GenerationsObserved uint64 `json:"generations_observed"` + OmittedGenerations uint64 `json:"omitted_generations"` + Evidence *llm.RequestCostEvidence `json:"evidence"` + Generations []costGeneration `json:"generations"` + Pending *pendingCostGeneration `json:"pending"` +} + +type costCheckpoint struct { + costCheckpointProjection + Digest string `json:"digest"` +} + +// RequestCostCheckpoint lives outside the resettable task directory but inside +// the caller/run-owned artifact root covered by existing retention/purge. +func (p Paths) RequestCostCheckpoint(taskID string) (string, error) { + if _, err := p.TaskDir(taskID); err != nil { + return "", err + } + // Keep suffixes and the atomic writer's .tmp inside NAME_MAX even when + // the existing encoded task directory already uses that component budget. + identity := sha256.Sum256([]byte(strings.TrimSpace(taskID))) + return filepath.Join(p.LLMTasksDir, ".request-cost", hex.EncodeToString(identity[:])+".json"), nil +} + +func taskCostLock(paths Paths, taskID string) (*runlock.Lock, error) { + p, err := paths.RequestCostCheckpoint(taskID) + if err != nil { + return nil, err + } + lock, err := runlock.Acquire(strings.TrimSuffix(p, ".json") + ".lock") + if err != nil { + return nil, fmt.Errorf("llmlifecycle: acquire task ownership: %w", err) + } + return lock, nil +} + +func artifactCostScope(paths Paths) (string, error) { + p, err := filepath.Abs(paths.LLMTasksDir) + if err != nil { + return "", err + } + h := sha256.Sum256([]byte(filepath.Clean(p))) + return hex.EncodeToString(h[:]), nil +} + +func newCostCheckpoint(paths Paths, taskID, runID string) (*costCheckpoint, error) { + scope, err := artifactCostScope(paths) + if err != nil { + return nil, err + } + if !llm.SafeCostString(strings.TrimSpace(taskID), 256) || (runID != "" && !llm.SafeCostString(runID, 256)) { + return nil, errors.New("llmlifecycle: invalid request-cost scope identity") + } + e := llm.UnknownRequestCostEvidence(llm.CostGapLegacy) + e.Scope = "task_artifact_history" + e.ArtifactScope = scope + e.TaskID = strings.TrimSpace(taskID) + e.RunID = runID + return &costCheckpoint{costCheckpointProjection: costCheckpointProjection{Version: 1, ArtifactScope: scope, TaskID: e.TaskID, RunID: runID, Evidence: e, Generations: []costGeneration{}}}, nil +} + +func costCheckpointBytes(c *costCheckpoint) ([]byte, error) { + return json.Marshal(c.costCheckpointProjection) +} +func checkpointDigest(c *costCheckpoint) (string, error) { + b, err := costCheckpointBytes(c) + if err != nil { + return "", err + } + h := sha256.New() + _, _ = h.Write([]byte("cr-request-cost-checkpoint/v1\n")) + _, _ = h.Write(b) + return hex.EncodeToString(h.Sum(nil)), nil +} +func costGap(c *costCheckpoint, gap llm.CostEvidenceGap) { + c.Evidence.Gaps = llm.AddCostGap(c.Evidence.Gaps, gap) + c.Evidence.AttemptCountExact = false +} +func isCostDigest(s string) bool { + if len(s) != 64 { + return false + } + _, err := hex.DecodeString(s) + return err == nil +} + +func readCostCheckpoint(paths Paths, taskID, runID string, anyRun bool) (*costCheckpoint, llm.CostEvidenceGap) { + p, err := paths.RequestCostCheckpoint(taskID) + if err != nil { + return nil, llm.CostGapCheckpoint + } + f, err := os.Open(p) // #nosec G304 -- path is derived from the caller-owned root. + if err != nil { + return nil, llm.CostGapCheckpoint + } + defer f.Close() + b, err := io.ReadAll(io.LimitReader(f, llm.MaxRequestCostEvidenceBytes+1)) + if err != nil { + return nil, llm.CostGapCheckpoint + } + if len(b) > llm.MaxRequestCostEvidenceBytes { + return nil, llm.CostGapOverflow + } + var c costCheckpoint + if !llm.RequestCostJSONValid(b) || json.Unmarshal(b, &c) != nil || c.Version != 1 || c.Evidence == nil { + return nil, llm.CostGapCheckpoint + } + var rawScope struct { + RunID *string `json:"run_id"` + } + if json.Unmarshal(b, &rawScope) != nil || rawScope.RunID == nil || (*rawScope.RunID != "" && !llm.SafeCostString(*rawScope.RunID, 256)) { + return nil, llm.CostGapScope + } + scope, err := artifactCostScope(paths) + if err != nil || c.ArtifactScope != scope || c.TaskID != strings.TrimSpace(taskID) || (!anyRun && c.RunID != runID) { + return nil, llm.CostGapScope + } + if len(c.Generations) > maxCostGenerations || len(c.Evidence.Attempts) > llm.MaxRequestCostAttempts || len(c.Evidence.Gaps) > llm.MaxRequestCostGaps { + return nil, llm.CostGapOverflow + } + if c.FreshOrigin || c.Evidence.Version != 1 || c.Evidence.Source != "openai_responses" || c.Evidence.Scope != "task_artifact_history" || c.Evidence.ArtifactScope != c.ArtifactScope || c.Evidence.RunID != c.RunID || c.Evidence.TaskID != c.TaskID { + return nil, llm.CostGapScope + } + digest, err := checkpointDigest(&c) + if err != nil || digest != c.Digest { + return nil, llm.CostGapConflict + } + c.Evidence = llm.NormalizeRequestCostEvidence(c.Evidence) + seen := map[string]bool{} + for _, g := range c.Generations { + if !llm.SafeCostString(g.GenerationID, 256) || (g.InputFingerprint != "" && !llm.SafeCostString(g.InputFingerprint, 256)) || (g.EvidenceDigest != "" && !isCostDigest(g.EvidenceDigest)) { + return nil, llm.CostGapInvalid + } + if seen[g.GenerationID] { + costGap(&c, llm.CostGapDuplicate) + } + seen[g.GenerationID] = true + } + if c.Pending != nil && (!llm.SafeCostString(c.Pending.GenerationID, 256) || !llm.SafeCostString(c.Pending.InputFingerprint, 256)) { + return nil, llm.CostGapInvalid + } + if c.Pending != nil && seen[c.Pending.GenerationID] { + costGap(&c, llm.CostGapDuplicate) + } + if c.GenerationsObserved < uint64(len(c.Generations)) { + costGap(&c, llm.CostGapInvalid) + } + validateCostAttemptIdentities(&c) + costGap(&c, llm.CostGapLegacy) + return &c, "" +} + +func validateCostAttemptIdentities(c *costCheckpoint) { + seen := map[string]bool{} + ids := map[string]string{} + ordinals := map[string]uint32{} + for _, a := range c.Evidence.Attempts { + key := fmt.Sprintf("%s/%d", a.GenerationID, a.Ordinal) + if a.GenerationID == "" || a.Ordinal == 0 || a.AdapterAttempt == 0 || (a.ValidationPhase != "initial" && a.ValidationPhase != "correction") || seen[key] || a.Ordinal != ordinals[a.GenerationID]+1 { + costGap(c, llm.CostGapDuplicate) + } + seen[key] = true + ordinals[a.GenerationID] = a.Ordinal + if a.ProviderResponseID != nil { + if old, ok := ids[*a.ProviderResponseID]; ok && old != key { + costGap(c, llm.CostGapDuplicate) + } + ids[*a.ProviderResponseID] = key + } + } +} +func sameCostAttempt(a, b llm.RequestCostAttempt) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) +} + +// Admit only new records into the retained prefix, leaving framing/pending +// capacity. A later generation must not evict an earlier observation to fit. +func appendCostAttempt(c *costCheckpoint, a llm.RequestCostAttempt) bool { + if len(c.Evidence.Attempts) >= llm.MaxRequestCostAttempts { + return false + } + c.Evidence.Attempts = append(c.Evidence.Attempts, a) + b, _ := costCheckpointBytes(c) + if len(b) > llm.MaxRequestCostEvidenceBytes-llm.RequestCostReservedBytes { + c.Evidence.Attempts = c.Evidence.Attempts[:len(c.Evidence.Attempts)-1] + return false + } + return true +} +func appendCostGeneration(c *costCheckpoint, g costGeneration) bool { + if len(c.Generations) >= maxCostGenerations || c.OmittedGenerations != 0 { + return false + } + c.Generations = append(c.Generations, g) + b, _ := costCheckpointBytes(c) + if len(b) > llm.MaxRequestCostEvidenceBytes-llm.RequestCostReservedBytes { + c.Generations = c.Generations[:len(c.Generations)-1] + return false + } + return true +} + +// Repeated snapshots are references, not new charges. A conflicting same-identity +// copy keeps the existing prefix and a sticky ambiguity/loss reason. +func mergeCostObservations(c *costCheckpoint, other *llm.RequestCostEvidence) { + if other == nil { + return + } + e := llm.NormalizeRequestCostEvidence(other) + for _, gap := range e.Gaps { + costGap(c, gap) + } + if e.Scope != "task_artifact_history" || e.ArtifactScope != c.ArtifactScope || e.RunID != c.RunID || e.TaskID != c.TaskID { + costGap(c, llm.CostGapScope) + return + } + index := map[string]llm.RequestCostAttempt{} + for _, a := range c.Evidence.Attempts { + index[fmt.Sprintf("%s/%d", a.GenerationID, a.Ordinal)] = a + } + blocked := c.Evidence.Truncated + var omitted uint64 + for _, a := range e.Attempts { + key := fmt.Sprintf("%s/%d", a.GenerationID, a.Ordinal) + if prior, ok := index[key]; ok { + if !sameCostAttempt(prior, a) { + costGap(c, llm.CostGapConflict) + } + continue + } + if blocked || !appendCostAttempt(c, a) { + blocked = true + c.Evidence.Truncated = true + omitted = llm.SaturatingCostAdd(omitted, 1) + costGap(c, llm.CostGapOverflow) + continue + } + index[key] = a + } + if omitted > c.Evidence.DroppedAttempts { + c.Evidence.DroppedAttempts = omitted + } + if e.AttemptsObserved > c.Evidence.AttemptsObserved { + c.Evidence.AttemptsObserved = e.AttemptsObserved + } + if e.DroppedAttempts > c.Evidence.DroppedAttempts { + c.Evidence.DroppedAttempts = e.DroppedAttempts + } + if c.Evidence.AttemptsObserved < uint64(len(c.Evidence.Attempts)) { + c.Evidence.AttemptsObserved = uint64(len(c.Evidence.Attempts)) + } + c.Evidence.Truncated = c.Evidence.Truncated || e.Truncated + validateCostAttemptIdentities(c) + reconcileKnownCostGenerations(c) +} + +// Metadata-only observations establish a lower bound on observed generations, +// but cannot recreate missing finalized fingerprints/digests. Keep those missing +// descriptors as omissions with a gap rather than reporting a zero count. +func reconcileKnownCostGenerations(c *costCheckpoint) { + known := map[string]bool{} + described := map[string]bool{} + for _, g := range c.Generations { + known[g.GenerationID] = true + described[g.GenerationID] = true + } + for _, a := range c.Evidence.Attempts { + if a.GenerationID != "" { + known[a.GenerationID] = true + } + } + var missing uint64 + for id := range known { + if !described[id] { + missing++ + } + } + if uint64(len(known)) > c.GenerationsObserved { + c.GenerationsObserved = uint64(len(known)) + } + if missing > 0 { + costGap(c, llm.CostGapMissing) + if missing > c.OmittedGenerations { + c.OmittedGenerations = missing + } + } +} + +// The same bounded metadata-plus-sidecar reconciliation is used for execution, +// cache load, and reset, so reset cannot erase the last known A/B observations. +func reconcileCostCheckpoint(paths Paths, taskID, runID string, meta *Metadata, anyRun bool) (*costCheckpoint, error) { + // Reset has no explicit RunID argument. Valid same-root/task metadata is + // authoritative for its known observations; a foreign-run sidecar must not + // win and cause that sole metadata prefix to be discarded before deletion. + if anyRun && meta != nil && meta.requestCostScopeValid && meta.RequestCostEvidence != nil { + e := meta.RequestCostEvidence + scope, err := artifactCostScope(paths) + if err == nil && e.Version == 1 && e.Source == "openai_responses" && e.Scope == "task_artifact_history" && e.ArtifactScope == scope && e.TaskID == strings.TrimSpace(taskID) { + runID = e.RunID + anyRun = false + } + } + // With no valid raw scope, retain an independently valid same-root/task + // sidecar first. A binding is only a reference to a checkpoint: syntactic + // validity alone cannot authorize replacing an existing checkpoint under + // another run identity, including with an invented empty/no-run scope. + c, gap := readCostCheckpoint(paths, taskID, runID, anyRun) + if c == nil { + if anyRun && meta != nil && meta.RequestCostCheckpoint != nil { + b := meta.RequestCostCheckpoint + scope, err := artifactCostScope(paths) + if err == nil && b.Version == 1 && b.ArtifactScope == scope && b.TaskID == strings.TrimSpace(taskID) && (b.RunID == "" || llm.SafeCostString(b.RunID, 256)) { + runID = b.RunID + } + } + var err error + c, err = newCostCheckpoint(paths, taskID, runID) + if err != nil { + return nil, err + } + costGap(c, gap) + } + + if meta != nil { + if strings.TrimSpace(meta.TaskID) != c.TaskID { + costGap(c, llm.CostGapScope) + } else { + b := meta.RequestCostCheckpoint + if meta.requestCostScopeValid && b != nil && b.Digest == c.Digest && b.Sequence == c.Sequence && meta.RequestCostEvidence != nil { + left, _ := llm.RequestCostEvidenceDigest(c.Evidence) + right, _ := llm.RequestCostEvidenceDigest(meta.RequestCostEvidence) + if left != right { + costGap(c, llm.CostGapConflict) + } + } + if meta.requestCostScopeValid { + mergeCostObservations(c, meta.RequestCostEvidence) + } else if meta.RequestCostEvidence != nil { + costGap(c, llm.CostGapScope) + for _, gap := range meta.RequestCostEvidence.Gaps { + costGap(c, gap) + } + } + if meta.RequestCostEvidence != nil && (b == nil || b.Version != 1 || b.ArtifactScope != c.ArtifactScope || b.RunID != c.RunID || b.TaskID != c.TaskID || b.Sequence != c.Sequence || b.Digest != c.Digest) { + costGap(c, llm.CostGapConflict) + } + } + } + if c.Pending != nil { + costGap(c, llm.CostGapInterrupted) + } + c.Evidence = llm.NormalizeRequestCostEvidence(c.Evidence) + return c, nil +} + +func boundCostCheckpoint(c *costCheckpoint) { + c.FreshOrigin = false + costGap(c, llm.CostGapLegacy) + c.Evidence = llm.NormalizeRequestCostEvidence(c.Evidence) + if c.Generations == nil { + c.Generations = []costGeneration{} + } + if len(c.Generations) > maxCostGenerations { + c.OmittedGenerations = llm.SaturatingCostAdd(c.OmittedGenerations, uint64(len(c.Generations[maxCostGenerations:]))) + c.Generations = c.Generations[:maxCostGenerations] + costGap(c, llm.CostGapOverflow) + } + for { + b, _ := costCheckpointBytes(c) + if len(b) <= llm.MaxRequestCostEvidenceBytes-128 { + break + } + costGap(c, llm.CostGapOverflow) + if len(c.Evidence.Attempts) > 0 { + c.Evidence.Attempts = c.Evidence.Attempts[:len(c.Evidence.Attempts)-1] + c.Evidence.DroppedAttempts = llm.SaturatingCostAdd(c.Evidence.DroppedAttempts, 1) + c.Evidence.Truncated = true + } else if len(c.Generations) > 0 { + c.Generations = c.Generations[:len(c.Generations)-1] + c.OmittedGenerations = llm.SaturatingCostAdd(c.OmittedGenerations, 1) + } else { + break + } + } +} + +func writeCostCheckpoint(paths Paths, c *costCheckpoint) error { + boundCostCheckpoint(c) + if c.Sequence == ^uint64(0) { + costGap(c, llm.CostGapOverflow) + } else { + c.Sequence++ + } + digest, err := checkpointDigest(c) + if err != nil { + return err + } + c.Digest = digest + b, err := json.Marshal(c) + if err != nil { + return err + } + if len(b)+1 > llm.MaxRequestCostEvidenceBytes { + return errors.New("llmlifecycle: request-cost checkpoint exceeds limit") + } + p, err := paths.RequestCostCheckpoint(c.TaskID) + if err != nil { + return err + } + // fsatomic is temp-file/rename without fsync. No power-loss claim is made. + return fsatomic.WriteFileAtomic(p, append(b, '\n'), 0o600) +} + +func costFingerprint(req Request) string { + if s := strings.TrimSpace(req.InputFingerprint); s != "" { + return s + } + return Fingerprint(adapterName(req.Adapter), req.TaskID, req.Phase, req.Model, req.Effort, req.Prompt, req.DependencyTaskIDs) +} + +func beginCostGeneration(req Request, id string) (*costCheckpoint, error) { + meta, ok, err := ReadMetadata(req.Paths, req.TaskID) + if err != nil { + return nil, err + } + var observed *Metadata + if ok { + observed = &meta + } + c, err := reconcileCostCheckpoint(req.Paths, req.TaskID, req.RunID, observed, false) + if err != nil { + return nil, err + } + fingerprint := costFingerprint(req) + if !llm.SafeCostString(id, 256) || !llm.SafeCostString(fingerprint, 256) { + return nil, errors.New("llmlifecycle: invalid cost generation identity") + } + for _, g := range c.Generations { + if g.GenerationID == id { + return nil, errors.New("llmlifecycle: duplicate cost generation identity") + } + } + for _, a := range c.Evidence.Attempts { + if a.GenerationID == id { + return nil, errors.New("llmlifecycle: duplicate cost generation identity") + } + } + if c.Pending != nil { + if c.Pending.GenerationID == id { + return nil, errors.New("llmlifecycle: duplicate pending cost generation identity") + } + c.OmittedGenerations = llm.SaturatingCostAdd(c.OmittedGenerations, 1) + costGap(c, llm.CostGapInterrupted) + } + c.Pending = &pendingCostGeneration{GenerationID: id, InputFingerprint: fingerprint} + if err := writeCostCheckpoint(req.Paths, c); err != nil { + return nil, fmt.Errorf("llmlifecycle: checkpoint before dispatch: %w", err) + } + return c, nil +} + +func finishCostGeneration(paths Paths, c *costCheckpoint, invocation *llm.RequestCostEvidence) error { + if c == nil || c.Pending == nil { + return errors.New("llmlifecycle: missing pending cost generation") + } + e := llm.NormalizeRequestCostEvidence(invocation) + if e == nil || e.Scope != "invocation" { + e = llm.UnknownRequestCostEvidence(llm.CostGapMissing) + } + for _, gap := range e.Gaps { + costGap(c, gap) + } + c.Evidence.AttemptsObserved = llm.SaturatingCostAdd(c.Evidence.AttemptsObserved, e.AttemptsObserved) + c.Evidence.DroppedAttempts = llm.SaturatingCostAdd(c.Evidence.DroppedAttempts, e.DroppedAttempts) + // Preserve the invocation's retained prefix even when that invocation was + // truncated; later attempts must not replace an already full task prefix. + wasTruncated := c.Evidence.Truncated + for _, a := range e.Attempts { + a.GenerationID = c.Pending.GenerationID + if wasTruncated || !appendCostAttempt(c, a) { + wasTruncated = true + c.Evidence.Truncated = true + c.Evidence.DroppedAttempts = llm.SaturatingCostAdd(c.Evidence.DroppedAttempts, 1) + costGap(c, llm.CostGapOverflow) + } + } + c.Evidence.Truncated = c.Evidence.Truncated || e.Truncated + digest, err := llm.RequestCostEvidenceDigest(e) + if err != nil { + return err + } + c.GenerationsObserved = llm.SaturatingCostAdd(c.GenerationsObserved, 1) + if !appendCostGeneration(c, costGeneration{GenerationID: c.Pending.GenerationID, InputFingerprint: c.Pending.InputFingerprint, EvidenceDigest: digest}) { + c.OmittedGenerations = llm.SaturatingCostAdd(c.OmittedGenerations, 1) + costGap(c, llm.CostGapOverflow) + } + c.Pending = nil + validateCostAttemptIdentities(c) + return writeCostCheckpoint(paths, c) +} + +func checkpointBinding(c *costCheckpoint) *RequestCostCheckpointBinding { + if c == nil { + return nil + } + return &RequestCostCheckpointBinding{Version: 1, ArtifactScope: c.ArtifactScope, RunID: c.RunID, TaskID: c.TaskID, Sequence: c.Sequence, Digest: c.Digest} +} + +func overlayCostMetadata(paths Paths, taskID, runID string, meta *Metadata) { + c, err := reconcileCostCheckpoint(paths, taskID, runID, meta, false) + if err != nil { + meta.RequestCostEvidence = llm.UnknownRequestCostEvidence(llm.CostGapCheckpoint) + return + } + meta.RequestCostEvidence = llm.CloneRequestCostEvidence(c.Evidence) +} + +// Both schema-4 readers share this decoder. Optional cost corruption must not +// invalidate valid output/tool fields. The existing whole metadata read remains +// unbounded; the nested cost envelope is bounded before typed nested decoding. +func decodeMetadata(data []byte, meta *Metadata) error { + type ordinary Metadata + var raw struct { + *ordinary + Cost json.RawMessage `json:"request_cost_evidence"` + Binding json.RawMessage `json:"request_cost_checkpoint"` + } + raw.ordinary = (*ordinary)(meta) + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + meta.requestCostScopeValid = validRawMetadataCostScope(raw.Cost) + meta.RequestCostEvidence = llm.DecodeRequestCostEvidence(raw.Cost) + meta.RequestCostCheckpoint = nil + if len(raw.Binding) > 0 && string(raw.Binding) != "null" { + var b RequestCostCheckpointBinding + if len(raw.Binding) > llm.RequestCostReservedBytes || !llm.RequestCostJSONValid(raw.Binding) || json.Unmarshal(raw.Binding, &b) != nil || b.Version != 1 || !isCostDigest(b.Digest) || !llm.SafeCostString(b.TaskID, 256) || !isCostDigest(b.ArtifactScope) || (b.RunID != "" && !llm.SafeCostString(b.RunID, 256)) { + if meta.RequestCostEvidence == nil { + meta.RequestCostEvidence = llm.UnknownRequestCostEvidence(llm.CostGapInvalid) + } else { + meta.RequestCostEvidence.Gaps = llm.AddCostGap(meta.RequestCostEvidence.Gaps, llm.CostGapInvalid) + meta.RequestCostEvidence.AttemptCountExact = false + } + } else { + meta.RequestCostCheckpoint = &b + } + } + if duplicateMetadataCostFields(data) { + meta.RequestCostEvidence = llm.UnknownRequestCostEvidence(llm.CostGapDuplicate) + meta.RequestCostCheckpoint = nil + meta.requestCostScopeValid = false + } + return nil +} + +func duplicateMetadataCostFields(data []byte) bool { + d := json.NewDecoder(bytes.NewReader(data)) + token, err := d.Token() + if err != nil || token != json.Delim('{') { + return false + } + seen := map[string]bool{} + for d.More() { + token, err = d.Token() + if err != nil { + return false + } + key, ok := token.(string) + if !ok { + return false + } + canonical := "" + if strings.EqualFold(key, "request_cost_evidence") { + canonical = "request_cost_evidence" + } + if strings.EqualFold(key, "request_cost_checkpoint") { + canonical = "request_cost_checkpoint" + } + if canonical != "" { + if seen[canonical] { + return true + } + seen[canonical] = true + } + var ignored json.RawMessage + if d.Decode(&ignored) != nil { + return false + } + } + return false +} + +// Scope validity is measured on the raw envelope before normalization can omit +// invalid strings. In particular, an omitted invalid RunID is not proof of an +// actual empty/no-run scope. Only this raw fact (or a valid binding) can guide +// reset's scope selection; it never establishes complete accounting history. +func validRawMetadataCostScope(raw json.RawMessage) bool { + if len(raw) == 0 || len(raw) > llm.MaxRequestCostEvidenceBytes || !llm.RequestCostJSONValid(raw) { + return false + } + var scope struct { + Version int `json:"version"` + Source string `json:"source"` + Scope string `json:"scope"` + ArtifactScope string `json:"artifact_scope"` + RunID *string `json:"run_id"` + TaskID string `json:"task_id"` + } + if json.Unmarshal(raw, &scope) != nil { + return false + } + return scope.Version == 1 && scope.Source == "openai_responses" && scope.Scope == "task_artifact_history" && isCostDigest(scope.ArtifactScope) && llm.SafeCostString(scope.TaskID, 256) && scope.RunID != nil && (*scope.RunID == "" || llm.SafeCostString(*scope.RunID, 256)) +} diff --git a/internal/llmlifecycle/request_cost_evidence_test.go b/internal/llmlifecycle/request_cost_evidence_test.go new file mode 100644 index 0000000..71da3fb --- /dev/null +++ b/internal/llmlifecycle/request_cost_evidence_test.go @@ -0,0 +1,814 @@ +package llmlifecycle + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/llm" + "github.com/open-cli-collective/codereview-cli/internal/runlock" +) + +type lifecycleCostAdapter struct{ *llm.FakeAdapter } + +func (*lifecycleCostAdapter) RequestCostEvidenceSource() string { return "openai_responses" } +func lifecycleCostPtr[T any](v T) *T { return &v } +func lifecycleCostResponse(id, text string) llm.Response { + return llm.Response{StructuredOutput: []byte(text), ReviewerToolEvidence: &llm.ReviewerToolEvidence{DiffStatus: llm.DiffToolStatusSucceeded}, RequestCostEvidence: &llm.RequestCostEvidence{ + Version: 1, Source: "openai_responses", Scope: "invocation", AttemptsObserved: 1, AttemptCountExact: true, + Attempts: []llm.RequestCostAttempt{{Ordinal: 1, AdapterAttempt: 1, ValidationPhase: "initial", RuntimeID: "openai-api-key", EndpointKind: "official_global", TransportMode: "single_send_v1", Dispatch: "possibly_dispatched", HTTPStatus: lifecycleCostPtr(200), BodyState: "complete", ProviderResponseID: lifecycleCostPtr(id), ObservedModel: lifecycleCostPtr("observed"), ObservedServiceTier: lifecycleCostPtr("default"), ResponseStatus: lifecycleCostPtr("completed"), RequestedModel: "model-1", Usage: &llm.OpenAIRequestUsage{InputTokens: lifecycleCostPtr(int64(10)), OutputTokens: lifecycleCostPtr(int64(3)), CacheReadTokens: lifecycleCostPtr(int64(2)), CacheWriteTokens: lifecycleCostPtr(int64(0))}}}, + }} +} +func newLifecycleCostAdapter() *lifecycleCostAdapter { + return &lifecycleCostAdapter{&llm.FakeAdapter{NameValue: "openai_api"}} +} +func requireCostAttempts(t *testing.T, e *llm.RequestCostEvidence, ids ...string) { + t.Helper() + if e == nil || len(e.Attempts) != len(ids) { + t.Fatalf("cost history=%#v, want %v", e, ids) + } + for i, id := range ids { + if e.Attempts[i].ProviderResponseID == nil || *e.Attempts[i].ProviderResponseID != id { + t.Fatalf("receipt %d=%#v, want %s", i, e.Attempts[i], id) + } + } + if e.AttemptCountExact || !slices.Contains(e.Gaps, llm.CostGapLegacy) { + t.Fatal("task history acquired an unestablished fresh origin") + } +} + +func TestRequestCostLifecycleSuccessCacheAndValidation(t *testing.T) { + for _, noRun := range []bool{false, true} { + t.Run(fmt.Sprint(noRun), func(t *testing.T) { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{SessionID: "provider-A", Response: lifecycleCostResponse("A", `invalid`)}) + a.Queue(llm.FakeResult{SessionID: "provider-B", Response: lifecycleCostResponse("B", `{"ok":true}`)}) + store := newLifecycleStore() + req := lifecycleRequest(t, store, a) + if noRun { + req.RunID = "" + req.AllowNoRunCache = true + req.Store = nil + } + first, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + requireCostAttempts(t, first.Draft.Response.RequestCostEvidence, "A", "B") + cached, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil || !cached.Cached { + t.Fatalf("cache err=%v", err) + } + requireCostAttempts(t, cached.Draft.Response.RequestCostEvidence, "A", "B") + if !reflect.DeepEqual(first.Draft.Response.RequestCostEvidence, cached.Draft.Response.RequestCostEvidence) || len(a.Requests()) != 2 { + t.Fatal("reload changed/counted history or called provider") + } + if !noRun && len(store.inserted) != 1 { + t.Fatal("cache inserted another session") + } + *cached.Draft.Response.RequestCostEvidence.Attempts[0].Usage.InputTokens = 900 + if *first.Draft.Response.RequestCostEvidence.Attempts[0].Usage.InputTokens != 10 { + t.Fatal("cache aliases response") + } + }) + } +} + +func TestRequestCostIsolatedFailureWithAndWithoutLedgerSession(t *testing.T) { + for _, sessionID := range []string{"", "provider-A"} { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{SessionID: sessionID, Response: lifecycleCostResponse("A", `invalid`)}) + a.Queue(llm.FakeResult{SessionID: sessionID, Response: lifecycleCostResponse("B", `invalid`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + req.FailureStatus = StatusFailedIsolated + first, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err == nil { + t.Fatal("invalid output succeeded") + } + requireCostAttempts(t, first.Draft.Response.RequestCostEvidence, "A", "B") + cached, ok, err := LoadStructured(context.Background(), req, decodeLifecyclePayload) + if !ok || err == nil || !cached.Cached { + t.Fatalf("isolated cache ok=%t err=%v", ok, err) + } + requireCostAttempts(t, cached.Draft.Response.RequestCostEvidence, "A", "B") + if len(a.Requests()) != 2 { + t.Fatal("isolated cache dispatched again") + } + } +} + +func TestRequestCostResetRetainsMetadataPrefixAfterMissingSidecar(t *testing.T) { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `invalid`)}) + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("B", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + first, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + requireCostAttempts(t, first.Draft.Response.RequestCostEvidence, "A", "B") + path, _ := req.Paths.RequestCostCheckpoint(req.TaskID) + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); err != nil { + t.Fatal(err) + } + preserved, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatalf("preserved checkpoint gap=%s", gap) + } + requireCostAttempts(t, preserved.Evidence, "A", "B") + if preserved.GenerationsObserved < 1 || preserved.OmittedGenerations < 1 { + t.Fatal("metadata-only known generation reported as zero") + } + req.InputFingerprint = "changed" + req.Model = "changed-model" + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("C", `{"ok":true}`)}) + last, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + requireCostAttempts(t, last.Draft.Response.RequestCostEvidence, "A", "B", "C") + if !slices.Contains(last.Draft.Response.RequestCostEvidence.Gaps, llm.CostGapCheckpoint) { + t.Fatal("missing sidecar gap healed") + } + final, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" || final.GenerationsObserved < 2 || final.OmittedGenerations < 1 { + t.Fatal("retained A/B/C generation lower bound lost") + } +} + +func TestRequestCostInterruptionAndFinalBeforeLedger(t *testing.T) { + a := newLifecycleCostAdapter() + req := lifecycleRequest(t, newLifecycleStore(), a) + if _, err := beginCostGeneration(req, "interrupted"); err != nil { + t.Fatal(err) + } + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("C", `{"ok":true}`)}) + got, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(got.Draft.Response.RequestCostEvidence.Gaps, llm.CostGapInterrupted) { + t.Fatal("pending generation erased") + } + b := newLifecycleCostAdapter() + b.Queue(llm.FakeResult{SessionID: "reported", Response: lifecycleCostResponse("D", `{"ok":true}`)}) + store := newLifecycleStore() + store.insertErr = errors.New("ledger failure") + req = lifecycleRequest(t, store, b) + got, err = RunStructured(context.Background(), req, decodeLifecyclePayload) + if err == nil { + t.Fatal("ledger failure hidden") + } + requireCostAttempts(t, got.Draft.Response.RequestCostEvidence, "D") + checkpoint, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" || checkpoint.Pending != nil { + t.Fatalf("final checkpoint missing before ledger: %s", gap) + } + requireCostAttempts(t, checkpoint.Evidence, "D") +} + +func TestRequestCostCheckpointFailurePreventsDispatchAndReset(t *testing.T) { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + path, _ := req.Paths.RequestCostCheckpoint(req.TaskID) + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err == nil || len(a.Requests()) != 0 { + t.Fatal("dispatched without pending checkpoint") + } + meta := Metadata{SchemaVersion: SchemaVersion, TaskID: req.TaskID, InputFingerprint: "old"} + if err := WriteMetadata(req.Paths, meta); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "new"); err == nil { + t.Fatal("reset without preservation") + } + if _, ok, _ := ReadMetadata(req.Paths, req.TaskID); !ok { + t.Fatal("reset erased unpreserved metadata") + } +} + +func TestRequestCostBothSchema4ReadersIsolateOptionalCorruption(t *testing.T) { + for _, raw := range []string{`"bad"`, `{"version":2}`, `{"version":1,"attempts":"bad"}`, `{"version":1,"version":1}`, `{"huge":"` + strings.Repeat("x", llm.MaxRequestCostEvidenceBytes) + `"}`} { + paths := Paths{LLMTasksDir: t.TempDir()} + path, _ := paths.Metadata("task") + data := []byte(`{"schema_version":4,"task_id":"task","status":"succeeded","reviewer_tool_evidence":{"diff_status":"succeeded","trace":{"version":1,"source":"pi_rpc","calls":[]}},"request_cost_evidence":` + raw + `}`) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + meta, ok, err := ReadMetadata(paths, "task") + if err != nil || !ok { + t.Fatalf("ReadMetadata rejected optional cost: %v", err) + } + all, err := ListMetadata(paths) + if err != nil || len(all) != 1 { + t.Fatalf("ListMetadata rejected optional cost: %v", err) + } + for _, m := range []Metadata{meta, all[0]} { + if m.ReviewerToolEvidence == nil || m.ReviewerToolEvidence.Trace == nil || m.ReviewerToolEvidence.DiffStatus != llm.DiffToolStatusSucceeded || m.RequestCostEvidence == nil || m.RequestCostEvidence.AttemptCountExact { + t.Fatal("cost corruption damaged valid tool metadata or became complete") + } + } + } + var meta Metadata + if decodeMetadata([]byte(`{"schema_version":4`), &meta) == nil { + t.Fatal("malformed whole metadata accepted") + } +} + +type blockingCostAdapter struct { + *lifecycleCostAdapter + entered chan struct{} + release chan struct{} +} + +func (a *blockingCostAdapter) Start(ctx context.Context, req llm.Request) (llm.Stream, error) { + close(a.entered) + <-a.release + return a.lifecycleCostAdapter.Start(ctx, req) +} +func TestRequestCostConcurrentRunNoRunAndResetOwnership(t *testing.T) { + for _, noRun := range []bool{false, true} { + t.Run(fmt.Sprint(noRun), func(t *testing.T) { + a := &blockingCostAdapter{newLifecycleCostAdapter(), make(chan struct{}), make(chan struct{})} + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + if noRun { + req.RunID = "" + req.AllowNoRunCache = true + req.Store = nil + } + finished := make(chan error, 1) + go func() { _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); finished <- err }() + <-a.entered + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); !errors.Is(err, runlock.ErrHeld) { + t.Errorf("concurrent run=%v", err) + } + if _, _, err := LoadStructured(context.Background(), req, decodeLifecyclePayload); !errors.Is(err, runlock.ErrHeld) { + t.Errorf("concurrent load=%v", err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); !errors.Is(err, runlock.ErrHeld) { + t.Errorf("active reset=%v", err) + } + close(a.release) + if err := <-finished; err != nil { + t.Fatal(err) + } + if len(a.Requests()) != 1 { + t.Fatal("multiple dispatch owners") + } + }) + } +} + +func TestRequestCostGenerationAndByteBoundsKeepPendingSpace(t *testing.T) { + paths := Paths{LLMTasksDir: t.TempDir()} + c, err := newCostCheckpoint(paths, "task", "run") + if err != nil { + t.Fatal(err) + } + for i := 0; i < 65; i++ { + c.Pending = &pendingCostGeneration{GenerationID: fmt.Sprint(i), InputFingerprint: "fingerprint"} + if err := writeCostCheckpoint(paths, c); err != nil { + t.Fatal(err) + } + if err := finishCostGeneration(paths, c, lifecycleCostResponse(fmt.Sprint(i), "").RequestCostEvidence); err != nil { + t.Fatal(err) + } + } + if len(c.Generations) != 64 || c.OmittedGenerations != 1 || len(c.Evidence.Attempts) != 64 || !c.Evidence.Truncated { + t.Fatalf("bounds=%#v", c) + } + c.Pending = &pendingCostGeneration{GenerationID: "still-pending", InputFingerprint: "fingerprint"} + if err := writeCostCheckpoint(paths, c); err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(c) + if len(b) > llm.MaxRequestCostEvidenceBytes || c.Pending == nil { + t.Fatal("full prefix prevented pending marker") + } + got, gap := readCostCheckpoint(paths, "task", "other-run", false) + if got != nil || gap != llm.CostGapScope { + t.Fatal("cross-run checkpoint imported") + } +} + +func TestRequestCostCheckpointDigestIncludesScopePendingAndFingerprint(t *testing.T) { + c, _ := newCostCheckpoint(Paths{LLMTasksDir: t.TempDir()}, "task", "run") + c.Pending = &pendingCostGeneration{GenerationID: "generation", InputFingerprint: "first"} + a, _ := checkpointDigest(c) + c.Pending.InputFingerprint = "second" + b, _ := checkpointDigest(c) + c.Pending = nil + d, _ := checkpointDigest(c) + c.RunID = "other" + e, _ := checkpointDigest(c) + if a == b || b == d || d == e { + t.Fatal("digest omitted scoped pending or fingerprint state") + } +} + +func TestRequestCostFinalizationFailureKeepsPendingCheckpoint(t *testing.T) { + req := lifecycleRequest(t, nil, newLifecycleCostAdapter()) + c, err := beginCostGeneration(req, "generation") + if err != nil { + t.Fatal(err) + } + path, _ := req.Paths.RequestCostCheckpoint(req.TaskID) + // Blocking the temporary destination models a failure before atomic rename. + if err := os.Mkdir(path+".tmp", 0o700); err != nil { + t.Fatal(err) + } + e := lifecycleCostResponse("A", "").RequestCostEvidence + e.Attempts[0].Ordinal = 1 + if err := finishCostGeneration(req.Paths, c, e); err == nil { + t.Fatal("finalization failure hidden") + } + persisted, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" || persisted.Pending == nil || persisted.Pending.GenerationID != "generation" { + t.Fatal("failed finalization overwrote pending marker") + } +} + +func TestRequestCostMetadataAndOutputFailuresKeepFinalCheckpoint(t *testing.T) { + for _, target := range []string{"metadata.json", "validated-output.json"} { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + dir, _ := req.Paths.TaskDir(req.TaskID) + // metadata itself must remain absent for pre-dispatch reads. A directory + // at its .tmp blocks only the later atomic commit. + if err := os.MkdirAll(filepath.Join(dir, target+".tmp"), 0o700); err != nil { + t.Fatal(err) + } + got, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err == nil { + t.Fatal("artifact failure hidden") + } + requireCostAttempts(t, got.Draft.Response.RequestCostEvidence, "A") + c, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" || c.Pending != nil { + t.Fatal("artifact error erased final cost checkpoint") + } + requireCostAttempts(t, c.Evidence, "A") + } +} + +func TestRequestCostReconciliationCorruptMissingConflictingAndLegacy(t *testing.T) { + for _, damage := range []string{"missing", "corrupt", "oversized", "wrong-scope"} { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err != nil { + t.Fatal(err) + } + path, _ := req.Paths.RequestCostCheckpoint(req.TaskID) + switch damage { + case "missing": + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + case "corrupt": + if err := os.WriteFile(path, []byte(`{`), 0o600); err != nil { + t.Fatal(err) + } + case "oversized": + if err := os.WriteFile(path, []byte(strings.Repeat("x", llm.MaxRequestCostEvidenceBytes+1)), 0o600); err != nil { + t.Fatal(err) + } + case "wrong-scope": + c, _ := newCostCheckpoint(req.Paths, "other", req.RunID) + c.Digest, _ = checkpointDigest(c) + b, _ := json.Marshal(c) + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatal(err) + } + } + got, ok, err := LoadStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil || !ok || !got.Value.OK { + t.Fatalf("cost damage broke output reuse: %s %v", damage, err) + } + requireCostAttempts(t, got.Draft.Response.RequestCostEvidence, "A") + if len(got.Draft.Response.RequestCostEvidence.Gaps) < 2 || len(a.Requests()) != 1 { + t.Fatal("gap repaired by assumption or provider call") + } + } + paths := Paths{LLMTasksDir: t.TempDir()} + c, _ := newCostCheckpoint(paths, "task", "run") + e := lifecycleCostResponse("A", "").RequestCostEvidence + e.Scope = "task_artifact_history" + e.ArtifactScope = c.ArtifactScope + e.TaskID = c.TaskID + e.RunID = c.RunID + e.Attempts[0].GenerationID = "g" + e.Attempts[0].Ordinal = 1 + mergeCostObservations(c, e) + mergeCostObservations(c, e) + if len(c.Evidence.Attempts) != 1 { + t.Fatal("duplicate cache reference counted twice") + } + conflict := llm.CloneRequestCostEvidence(e) + *conflict.Attempts[0].Usage.InputTokens = 99 + mergeCostObservations(c, conflict) + if len(c.Evidence.Attempts) != 1 || !slices.Contains(c.Evidence.Gaps, llm.CostGapConflict) { + t.Fatal("same identity conflict silently selected") + } + e.Attempts[0].GenerationID = "other-generation" + mergeCostObservations(c, e) + if !slices.Contains(c.Evidence.Gaps, llm.CostGapDuplicate) { + t.Fatal("repeated provider response ID silently deduplicated") + } + freshLooking, _ := reconcileCostCheckpoint(Paths{LLMTasksDir: filepath.Join(t.TempDir(), "absent")}, "task", "run", nil, false) + if freshLooking.Evidence.AttemptCountExact || !slices.Contains(freshLooking.Evidence.Gaps, llm.CostGapLegacy) { + t.Fatal("directory absence became origin proof") + } +} + +func TestRequestCostCheckpointCanonicalGolden(t *testing.T) { + want, err := os.ReadFile("testdata/request-cost-checkpoint-v1.golden.json") + if err != nil { + t.Fatal(err) + } + var c costCheckpointProjection + if json.Unmarshal(want, &c) != nil { + t.Fatal("golden fixture") + } + checkpoint := &costCheckpoint{costCheckpointProjection: c} + got, err := costCheckpointBytes(checkpoint) + if err != nil || string(got) != string(want) { + t.Fatalf("canonical bytes changed: %s %v", got, err) + } + digest, err := checkpointDigest(checkpoint) + if err != nil || digest != "3b909dbf6714f12b12b8a5a508f7fd63d4af3d640907d729c37a57a5abb74cc8" { + t.Fatalf("digest=%s err=%v", digest, err) + } +} + +func TestRequestCostResetForeignRunSidecarPreservesMetadataPrefix(t *testing.T) { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `invalid`)}) + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("B", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err != nil { + t.Fatal(err) + } + foreign, _ := newCostCheckpoint(req.Paths, req.TaskID, "foreign-run") + if err := writeCostCheckpoint(req.Paths, foreign); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); err != nil { + t.Fatal(err) + } + preserved, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatal(gap) + } + requireCostAttempts(t, preserved.Evidence, "A", "B") + if !slices.Contains(preserved.Evidence.Gaps, llm.CostGapScope) { + t.Fatal("foreign-run mismatch gap missing") + } + req.InputFingerprint = "changed" + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("C", `{"ok":true}`)}) + got, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + requireCostAttempts(t, got.Draft.Response.RequestCostEvidence, "A", "B", "C") +} + +func TestRequestCostNearByteLimitMetadataReadersAndReset(t *testing.T) { + paths := Paths{LLMTasksDir: t.TempDir()} + c, _ := newCostCheckpoint(paths, "task", "run") + for i := 0; i < 64; i++ { + a := lifecycleCostResponse(fmt.Sprint(i), "").RequestCostEvidence.Attempts[0] + a.GenerationID = fmt.Sprintf("%02d", i) + strings.Repeat("<", 254) + a.RequestedModel = strings.Repeat("<", 256) + a.ProviderResponseID = lifecycleCostPtr(fmt.Sprintf("%02d", i) + strings.Repeat("<", 254)) + a.ObservedModel = lifecycleCostPtr(strings.Repeat("<", 256)) + a.RequestedServiceTier = lifecycleCostPtr(strings.Repeat("<", 64)) + a.ObservedServiceTier = lifecycleCostPtr(strings.Repeat("<", 64)) + if !appendCostAttempt(c, a) { + break + } + c.Evidence.AttemptsObserved++ + } + if len(c.Evidence.Attempts) < 2 { + t.Fatal("near-cap fixture did not retain observations") + } + if err := writeCostCheckpoint(paths, c); err != nil { + t.Fatal(err) + } + meta := Metadata{SchemaVersion: SchemaVersion, TaskID: "task", InputFingerprint: "before", ReviewerToolEvidence: &llm.ReviewerToolEvidence{DiffStatus: llm.DiffToolStatusSucceeded}, RequestCostEvidence: llm.CloneRequestCostEvidence(c.Evidence), RequestCostCheckpoint: checkpointBinding(c)} + pretty, err := json.MarshalIndent(meta, "", " ") + if err != nil { + t.Fatal(err) + } + var prettyFields map[string]json.RawMessage + if err := json.Unmarshal(pretty, &prettyFields); err != nil { + t.Fatal(err) + } + if len(prettyFields["request_cost_evidence"]) <= llm.MaxRequestCostEvidenceBytes { + t.Fatal("near-cap fixture does not expose the historical indented-envelope overflow") + } + if err := WriteMetadata(paths, meta); err != nil { + t.Fatal(err) + } + read, ok, err := ReadMetadata(paths, "task") + if err != nil || !ok { + t.Fatal(err) + } + listed, err := ListMetadata(paths) + if err != nil || len(listed) != 1 { + t.Fatal(err) + } + for _, m := range []Metadata{read, listed[0]} { + if m.RequestCostEvidence == nil || len(m.RequestCostEvidence.Attempts) != len(c.Evidence.Attempts) || m.ReviewerToolEvidence == nil { + t.Fatal("durable formatting inflated cost envelope beyond reader bound") + } + } + path, _ := paths.RequestCostCheckpoint("task") + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(paths, "task", "after"); err != nil { + t.Fatal(err) + } + preserved, gap := readCostCheckpoint(paths, "task", "run", false) + if gap != "" || len(preserved.Evidence.Attempts) != len(c.Evidence.Attempts) { + t.Fatal("near-cap metadata prefix lost at reset") + } +} + +func TestRequestCostReadersRejectCaseFoldedDuplicateCostFields(t *testing.T) { + for _, tc := range []struct { + fields string + gap llm.CostEvidenceGap + }{ + {`"request_cost_evidence":{"version":1},"REQUEST_COST_EVIDENCE":{"version":2}`, llm.CostGapDuplicate}, + {`"request_cost_evidence":{"version":1,"VERSION":1,"source":"openai_responses","scope":"invocation","attempts_observed":0,"attempt_count_exact":true,"gaps":[],"attempts":[]}`, llm.CostGapInvalid}, + {`"request_cost_checkpoint":null,"REQUEST_COST_CHECKPOINT":null`, llm.CostGapDuplicate}, + } { + paths := Paths{LLMTasksDir: t.TempDir()} + path, _ := paths.Metadata("task") + body := []byte(`{"schema_version":4,"task_id":"task","reviewer_tool_evidence":{"diff_status":"succeeded"},` + tc.fields + `}`) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + meta, ok, err := ReadMetadata(paths, "task") + if err != nil || !ok { + t.Fatal(err) + } + listed, err := ListMetadata(paths) + if err != nil || len(listed) != 1 { + t.Fatal(err) + } + for _, m := range []Metadata{meta, listed[0]} { + if m.ReviewerToolEvidence == nil || m.RequestCostEvidence == nil || !slices.Contains(m.RequestCostEvidence.Gaps, tc.gap) || m.RequestCostEvidence.AttemptCountExact { + t.Fatalf("case-folded duplicate did not produce expected %s gap", tc.gap) + } + } + } +} + +func TestRequestCostSidecarNamesPreserveExistingLongTaskIDs(t *testing.T) { + for _, length := range []int{234, 238, 242} { + for _, direct := range []bool{false, true} { + fake := &llm.FakeAdapter{} + var adapter llm.Adapter = fake + if direct { + adapter = &lifecycleCostAdapter{fake} + } + fake.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), adapter) + req.TaskID = strings.Repeat("a", length) + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err != nil { + t.Fatalf("length=%d direct=%t error=%v", length, direct, err) + } + path, err := req.Paths.RequestCostCheckpoint(req.TaskID) + if err != nil || len(filepath.Base(path)+".tmp") > 255 { + t.Fatal("sidecar suffix exceeds filename budget") + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); err != nil { + t.Fatal(err) + } + } + } +} + +func TestRequestCostResetInvalidMetadataRunIDKeepsNewerValidSidecar(t *testing.T) { + for _, badRunID := range []string{"bad\nrun", strings.Repeat("x", 257)} { + a := newLifecycleCostAdapter() + a.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), a) + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err != nil { + t.Fatal(err) + } + metadataPath, _ := req.Paths.Metadata(req.TaskID) + metadataRoot, err := os.OpenRoot(filepath.Dir(metadataPath)) + if err != nil { + t.Fatal(err) + } + data, err := metadataRoot.ReadFile(filepath.Base(metadataPath)) + _ = metadataRoot.Close() + if err != nil { + t.Fatal(err) + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatal(err) + } + var evidence map[string]json.RawMessage + if err := json.Unmarshal(raw["request_cost_evidence"], &evidence); err != nil { + t.Fatal(err) + } + evidence["run_id"], _ = json.Marshal(badRunID) + raw["request_cost_evidence"], _ = json.Marshal(evidence) + corrupt, _ := json.Marshal(raw) + if err := os.WriteFile(metadataPath, corrupt, 0o600); err != nil { + t.Fatal(err) + } + checkpoint, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatal(gap) + } + checkpoint.Pending = &pendingCostGeneration{GenerationID: "newer-generation", InputFingerprint: req.InputFingerprint} + if err := finishCostGeneration(req.Paths, checkpoint, lifecycleCostResponse("B", "").RequestCostEvidence); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); err != nil { + t.Fatal(err) + } + preserved, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatal(gap) + } + requireCostAttempts(t, preserved.Evidence, "A", "B") + if preserved.RunID != req.RunID || !slices.Contains(preserved.Evidence.Gaps, llm.CostGapScope) { + t.Fatal("invalid metadata RunID became authoritative no-run scope") + } + } +} + +func TestRequestCostResetUntrustedBindingCannotDisplaceNewerSidecar(t *testing.T) { + for _, rawScope := range []string{"invalid", "overlong", "null", "missing"} { + for _, actualRunID := range []string{"run-1", ""} { + for _, bindingRun := range []string{"foreign-run", ""} { + runLabel := actualRunID + if runLabel == "" { + runLabel = "no-run" + } + t.Run(rawScope+"/"+runLabel+"/"+bindingRun, func(t *testing.T) { + adapter := newLifecycleCostAdapter() + adapter.Queue(llm.FakeResult{Response: lifecycleCostResponse("A", `{"ok":true}`)}) + req := lifecycleRequest(t, newLifecycleStore(), adapter) + req.RunID = actualRunID + if actualRunID == "" { + req.AllowNoRunCache = true + req.Store = nil + } + if _, err := RunStructured(context.Background(), req, decodeLifecyclePayload); err != nil { + t.Fatal(err) + } + metadataPath, _ := req.Paths.Metadata(req.TaskID) + metadataRoot, err := os.OpenRoot(filepath.Dir(metadataPath)) + if err != nil { + t.Fatal(err) + } + data, err := metadataRoot.ReadFile(filepath.Base(metadataPath)) + _ = metadataRoot.Close() + if err != nil { + t.Fatal(err) + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatal(err) + } + var evidence map[string]json.RawMessage + if err := json.Unmarshal(raw["request_cost_evidence"], &evidence); err != nil { + t.Fatal(err) + } + switch rawScope { + case "invalid": + evidence["run_id"], _ = json.Marshal("bad\nrun") + case "overlong": + evidence["run_id"], _ = json.Marshal(strings.Repeat("x", 257)) + case "null": + evidence["run_id"] = json.RawMessage(`null`) + case "missing": + delete(evidence, "run_id") + } + var binding map[string]json.RawMessage + if err := json.Unmarshal(raw["request_cost_checkpoint"], &binding); err != nil { + t.Fatal(err) + } + binding["run_id"], _ = json.Marshal(bindingRun) + raw["request_cost_evidence"], _ = json.Marshal(evidence) + raw["request_cost_checkpoint"], _ = json.Marshal(binding) + corrupt, _ := json.Marshal(raw) + if err := os.WriteFile(metadataPath, corrupt, 0o600); err != nil { + t.Fatal(err) + } + checkpoint, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatal(gap) + } + checkpoint.Pending = &pendingCostGeneration{GenerationID: "newer-generation", InputFingerprint: req.InputFingerprint} + if err := finishCostGeneration(req.Paths, checkpoint, lifecycleCostResponse("B", "").RequestCostEvidence); err != nil { + t.Fatal(err) + } + if err := ResetIfInputFingerprintChanged(req.Paths, req.TaskID, "changed"); err != nil { + t.Fatal(err) + } + preserved, gap := readCostCheckpoint(req.Paths, req.TaskID, req.RunID, false) + if gap != "" { + t.Fatal(gap) + } + requireCostAttempts(t, preserved.Evidence, "A", "B") + if preserved.RunID != req.RunID || !slices.Contains(preserved.Evidence.Gaps, llm.CostGapScope) || !slices.Contains(preserved.Evidence.Gaps, llm.CostGapConflict) { + t.Fatal("untrusted binding displaced the newer sidecar or hid ambiguity") + } + req.InputFingerprint = "changed" + adapter.Queue(llm.FakeResult{Response: lifecycleCostResponse("C", `{"ok":true}`)}) + result, err := RunStructured(context.Background(), req, decodeLifecyclePayload) + if err != nil { + t.Fatal(err) + } + requireCostAttempts(t, result.Draft.Response.RequestCostEvidence, "A", "B", "C") + }) + } + } + } +} + +func TestRequestCostSidecarOwnRawRunScopeMustBeValid(t *testing.T) { + for _, mutation := range []string{"invalid", "overlong", "missing", "null", "actual-empty"} { + paths := Paths{LLMTasksDir: t.TempDir()} + checkpoint, err := newCostCheckpoint(paths, "task", "") + if err != nil { + t.Fatal(err) + } + if mutation == "invalid" { + checkpoint.RunID = "bad\nrun" + checkpoint.Evidence.RunID = checkpoint.RunID + } + if mutation == "overlong" { + checkpoint.RunID = strings.Repeat("x", 257) + checkpoint.Evidence.RunID = checkpoint.RunID + } + checkpoint.Digest, err = checkpointDigest(checkpoint) + if err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(checkpoint) + if err != nil { + t.Fatal(err) + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(encoded, &raw); err != nil { + t.Fatal(err) + } + if mutation == "missing" { + delete(raw, "run_id") + } + if mutation == "null" { + raw["run_id"] = json.RawMessage(`null`) + } + encoded, err = json.Marshal(raw) + if err != nil { + t.Fatal(err) + } + path, _ := paths.RequestCostCheckpoint("task") + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, encoded, 0o600); err != nil { + t.Fatal(err) + } + got, gap := readCostCheckpoint(paths, "task", "", true) + if mutation == "actual-empty" { + if got == nil || gap != "" { + t.Fatal("genuine empty no-run scope rejected") + } + } else if got != nil || gap != llm.CostGapScope { + t.Fatal("invalid sidecar run scope became authoritative") + } + } +} diff --git a/internal/llmlifecycle/testdata/request-cost-checkpoint-v1.golden.json b/internal/llmlifecycle/testdata/request-cost-checkpoint-v1.golden.json new file mode 100644 index 0000000..f56452a --- /dev/null +++ b/internal/llmlifecycle/testdata/request-cost-checkpoint-v1.golden.json @@ -0,0 +1 @@ +{"version":1,"artifact_scope":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","run_id":"run","task_id":"task","sequence":7,"fresh_origin":false,"generations_observed":1,"omitted_generations":0,"evidence":{"version":1,"source":"openai_responses","scope":"task_artifact_history","artifact_scope":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","run_id":"run","task_id":"task","attempts_observed":1,"attempt_count_exact":false,"truncated":false,"dropped_attempts":0,"gaps":["legacy_history_unknown"],"attempts":[{"generation_id":"g","ordinal":1,"validation_phase":"initial","adapter_attempt":1,"runtime_id":"openai-api-key","endpoint_kind":"official_global","transport_mode":"single_send_v1","dispatch":"not_dispatched","http_status":null,"body_state":"missing","provider_response_id":null,"observed_model":null,"observed_service_tier":null,"response_status":null,"requested_model":"requested\u003cmodel\u003e","requested_service_tier":null,"usage":null,"issues":[]}]},"generations":[{"generation_id":"g","input_fingerprint":"fingerprint","evidence_digest":"40a0d89467879b363d192692bb0df2599c92fb476b4e1f7db8ad88368ad842d0"}],"pending":null} \ No newline at end of file diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index 4c27c49..9789333 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -2869,20 +2869,26 @@ func (opts Options) buildRunSummary(req Request, inputs planRunInputs) (reviewpl // several provider sessions, so they are deliberately not a sessionDraft: there // is no single session they could be written back to. type workstreamTotals struct { - model string - usage llm.Usage - durationMS int64 - startedAt time.Time - completedAt time.Time + requestHistories []*llm.RequestCostEvidence + blockLegacyEstimate bool + unknownRequestCost bool + model string + usage llm.Usage + durationMS int64 + startedAt time.Time + completedAt time.Time } func draftWorkstreamTotals(draft sessionDraft) workstreamTotals { return workstreamTotals{ - model: draft.Model, - usage: draft.Response.Usage, - durationMS: draft.Response.DurationMS, - startedAt: draft.StartedAt, - completedAt: draft.CompletedAt, + requestHistories: []*llm.RequestCostEvidence{llm.CloneRequestCostEvidence(draft.Response.RequestCostEvidence)}, + blockLegacyEstimate: draft.Adapter == "openai_api" || draft.Adapter == "codex_cli" || draft.Response.RequestCostEvidence != nil, + unknownRequestCost: draft.Adapter == "openai_api" || draft.Response.RequestCostEvidence != nil, + model: draft.Model, + usage: draft.Response.Usage, + durationMS: draft.Response.DurationMS, + startedAt: draft.StartedAt, + completedAt: draft.CompletedAt, } } @@ -2904,6 +2910,10 @@ func combineReviewerWorkstreamTotals(drafts []sessionDraft) workstreamTotals { allStandard := true anySpeed := false for _, draft := range drafts { + provenance := draftWorkstreamTotals(draft) + combined.requestHistories = append(combined.requestHistories, provenance.requestHistories...) + combined.blockLegacyEstimate = combined.blockLegacyEstimate || provenance.blockLegacyEstimate + combined.unknownRequestCost = combined.unknownRequestCost || provenance.unknownRequestCost combined.durationMS += draft.Response.DurationMS if combined.startedAt.IsZero() || !draft.StartedAt.IsZero() && draft.StartedAt.Before(combined.startedAt) { combined.startedAt = draft.StartedAt @@ -3444,10 +3454,16 @@ func workstreamUsageFromTotalsForCatalog(name string, totals workstreamTotals, c CacheCreate1h: usage.CacheCreate1h, CostUSD: usage.CostUSD, } + // Collection-only: direct OpenAI history and legacy totals remain unpriced. + // A custom all-band price row cannot bypass request evidence gates. Codex + // retains reported costs but cannot borrow a direct-API estimate. + if totals.unknownRequestCost { + workstream.CostUSD = nil + } // When the adapter reports no cost (e.g. subscription auth), estimate it from // tokens at public list prices — only for models the price table knows, so an // agent's unpriced model leaves cost unavailable rather than wrong. - if workstream.CostUSD == nil { + if workstream.CostUSD == nil && !totals.blockLegacyEstimate { priceUsage := pricing.Usage{ TokensIn: usage.TokensIn, TokensOut: usage.TokensOut, CacheRead: usage.CacheRead, CacheCreate5m: usage.CacheCreate5m, CacheCreate1h: usage.CacheCreate1h, diff --git a/internal/pipeline/request_cost_evidence_test.go b/internal/pipeline/request_cost_evidence_test.go new file mode 100644 index 0000000..422865b --- /dev/null +++ b/internal/pipeline/request_cost_evidence_test.go @@ -0,0 +1,50 @@ +package pipeline + +import ( + "testing" + + "github.com/open-cli-collective/codereview-cli/internal/llm" +) + +func TestDirectOpenAIAndCodexCannotUseLegacyAggregateEstimator(t *testing.T) { + catalog := pipelineCatalogWithRevision(t, "synthetic-request-cost-catalog") + in, out := 1_000_000, 1_000_000 + // The synthetic choice is deliberately priced by the legacy all-band + // catalog: runtime provenance, rather than a model-prefix guess, must block it. + for _, adapter := range []string{"openai_api", "codex_cli"} { + draft := sessionDraft{Adapter: adapter, Model: "claude-sonnet-5", Response: llm.Response{Usage: llm.Usage{TokensIn: &in, TokensOut: &out, Speed: "standard"}}} + got := workstreamUsageFromTotalsForCatalog("reviewer", draftWorkstreamTotals(draft), catalog) + if got.CostUSD != nil || got.CostEstimated { + t.Fatalf("%s gained legacy pricing: %#v", adapter, got) + } + } +} + +func TestRequestCostPrimaryRepairHistoriesStayDetachedAndUnpriced(t *testing.T) { + known := 0.01 + one := &llm.RequestCostEvidence{Version: 1, Source: "openai_responses", Scope: "task_artifact_history", TaskID: "primary", AttemptsObserved: 1, Attempts: []llm.RequestCostAttempt{{GenerationID: "one", Ordinal: 1}}} + two := llm.CloneRequestCostEvidence(one) + two.TaskID = "repair" + two.Gaps = []llm.CostEvidenceGap{llm.CostGapInterrupted} + drafts := []sessionDraft{ + {Adapter: "openai_api", Model: "first-model", Response: llm.Response{Usage: llm.Usage{CostUSD: &known}, RequestCostEvidence: one}}, + {Adapter: "openai_api", Model: "second-model", Response: llm.Response{RequestCostEvidence: two}}, + } + totals := combineReviewerWorkstreamTotals(drafts) + if len(totals.requestHistories) != 2 || totals.requestHistories[0].TaskID != "primary" || totals.requestHistories[1].TaskID != "repair" { + t.Fatal("task identities merged") + } + one.Attempts[0].GenerationID = "mutated" + if totals.requestHistories[0].Attempts[0].GenerationID != "one" { + t.Fatal("workstream history aliases source") + } + got := workstreamUsageFromTotalsForCatalog("reviewer", totals, nil) + if got.CostUSD != nil || got.CostEstimated { + t.Fatal("known subset masked unknown task") + } + // A duplicate reference is never sent through a price selector in this slice. + duplicate := combineReviewerWorkstreamTotals([]sessionDraft{drafts[0], drafts[0]}) + if workstreamUsageFromTotalsForCatalog("reviewer", duplicate, nil).CostUSD != nil { + t.Fatal("duplicate snapshot charged") + } +}