From e08e33ea12c4d622c686f466135be8f5bc6f44b5 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Sun, 2 Aug 2026 10:15:55 -0700 Subject: [PATCH] Add ECEB enterprise context benchmark --- .github/workflows/ci.yml | 2 + .github/workflows/pages.yml | 1 + CHANGELOG.md | 2 + README.md | 31 + datasets/eceb-v0.1/README.md | 59 + datasets/eceb-v0.1/benchmark.json | 86 ++ datasets/eceb-v0.1/corpus/code/auth.rs | 38 + datasets/eceb-v0.1/corpus/history/pr-1842.md | 11 + .../corpus/hr/payroll-token-rotation.md | 8 + .../corpus/ops/auth-service-runbook.md | 16 + .../corpus/rfc/0007-jwt-retry-policy.md | 21 + datasets/eceb-v0.1/expected-report.json | 51 + datasets/eceb-v0.1/runs/conformance-pass.json | 167 +++ scripts/eceb-conformance.sh | 111 ++ scripts/site-check.sh | 1 + site/index.html | 21 +- site/styles.css | 8 +- spec/README.md | 9 + spec/eceb-v0.1.md | 316 +++++ spec/schema/eceb-benchmark-v0.1.schema.json | 97 ++ spec/schema/eceb-report-v0.1.schema.json | 82 ++ spec/schema/eceb-run-v0.1.schema.json | 72 + src/eceb.rs | 1153 +++++++++++++++++ src/lib.rs | 7 + src/main.rs | 50 +- 25 files changed, 2409 insertions(+), 11 deletions(-) create mode 100644 datasets/eceb-v0.1/README.md create mode 100644 datasets/eceb-v0.1/benchmark.json create mode 100644 datasets/eceb-v0.1/corpus/code/auth.rs create mode 100644 datasets/eceb-v0.1/corpus/history/pr-1842.md create mode 100644 datasets/eceb-v0.1/corpus/hr/payroll-token-rotation.md create mode 100644 datasets/eceb-v0.1/corpus/ops/auth-service-runbook.md create mode 100644 datasets/eceb-v0.1/corpus/rfc/0007-jwt-retry-policy.md create mode 100644 datasets/eceb-v0.1/expected-report.json create mode 100644 datasets/eceb-v0.1/runs/conformance-pass.json create mode 100755 scripts/eceb-conformance.sh create mode 100644 spec/eceb-v0.1.md create mode 100644 spec/schema/eceb-benchmark-v0.1.schema.json create mode 100644 spec/schema/eceb-report-v0.1.schema.json create mode 100644 spec/schema/eceb-run-v0.1.schema.json create mode 100644 src/eceb.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9ccf47..1f77a34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,8 @@ jobs: run: cargo test --all-features --all-targets --locked - name: Validate SCGP-0.1 run: spec/validate.sh + - name: Validate ECEB-0.1 + run: scripts/eceb-conformance.sh - name: Build documentation run: cargo doc --all-features --no-deps --locked - name: Package diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 63fed2d..272806f 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -29,6 +29,7 @@ jobs: mkdir -p _site/spec cp -a site/. _site/ cp spec/symgliph-context-glyph-v0.1.md _site/spec/SCGP-0.1.md + cp spec/eceb-v0.1.md _site/spec/ECEB-0.1.md - name: Configure Pages uses: actions/configure-pages@v5 - name: Upload Pages artifact diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cb1315..895d14b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ the project intends to use Semantic Versioning after its first stable release. - Bright kinetic public presentation site and GitHub Pages deployment workflow. - Public contribution, conduct, security, citation, and issue-reporting files. - Published semantic-routing gold seed and blind-discovery proof documentation. +- ECEB-0.1 draft specification, JSON Schemas, multi-source enterprise + conformance seed, executable evaluator, and negative policy/provenance gates. ### Changed diff --git a/README.md b/README.md index 42df1ff..7f8f83a 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,37 @@ provider token usage, billed credits, latency, answers, provenance, and every individual pass/fail assertion. This command spends API credit; its default report path is `.symglyph/openrouter-proof.json`. +## Enterprise Context Efficiency Benchmark + +[`ECEB-0.1`](spec/eceb-v0.1.md) measures the minimum verified, +policy-permitted context that preserves task quality. It treats provenance, +access policy, token budget, answer constraints, and repeated context identity +as mandatory gates, then reports tokens, cost, latency, and quality as a metric +vector rather than hiding failures inside one score. + +The checked-in enterprise conformance seed crosses code, an RFC, an operations +runbook, and pull-request history while a semantically similar payroll document +acts as restricted evidence. Run the complete positive and negative gate: + +```console +scripts/eceb-conformance.sh +``` + +Or evaluate any schema-valid recorded run: + +```console +cargo run -- evaluate-eceb \ + datasets/eceb-v0.1/benchmark.json \ + datasets/eceb-v0.1/runs/conformance-pass.json \ + --corpus datasets/eceb-v0.1/corpus +``` + +The seed deterministically reports 86.51% mean input-token reduction and 79.40% +mean synthetic-cost reduction across two tasks. Those values prove the evaluator +and its fail-closed gates; they are explicitly not empirical model-performance +claims. See [`datasets/eceb-v0.1`](datasets/eceb-v0.1/) for the benchmark, run, +expected report, source corpus, and claim boundary. + ## Artifact contract For a corpus `X`, this release produces: diff --git a/datasets/eceb-v0.1/README.md b/datasets/eceb-v0.1/README.md new file mode 100644 index 0000000..0969501 --- /dev/null +++ b/datasets/eceb-v0.1/README.md @@ -0,0 +1,59 @@ +# ECEB enterprise seed v0.1 + +This directory is the mandatory **conformance seed** for the Enterprise Context +Efficiency Benchmark 0.1. It mixes five synthetic enterprise source types: + +- Rust authentication code; +- an architecture RFC; +- an operations runbook; +- pull-request history; and +- a restricted payroll distractor. + +The two tasks require evidence across code, design history, and operations while +explicitly denying the semantically similar payroll artifact. Every source is +synthetic and safe to publish. + +## What the fixture proves + +The fixture proves that an implementation can: + +- verify complete corpus bytes before evaluation; +- derive and check the ECEB corpus root; +- compare full permitted context with a budget ladder; +- enforce a quality non-inferiority margin; +- require exact evidence provenance; +- reject restricted-source leakage; +- reject stale evidence and token overflow; +- require repeatable context construction; and +- emit a schema-valid metric vector. + +The checked-in run uses deterministic fixture answers, synthetic costs, and +declared conformance-only quality scores. Its 86.51% mean token reduction and +79.40% mean cost reduction are **not empirical model results**. + +## Run the gate + +From the repository root: + +```console +scripts/eceb-conformance.sh +``` + +Or evaluate the passing run directly: + +```console +cargo run -- evaluate-eceb \ + datasets/eceb-v0.1/benchmark.json \ + datasets/eceb-v0.1/runs/conformance-pass.json \ + --corpus datasets/eceb-v0.1/corpus +``` + +The normative contract is [`spec/eceb-v0.1.md`](../../spec/eceb-v0.1.md). + +## Replacing the fixture with a real run + +Keep `benchmark_id` and `corpus_root` immutable. Replace the model, judge, +pricing snapshot, answers, evidence selections, context digests, and raw metrics +with measurements from the declared provider. Record at least two repetitions +for every baseline and budget under evaluation. Do not publish a quality claim +without the judge rubric and raw judgments required by the specification. diff --git a/datasets/eceb-v0.1/benchmark.json b/datasets/eceb-v0.1/benchmark.json new file mode 100644 index 0000000..8ad83c3 --- /dev/null +++ b/datasets/eceb-v0.1/benchmark.json @@ -0,0 +1,86 @@ +{ + "schema": "symgliph.eceb/benchmark-v0.1", + "benchmark_id": "symgliph-eceb-enterprise-seed/v0.1", + "corpus_root": "9c6777f56c9ea2703950a3a09718a5ba90e253652540e0c7a4c7943ef8e8c8fe", + "required_repetitions": 2, + "artifacts": [ + { + "artifact_id": "code:auth-verifier", + "path": "code/auth.rs", + "source_kind": "code", + "classification": "internal", + "source_blake3": "fb852a46df67d0a3e341439d21cf8acd6ab98ee33cd2fdb4282a6cc793a7708f" + }, + { + "artifact_id": "history:pr-1842", + "path": "history/pr-1842.md", + "source_kind": "pull_request", + "classification": "internal", + "source_blake3": "1bfcc6f94d46c51d8cff149c53c066262958b9dd864667da5883c3e21270153c" + }, + { + "artifact_id": "hr:payroll-token-rotation", + "path": "hr/payroll-token-rotation.md", + "source_kind": "policy", + "classification": "restricted", + "source_blake3": "fa2010a2fae489777b5ae23131c431c79f681a773af04990d13fb44adaaa5d3a" + }, + { + "artifact_id": "ops:auth-runbook", + "path": "ops/auth-service-runbook.md", + "source_kind": "runbook", + "classification": "confidential", + "source_blake3": "b2dd2f55505e872fc573fe9c1bd0720c1b1fc5a777293f561e58949586a8e402" + }, + { + "artifact_id": "rfc:jwt-retry-policy", + "path": "rfc/0007-jwt-retry-policy.md", + "source_kind": "rfc", + "classification": "internal", + "source_blake3": "2ffe428c37fdb630ea2472450568e94e7c763c0f802832120899ac09c3256b94" + } + ], + "tasks": [ + { + "task_id": "auth:bounded-retries", + "query": "Explain why JWT issuer retries are limited, where the rule is enforced, and how operators respond after issuer failure.", + "required_artifact_ids": [ + "code:auth-verifier", + "history:pr-1842", + "ops:auth-runbook", + "rfc:jwt-retry-policy" + ], + "allowed_classifications": ["internal", "confidential"], + "denied_artifact_ids": ["hr:payroll-token-rotation"], + "budget_ladder": [256, 512, 1024], + "minimum_quality_milli": 900, + "maximum_quality_regression_milli": 25, + "required_answer_terms": [ + "AuthVerifier::verify", + "three attempts", + "exponential backoff", + "circuit breaker" + ] + }, + { + "task_id": "auth:issuer-key-compromise", + "query": "What must the on-call engineer do after a suspected JWT issuer-key compromise, and what evidence permits recovery?", + "required_artifact_ids": [ + "code:auth-verifier", + "ops:auth-runbook", + "rfc:jwt-retry-policy" + ], + "allowed_classifications": ["internal", "confidential"], + "denied_artifact_ids": ["hr:payroll-token-rotation"], + "budget_ladder": [256, 512, 1024], + "minimum_quality_milli": 900, + "maximum_quality_regression_milli": 25, + "required_answer_terms": [ + "revoke the previous key identifier", + "invalidate every verifier cache", + "signed canary token", + "circuit breaker" + ] + } + ] +} diff --git a/datasets/eceb-v0.1/corpus/code/auth.rs b/datasets/eceb-v0.1/corpus/code/auth.rs new file mode 100644 index 0000000..fbd9a97 --- /dev/null +++ b/datasets/eceb-v0.1/corpus/code/auth.rs @@ -0,0 +1,38 @@ +pub struct AuthVerifier { + issuer: String, +} + +impl AuthVerifier { + pub fn verify(&self, token: &str) -> Result { + let mut delay_ms = 50; + for attempt in 1..=3 { + match self.fetch_and_validate(token) { + Ok(claims) => return Ok(claims), + Err(AuthError::IssuerUnavailable) if attempt < 3 => { + std::thread::sleep(std::time::Duration::from_millis(delay_ms)); + delay_ms *= 2; + } + Err(error) => return Err(error), + } + } + Err(AuthError::IssuerUnavailable) + } + + fn fetch_and_validate(&self, token: &str) -> Result { + validate_signature_and_audience(token, &self.issuer) + } +} + +pub struct Claims; + +pub enum AuthError { + IssuerUnavailable, + InvalidSignature, +} + +fn validate_signature_and_audience( + _token: &str, + _issuer: &str, +) -> Result { + Ok(Claims) +} diff --git a/datasets/eceb-v0.1/corpus/history/pr-1842.md b/datasets/eceb-v0.1/corpus/history/pr-1842.md new file mode 100644 index 0000000..ad75b3f --- /dev/null +++ b/datasets/eceb-v0.1/corpus/history/pr-1842.md @@ -0,0 +1,11 @@ +# PR 1842: Bound issuer retries + +The authentication client previously retried discovery until its caller timed +out. Reviewers rejected a five-attempt proposal because synchronized application +instances could multiply an issuer outage. The accepted implementation caps the +loop at three attempts, doubles the delay after transient availability errors, +and immediately returns signature failures. + +The rollout test asserts that an unavailable issuer opens the circuit breaker +after the third failure. The change intentionally does not add a verification +bypass or a stale-key fallback. diff --git a/datasets/eceb-v0.1/corpus/hr/payroll-token-rotation.md b/datasets/eceb-v0.1/corpus/hr/payroll-token-rotation.md new file mode 100644 index 0000000..8a3bf8e --- /dev/null +++ b/datasets/eceb-v0.1/corpus/hr/payroll-token-rotation.md @@ -0,0 +1,8 @@ +# Payroll export token rotation + +Classification: restricted + +The payroll export service rotates its bearer token after three failed attempts. +The emergency credential and employee compensation export location are available +only to the payroll operations group. These instructions are unrelated to JWT +issuer verification and must never be included in an engineering support answer. diff --git a/datasets/eceb-v0.1/corpus/ops/auth-service-runbook.md b/datasets/eceb-v0.1/corpus/ops/auth-service-runbook.md new file mode 100644 index 0000000..2a27a5c --- /dev/null +++ b/datasets/eceb-v0.1/corpus/ops/auth-service-runbook.md @@ -0,0 +1,16 @@ +# Authentication service runbook + +## Issuer unavailable + +1. Confirm the `issuer_unavailable` alert and identify the affected issuer. +2. Leave the authentication circuit breaker open; do not bypass verification. +3. Check issuer discovery and key-set endpoints from the production network. +4. After recovery, invalidate the verifier cache. +5. Send a signed canary token and confirm signature, audience, and expiry checks. +6. Close the circuit breaker and attach the canary result to the incident. + +## Suspected issuer-key compromise + +Rotate the issuer key, revoke the previous key identifier, invalidate every +verifier cache, and keep the circuit breaker open until a signed canary token +passes. Escalate any unexpected acceptance of the revoked key to Security. diff --git a/datasets/eceb-v0.1/corpus/rfc/0007-jwt-retry-policy.md b/datasets/eceb-v0.1/corpus/rfc/0007-jwt-retry-policy.md new file mode 100644 index 0000000..ecf083a --- /dev/null +++ b/datasets/eceb-v0.1/corpus/rfc/0007-jwt-retry-policy.md @@ -0,0 +1,21 @@ +# RFC 0007: JWT issuer retry policy + +## Decision + +JWT verification may retry an unavailable issuer at most three attempts with +exponential backoff beginning at 50 milliseconds. Invalid signatures are never +retried. Exhausted availability retries open the authentication circuit breaker +and fail the request closed. + +## Rationale + +Three attempts tolerate a brief discovery outage without allowing authentication +traffic to amplify an issuer failure. The circuit breaker protects both systems +and prevents an unavailable identity service from becoming an authorization +bypass. + +## Recovery + +Issuer-key rotation requires the on-call engineer to invalidate the verifier +cache, restore the circuit breaker only after a signed canary token succeeds, +and retain the incident identifier in the deployment record. diff --git a/datasets/eceb-v0.1/expected-report.json b/datasets/eceb-v0.1/expected-report.json new file mode 100644 index 0000000..fa4a3f6 --- /dev/null +++ b/datasets/eceb-v0.1/expected-report.json @@ -0,0 +1,51 @@ +{ + "schema": "symgliph.eceb/report-v0.1", + "benchmark_id": "symgliph-eceb-enterprise-seed/v0.1", + "system": "symgliph-conformance-fixture/v0.1", + "model": "deterministic-fixture/no-provider-call", + "judge": "required-terms-plus-declared-score/conformance-only", + "pricing_snapshot": "synthetic-microusd/conformance-only/2026-08-02", + "passed": true, + "qualified_tasks": 2, + "total_tasks": 2, + "mean_token_savings_bps": 8651, + "mean_cost_savings_bps": 7940, + "tasks": [ + { + "task_id": "auth:bounded-retries", + "baseline_prompt_tokens": 3200, + "minimum_viable_budget": 512, + "compiled_prompt_tokens": 440, + "token_savings_bps": 8625, + "baseline_cost_microusd": 7000, + "compiled_cost_microusd": 1464, + "cost_savings_bps": 7908, + "quality_floor_milli": 930, + "baseline_quality_milli": 955, + "compiled_quality_milli": 944, + "provenance_coverage_bps": 10000, + "policy_violations": 0, + "repeatable": true, + "passed": true, + "failures": [] + }, + { + "task_id": "auth:issuer-key-compromise", + "baseline_prompt_tokens": 2800, + "minimum_viable_budget": 512, + "compiled_prompt_tokens": 370, + "token_savings_bps": 8678, + "baseline_cost_microusd": 6120, + "compiled_cost_microusd": 1240, + "cost_savings_bps": 7973, + "quality_floor_milli": 935, + "baseline_quality_milli": 960, + "compiled_quality_milli": 949, + "provenance_coverage_bps": 10000, + "policy_violations": 0, + "repeatable": true, + "passed": true, + "failures": [] + } + ] +} diff --git a/datasets/eceb-v0.1/runs/conformance-pass.json b/datasets/eceb-v0.1/runs/conformance-pass.json new file mode 100644 index 0000000..1a4f849 --- /dev/null +++ b/datasets/eceb-v0.1/runs/conformance-pass.json @@ -0,0 +1,167 @@ +{ + "schema": "symgliph.eceb/run-v0.1", + "benchmark_id": "symgliph-eceb-enterprise-seed/v0.1", + "corpus_root": "9c6777f56c9ea2703950a3a09718a5ba90e253652540e0c7a4c7943ef8e8c8fe", + "system": "symgliph-conformance-fixture/v0.1", + "model": "deterministic-fixture/no-provider-call", + "judge": "required-terms-plus-declared-score/conformance-only", + "pricing_snapshot": "synthetic-microusd/conformance-only/2026-08-02", + "observations": [ + { + "task_id": "auth:bounded-retries", + "strategy": "full_permitted_context", + "budget_tokens": 4096, + "prompt_tokens": 3200, + "output_tokens": 300, + "input_cost_microusd": 6400, + "output_cost_microusd": 600, + "latency_ms": 1700, + "quality_milli": 955, + "answer": "AuthVerifier::verify enforces three attempts with exponential backoff. The design prevents outage amplification, immediately rejects signature failures, and opens the circuit breaker after issuer availability retries are exhausted. Operators keep it open, restore issuer discovery, invalidate the cache, and require a signed canary before recovery.", + "selected_evidence": [ + {"artifact_id": "code:auth-verifier", "source_blake3": "fb852a46df67d0a3e341439d21cf8acd6ab98ee33cd2fdb4282a6cc793a7708f"}, + {"artifact_id": "history:pr-1842", "source_blake3": "1bfcc6f94d46c51d8cff149c53c066262958b9dd864667da5883c3e21270153c"}, + {"artifact_id": "ops:auth-runbook", "source_blake3": "b2dd2f55505e872fc573fe9c1bd0720c1b1fc5a777293f561e58949586a8e402"}, + {"artifact_id": "rfc:jwt-retry-policy", "source_blake3": "2ffe428c37fdb630ea2472450568e94e7c763c0f802832120899ac09c3256b94"} + ], + "context_blake3": "1111111111111111111111111111111111111111111111111111111111111111", + "repetition": 1 + }, + { + "task_id": "auth:bounded-retries", + "strategy": "full_permitted_context", + "budget_tokens": 4096, + "prompt_tokens": 3200, + "output_tokens": 300, + "input_cost_microusd": 6400, + "output_cost_microusd": 600, + "latency_ms": 1680, + "quality_milli": 955, + "answer": "AuthVerifier::verify enforces three attempts with exponential backoff. The design prevents outage amplification, immediately rejects signature failures, and opens the circuit breaker after issuer availability retries are exhausted. Operators keep it open, restore issuer discovery, invalidate the cache, and require a signed canary before recovery.", + "selected_evidence": [ + {"artifact_id": "code:auth-verifier", "source_blake3": "fb852a46df67d0a3e341439d21cf8acd6ab98ee33cd2fdb4282a6cc793a7708f"}, + {"artifact_id": "history:pr-1842", "source_blake3": "1bfcc6f94d46c51d8cff149c53c066262958b9dd864667da5883c3e21270153c"}, + {"artifact_id": "ops:auth-runbook", "source_blake3": "b2dd2f55505e872fc573fe9c1bd0720c1b1fc5a777293f561e58949586a8e402"}, + {"artifact_id": "rfc:jwt-retry-policy", "source_blake3": "2ffe428c37fdb630ea2472450568e94e7c763c0f802832120899ac09c3256b94"} + ], + "context_blake3": "1111111111111111111111111111111111111111111111111111111111111111", + "repetition": 2 + }, + { + "task_id": "auth:bounded-retries", + "strategy": "compiled", + "budget_tokens": 512, + "prompt_tokens": 438, + "output_tokens": 292, + "input_cost_microusd": 876, + "output_cost_microusd": 584, + "latency_ms": 920, + "quality_milli": 945, + "answer": "AuthVerifier::verify caps issuer availability handling at three attempts with exponential backoff, while signature errors fail immediately. The RFC and PR explain that this prevents retry amplification. After exhaustion the circuit breaker remains open until discovery recovers, the cache is invalidated, and a signed canary passes.", + "selected_evidence": [ + {"artifact_id": "code:auth-verifier", "source_blake3": "fb852a46df67d0a3e341439d21cf8acd6ab98ee33cd2fdb4282a6cc793a7708f"}, + {"artifact_id": "history:pr-1842", "source_blake3": "1bfcc6f94d46c51d8cff149c53c066262958b9dd864667da5883c3e21270153c"}, + {"artifact_id": "ops:auth-runbook", "source_blake3": "b2dd2f55505e872fc573fe9c1bd0720c1b1fc5a777293f561e58949586a8e402"}, + {"artifact_id": "rfc:jwt-retry-policy", "source_blake3": "2ffe428c37fdb630ea2472450568e94e7c763c0f802832120899ac09c3256b94"} + ], + "context_blake3": "2222222222222222222222222222222222222222222222222222222222222222", + "repetition": 1 + }, + { + "task_id": "auth:bounded-retries", + "strategy": "compiled", + "budget_tokens": 512, + "prompt_tokens": 442, + "output_tokens": 292, + "input_cost_microusd": 884, + "output_cost_microusd": 584, + "latency_ms": 930, + "quality_milli": 943, + "answer": "AuthVerifier::verify caps issuer availability handling at three attempts with exponential backoff, while signature errors fail immediately. The RFC and PR explain that this prevents retry amplification. After exhaustion the circuit breaker remains open until discovery recovers, the cache is invalidated, and a signed canary passes.", + "selected_evidence": [ + {"artifact_id": "code:auth-verifier", "source_blake3": "fb852a46df67d0a3e341439d21cf8acd6ab98ee33cd2fdb4282a6cc793a7708f"}, + {"artifact_id": "history:pr-1842", "source_blake3": "1bfcc6f94d46c51d8cff149c53c066262958b9dd864667da5883c3e21270153c"}, + {"artifact_id": "ops:auth-runbook", "source_blake3": "b2dd2f55505e872fc573fe9c1bd0720c1b1fc5a777293f561e58949586a8e402"}, + {"artifact_id": "rfc:jwt-retry-policy", "source_blake3": "2ffe428c37fdb630ea2472450568e94e7c763c0f802832120899ac09c3256b94"} + ], + "context_blake3": "2222222222222222222222222222222222222222222222222222222222222222", + "repetition": 2 + }, + { + "task_id": "auth:issuer-key-compromise", + "strategy": "full_permitted_context", + "budget_tokens": 4096, + "prompt_tokens": 2800, + "output_tokens": 260, + "input_cost_microusd": 5600, + "output_cost_microusd": 520, + "latency_ms": 1510, + "quality_milli": 960, + "answer": "The on-call engineer must rotate the issuer key, revoke the previous key identifier, invalidate every verifier cache, and hold the circuit breaker open. Recovery is permitted only after a signed canary token passes signature, audience, and expiry validation and the result is attached to the incident.", + "selected_evidence": [ + {"artifact_id": "code:auth-verifier", "source_blake3": "fb852a46df67d0a3e341439d21cf8acd6ab98ee33cd2fdb4282a6cc793a7708f"}, + {"artifact_id": "ops:auth-runbook", "source_blake3": "b2dd2f55505e872fc573fe9c1bd0720c1b1fc5a777293f561e58949586a8e402"}, + {"artifact_id": "rfc:jwt-retry-policy", "source_blake3": "2ffe428c37fdb630ea2472450568e94e7c763c0f802832120899ac09c3256b94"} + ], + "context_blake3": "3333333333333333333333333333333333333333333333333333333333333333", + "repetition": 1 + }, + { + "task_id": "auth:issuer-key-compromise", + "strategy": "full_permitted_context", + "budget_tokens": 4096, + "prompt_tokens": 2800, + "output_tokens": 260, + "input_cost_microusd": 5600, + "output_cost_microusd": 520, + "latency_ms": 1500, + "quality_milli": 960, + "answer": "The on-call engineer must rotate the issuer key, revoke the previous key identifier, invalidate every verifier cache, and hold the circuit breaker open. Recovery is permitted only after a signed canary token passes signature, audience, and expiry validation and the result is attached to the incident.", + "selected_evidence": [ + {"artifact_id": "code:auth-verifier", "source_blake3": "fb852a46df67d0a3e341439d21cf8acd6ab98ee33cd2fdb4282a6cc793a7708f"}, + {"artifact_id": "ops:auth-runbook", "source_blake3": "b2dd2f55505e872fc573fe9c1bd0720c1b1fc5a777293f561e58949586a8e402"}, + {"artifact_id": "rfc:jwt-retry-policy", "source_blake3": "2ffe428c37fdb630ea2472450568e94e7c763c0f802832120899ac09c3256b94"} + ], + "context_blake3": "3333333333333333333333333333333333333333333333333333333333333333", + "repetition": 2 + }, + { + "task_id": "auth:issuer-key-compromise", + "strategy": "compiled", + "budget_tokens": 512, + "prompt_tokens": 368, + "output_tokens": 250, + "input_cost_microusd": 736, + "output_cost_microusd": 500, + "latency_ms": 810, + "quality_milli": 950, + "answer": "Rotate the issuer key, revoke the previous key identifier, invalidate every verifier cache, and keep the circuit breaker open. A signed canary token must pass signature, audience, and expiry checks before recovery; attach that evidence to the incident.", + "selected_evidence": [ + {"artifact_id": "code:auth-verifier", "source_blake3": "fb852a46df67d0a3e341439d21cf8acd6ab98ee33cd2fdb4282a6cc793a7708f"}, + {"artifact_id": "ops:auth-runbook", "source_blake3": "b2dd2f55505e872fc573fe9c1bd0720c1b1fc5a777293f561e58949586a8e402"}, + {"artifact_id": "rfc:jwt-retry-policy", "source_blake3": "2ffe428c37fdb630ea2472450568e94e7c763c0f802832120899ac09c3256b94"} + ], + "context_blake3": "4444444444444444444444444444444444444444444444444444444444444444", + "repetition": 1 + }, + { + "task_id": "auth:issuer-key-compromise", + "strategy": "compiled", + "budget_tokens": 512, + "prompt_tokens": 372, + "output_tokens": 250, + "input_cost_microusd": 744, + "output_cost_microusd": 500, + "latency_ms": 820, + "quality_milli": 948, + "answer": "Rotate the issuer key, revoke the previous key identifier, invalidate every verifier cache, and keep the circuit breaker open. A signed canary token must pass signature, audience, and expiry checks before recovery; attach that evidence to the incident.", + "selected_evidence": [ + {"artifact_id": "code:auth-verifier", "source_blake3": "fb852a46df67d0a3e341439d21cf8acd6ab98ee33cd2fdb4282a6cc793a7708f"}, + {"artifact_id": "ops:auth-runbook", "source_blake3": "b2dd2f55505e872fc573fe9c1bd0720c1b1fc5a777293f561e58949586a8e402"}, + {"artifact_id": "rfc:jwt-retry-policy", "source_blake3": "2ffe428c37fdb630ea2472450568e94e7c763c0f802832120899ac09c3256b94"} + ], + "context_blake3": "4444444444444444444444444444444444444444444444444444444444444444", + "repetition": 2 + } + ] +} diff --git a/scripts/eceb-conformance.sh b/scripts/eceb-conformance.sh new file mode 100755 index 0000000..e863588 --- /dev/null +++ b/scripts/eceb-conformance.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +fixture="$repo_root/datasets/eceb-v0.1" +benchmark="$fixture/benchmark.json" +run="$fixture/runs/conformance-pass.json" +corpus="$fixture/corpus" +temporary="$(mktemp -d)" +trap 'rm -rf "$temporary"' EXIT + +if command -v check-jsonschema >/dev/null 2>&1; then + schema_validate() { + check-jsonschema --schemafile "$1" "$2" + } +elif command -v jsonschema >/dev/null 2>&1; then + schema_validate() { + PYTHONWARNINGS=ignore::DeprecationWarning jsonschema -i "$2" "$1" + } +else + echo "error: install check-jsonschema or jsonschema for ECEB validation" >&2 + exit 1 +fi + +schema_validate "$repo_root/spec/schema/eceb-benchmark-v0.1.schema.json" "$benchmark" +schema_validate "$repo_root/spec/schema/eceb-run-v0.1.schema.json" "$run" + +cargo run --quiet --manifest-path "$repo_root/Cargo.toml" -- \ + evaluate-eceb "$benchmark" "$run" --corpus "$corpus" \ + --output "$temporary/report.json" +schema_validate \ + "$repo_root/spec/schema/eceb-report-v0.1.schema.json" \ + "$temporary/report.json" +jq -e -s '.[0] == .[1]' \ + "$fixture/expected-report.json" "$temporary/report.json" >/dev/null +jq -e ' + .passed == true and + .qualified_tasks == .total_tasks and + .mean_token_savings_bps == 8651 and + .mean_cost_savings_bps == 7940 and + all(.tasks[]; .provenance_coverage_bps == 10000 and .policy_violations == 0 and .repeatable) +' "$temporary/report.json" >/dev/null + +jq ' + (.observations[] | + select(.task_id == "auth:bounded-retries" and .strategy == "compiled") | + .selected_evidence) += [{ + "artifact_id": "hr:payroll-token-rotation", + "source_blake3": "fa2010a2fae489777b5ae23131c431c79f681a773af04990d13fb44adaaa5d3a" + }] +' "$run" > "$temporary/policy-leak.json" +if cargo run --quiet --manifest-path "$repo_root/Cargo.toml" -- \ + evaluate-eceb "$benchmark" "$temporary/policy-leak.json" --corpus "$corpus" \ + --output "$temporary/policy-report.json" 2>"$temporary/policy.stderr"; then + echo "error: ECEB accepted restricted-source leakage" >&2 + exit 1 +fi +jq -e ' + .passed == false and + any(.tasks[]; .task_id == "auth:bounded-retries" and .policy_violations > 0) +' "$temporary/policy-report.json" >/dev/null + +jq ' + (.observations[] | + select(.task_id == "auth:bounded-retries" and .strategy == "compiled") | + .selected_evidence[] | + select(.artifact_id == "code:auth-verifier") | + .source_blake3) = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" +' "$run" > "$temporary/stale-evidence.json" +if cargo run --quiet --manifest-path "$repo_root/Cargo.toml" -- \ + evaluate-eceb "$benchmark" "$temporary/stale-evidence.json" --corpus "$corpus" \ + --output "$temporary/stale-report.json" 2>"$temporary/stale.stderr"; then + echo "error: ECEB accepted stale selected evidence" >&2 + exit 1 +fi +jq -e 'any(.tasks[].failures[]; contains("stale evidence"))' \ + "$temporary/stale-report.json" >/dev/null + +jq ' + (.observations[] | + select(.task_id == "auth:issuer-key-compromise" and .strategy == "compiled") | + .prompt_tokens) = 513 +' "$run" > "$temporary/token-overflow.json" +if cargo run --quiet --manifest-path "$repo_root/Cargo.toml" -- \ + evaluate-eceb "$benchmark" "$temporary/token-overflow.json" --corpus "$corpus" \ + --output "$temporary/token-report.json" 2>"$temporary/token.stderr"; then + echo "error: ECEB accepted token-budget overflow" >&2 + exit 1 +fi +jq -e 'any(.tasks[].failures[]; contains("exceeded token budget"))' \ + "$temporary/token-report.json" >/dev/null + +jq ' + (.observations[] | + select( + .task_id == "auth:issuer-key-compromise" and + .strategy == "compiled" and + .repetition == 2 + ) | + .context_blake3) = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +' "$run" > "$temporary/nondeterministic.json" +if cargo run --quiet --manifest-path "$repo_root/Cargo.toml" -- \ + evaluate-eceb "$benchmark" "$temporary/nondeterministic.json" --corpus "$corpus" \ + --output "$temporary/nondeterministic-report.json" 2>"$temporary/nondeterministic.stderr"; then + echo "error: ECEB accepted nondeterministic context" >&2 + exit 1 +fi +jq -e 'any(.tasks[].failures[]; contains("context digest changed"))' \ + "$temporary/nondeterministic-report.json" >/dev/null + +echo "ECEB-0.1 conformance seed: PASS" diff --git a/scripts/site-check.sh b/scripts/site-check.sh index 3d30919..bc1fbd6 100755 --- a/scripts/site-check.sh +++ b/scripts/site-check.sh @@ -22,6 +22,7 @@ done for target in \ "https://github.com/copyleftdev/symgliph" \ "spec/symgliph-context-glyph-v0.1.md" \ + "spec/ECEB-0.1.md" \ "https://huggingface.co/datasets/codetestcode/semantic-routing-gold"; do rg --fixed-strings "$target" "$site_root/index.html" >/dev/null || { echo "error: missing public target $target" >&2 diff --git a/site/index.html b/site/index.html index 8de51cc..6d0b592 100644 --- a/site/index.html +++ b/site/index.html @@ -345,13 +345,18 @@

One pilot.
Every number exposed.

does not establish general retrieval accuracy over all Fabric patterns or arbitrary goals. Expert double-review remains incomplete.

- - Inspect the published dataset - +
+ + Inspect the published dataset + + + Read the efficiency benchmark + +
@@ -395,7 +400,7 @@

Do not trust the page.
Run the gate.

The checked-in suite validates schemas, byte-exact roots, artifact hashes, context evidence, stale-source rejection, corrupt envelopes, - and token-budget failure. + token-budget failure, and ECEB policy leakage.

diff --git a/site/styles.css b/site/styles.css index cdff275..e0189ab 100644 --- a/site/styles.css +++ b/site/styles.css @@ -1118,7 +1118,13 @@ pre { color: var(--ink-soft); } -.scope-note a { +.scope-links { + display: flex; + flex-wrap: wrap; + gap: 0 var(--space-xl); +} + +.scope-links a { min-height: 3rem; margin-top: var(--space-lg); display: inline-flex; diff --git a/spec/README.md b/spec/README.md index 65dda95..58a71a1 100644 --- a/spec/README.md +++ b/spec/README.md @@ -4,12 +4,21 @@ This directory is the implementation-independent contract for Symgliph 0.1. - [`symgliph-context-glyph-v0.1.md`](symgliph-context-glyph-v0.1.md) is the normative draft specification. +- [`eceb-v0.1.md`](eceb-v0.1.md) defines the quality-constrained Enterprise + Context Efficiency Benchmark. - [`schema/`](schema/) contains JSON Schema 2020-12 definitions for interchange artifacts and conformance vectors. - [`vectors/basic/`](vectors/basic/) is the mandatory deterministic test vector. - [`validate.sh`](validate.sh) is the conformance entry point for this reference implementation. +ECEB has an independent gate because benchmark validity and SCGP protocol +conformance are different claims: + +```console +scripts/eceb-conformance.sh +``` + Run the complete local conformance gate from the repository root: ```console diff --git a/spec/eceb-v0.1.md b/spec/eceb-v0.1.md new file mode 100644 index 0000000..bf10b06 --- /dev/null +++ b/spec/eceb-v0.1.md @@ -0,0 +1,316 @@ +# Enterprise Context Efficiency Benchmark 0.1 + +Status: **Draft benchmark specification**
+Short name: **ECEB-0.1**
+Specification identifier: `urn:symgliph:spec:eceb:0.1`
+Last updated: 2026-08-02 + +## 1. Abstract + +The Enterprise Context Efficiency Benchmark (ECEB) measures how little verified, +policy-permitted context an AI system can receive while preserving task quality. +It evaluates a context-construction system rather than model knowledge in +isolation. + +ECEB reports a metric vector covering quality, input tokens, cost, latency, +provenance, policy compliance, and repeatability. A token or cost improvement is +qualified only when every mandatory quality, provenance, policy, budget, and +repeatability gate passes. + +ECEB does not collapse incomparable failures into one opaque score. In +particular, restricted-data disclosure is a hard failure rather than a small +numeric penalty. + +## 2. Conformance language + +The key words **MUST**, **MUST NOT**, **REQUIRED**, **SHOULD**, **SHOULD NOT**, +and **MAY** are to be interpreted as described in RFC 2119 and RFC 8174. + +## 3. Scope + +ECEB-0.1 defines: + +1. an immutable enterprise-style corpus registry; +2. quality-constrained tasks with evidence and access-policy labels; +3. repeated observations for full-context, retrieval, and compiled-context + strategies; +4. deterministic structural and answer-constraint gates; +5. a declared external-judge trust boundary; and +6. comparable token, cost, latency, provenance, policy, and repeatability + measurements. + +ECEB-0.1 does not define a universal model judge, provider pricing schedule, +connector protocol, or enterprise authorization system. + +## 4. Registered schemas + +| Artifact | Schema identifier | +|---|---| +| Benchmark definition | `symgliph.eceb/benchmark-v0.1` | +| Recorded run | `symgliph.eceb/run-v0.1` | +| Evaluation report | `symgliph.eceb/report-v0.1` | + +JSON interchange artifacts MUST validate against the corresponding JSON Schema +in [`schema/`](schema/). + +## 5. Corpus and provenance + +Every corpus artifact records: + +- a stable `artifact_id`; +- a portable corpus-relative `path`; +- an enterprise source kind; +- an information classification; and +- the BLAKE3 digest of its complete source bytes. + +The reference source kinds are code, RFC, runbook, pull request, chat, issue, +API specification, infrastructure, and policy. + +The ECEB corpus root is: + +```text +BLAKE3( + "symgliph:eceb:corpus:v0.1\0" || + canonical_json(sorted_artifact_1) || "\n" || + ... +) +``` + +Artifacts are sorted by their complete record ordering. The corpus root commits +to identity, path, kind, classification, and source digest. Before evaluation, +an implementation MUST re-hash every registered file and reject missing or stale +source bytes. + +## 6. Task definition + +Each task declares: + +- the user query; +- the artifacts required to support an acceptable answer; +- allowed information classifications; +- explicitly denied artifacts; +- an ascending token-budget ladder; +- an absolute quality floor; +- a maximum regression from the full permitted-context baseline; and +- deterministic answer terms. + +Required evidence and denied evidence are independent labels. A required artifact +MUST exist in the benchmark registry. Selecting an unknown artifact is a policy +violation and a provenance failure. + +ECEB-0.1 uses a linear four-value classification vocabulary: `public`, +`internal`, `confidential`, and `restricted`. A task lists permitted values +explicitly; the evaluator MUST NOT infer access from ordinal position. + +## 7. Strategies + +ECEB-0.1 registers three context strategies: + +`full_permitted_context` +: The maximum policy-permitted baseline context. It excludes sources the actor + may not receive and therefore is not equivalent to an unrestricted corpus + dump. + +`naive_top_k` +: A declared fixed retrieval baseline. ECEB records it but does not require it + for conformance. + +`compiled` +: A budget-aware context packet produced by the evaluated system. + +Every comparative run MUST include the full permitted-context baseline and at +least one compiled budget for every task. All strategies MUST use the same model, +judge profile, corpus root, task text, decoding parameters, and pricing snapshot +unless a report explicitly declares a factorial experiment. + +## 8. Recorded observations + +An observation binds one task, strategy, budget, and repetition to: + +- provider-reported input and output tokens; +- input and output cost in micro-US-dollars; +- end-to-end latency in milliseconds; +- judge quality in thousandths; +- the complete answer; +- selected artifact identifiers and observed source digests; and +- a BLAKE3 digest of the exact rendered model context. + +The model and judge fields SHOULD include immutable provider revisions when the +provider exposes them. The pricing snapshot MUST identify its source and date. +Cached-input discounts, embedding requests, tool calls, and retries MUST be +included or reported as separate observations; they MUST NOT silently disappear +from the compiled strategy's cost. + +## 9. Trust boundary + +The reference evaluator verifies structure, budgets, source provenance, evidence +coverage, access policy, deterministic answer constraints, and repeated context +identity. It does not independently reproduce a model-judge score recorded in +`quality_milli`. + +A publishable model-quality claim MUST therefore provide the judge prompt, +rubric, model identity, raw judgments, and human-review status. A conformance +fixture MAY use synthetic scores when it is labeled as non-empirical. + +Required answer terms are a deterministic floor, not a semantic-quality metric. +They prevent a high declared judge score from passing an answer that omits an +explicit benchmark invariant. + +## 10. Quality-constrained minimum budget + +For task `q`, quality threshold `τ`, and compiled context under budget `B`: + +```text +B*(q, τ) = min B +``` + +subject to: + +```text +quality(answer(context_B(q))) >= τ +provenance_coverage == 100% +policy_violations == 0 +prompt_tokens <= B +repeatability == true +answer_constraints == satisfied +``` + +The dynamic quality threshold is: + +```text +τ = max( + task.minimum_quality_milli, + baseline_quality_milli - task.maximum_quality_regression_milli +) +``` + +The evaluator walks the declared budget ladder in ascending order. The first +budget whose required repetitions all pass is the minimum viable budget. Missing +observations at a budget are failures for that budget, not implied results. + +## 11. Mandatory gates + +A strategy-budget group passes only when: + +1. it contains at least `required_repetitions` observations; +2. every prompt fits its declared token budget; +3. every quality score meets the dynamic floor; +4. every required answer term occurs case-insensitively in every answer; +5. every required artifact is selected with its current complete-source digest; +6. no selected artifact is denied, disallowed, or unknown; and +7. every repetition has the same exact-context BLAKE3 digest. + +An implementation MUST evaluate all applicable failures and SHOULD report them +together. It MUST NOT stop at the first favorable budget while concealing a +policy violation observed at that same budget. + +## 12. Metrics + +For baseline tokens `T_full` and minimum viable compiled tokens `T_compiled`: + +```text +token_savings_bps = floor(10,000 × (T_full - T_compiled) / T_full) +``` + +Cost savings use the same equation over total input plus output micro-US-dollar +cost. Negative savings saturate at zero in the v0.1 report. + +The report MUST retain, per task: + +- baseline and compiled prompt tokens; +- minimum viable declared budget; +- token savings; +- baseline and compiled cost; +- cost savings; +- baseline, floor, and compiled quality; +- provenance coverage; +- policy-violation count; +- repeatability; and +- every failed assertion. + +The aggregate report MAY average token and cost savings across passing tasks. +It MUST also report the numerator and denominator of passing tasks. Failed tasks +MUST NOT be dropped from the denominator. + +## 13. Repeatability + +`required_repetitions` MUST be positive. All observations for the same task, +strategy, and budget MUST carry a unique one-based repetition number. + +ECEB-0.1 defines context repeatability as equality of the exact rendered-context +digest. Answer text MAY vary because model decoding can be nondeterministic, but +every repetition must independently pass quality and answer constraints. + +## 14. Policy failures + +The evaluator increments `policy_violations` when selected evidence: + +- has an unknown artifact identifier; +- has a classification absent from the task's allow-list; or +- is explicitly denied for the task. + +A stale digest is a provenance failure even when the artifact would otherwise be +permitted. Policy and provenance are reported separately because they require +different remediation. + +No numeric gain can compensate for a policy violation. A task with one or more +policy violations at its candidate budget does not qualify. + +## 15. Claim classes + +ECEB reports SHOULD label claims using one of these classes: + +`conformance_seed` +: Synthetic or hand-authored records proving evaluator behavior. It is not a + model-performance claim. + +`public_reproduction` +: A public, immutable corpus and replayable model/judge configuration. + +`enterprise_proxy` +: A public corpus constructed to resemble enterprise source mixtures and policy + boundaries. + +`enterprise_validated` +: A private or public real-enterprise evaluation with declared sampling, + authorization, human review, and reproducibility controls. + +Results from one class MUST NOT be represented as evidence for a stronger class. + +## 16. Publication requirements + +A comparative ECEB result MUST publish or escrow: + +- benchmark, run, and report artifacts; +- immutable corpus and task roots; +- source licenses and redistribution status; +- the context-construction implementation revision; +- model, judge, tokenizer, and pricing identities; +- raw per-repetition measurements; +- every exclusion, timeout, retry, and failed task; +- uncertainty or a statement that it was not estimated; and +- human-review status. + +Provider-native token counts are authoritative for cost comparison. Local token +counts MAY be included for deterministic budgeting but MUST be labeled separately. + +## 17. Security considerations + +An ECEB corpus can contain material intentionally designed to resemble secrets, +PII, and access-controlled records. Public fixtures MUST use synthetic values and +MUST NOT contain operational credentials or personal data. + +Benchmark runners SHOULD isolate corpus connectors, redact provider logs, avoid +placing credentials in run artifacts, and treat model output as a possible data +exfiltration channel. A system that refuses a task because no policy-permitted +answer exists SHOULD receive an abstention task rather than be forced to leak. + +## 18. Limitations of v0.1 + +ECEB-0.1 uses explicit artifact-level evidence and classification allow-lists. +It does not yet standardize field-level redaction, attribute-based access control, +temporal permissions, prompt-injection scoring, confidence intervals, or a sealed +private test set. These are candidates for later profiles. + +The checked-in enterprise seed is deliberately small and synthetic. Its passing +numbers prove the evaluator and negative gates only. diff --git a/spec/schema/eceb-benchmark-v0.1.schema.json b/spec/schema/eceb-benchmark-v0.1.schema.json new file mode 100644 index 0000000..971b258 --- /dev/null +++ b/spec/schema/eceb-benchmark-v0.1.schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://symgliph.org/schema/eceb-benchmark-v0.1.schema.json", + "title": "Symgliph ECEB benchmark v0.1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "benchmark_id", "corpus_root", "required_repetitions", "artifacts", "tasks"], + "properties": { + "schema": {"const": "symgliph.eceb/benchmark-v0.1"}, + "benchmark_id": {"type": "string", "minLength": 1}, + "corpus_root": {"$ref": "#/$defs/digest"}, + "required_repetitions": {"type": "integer", "minimum": 1}, + "artifacts": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/artifact"} + }, + "tasks": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/task"} + } + }, + "$defs": { + "digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "classification": { + "enum": ["public", "internal", "confidential", "restricted"] + }, + "source_kind": { + "enum": ["code", "rfc", "runbook", "pull_request", "chat", "issue", "api_spec", "infrastructure", "policy"] + }, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": ["artifact_id", "path", "source_kind", "classification", "source_blake3"], + "properties": { + "artifact_id": {"type": "string", "minLength": 1}, + "path": {"type": "string", "minLength": 1, "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$"}, + "source_kind": {"$ref": "#/$defs/source_kind"}, + "classification": {"$ref": "#/$defs/classification"}, + "source_blake3": {"$ref": "#/$defs/digest"} + } + }, + "task": { + "type": "object", + "additionalProperties": false, + "required": [ + "task_id", + "query", + "required_artifact_ids", + "allowed_classifications", + "denied_artifact_ids", + "budget_ladder", + "minimum_quality_milli", + "maximum_quality_regression_milli", + "required_answer_terms" + ], + "properties": { + "task_id": {"type": "string", "minLength": 1}, + "query": {"type": "string", "minLength": 1}, + "required_artifact_ids": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "allowed_classifications": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/classification"} + }, + "denied_artifact_ids": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "budget_ladder": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "integer", "minimum": 1} + }, + "minimum_quality_milli": {"type": "integer", "minimum": 0, "maximum": 1000}, + "maximum_quality_regression_milli": {"type": "integer", "minimum": 0, "maximum": 1000}, + "required_answer_terms": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + } + } + } + } +} diff --git a/spec/schema/eceb-report-v0.1.schema.json b/spec/schema/eceb-report-v0.1.schema.json new file mode 100644 index 0000000..966272e --- /dev/null +++ b/spec/schema/eceb-report-v0.1.schema.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://symgliph.org/schema/eceb-report-v0.1.schema.json", + "title": "Symgliph ECEB report v0.1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "benchmark_id", + "system", + "model", + "judge", + "pricing_snapshot", + "passed", + "qualified_tasks", + "total_tasks", + "mean_token_savings_bps", + "mean_cost_savings_bps", + "tasks" + ], + "properties": { + "schema": {"const": "symgliph.eceb/report-v0.1"}, + "benchmark_id": {"type": "string", "minLength": 1}, + "system": {"type": "string", "minLength": 1}, + "model": {"type": "string", "minLength": 1}, + "judge": {"type": "string", "minLength": 1}, + "pricing_snapshot": {"type": "string", "minLength": 1}, + "passed": {"type": "boolean"}, + "qualified_tasks": {"type": "integer", "minimum": 0}, + "total_tasks": {"type": "integer", "minimum": 0}, + "mean_token_savings_bps": {"type": ["integer", "null"], "minimum": 0, "maximum": 10000}, + "mean_cost_savings_bps": {"type": ["integer", "null"], "minimum": 0, "maximum": 10000}, + "tasks": { + "type": "array", + "items": {"$ref": "#/$defs/task_report"} + } + }, + "$defs": { + "nullable_nonnegative_integer": {"type": ["integer", "null"], "minimum": 0}, + "nullable_bps": {"type": ["integer", "null"], "minimum": 0, "maximum": 10000}, + "task_report": { + "type": "object", + "additionalProperties": false, + "required": [ + "task_id", + "baseline_prompt_tokens", + "minimum_viable_budget", + "compiled_prompt_tokens", + "token_savings_bps", + "baseline_cost_microusd", + "compiled_cost_microusd", + "cost_savings_bps", + "quality_floor_milli", + "baseline_quality_milli", + "compiled_quality_milli", + "provenance_coverage_bps", + "policy_violations", + "repeatable", + "passed", + "failures" + ], + "properties": { + "task_id": {"type": "string", "minLength": 1}, + "baseline_prompt_tokens": {"type": "integer", "minimum": 0}, + "minimum_viable_budget": {"$ref": "#/$defs/nullable_nonnegative_integer"}, + "compiled_prompt_tokens": {"$ref": "#/$defs/nullable_nonnegative_integer"}, + "token_savings_bps": {"$ref": "#/$defs/nullable_bps"}, + "baseline_cost_microusd": {"type": "integer", "minimum": 0}, + "compiled_cost_microusd": {"$ref": "#/$defs/nullable_nonnegative_integer"}, + "cost_savings_bps": {"$ref": "#/$defs/nullable_bps"}, + "quality_floor_milli": {"type": "integer", "minimum": 0, "maximum": 1000}, + "baseline_quality_milli": {"type": "integer", "minimum": 0, "maximum": 1000}, + "compiled_quality_milli": {"type": ["integer", "null"], "minimum": 0, "maximum": 1000}, + "provenance_coverage_bps": {"type": "integer", "minimum": 0, "maximum": 10000}, + "policy_violations": {"type": "integer", "minimum": 0}, + "repeatable": {"type": "boolean"}, + "passed": {"type": "boolean"}, + "failures": {"type": "array", "items": {"type": "string"}} + } + } + } +} diff --git a/spec/schema/eceb-run-v0.1.schema.json b/spec/schema/eceb-run-v0.1.schema.json new file mode 100644 index 0000000..c0e35bc --- /dev/null +++ b/spec/schema/eceb-run-v0.1.schema.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://symgliph.org/schema/eceb-run-v0.1.schema.json", + "title": "Symgliph ECEB run v0.1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "benchmark_id", "corpus_root", "system", "model", "judge", "pricing_snapshot", "observations"], + "properties": { + "schema": {"const": "symgliph.eceb/run-v0.1"}, + "benchmark_id": {"type": "string", "minLength": 1}, + "corpus_root": {"$ref": "#/$defs/digest"}, + "system": {"type": "string", "minLength": 1}, + "model": {"type": "string", "minLength": 1}, + "judge": {"type": "string", "minLength": 1}, + "pricing_snapshot": {"type": "string", "minLength": 1}, + "observations": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/observation"} + } + }, + "$defs": { + "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": ["artifact_id", "source_blake3"], + "properties": { + "artifact_id": {"type": "string", "minLength": 1}, + "source_blake3": {"$ref": "#/$defs/digest"} + } + }, + "observation": { + "type": "object", + "additionalProperties": false, + "required": [ + "task_id", + "strategy", + "budget_tokens", + "prompt_tokens", + "output_tokens", + "input_cost_microusd", + "output_cost_microusd", + "latency_ms", + "quality_milli", + "answer", + "selected_evidence", + "context_blake3", + "repetition" + ], + "properties": { + "task_id": {"type": "string", "minLength": 1}, + "strategy": {"enum": ["full_permitted_context", "naive_top_k", "compiled"]}, + "budget_tokens": {"type": "integer", "minimum": 1}, + "prompt_tokens": {"type": "integer", "minimum": 0}, + "output_tokens": {"type": "integer", "minimum": 0}, + "input_cost_microusd": {"type": "integer", "minimum": 0}, + "output_cost_microusd": {"type": "integer", "minimum": 0}, + "latency_ms": {"type": "integer", "minimum": 0}, + "quality_milli": {"type": "integer", "minimum": 0, "maximum": 1000}, + "answer": {"type": "string"}, + "selected_evidence": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/evidence"} + }, + "context_blake3": {"$ref": "#/$defs/digest"}, + "repetition": {"type": "integer", "minimum": 1} + } + } + } +} diff --git a/src/eceb.rs b/src/eceb.rs new file mode 100644 index 0000000..725a6f6 --- /dev/null +++ b/src/eceb.rs @@ -0,0 +1,1153 @@ +//! Enterprise Context Efficiency Benchmark (ECEB) records and evaluator. + +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt, fs, + path::Path, +}; + +use serde::{Deserialize, Serialize}; + +/// ECEB benchmark definition schema. +pub const ECEB_BENCHMARK_SCHEMA_V01: &str = "symgliph.eceb/benchmark-v0.1"; +/// ECEB recorded run schema. +pub const ECEB_RUN_SCHEMA_V01: &str = "symgliph.eceb/run-v0.1"; +/// ECEB evaluation report schema. +pub const ECEB_REPORT_SCHEMA_V01: &str = "symgliph.eceb/report-v0.1"; + +/// Enterprise source category represented by one artifact. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EnterpriseSourceKind { + /// Application or library source code. + Code, + /// Architecture or policy RFC. + Rfc, + /// Operations runbook. + Runbook, + /// Pull-request or design-review history. + PullRequest, + /// Chat or collaboration history. + Chat, + /// Issue tracker record. + Issue, + /// API specification. + ApiSpec, + /// Infrastructure-as-code configuration. + Infrastructure, + /// Administrative policy document. + Policy, +} + +/// Linear information classification used by the ECEB v0.1 policy profile. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum InformationClassification { + /// Publicly distributable information. + Public, + /// Organization-internal information. + Internal, + /// Confidential business information. + Confidential, + /// Explicitly restricted information. + Restricted, +} + +/// One immutable corpus artifact referenced by benchmark tasks. +#[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +pub struct EcebArtifact { + /// Stable artifact identifier. + pub artifact_id: String, + /// Portable path beneath the benchmark corpus directory. + pub path: String, + /// Enterprise source category. + pub source_kind: EnterpriseSourceKind, + /// Information classification enforced during evaluation. + pub classification: InformationClassification, + /// BLAKE3 digest of the complete source bytes. + pub source_blake3: String, +} + +/// One quality-constrained context-efficiency task. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct EcebTask { + /// Stable task identifier. + pub task_id: String, + /// User question presented to every strategy. + pub query: String, + /// Artifacts that must be recovered with valid provenance. + pub required_artifact_ids: Vec, + /// Classifications the requesting actor may receive. + pub allowed_classifications: Vec, + /// Artifacts prohibited even when their classification is otherwise allowed. + pub denied_artifact_ids: Vec, + /// Candidate token ceilings, in ascending order. + pub budget_ladder: Vec, + /// Absolute minimum judge score in thousandths. + pub minimum_quality_milli: u16, + /// Maximum permitted regression from the full-context baseline. + pub maximum_quality_regression_milli: u16, + /// Deterministic answer constraints used in addition to the declared judge. + pub required_answer_terms: Vec, +} + +/// Versioned ECEB benchmark definition. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct EcebBenchmark { + /// Schema identifier. + pub schema: String, + /// Stable benchmark identifier. + pub benchmark_id: String, + /// Domain-separated root over canonical artifact metadata. + pub corpus_root: String, + /// Minimum repetitions required for a strategy and budget. + pub required_repetitions: usize, + /// Immutable artifact registry. + pub artifacts: Vec, + /// Evaluation tasks. + pub tasks: Vec, +} + +/// Context-construction strategy recorded by an observation. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EcebStrategy { + /// Every policy-permitted source is provided as a baseline. + FullPermittedContext, + /// A fixed top-k retrieval baseline. + NaiveTopK, + /// Budget-aware compiled context. + Compiled, +} + +/// One selected source and the digest observed by the evaluated system. +#[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +pub struct EcebEvidenceSelection { + /// Selected artifact identifier. + pub artifact_id: String, + /// Complete-source BLAKE3 digest observed at retrieval time. + pub source_blake3: String, +} + +/// One repeated observation for a task, strategy, and budget. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct EcebObservation { + /// Task identifier. + pub task_id: String, + /// Context-construction strategy. + pub strategy: EcebStrategy, + /// Declared input-token ceiling. + pub budget_tokens: usize, + /// Provider-reported input tokens. + pub prompt_tokens: usize, + /// Provider-reported output tokens. + pub output_tokens: usize, + /// Input cost in millionths of a US dollar. + pub input_cost_microusd: u64, + /// Output cost in millionths of a US dollar. + pub output_cost_microusd: u64, + /// End-to-end latency in milliseconds. + pub latency_ms: u64, + /// Declared judge score in thousandths. + pub quality_milli: u16, + /// Complete answer used for deterministic constraint checks. + pub answer: String, + /// Exact sources selected into model context. + pub selected_evidence: Vec, + /// Digest of the exact rendered model context. + pub context_blake3: String, + /// One-based repetition number. + pub repetition: usize, +} + +/// A complete, replayable model run. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct EcebRun { + /// Schema identifier. + pub schema: String, + /// Benchmark identifier. + pub benchmark_id: String, + /// Corpus root used for this run. + pub corpus_root: String, + /// Evaluated system and version. + pub system: String, + /// Model provider, model, and immutable version when available. + pub model: String, + /// Judge identity and evaluation profile. + pub judge: String, + /// Pricing source and observation date. + pub pricing_snapshot: String, + /// Per-task repeated observations. + pub observations: Vec, +} + +/// Per-task quality-constrained efficiency result. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct EcebTaskReport { + /// Task identifier. + pub task_id: String, + /// Mean full-context input tokens. + pub baseline_prompt_tokens: usize, + /// Lowest declared budget whose repeated compiled observations passed. + pub minimum_viable_budget: Option, + /// Mean compiled prompt tokens at the minimum viable budget. + pub compiled_prompt_tokens: Option, + /// Input-token reduction relative to the full-context baseline. + pub token_savings_bps: Option, + /// Mean full-context total cost. + pub baseline_cost_microusd: u64, + /// Mean compiled total cost at the minimum viable budget. + pub compiled_cost_microusd: Option, + /// Cost reduction relative to the full-context baseline. + pub cost_savings_bps: Option, + /// Dynamic quality floor after applying the non-inferiority margin. + pub quality_floor_milli: u16, + /// Mean full-context judge score. + pub baseline_quality_milli: u16, + /// Mean compiled judge score at the minimum viable budget. + pub compiled_quality_milli: Option, + /// Required-evidence coverage with valid source digests. + pub provenance_coverage_bps: u32, + /// Count of denied, disallowed, or unknown selected artifacts. + pub policy_violations: usize, + /// Whether repeated context digests were identical. + pub repeatable: bool, + /// Whether every mandatory task gate passed. + pub passed: bool, + /// Human-readable failed assertions. + pub failures: Vec, +} + +/// Machine-readable ECEB result vector. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct EcebReport { + /// Schema identifier. + pub schema: String, + /// Benchmark identifier. + pub benchmark_id: String, + /// Evaluated system. + pub system: String, + /// Model identity. + pub model: String, + /// Judge identity. + pub judge: String, + /// Pricing snapshot. + pub pricing_snapshot: String, + /// Whether every task passed every mandatory gate. + pub passed: bool, + /// Number of passing tasks. + pub qualified_tasks: usize, + /// Total task count. + pub total_tasks: usize, + /// Mean token reduction across passing tasks. + pub mean_token_savings_bps: Option, + /// Mean cost reduction across passing tasks. + pub mean_cost_savings_bps: Option, + /// Complete per-task metric vectors and failures. + pub tasks: Vec, +} + +/// Structural error that prevents an ECEB run from being evaluated. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct EcebValidationError { + /// Every structural error detected in one validation pass. + pub errors: Vec, +} + +impl fmt::Display for EcebValidationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "ECEB validation failed: {}", + self.errors.join("; ") + ) + } +} + +impl std::error::Error for EcebValidationError {} + +#[derive(Clone, Debug)] +struct GroupEvaluation { + passed: bool, + prompt_tokens: usize, + total_cost_microusd: u64, + quality_milli: u16, + provenance_coverage_bps: u32, + policy_violations: usize, + repeatable: bool, + failures: Vec, +} + +#[derive(Clone, Debug)] +struct CandidateSearch { + selected: Option<(usize, GroupEvaluation)>, + failures: Vec, + policy_violations: usize, + provenance_coverage_bps: u32, + repeatable: bool, +} + +#[derive(Clone, Copy, Debug)] +struct ObservationEvaluation { + provenance_coverage_bps: u32, + policy_violations: usize, +} + +impl EcebBenchmark { + /// Compute the domain-separated root over canonical artifact metadata. + /// + /// # Errors + /// + /// Returns an error when artifact metadata cannot be serialized. + pub fn canonical_corpus_root(&self) -> Result { + let mut artifacts = self.artifacts.clone(); + artifacts.sort(); + let mut hasher = blake3::Hasher::new(); + hasher.update(b"symgliph:eceb:corpus:v0.1\0"); + for artifact in artifacts { + hasher.update(&serde_json::to_vec(&artifact)?); + hasher.update(b"\n"); + } + Ok(hasher.finalize().to_hex().to_string()) + } + + /// Verify every registered source file against its declared digest. + /// + /// # Errors + /// + /// Returns all missing, unreadable, non-portable, and stale-source failures. + pub fn verify_corpus(&self, root: &Path) -> Result<(), EcebValidationError> { + let mut errors = Vec::new(); + for artifact in &self.artifacts { + if artifact.path.starts_with('/') + || artifact.path.split('/').any(|component| component == "..") + { + errors.push(format!( + "artifact {} uses non-portable path {}", + artifact.artifact_id, artifact.path + )); + continue; + } + let path = root.join(&artifact.path); + match fs::read(&path) { + Ok(bytes) => { + let actual = blake3::hash(&bytes).to_hex().to_string(); + if actual != artifact.source_blake3 { + errors.push(format!( + "artifact {} digest mismatch: expected {}, found {}", + artifact.artifact_id, artifact.source_blake3, actual + )); + } + } + Err(error) => errors.push(format!( + "artifact {} cannot be read at {}: {error}", + artifact.artifact_id, + path.display() + )), + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(EcebValidationError { errors }) + } + } +} + +/// Evaluate a recorded run against quality, budget, provenance, policy, and repeatability gates. +/// +/// # Errors +/// +/// Returns all structural benchmark and run errors before scoring any task. +pub fn evaluate_eceb( + benchmark: &EcebBenchmark, + run: &EcebRun, +) -> Result { + validate_structure(benchmark, run)?; + let artifacts: BTreeMap<&str, &EcebArtifact> = benchmark + .artifacts + .iter() + .map(|artifact| (artifact.artifact_id.as_str(), artifact)) + .collect(); + let mut reports = Vec::new(); + for task in &benchmark.tasks { + reports.push(evaluate_task(benchmark, task, run, &artifacts)); + } + let passing: Vec<&EcebTaskReport> = reports.iter().filter(|report| report.passed).collect(); + let mean_token_savings_bps = + mean_optional(passing.iter().filter_map(|report| report.token_savings_bps)); + let mean_cost_savings_bps = + mean_optional(passing.iter().filter_map(|report| report.cost_savings_bps)); + Ok(EcebReport { + schema: ECEB_REPORT_SCHEMA_V01.to_owned(), + benchmark_id: benchmark.benchmark_id.clone(), + system: run.system.clone(), + model: run.model.clone(), + judge: run.judge.clone(), + pricing_snapshot: run.pricing_snapshot.clone(), + passed: passing.len() == reports.len(), + qualified_tasks: passing.len(), + total_tasks: reports.len(), + mean_token_savings_bps, + mean_cost_savings_bps, + tasks: reports, + }) +} + +fn validate_structure(benchmark: &EcebBenchmark, run: &EcebRun) -> Result<(), EcebValidationError> { + let mut errors = Vec::new(); + if benchmark.schema != ECEB_BENCHMARK_SCHEMA_V01 { + errors.push(format!("unsupported benchmark schema {}", benchmark.schema)); + } + if run.schema != ECEB_RUN_SCHEMA_V01 { + errors.push(format!("unsupported run schema {}", run.schema)); + } + if benchmark.benchmark_id != run.benchmark_id { + errors.push("benchmark identifier mismatch".to_owned()); + } + if benchmark.corpus_root != run.corpus_root { + errors.push("run corpus root does not match benchmark".to_owned()); + } + let (artifact_ids, task_ids) = validate_benchmark(benchmark, &mut errors); + validate_run(run, &task_ids, &mut errors); + for task in &benchmark.tasks { + validate_task(task, &artifact_ids, &mut errors); + } + if errors.is_empty() { + Ok(()) + } else { + Err(EcebValidationError { errors }) + } +} + +fn validate_benchmark<'a>( + benchmark: &'a EcebBenchmark, + errors: &mut Vec, +) -> (BTreeSet<&'a str>, BTreeSet<&'a str>) { + match benchmark.canonical_corpus_root() { + Ok(root) if root != benchmark.corpus_root => errors.push(format!( + "benchmark corpus root mismatch: expected {root}, found {}", + benchmark.corpus_root + )), + Err(error) => errors.push(format!("cannot calculate benchmark corpus root: {error}")), + Ok(_) => {} + } + if benchmark.required_repetitions == 0 { + errors.push("required_repetitions must be positive".to_owned()); + } + let artifact_ids = unique_ids( + benchmark + .artifacts + .iter() + .map(|artifact| artifact.artifact_id.as_str()), + "artifact", + errors, + ); + let task_ids = unique_ids( + benchmark.tasks.iter().map(|task| task.task_id.as_str()), + "task", + errors, + ); + for artifact in &benchmark.artifacts { + if !is_digest(&artifact.source_blake3) { + errors.push(format!( + "artifact {} has invalid BLAKE3 digest", + artifact.artifact_id + )); + } + } + (artifact_ids, task_ids) +} + +fn validate_task(task: &EcebTask, artifact_ids: &BTreeSet<&str>, errors: &mut Vec) { + if task.query.trim().is_empty() { + errors.push(format!("task {} has an empty query", task.task_id)); + } + if task.minimum_quality_milli > 1_000 || task.maximum_quality_regression_milli > 1_000 { + errors.push(format!( + "task {} has quality values above 1000", + task.task_id + )); + } + if task.required_artifact_ids.is_empty() { + errors.push(format!("task {} has no required evidence", task.task_id)); + } + if task.budget_ladder.is_empty() + || task.budget_ladder.contains(&0) + || !task.budget_ladder.windows(2).all(|pair| pair[0] < pair[1]) + { + errors.push(format!( + "task {} budget ladder must be positive and strictly increasing", + task.task_id + )); + } + for artifact_id in task + .required_artifact_ids + .iter() + .chain(&task.denied_artifact_ids) + { + if !artifact_ids.contains(artifact_id.as_str()) { + errors.push(format!( + "task {} references unknown artifact {artifact_id}", + task.task_id + )); + } + } +} + +fn validate_run(run: &EcebRun, task_ids: &BTreeSet<&str>, errors: &mut Vec) { + let mut observation_keys = BTreeSet::new(); + for observation in &run.observations { + if !task_ids.contains(observation.task_id.as_str()) { + errors.push(format!( + "observation references unknown task {}", + observation.task_id + )); + } + if observation.quality_milli > 1_000 { + errors.push(format!( + "observation {} has quality above 1000", + observation.task_id + )); + } + if observation.repetition == 0 { + errors.push(format!( + "observation {} has zero repetition number", + observation.task_id + )); + } + if !is_digest(&observation.context_blake3) { + errors.push(format!( + "observation {} has invalid context digest", + observation.task_id + )); + } + let key = ( + observation.task_id.as_str(), + observation.strategy, + observation.budget_tokens, + observation.repetition, + ); + if !observation_keys.insert(key) { + errors.push(format!( + "duplicate observation for task {}, strategy {:?}, budget {}, repetition {}", + observation.task_id, + observation.strategy, + observation.budget_tokens, + observation.repetition + )); + } + } +} + +fn evaluate_task( + benchmark: &EcebBenchmark, + task: &EcebTask, + run: &EcebRun, + artifacts: &BTreeMap<&str, &EcebArtifact>, +) -> EcebTaskReport { + let baseline: Vec<&EcebObservation> = run + .observations + .iter() + .filter(|observation| { + observation.task_id == task.task_id + && observation.strategy == EcebStrategy::FullPermittedContext + }) + .collect(); + let baseline_quality = mean_u16(baseline.iter().map(|observation| observation.quality_milli)); + let quality_floor = task + .minimum_quality_milli + .max(baseline_quality.saturating_sub(task.maximum_quality_regression_milli)); + let baseline_evaluation = evaluate_group( + benchmark, + task, + &baseline, + artifacts, + task.minimum_quality_milli, + None, + ); + let candidate_search = find_compiled_candidate(benchmark, task, run, artifacts, quality_floor); + build_task_report(task, quality_floor, &baseline_evaluation, candidate_search) +} + +fn find_compiled_candidate( + benchmark: &EcebBenchmark, + task: &EcebTask, + run: &EcebRun, + artifacts: &BTreeMap<&str, &EcebArtifact>, + quality_floor: u16, +) -> CandidateSearch { + let mut compiled_groups: BTreeMap> = BTreeMap::new(); + for observation in run.observations.iter().filter(|observation| { + observation.task_id == task.task_id && observation.strategy == EcebStrategy::Compiled + }) { + compiled_groups + .entry(observation.budget_tokens) + .or_default() + .push(observation); + } + let mut selected = None; + let mut candidate_failures = Vec::new(); + let mut candidate_policy_violations = 0; + let mut candidate_provenance_coverage_bps = 0; + let mut candidate_repeatable = false; + for budget in &task.budget_ladder { + let observations = compiled_groups.get(budget).map_or(&[][..], Vec::as_slice); + let evaluation = evaluate_group( + benchmark, + task, + observations, + artifacts, + quality_floor, + Some(*budget), + ); + if evaluation.passed { + selected = Some((*budget, evaluation)); + break; + } + candidate_policy_violations = candidate_policy_violations.max(evaluation.policy_violations); + candidate_provenance_coverage_bps = + candidate_provenance_coverage_bps.max(evaluation.provenance_coverage_bps); + candidate_repeatable |= evaluation.repeatable; + for failure in evaluation.failures { + candidate_failures.push(format!("budget {budget}: {failure}")); + } + } + CandidateSearch { + selected, + failures: candidate_failures, + policy_violations: candidate_policy_violations, + provenance_coverage_bps: candidate_provenance_coverage_bps, + repeatable: candidate_repeatable, + } +} + +fn build_task_report( + task: &EcebTask, + quality_floor: u16, + baseline: &GroupEvaluation, + candidate_search: CandidateSearch, +) -> EcebTaskReport { + let mut failures = baseline.failures.clone(); + if !baseline.passed { + failures.insert( + 0, + "full permitted-context baseline failed mandatory gates".to_owned(), + ); + } + let ( + minimum_viable_budget, + compiled_prompt_tokens, + token_savings_bps, + compiled_cost_microusd, + cost_savings_bps, + compiled_quality_milli, + provenance_coverage_bps, + policy_violations, + repeatable, + ) = if let Some((budget, candidate)) = candidate_search.selected { + ( + Some(budget), + Some(candidate.prompt_tokens), + Some(savings_bps( + baseline.prompt_tokens as u64, + candidate.prompt_tokens as u64, + )), + Some(candidate.total_cost_microusd), + Some(savings_bps( + baseline.total_cost_microusd, + candidate.total_cost_microusd, + )), + Some(candidate.quality_milli), + candidate.provenance_coverage_bps, + candidate.policy_violations, + candidate.repeatable, + ) + } else { + failures.push("no compiled budget passed every mandatory gate".to_owned()); + failures.extend(candidate_search.failures); + ( + None, + None, + None, + None, + None, + None, + candidate_search.provenance_coverage_bps, + candidate_search.policy_violations, + candidate_search.repeatable, + ) + }; + let passed = baseline.passed && minimum_viable_budget.is_some(); + EcebTaskReport { + task_id: task.task_id.clone(), + baseline_prompt_tokens: baseline.prompt_tokens, + minimum_viable_budget, + compiled_prompt_tokens, + token_savings_bps, + baseline_cost_microusd: baseline.total_cost_microusd, + compiled_cost_microusd, + cost_savings_bps, + quality_floor_milli: quality_floor, + baseline_quality_milli: baseline.quality_milli, + compiled_quality_milli, + provenance_coverage_bps, + policy_violations, + repeatable, + passed, + failures, + } +} + +fn evaluate_group( + benchmark: &EcebBenchmark, + task: &EcebTask, + observations: &[&EcebObservation], + artifacts: &BTreeMap<&str, &EcebArtifact>, + quality_floor: u16, + expected_budget: Option, +) -> GroupEvaluation { + let mut failures = Vec::new(); + if observations.len() < benchmark.required_repetitions { + failures.push(format!( + "requires {} repetitions, found {}", + benchmark.required_repetitions, + observations.len() + )); + } + let digests: BTreeSet<&str> = observations + .iter() + .map(|observation| observation.context_blake3.as_str()) + .collect(); + let repeatable = !observations.is_empty() && digests.len() == 1; + if !repeatable { + failures.push("context digest changed between repetitions".to_owned()); + } + let required: BTreeSet<&str> = task + .required_artifact_ids + .iter() + .map(String::as_str) + .collect(); + let allowed: BTreeSet = + task.allowed_classifications.iter().copied().collect(); + let denied: BTreeSet<&str> = task + .denied_artifact_ids + .iter() + .map(String::as_str) + .collect(); + let mut minimum_coverage = 10_000; + let mut total_policy_violations = 0; + for observation in observations { + let evaluation = evaluate_observation( + task, + observation, + artifacts, + &required, + &allowed, + &denied, + quality_floor, + expected_budget, + &mut failures, + ); + minimum_coverage = minimum_coverage.min(evaluation.provenance_coverage_bps); + total_policy_violations += evaluation.policy_violations; + } + GroupEvaluation { + passed: failures.is_empty(), + prompt_tokens: mean_usize(observations.iter().map(|value| value.prompt_tokens)), + total_cost_microusd: mean_u64(observations.iter().map(|value| { + value + .input_cost_microusd + .saturating_add(value.output_cost_microusd) + })), + quality_milli: mean_u16(observations.iter().map(|value| value.quality_milli)), + provenance_coverage_bps: if observations.is_empty() { + 0 + } else { + minimum_coverage + }, + policy_violations: total_policy_violations, + repeatable, + failures, + } +} + +#[allow(clippy::too_many_arguments)] +fn evaluate_observation( + task: &EcebTask, + observation: &EcebObservation, + artifacts: &BTreeMap<&str, &EcebArtifact>, + required: &BTreeSet<&str>, + allowed: &BTreeSet, + denied: &BTreeSet<&str>, + quality_floor: u16, + expected_budget: Option, + failures: &mut Vec, +) -> ObservationEvaluation { + if observation.prompt_tokens > observation.budget_tokens { + failures.push(format!( + "repetition {} exceeded token budget: {} > {}", + observation.repetition, observation.prompt_tokens, observation.budget_tokens + )); + } + if expected_budget.is_some_and(|budget| observation.budget_tokens != budget) { + failures.push(format!( + "repetition {} declared unexpected budget {}", + observation.repetition, observation.budget_tokens + )); + } + if observation.quality_milli < quality_floor { + failures.push(format!( + "repetition {} quality {} is below floor {quality_floor}", + observation.repetition, observation.quality_milli + )); + } + let answer = observation.answer.to_lowercase(); + for term in &task.required_answer_terms { + if !answer.contains(&term.to_lowercase()) { + failures.push(format!( + "repetition {} answer omits required term {term:?}", + observation.repetition + )); + } + } + let (valid_selected, policy_violations) = + evaluate_selected_evidence(observation, artifacts, allowed, denied, failures); + let recovered = required.intersection(&valid_selected).count(); + let provenance_coverage_bps = ratio_bps(recovered as u64, required.len() as u64); + if provenance_coverage_bps != 10_000 { + failures.push(format!( + "repetition {} provenance coverage is {provenance_coverage_bps} bps", + observation.repetition + )); + } + ObservationEvaluation { + provenance_coverage_bps, + policy_violations, + } +} + +fn evaluate_selected_evidence<'a>( + observation: &'a EcebObservation, + artifacts: &BTreeMap<&str, &'a EcebArtifact>, + allowed: &BTreeSet, + denied: &BTreeSet<&str>, + failures: &mut Vec, +) -> (BTreeSet<&'a str>, usize) { + let mut valid_selected = BTreeSet::new(); + let mut policy_violations = 0; + for selection in &observation.selected_evidence { + let Some(artifact) = artifacts.get(selection.artifact_id.as_str()) else { + policy_violations += 1; + failures.push(format!( + "repetition {} selected unknown artifact {}", + observation.repetition, selection.artifact_id + )); + continue; + }; + if selection.source_blake3 != artifact.source_blake3 { + failures.push(format!( + "repetition {} selected stale evidence {}", + observation.repetition, selection.artifact_id + )); + continue; + } + valid_selected.insert(selection.artifact_id.as_str()); + if denied.contains(selection.artifact_id.as_str()) + || !allowed.contains(&artifact.classification) + { + policy_violations += 1; + failures.push(format!( + "repetition {} violated policy with artifact {}", + observation.repetition, selection.artifact_id + )); + } + } + (valid_selected, policy_violations) +} + +fn unique_ids<'a>( + values: impl Iterator, + kind: &str, + errors: &mut Vec, +) -> BTreeSet<&'a str> { + let mut identifiers = BTreeSet::new(); + for identifier in values { + if identifier.is_empty() { + errors.push(format!("empty {kind} identifier")); + } else if !identifiers.insert(identifier) { + errors.push(format!("duplicate {kind} identifier {identifier}")); + } + } + identifiers +} + +fn is_digest(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn ratio_bps(numerator: u64, denominator: u64) -> u32 { + if denominator == 0 { + return 0; + } + u32::try_from( + u128::from(numerator) + .saturating_mul(10_000) + .checked_div(u128::from(denominator)) + .unwrap_or(0) + .min(10_000), + ) + .unwrap_or(10_000) +} + +fn savings_bps(baseline: u64, candidate: u64) -> u32 { + if baseline == 0 { + return 0; + } + ratio_bps(baseline.saturating_sub(candidate), baseline) +} + +fn mean_optional(values: impl Iterator) -> Option { + let values: Vec = values.collect(); + if values.is_empty() { + None + } else { + Some( + u32::try_from( + values.iter().map(|value| u64::from(*value)).sum::() + / u64::try_from(values.len()).unwrap_or(u64::MAX), + ) + .unwrap_or(u32::MAX), + ) + } +} + +fn mean_usize(values: impl Iterator) -> usize { + let values: Vec = values.collect(); + if values.is_empty() { + 0 + } else { + values.iter().sum::() / values.len() + } +} + +fn mean_u64(values: impl Iterator) -> u64 { + let values: Vec = values.collect(); + if values.is_empty() { + 0 + } else { + values.iter().sum::() / u64::try_from(values.len()).unwrap_or(u64::MAX) + } +} + +fn mean_u16(values: impl Iterator) -> u16 { + let values: Vec = values.collect(); + if values.is_empty() { + 0 + } else { + u16::try_from( + values.iter().map(|value| u64::from(*value)).sum::() + / u64::try_from(values.len()).unwrap_or(u64::MAX), + ) + .unwrap_or(u16::MAX) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn artifact( + artifact_id: &str, + classification: InformationClassification, + byte: u8, + ) -> EcebArtifact { + EcebArtifact { + artifact_id: artifact_id.to_owned(), + path: format!("{artifact_id}.md"), + source_kind: EnterpriseSourceKind::Policy, + classification, + source_blake3: format!("{byte:02x}").repeat(32), + } + } + + fn selection(artifact: &EcebArtifact) -> EcebEvidenceSelection { + EcebEvidenceSelection { + artifact_id: artifact.artifact_id.clone(), + source_blake3: artifact.source_blake3.clone(), + } + } + + fn observation( + artifact: &EcebArtifact, + strategy: EcebStrategy, + budget_tokens: usize, + repetition: usize, + ) -> EcebObservation { + EcebObservation { + task_id: "task:auth".to_owned(), + strategy, + budget_tokens, + prompt_tokens: if strategy == EcebStrategy::Compiled { + 50 + } else { + 200 + }, + output_tokens: 20, + input_cost_microusd: if strategy == EcebStrategy::Compiled { + 50 + } else { + 200 + }, + output_cost_microusd: 20, + latency_ms: 100, + quality_milli: 950, + answer: "Use three attempts and then open the circuit breaker.".to_owned(), + selected_evidence: vec![selection(artifact)], + context_blake3: "ab".repeat(32), + repetition, + } + } + + fn fixture() -> (EcebBenchmark, EcebRun) { + let required = artifact("source:auth", InformationClassification::Internal, 0x11); + let denied = artifact("hr:payroll", InformationClassification::Restricted, 0x22); + let mut benchmark = EcebBenchmark { + schema: ECEB_BENCHMARK_SCHEMA_V01.to_owned(), + benchmark_id: "eceb:test".to_owned(), + corpus_root: String::new(), + required_repetitions: 2, + artifacts: vec![required.clone(), denied], + tasks: vec![EcebTask { + task_id: "task:auth".to_owned(), + query: "How do retries fail closed?".to_owned(), + required_artifact_ids: vec![required.artifact_id.clone()], + allowed_classifications: vec![InformationClassification::Internal], + denied_artifact_ids: vec!["hr:payroll".to_owned()], + budget_ladder: vec![100, 200], + minimum_quality_milli: 900, + maximum_quality_regression_milli: 25, + required_answer_terms: vec![ + "three attempts".to_owned(), + "circuit breaker".to_owned(), + ], + }], + }; + benchmark.corpus_root = benchmark.canonical_corpus_root().unwrap(); + let mut observations = Vec::new(); + for repetition in 1..=2 { + observations.push(observation( + &required, + EcebStrategy::FullPermittedContext, + 300, + repetition, + )); + observations.push(observation( + &required, + EcebStrategy::Compiled, + 100, + repetition, + )); + } + let run = EcebRun { + schema: ECEB_RUN_SCHEMA_V01.to_owned(), + benchmark_id: benchmark.benchmark_id.clone(), + corpus_root: benchmark.corpus_root.clone(), + system: "fixture".to_owned(), + model: "fixture-model".to_owned(), + judge: "fixture-judge".to_owned(), + pricing_snapshot: "fixture-pricing".to_owned(), + observations, + }; + (benchmark, run) + } + + #[test] + fn finds_the_minimum_quality_preserving_budget() { + let (benchmark, run) = fixture(); + let report = evaluate_eceb(&benchmark, &run).unwrap(); + assert!(report.passed); + assert_eq!(report.tasks[0].minimum_viable_budget, Some(100)); + assert_eq!(report.tasks[0].token_savings_bps, Some(7_500)); + assert_eq!(report.tasks[0].provenance_coverage_bps, 10_000); + } + + #[test] + fn policy_leakage_fails_the_candidate() { + let (benchmark, mut run) = fixture(); + let denied = benchmark + .artifacts + .iter() + .find(|artifact| artifact.artifact_id == "hr:payroll") + .unwrap(); + for observation in run + .observations + .iter_mut() + .filter(|observation| observation.strategy == EcebStrategy::Compiled) + { + observation.selected_evidence.push(selection(denied)); + } + let report = evaluate_eceb(&benchmark, &run).unwrap(); + assert!(!report.passed); + assert_eq!(report.tasks[0].policy_violations, 2); + assert!( + report.tasks[0] + .failures + .iter() + .any(|failure| failure.contains("violated policy")) + ); + } + + #[test] + fn stale_provenance_fails_the_candidate() { + let (benchmark, mut run) = fixture(); + for observation in run + .observations + .iter_mut() + .filter(|observation| observation.strategy == EcebStrategy::Compiled) + { + observation.selected_evidence[0].source_blake3 = "ff".repeat(32); + } + let report = evaluate_eceb(&benchmark, &run).unwrap(); + assert!(!report.passed); + assert!( + report.tasks[0] + .failures + .iter() + .any(|failure| failure.contains("stale evidence")) + ); + } + + #[test] + fn token_overflow_fails_the_candidate() { + let (benchmark, mut run) = fixture(); + for observation in run + .observations + .iter_mut() + .filter(|observation| observation.strategy == EcebStrategy::Compiled) + { + observation.prompt_tokens = observation.budget_tokens + 1; + } + let report = evaluate_eceb(&benchmark, &run).unwrap(); + assert!(!report.passed); + assert!( + report.tasks[0] + .failures + .iter() + .any(|failure| failure.contains("exceeded token budget")) + ); + } + + #[test] + fn corpus_root_tampering_is_a_structural_error() { + let (mut benchmark, run) = fixture(); + benchmark.artifacts[0].classification = InformationClassification::Public; + let error = evaluate_eceb(&benchmark, &run).unwrap_err(); + assert!( + error + .errors + .iter() + .any(|failure| failure.contains("benchmark corpus root mismatch")) + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index a4bbaf1..018e843 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,7 @@ mod cargo_context; mod context; mod corpus; +mod eceb; mod error; mod glyph; mod golden; @@ -28,6 +29,12 @@ pub use context::{ FullContext, }; pub use corpus::{CorpusBuilder, verify}; +pub use eceb::{ + ECEB_BENCHMARK_SCHEMA_V01, ECEB_REPORT_SCHEMA_V01, ECEB_RUN_SCHEMA_V01, EcebArtifact, + EcebBenchmark, EcebEvidenceSelection, EcebObservation, EcebReport, EcebRun, EcebStrategy, + EcebTask, EcebTaskReport, EcebValidationError, EnterpriseSourceKind, InformationClassification, + evaluate_eceb, +}; pub use error::{Error, Result}; pub use glyph::{GLYPH_PROFILE_SVG_V1, GlyphOptions, GlyphRenderer}; pub use golden::{ diff --git a/src/main.rs b/src/main.rs index 228d18a..2d94d53 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,8 +9,8 @@ use std::{ use clap::{Parser, Subcommand}; use symgliph::{ - ContextEngine, ContextRequest, CorpusBuilder, CorpusManifest, GlyphRenderer, serve_jsonl, - verify, + ContextEngine, ContextRequest, CorpusBuilder, CorpusManifest, EcebBenchmark, EcebRun, + GlyphRenderer, evaluate_eceb, serve_jsonl, verify, }; #[derive(Debug, Parser)] @@ -125,6 +125,19 @@ enum Command { #[arg(short, long)] manifest: Option, }, + /// Evaluate a recorded run under the ECEB quality, policy, and budget gates. + EvaluateEceb { + /// ECEB benchmark definition. + benchmark: PathBuf, + /// Recorded baseline and compiled-context observations. + run: PathBuf, + /// Directory containing the immutable benchmark corpus. + #[arg(long)] + corpus: PathBuf, + /// Optional report destination. Prints JSON to stdout when omitted. + #[arg(short, long)] + output: Option, + }, } fn main() -> ExitCode { @@ -173,6 +186,12 @@ fn run(cli: Cli) -> Result> { max_ratio_bps, } => prove_context(&root, manifest, query, max_tokens, max_nodes, max_ratio_bps), Command::Serve { root, manifest } => serve(&root, manifest), + Command::EvaluateEceb { + benchmark, + run, + corpus, + output, + } => evaluate_eceb_command(&benchmark, &run, &corpus, output.as_deref()), } } @@ -321,6 +340,33 @@ fn serve(root: &Path, manifest: Option) -> CliResult { Ok(ExitCode::SUCCESS) } +fn evaluate_eceb_command( + benchmark_path: &Path, + run_path: &Path, + corpus: &Path, + output: Option<&Path>, +) -> CliResult { + let benchmark: EcebBenchmark = serde_json::from_slice(&fs::read(benchmark_path)?)?; + let run: EcebRun = serde_json::from_slice(&fs::read(run_path)?)?; + benchmark.verify_corpus(corpus)?; + let report = evaluate_eceb(&benchmark, &run)?; + let encoded = serde_json::to_vec_pretty(&report)?; + if let Some(path) = output { + fs::write(path, &encoded)?; + } else { + println!("{}", String::from_utf8(encoded)?); + } + if report.passed { + Ok(ExitCode::SUCCESS) + } else { + eprintln!( + "ECEB failed: qualified_tasks={}/{}", + report.qualified_tasks, report.total_tasks + ); + Ok(ExitCode::from(2)) + } +} + fn context_engine( root: &Path, manifest: Option,