From 39da711f7d495c4349e870aa57a827f6481cf411 Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:51:18 +0000 Subject: [PATCH 1/6] Add basic direct report provenance for 0.3.0 --- CHANGELOG.md | 2 + examples/with-provenance-enriched.json | 43 ++++ examples/with-provenance-filtered.json | 38 ++++ examples/with-provenance.json | 46 +++++ schema/ctrf.schema.json | 35 ++++ spec/ctrf.md | 264 ++++++++++++++++++++++++- 6 files changed, 419 insertions(+), 9 deletions(-) create mode 100644 examples/with-provenance-enriched.json create mode 100644 examples/with-provenance-filtered.json create mode 100644 examples/with-provenance.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c3bfa57..6fbc235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Optional document `provenance` in specification 0.3.0, recording direct source report identities for derived reports, with explicit lineage and interpretation rules. + - Optional `results.errors` in specification 0.3.0 for global error messages, traces and metadata, separate from individual test results and counts. - Optional `assertionFailures` on final test results and retry history, with producer-formatted comparisons, individual diagnostics and `extra` metadata. diff --git a/examples/with-provenance-enriched.json b/examples/with-provenance-enriched.json new file mode 100644 index 0000000..19a3f93 --- /dev/null +++ b/examples/with-provenance-enriched.json @@ -0,0 +1,43 @@ +{ + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440997", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "example-dashboard", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440998" + } + ], + "extra": { + "example.dashboard": { + "label": "triage" + } + } + }, + "runId": "coordinated-checkout-run" +} diff --git a/examples/with-provenance-filtered.json b/examples/with-provenance-filtered.json new file mode 100644 index 0000000..e33acc3 --- /dev/null +++ b/examples/with-provenance-filtered.json @@ -0,0 +1,38 @@ +{ + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440998", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "example-dashboard", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440999" + } + ] + }, + "runId": "coordinated-checkout-run" +} diff --git a/examples/with-provenance.json b/examples/with-provenance.json new file mode 100644 index 0000000..2925637 --- /dev/null +++ b/examples/with-provenance.json @@ -0,0 +1,46 @@ +{ + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + }, + { + "reportId": "550e8400-e29b-41d4-a716-446655440101" + } + ] + }, + "runId": "coordinated-checkout-run" +} diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 9ce1fbe..3f94349 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -34,6 +34,41 @@ "description": "Tool or system that produced this CTRF document", "type": "string" }, + "provenance": { + "description": "Direct CTRF document sources from which this report was derived; does not establish completeness or supersession", + "type": "object", + "required": [ "inputs" ], + "properties": { + "inputs": { + "description": "All distinct direct contributing CTRF source documents, without flattening their ancestry; order has no meaning", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "object", + "required": [ "reportId" ], + "propertyNames": { + "const": "reportId" + }, + "properties": { + "reportId": { + "description": "The reportId of the contributing document", + "type": "string", + "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", + "format": "uuid", + "minLength": 1 + } + }, + "additionalProperties": false + } + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + }, "extra": { "description": "Extension point for arbitrary metadata", "type": "object" diff --git a/spec/ctrf.md b/spec/ctrf.md index f8e1e89..733fe9c 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -145,7 +145,7 @@ Consumers MAY reconstruct hierarchical views for presentation purposes, but such ### 2.4. Identity Model -CTRF provides a layered identity model that supports stable identifiers for test cases, correlation of documents belonging to the same logical run, deduplication, aggregation, and future provenance support. +CTRF provides a layered identity model that supports stable identifiers for test cases, correlation of documents belonging to the same logical run, deduplication, aggregation, and document provenance. Identity fields are OPTIONAL. When provided, they enable: @@ -343,6 +343,9 @@ A structure describing a metric’s `current` value, the corresponding value fro **Attachment**: An external file or resource referenced by a test case or attempt history entry, such as a screenshot, log file, or other artifact. +**Derived Report**: +A CTRF document produced by transforming or combining content from one or more existing CTRF documents, including content used to compute derived values. + --- ## 4. Overall Document Structure @@ -574,7 +577,7 @@ Each layer addresses a distinct identification concern: All identity fields are OPTIONAL. Producers SHOULD assign them when the corresponding correlation is meaningful for their workflow. -This identity model provides primitives that can be used by future provenance and lineage extensions. It does not itself define derivation, parent/child report relationships, transformation history, or trust metadata. +The identity model provides the identifiers used by the optional document provenance model in Section 20. Identity alone does not establish derivation, transformation history, or trust. #### 4.9.1. Uniqueness @@ -746,7 +749,18 @@ If present, it MUST follow the structure described in Section 19 (Baseline Objec --- -### 5.10. `extra` +### 5.10. `provenance` + +**Description:** +Identifies the direct CTRF source documents from which this document was derived. + +**Requirements:** +`provenance` is OPTIONAL. +If present, it MUST follow the structure described in Section 20 (Provenance Object). + +--- + +### 5.11. `extra` **Description:** An extensibility object containing arbitrary metadata. @@ -2288,7 +2302,49 @@ An extensibility object containing arbitrary metadata related to the baseline. It MAY contain arbitrary fields. It is the ONLY permitted extension point within the baseline object. -## 20. Versioning +## 20. Provenance Object + +The optional top-level `provenance` object records direct document lineage for a derived CTRF report. Its `inputs` identify the existing CTRF source documents whose content contributed to the output, including content used to compute derived values. A report merely consulted without contributing content is not a provenance input. + +Provenance describes observed derivation. It does not assert that all expected shards or reports arrived, that the output replaces its inputs, or that overlapping inputs can safely be counted together. It does not attribute individual tests, attempts, attachments, or errors to a source. A schema-valid report is not evidence of complete or verified lineage. + +Unknown fields MUST NOT appear in `provenance` except within its `extra` object. + +### 20.1. `inputs` + +**Description:** +All distinct direct CTRF source documents from which the current document was derived. + +**Requirements:** +`inputs` MUST be present when `provenance` is present. It MUST be a non-empty array of objects. Each input object MUST contain only a required `reportId`, which MUST be a non-empty string and a valid UUID. + +Input order has no semantic meaning. Each distinct source document MUST appear exactly once, regardless of how many times its content was used. UUID comparison MUST ignore letter case. An input MUST NOT identify the current document itself when the current document has a `reportId`. + +Producers MUST list all contributing direct CTRF source documents. If any such source lacks a `reportId`, producers MUST omit `provenance`; they MUST NOT invent an identity for an already-emitted source or list only the identified subset. Omission does not establish that a report is an original source. + +Producers MUST reference the immediate source documents rather than copying their `provenance.inputs` or flattening their ancestry. If reports A and B produce C, and C produces D, C lists A and B; D lists C. If A also contributes directly to D, D lists both C and A. These links do not resolve overlapping content or prescribe merge/deduplication behavior. + +A derived report SHOULD receive a new `reportId`, distinct from its inputs, consistent with Section 5.3. Reports without provenance or document identity remain valid. Retransmission of the same unchanged artifact SHOULD preserve its `reportId` and provenance. Logical-run correlation remains separate; provenance does not change the single-logical-run scope of `results`. + +Input references neither embed nor locate their sources. Following a chain requires access to the referenced documents through a consumer's storage or lookup mechanism. An unavailable ancestor does not invalidate a direct reference. Consumers MUST NOT interpret an input reference alone as proof of completeness, supersession, authenticity, or artifact integrity. A UUID is an identity, not a content digest or retrieval URL. + +The JSON Schema checks the object structure, UUID format, non-empty input list and identical duplicate entries. Case-insensitive identity uniqueness and self-reference require semantic checks in addition to JSON Schema validation. + +### 20.2. `extra` + +**Description:** +An optional object for provenance-related extension metadata, following Section 4's extension rules. + +**Requirements:** +`extra` MAY contain arbitrary fields. It is the ONLY permitted extension point directly within `provenance`. Input entries do not provide an extension point in this version. + +This model is CTRF-native and does not claim conformance to an external provenance or attestation standard. Future integrations can supply additional metadata or separate linked documents without changing the meaning of direct inputs. Expected-input manifests, completeness, supersession, activity details, source lookup and verification are outside this version's provenance contract. + +See Appendix D.9 for merge, single-source transformation and multi-hop examples. + +--- + +## 21. Versioning CTRF follows **Semantic Versioning**. @@ -2320,15 +2376,16 @@ The `specVersion` field in a CTRF document MUST follow these versioning rules. --- -## 21. Conformance Requirements +## 22. Conformance Requirements -### 21.1. Producer Conformance +### 22.1. Producer Conformance Producers: - MUST output UTF-8 JSON - MUST set `reportFormat = "CTRF"` - MUST include all required fields +- MUST follow the provenance requirements in Section 20 when emitting provenance - MUST follow rules for retry histories - MUST NOT introduce fields outside `extra` objects - MUST NOT use extensions to redefine, weaken, or contradict fields defined in this specification @@ -2351,7 +2408,7 @@ Producers SHOULD: --- -## 21.2. Consumer Conformance +## 22.2. Consumer Conformance Consumers: @@ -2374,7 +2431,9 @@ Consumers: --- -## 22. Security Considerations +Consumers MAY follow `provenance.inputs` to inspect document lineage when source reports are available. They MUST apply the interpretation limits in Section 20; lineage alone is not a consumer selection or deduplication policy. + +## 23. Security Considerations CTRF documents may contain sensitive information such as: @@ -2391,7 +2450,7 @@ Consumers SHOULD consider size limits when processing attachments or screenshots --- -## 23. IANA Considerations +## 24. IANA Considerations This document registers **no** IANA resources. @@ -2442,6 +2501,41 @@ to this specification. "description": "Tool or system that produced this CTRF document", "type": "string" }, + "provenance": { + "description": "Direct CTRF document sources from which this report was derived; does not establish completeness or supersession", + "type": "object", + "required": [ "inputs" ], + "properties": { + "inputs": { + "description": "All distinct direct contributing CTRF source documents, without flattening their ancestry; order has no meaning", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "object", + "required": [ "reportId" ], + "propertyNames": { + "const": "reportId" + }, + "properties": { + "reportId": { + "description": "The reportId of the contributing document", + "type": "string", + "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", + "format": "uuid", + "minLength": 1 + } + }, + "additionalProperties": false + } + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + }, "extra": { "description": "Extension point for arbitrary metadata", "type": "object" @@ -4206,3 +4300,155 @@ Each block below matches its named standalone example. } } ``` + +### D.9. Direct Document Provenance + +The following reports illustrate a merge followed by filtering and enrichment. The source reports identified by the merge inputs are not embedded. Their absence here does not establish whether other shards were expected. + +#### D.9.1. Merge of two source reports + +GitHub combines reports ending in `0100` and `0101` into a new report ending in `0999`. It records both direct inputs. This example corresponds to `examples/with-provenance.json`. + +```json +{ + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + }, + { + "reportId": "550e8400-e29b-41d4-a716-446655440101" + } + ] + }, + "runId": "coordinated-checkout-run" +} +``` + +#### D.9.2. Single-source filtering + +A dashboard reads the merged report `0999` and emits report `0998` containing only the failed test. It references `0999`, not that report's ancestors. This example corresponds to `examples/with-provenance-filtered.json`. + +```json +{ + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440998", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "example-dashboard", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440999" + } + ] + }, + "runId": "coordinated-checkout-run" +} +``` + +#### D.9.3. A further derivation + +Enrichment of `0998` produces `0997` with a dashboard label. It references only `0998`. Following that report leads to `0999` and, when available, the original sources. This example corresponds to `examples/with-provenance-enriched.json`. + +```json +{ + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440997", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "example-dashboard", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 1, + "passed": 0, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440998" + } + ], + "extra": { + "example.dashboard": { + "label": "triage" + } + } + }, + "runId": "coordinated-checkout-run" +} +``` From d6a64f0dd2fdfdb86f3fadadd5f35ecf046ed3a7 Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:51:18 +0000 Subject: [PATCH 2/6] Verify provenance structure and identity semantics --- .github/workflows/test.yml | 3 + CONTRIBUTING.md | 7 + tests/ctrf.test.json | 15 + .../normative/additional-properties.test.json | 71 ++ tests/normative/provenance.test.json | 753 ++++++++++++++++++ tests/normative/required-properties.test.json | 35 + tests/normative/string-constraints.test.json | 37 + tests/normative/type-validation.test.json | 103 +++ tests/normative/valid-documents.test.json | 15 + tests/provenance_semantics.py | 93 +++ 10 files changed, 1132 insertions(+) create mode 100644 tests/normative/provenance.test.json create mode 100644 tests/provenance_semantics.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bb7d4b2..3305f36 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,9 @@ jobs: - name: Run conformance tests (normative) run: jsonschema test tests/normative --json > ctrf-normative.json + - name: Check provenance identity semantics + run: python3 tests/provenance_semantics.py + - name: Run informative tests run: jsonschema test tests/informative --json > ctrf-informative.json continue-on-error: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2821b57..c6c276a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -150,6 +150,13 @@ jsonschema validate schema/ctrf.schema.json examples/*.json The `--exclude top_level_examples` flag is used because CTRF maintains examples in the `examples/` directory rather than embedding them in the schema itself. +The provenance identity checks supplement JSON Schema validation with UUID +identity comparison and self-reference checks from Section 20. Run them with: + +```bash +python3 tests/provenance_semantics.py +``` + ### Adding Tests for Changes When contributing changes, you SHOULD add or update tests: diff --git a/tests/ctrf.test.json b/tests/ctrf.test.json index 4161760..7171329 100644 --- a/tests/ctrf.test.json +++ b/tests/ctrf.test.json @@ -208,6 +208,21 @@ "description": "Merged coordinated retry report", "valid": true, "dataPath": "../examples/coordinated-retry-merged.json" + }, + { + "description": "Valid direct provenance: with-provenance.json", + "valid": true, + "dataPath": "../examples/with-provenance.json" + }, + { + "description": "Valid direct provenance: with-provenance-filtered.json", + "valid": true, + "dataPath": "../examples/with-provenance-filtered.json" + }, + { + "description": "Valid direct provenance: with-provenance-enriched.json", + "valid": true, + "dataPath": "../examples/with-provenance-enriched.json" } ] } diff --git a/tests/normative/additional-properties.test.json b/tests/normative/additional-properties.test.json index a8782c5..96664f3 100644 --- a/tests/normative/additional-properties.test.json +++ b/tests/normative/additional-properties.test.json @@ -255,6 +255,77 @@ ] } } + }, + { + "description": "Provenance object with unknown property", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + }, + "provenance": { + "unknownField": true + } + } + }, + { + "description": "Provenance input object with unknown property", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440000", + "unknownField": true + } + ] + } + } } ] } diff --git a/tests/normative/provenance.test.json b/tests/normative/provenance.test.json new file mode 100644 index 0000000..171b1df --- /dev/null +++ b/tests/normative/provenance.test.json @@ -0,0 +1,753 @@ +{ + "target": "../../schema/ctrf.schema.json", + "$comment": "Provenance schema constraints; semantic identity rules are checked separately by tests/provenance_semantics.py", + "tests": [ + { + "description": "Provenance requires inputs", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": {}, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Extra alone does not replace required inputs", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "extra": { + "example.tool": {} + } + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Inputs cannot be empty", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Duplicate identical input references are rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + }, + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Multiple roots are represented by distinct direct inputs", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + }, + { + "reportId": "550e8400-e29b-41d4-a716-446655440101" + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "A single immediate source is sufficient", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440999" + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Unknown input fields are rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100", + "uri": "https://example.com/source" + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Per-input extra is not defined in this version", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100", + "extra": {} + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Unknown provenance fields are rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + } + ], + "operation": "merge" + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Single root is not part of the basic model", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + } + ], + "rootReportId": "550e8400-e29b-41d4-a716-446655440100" + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Null input is rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + null + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "String input is rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ "550e8400-e29b-41d4-a716-446655440100" ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Empty input object is rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + {} + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Invalid UUID is rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "not-a-uuid" + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Empty UUID is rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "" + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Provenance extra is optional and namespaced", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + } + ], + "extra": { + "example.tool": { + "note": "derived" + } + } + }, + "runId": "coordinated-checkout-run" + } + } + ] +} diff --git a/tests/normative/required-properties.test.json b/tests/normative/required-properties.test.json index fa314f2..15ab30a 100644 --- a/tests/normative/required-properties.test.json +++ b/tests/normative/required-properties.test.json @@ -398,6 +398,41 @@ }] } } + }, + { + "description": "Provenance input object missing required reportId property", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + }, + "provenance": { + "inputs": [ + {} + ] + } + } } ] } diff --git a/tests/normative/string-constraints.test.json b/tests/normative/string-constraints.test.json index dff9e85..b580aea 100644 --- a/tests/normative/string-constraints.test.json +++ b/tests/normative/string-constraints.test.json @@ -231,6 +231,43 @@ ] } } + }, + { + "description": "Provenance input with empty reportId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "" + } + ] + } + } } ] } diff --git a/tests/normative/type-validation.test.json b/tests/normative/type-validation.test.json index 8ab0c1d..d57c499 100644 --- a/tests/normative/type-validation.test.json +++ b/tests/normative/type-validation.test.json @@ -1419,6 +1419,109 @@ ] } } + }, + { + "description": "Document with provenance as array instead of object", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "provenance": [], + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + } + } + }, + { + "description": "Provenance with inputs as object instead of array", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "provenance": { + "inputs": { + "reportId": "550e8400-e29b-41d4-a716-446655440000" + } + }, + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + } + } + }, + { + "description": "Provenance input with reportId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "provenance": { + "inputs": [ + { + "reportId": 12345 + } + ] + }, + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + } + } } ] } diff --git a/tests/normative/valid-documents.test.json b/tests/normative/valid-documents.test.json index 1fd4a72..e6243d6 100644 --- a/tests/normative/valid-documents.test.json +++ b/tests/normative/valid-documents.test.json @@ -169,6 +169,21 @@ ] } } + }, + { + "description": "Valid direct provenance: with-provenance.json", + "valid": true, + "dataPath": "../../examples/with-provenance.json" + }, + { + "description": "Valid direct provenance: with-provenance-filtered.json", + "valid": true, + "dataPath": "../../examples/with-provenance-filtered.json" + }, + { + "description": "Valid direct provenance: with-provenance-enriched.json", + "valid": true, + "dataPath": "../../examples/with-provenance-enriched.json" } ] } diff --git a/tests/provenance_semantics.py b/tests/provenance_semantics.py new file mode 100644 index 0000000..aa76929 --- /dev/null +++ b/tests/provenance_semantics.py @@ -0,0 +1,93 @@ +"""Reference checks for Section 20 identity rules, after JSON Schema validation. + +These checks complement the normative JSON Schema corpus. They do not verify +source availability, all-input inclusion, graph completeness or authenticity. +Run with: python3 tests/provenance_semantics.py +""" + +import copy +import json +import unittest +from pathlib import Path +from uuid import UUID + + +def check_provenance_identities(report): + """Reject duplicate UUID identities and self-reference in a schema-valid report.""" + if "provenance" not in report: + return + own_id = UUID(report["reportId"]) if "reportId" in report else None + seen = set() + for source in report["provenance"]["inputs"]: + source_id = UUID(source["reportId"]) + if source_id in seen: + raise ValueError("duplicate provenance input identity") + if source_id == own_id: + raise ValueError("provenance input identifies the current document") + seen.add(source_id) + + +class ProvenanceIdentityTests(unittest.TestCase): + def setUp(self): + examples = Path(__file__).resolve().parents[1] / "examples" + self.merge = json.loads((examples / "with-provenance.json").read_text()) + self.filtered = json.loads((examples / "with-provenance-filtered.json").read_text()) + self.enriched = json.loads((examples / "with-provenance-enriched.json").read_text()) + + def test_omission_is_valid(self): + report = copy.deepcopy(self.merge) + del report["provenance"] + check_provenance_identities(report) + + def test_output_identity_remains_optional(self): + report = copy.deepcopy(self.merge) + del report["reportId"] + check_provenance_identities(report) + + def test_distinct_direct_sources_and_unavailable_ancestors_are_valid(self): + check_provenance_identities(self.merge) + + def test_input_order_has_no_meaning(self): + self.merge["provenance"]["inputs"].reverse() + check_provenance_identities(self.merge) + + def test_exact_duplicate_is_invalid(self): + inputs = self.merge["provenance"]["inputs"] + inputs.append(copy.deepcopy(inputs[0])) + with self.assertRaisesRegex(ValueError, "duplicate"): + check_provenance_identities(self.merge) + + def test_case_variant_duplicate_is_invalid(self): + inputs = self.merge["provenance"]["inputs"] + inputs.append({"reportId": inputs[0]["reportId"].upper()}) + with self.assertRaisesRegex(ValueError, "duplicate"): + check_provenance_identities(self.merge) + + def test_self_reference_is_invalid(self): + self.merge["provenance"]["inputs"] = [{"reportId": self.merge["reportId"]}] + with self.assertRaisesRegex(ValueError, "current document"): + check_provenance_identities(self.merge) + + def test_case_variant_self_reference_is_invalid(self): + self.merge["provenance"]["inputs"] = [{"reportId": self.merge["reportId"].upper()}] + with self.assertRaisesRegex(ValueError, "current document"): + check_provenance_identities(self.merge) + + def test_multi_hop_examples_reference_immediate_sources(self): + self.assertEqual(self.filtered["provenance"]["inputs"], + [{"reportId": self.merge["reportId"]}]) + self.assertEqual(self.enriched["provenance"]["inputs"], + [{"reportId": self.filtered["reportId"]}]) + self.assertEqual(len({r["reportId"] for r in + (self.merge, self.filtered, self.enriched)}), 3) + for report in (self.merge, self.filtered, self.enriched): + check_provenance_identities(report) + + def test_ancestor_and_parent_may_both_contribute_directly(self): + report = copy.deepcopy(self.enriched) + report["provenance"]["inputs"].append({"reportId": self.merge["reportId"]}) + check_provenance_identities(report) + + +if __name__ == "__main__": + unittest.main() From cb944891086e30cb37aed552cd84bba25fbc7289 Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:01:50 +0000 Subject: [PATCH 3/6] Show provenance in the existing merged report example --- examples/coordinated-retry-merged.json | 10 ++ examples/with-provenance-enriched.json | 43 ------ examples/with-provenance-filtered.json | 38 ------ examples/with-provenance.json | 46 ------- spec/ctrf.md | 152 +--------------------- tests/ctrf.test.json | 15 --- tests/normative/valid-documents.test.json | 15 --- tests/provenance_semantics.py | 33 ++--- 8 files changed, 34 insertions(+), 318 deletions(-) delete mode 100644 examples/with-provenance-enriched.json delete mode 100644 examples/with-provenance-filtered.json delete mode 100644 examples/with-provenance.json diff --git a/examples/coordinated-retry-merged.json b/examples/coordinated-retry-merged.json index 2e2a6c4..bdeccc1 100644 --- a/examples/coordinated-retry-merged.json +++ b/examples/coordinated-retry-merged.json @@ -53,5 +53,15 @@ ] } ] + }, + "provenance": { + "inputs": [ + { + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + }, + { + "reportId": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" + } + ] } } diff --git a/examples/with-provenance-enriched.json b/examples/with-provenance-enriched.json deleted file mode 100644 index 19a3f93..0000000 --- a/examples/with-provenance-enriched.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "reportFormat": "CTRF", - "specVersion": "0.3.0", - "reportId": "550e8400-e29b-41d4-a716-446655440997", - "timestamp": "2025-11-24T12:30:00Z", - "generatedBy": "example-dashboard", - "results": { - "tool": { - "name": "playwright" - }, - "summary": { - "tests": 1, - "passed": 0, - "failed": 1, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459220000 - }, - "tests": [ - { - "name": "dashboard renders metrics", - "status": "failed", - "duration": 980, - "message": "Element not found: .dashboard-metrics" - } - ] - }, - "provenance": { - "inputs": [ - { - "reportId": "550e8400-e29b-41d4-a716-446655440998" - } - ], - "extra": { - "example.dashboard": { - "label": "triage" - } - } - }, - "runId": "coordinated-checkout-run" -} diff --git a/examples/with-provenance-filtered.json b/examples/with-provenance-filtered.json deleted file mode 100644 index e33acc3..0000000 --- a/examples/with-provenance-filtered.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "reportFormat": "CTRF", - "specVersion": "0.3.0", - "reportId": "550e8400-e29b-41d4-a716-446655440998", - "timestamp": "2025-11-24T12:30:00Z", - "generatedBy": "example-dashboard", - "results": { - "tool": { - "name": "playwright" - }, - "summary": { - "tests": 1, - "passed": 0, - "failed": 1, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459220000 - }, - "tests": [ - { - "name": "dashboard renders metrics", - "status": "failed", - "duration": 980, - "message": "Element not found: .dashboard-metrics" - } - ] - }, - "provenance": { - "inputs": [ - { - "reportId": "550e8400-e29b-41d4-a716-446655440999" - } - ] - }, - "runId": "coordinated-checkout-run" -} diff --git a/examples/with-provenance.json b/examples/with-provenance.json deleted file mode 100644 index 2925637..0000000 --- a/examples/with-provenance.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "reportFormat": "CTRF", - "specVersion": "0.3.0", - "reportId": "550e8400-e29b-41d4-a716-446655440999", - "timestamp": "2025-11-24T12:30:00Z", - "generatedBy": "github-test-reporter", - "results": { - "tool": { - "name": "playwright" - }, - "summary": { - "tests": 2, - "passed": 1, - "failed": 1, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459220000 - }, - "tests": [ - { - "name": "checkout completes", - "status": "passed", - "duration": 240 - }, - { - "name": "dashboard renders metrics", - "status": "failed", - "duration": 980, - "message": "Element not found: .dashboard-metrics" - } - ] - }, - "provenance": { - "inputs": [ - { - "reportId": "550e8400-e29b-41d4-a716-446655440100" - }, - { - "reportId": "550e8400-e29b-41d4-a716-446655440101" - } - ] - }, - "runId": "coordinated-checkout-run" -} diff --git a/spec/ctrf.md b/spec/ctrf.md index 733fe9c..67f25f5 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -2340,7 +2340,7 @@ An optional object for provenance-related extension metadata, following Section This model is CTRF-native and does not claim conformance to an external provenance or attestation standard. Future integrations can supply additional metadata or separate linked documents without changing the meaning of direct inputs. Expected-input manifests, completeness, supersession, activity details, source lookup and verification are outside this version's provenance contract. -See Appendix D.9 for merge, single-source transformation and multi-hop examples. +See Appendix D.8 for a merged report whose provenance references both retained source reports. --- @@ -4144,6 +4144,8 @@ A coordinator executes cases A and B. A passes; B fails. It then starts another All three documents share one `runId` and have distinct `reportId` values. Case B retains one `testId` and `executionId` across processes; each attempt has its own `attemptId`. The merged result preserves those attempt identities. +The merged report’s provenance lists the two raw reports as its direct inputs. It records their contribution without claiming completeness or supersession. A subsequent report derived only from this merged document would reference the merged document, rather than copying its input list. + The raw reports contain the attempts observed by each process. The retry process need not repeat history it did not observe; its optional `retries`, `retryAttempts` and `flaky` fields are omitted here. The coordinator knows the relationship between the attempts and records that history in the merged report. The merged report contains two test results, rather than the three entries obtained by concatenating the raw reports. Case B's final duration is 150 ms, with the earlier 200 ms attempt in history, rather than a summed duration of 350 ms. It is flaky because it passed after a failed attempt. @@ -4297,158 +4299,16 @@ Each block below matches its named standalone example. ] } ] - } -} -``` - -### D.9. Direct Document Provenance - -The following reports illustrate a merge followed by filtering and enrichment. The source reports identified by the merge inputs are not embedded. Their absence here does not establish whether other shards were expected. - -#### D.9.1. Merge of two source reports - -GitHub combines reports ending in `0100` and `0101` into a new report ending in `0999`. It records both direct inputs. This example corresponds to `examples/with-provenance.json`. - -```json -{ - "reportFormat": "CTRF", - "specVersion": "0.3.0", - "reportId": "550e8400-e29b-41d4-a716-446655440999", - "timestamp": "2025-11-24T12:30:00Z", - "generatedBy": "github-test-reporter", - "results": { - "tool": { - "name": "playwright" - }, - "summary": { - "tests": 2, - "passed": 1, - "failed": 1, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459220000 - }, - "tests": [ - { - "name": "checkout completes", - "status": "passed", - "duration": 240 - }, - { - "name": "dashboard renders metrics", - "status": "failed", - "duration": 980, - "message": "Element not found: .dashboard-metrics" - } - ] }, "provenance": { "inputs": [ { - "reportId": "550e8400-e29b-41d4-a716-446655440100" + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" }, { - "reportId": "550e8400-e29b-41d4-a716-446655440101" - } - ] - }, - "runId": "coordinated-checkout-run" -} -``` - -#### D.9.2. Single-source filtering - -A dashboard reads the merged report `0999` and emits report `0998` containing only the failed test. It references `0999`, not that report's ancestors. This example corresponds to `examples/with-provenance-filtered.json`. - -```json -{ - "reportFormat": "CTRF", - "specVersion": "0.3.0", - "reportId": "550e8400-e29b-41d4-a716-446655440998", - "timestamp": "2025-11-24T12:30:00Z", - "generatedBy": "example-dashboard", - "results": { - "tool": { - "name": "playwright" - }, - "summary": { - "tests": 1, - "passed": 0, - "failed": 1, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459220000 - }, - "tests": [ - { - "name": "dashboard renders metrics", - "status": "failed", - "duration": 980, - "message": "Element not found: .dashboard-metrics" - } - ] - }, - "provenance": { - "inputs": [ - { - "reportId": "550e8400-e29b-41d4-a716-446655440999" + "reportId": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" } ] - }, - "runId": "coordinated-checkout-run" -} -``` - -#### D.9.3. A further derivation - -Enrichment of `0998` produces `0997` with a dashboard label. It references only `0998`. Following that report leads to `0999` and, when available, the original sources. This example corresponds to `examples/with-provenance-enriched.json`. - -```json -{ - "reportFormat": "CTRF", - "specVersion": "0.3.0", - "reportId": "550e8400-e29b-41d4-a716-446655440997", - "timestamp": "2025-11-24T12:30:00Z", - "generatedBy": "example-dashboard", - "results": { - "tool": { - "name": "playwright" - }, - "summary": { - "tests": 1, - "passed": 0, - "failed": 1, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459220000 - }, - "tests": [ - { - "name": "dashboard renders metrics", - "status": "failed", - "duration": 980, - "message": "Element not found: .dashboard-metrics" - } - ] - }, - "provenance": { - "inputs": [ - { - "reportId": "550e8400-e29b-41d4-a716-446655440998" - } - ], - "extra": { - "example.dashboard": { - "label": "triage" - } - } - }, - "runId": "coordinated-checkout-run" + } } ``` diff --git a/tests/ctrf.test.json b/tests/ctrf.test.json index 7171329..4161760 100644 --- a/tests/ctrf.test.json +++ b/tests/ctrf.test.json @@ -208,21 +208,6 @@ "description": "Merged coordinated retry report", "valid": true, "dataPath": "../examples/coordinated-retry-merged.json" - }, - { - "description": "Valid direct provenance: with-provenance.json", - "valid": true, - "dataPath": "../examples/with-provenance.json" - }, - { - "description": "Valid direct provenance: with-provenance-filtered.json", - "valid": true, - "dataPath": "../examples/with-provenance-filtered.json" - }, - { - "description": "Valid direct provenance: with-provenance-enriched.json", - "valid": true, - "dataPath": "../examples/with-provenance-enriched.json" } ] } diff --git a/tests/normative/valid-documents.test.json b/tests/normative/valid-documents.test.json index e6243d6..1fd4a72 100644 --- a/tests/normative/valid-documents.test.json +++ b/tests/normative/valid-documents.test.json @@ -169,21 +169,6 @@ ] } } - }, - { - "description": "Valid direct provenance: with-provenance.json", - "valid": true, - "dataPath": "../../examples/with-provenance.json" - }, - { - "description": "Valid direct provenance: with-provenance-filtered.json", - "valid": true, - "dataPath": "../../examples/with-provenance-filtered.json" - }, - { - "description": "Valid direct provenance: with-provenance-enriched.json", - "valid": true, - "dataPath": "../../examples/with-provenance-enriched.json" } ] } diff --git a/tests/provenance_semantics.py b/tests/provenance_semantics.py index aa76929..747e0e1 100644 --- a/tests/provenance_semantics.py +++ b/tests/provenance_semantics.py @@ -30,9 +30,9 @@ def check_provenance_identities(report): class ProvenanceIdentityTests(unittest.TestCase): def setUp(self): examples = Path(__file__).resolve().parents[1] / "examples" - self.merge = json.loads((examples / "with-provenance.json").read_text()) - self.filtered = json.loads((examples / "with-provenance-filtered.json").read_text()) - self.enriched = json.loads((examples / "with-provenance-enriched.json").read_text()) + self.merge = json.loads((examples / "coordinated-retry-merged.json").read_text()) + self.initial = json.loads((examples / "coordinated-retry-initial.json").read_text()) + self.rerun = json.loads((examples / "coordinated-retry-rerun.json").read_text()) def test_omission_is_valid(self): report = copy.deepcopy(self.merge) @@ -45,6 +45,9 @@ def test_output_identity_remains_optional(self): check_provenance_identities(report) def test_distinct_direct_sources_and_unavailable_ancestors_are_valid(self): + self.assertEqual(self.merge["provenance"]["inputs"], + [{"reportId": self.initial["reportId"]}, + {"reportId": self.rerun["reportId"]}]) check_provenance_identities(self.merge) def test_input_order_has_no_meaning(self): @@ -73,20 +76,20 @@ def test_case_variant_self_reference_is_invalid(self): with self.assertRaisesRegex(ValueError, "current document"): check_provenance_identities(self.merge) - def test_multi_hop_examples_reference_immediate_sources(self): - self.assertEqual(self.filtered["provenance"]["inputs"], - [{"reportId": self.merge["reportId"]}]) - self.assertEqual(self.enriched["provenance"]["inputs"], - [{"reportId": self.filtered["reportId"]}]) - self.assertEqual(len({r["reportId"] for r in - (self.merge, self.filtered, self.enriched)}), 3) - for report in (self.merge, self.filtered, self.enriched): - check_provenance_identities(report) + def test_multi_hop_uses_immediate_source(self): + downstream = copy.deepcopy(self.merge) + downstream["reportId"] = "dddddddd-dddd-4ddd-8ddd-dddddddddddd" + downstream["provenance"]["inputs"] = [ + {"reportId": "cccccccc-cccc-4ccc-8ccc-cccccccccccc"} + ] + check_provenance_identities(downstream) def test_ancestor_and_parent_may_both_contribute_directly(self): - report = copy.deepcopy(self.enriched) - report["provenance"]["inputs"].append({"reportId": self.merge["reportId"]}) - check_provenance_identities(report) + downstream = copy.deepcopy(self.merge) + downstream["reportId"] = "dddddddd-dddd-4ddd-8ddd-dddddddddddd" + downstream["provenance"]["inputs"].append( + {"reportId": "cccccccc-cccc-4ccc-8ccc-cccccccccccc"}) + check_provenance_identities(downstream) if __name__ == "__main__": From c5ef4e0c1a90b242e7345c6cb059a4ddaa21e859 Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:09:18 +0000 Subject: [PATCH 4/6] Allow input provenance metadata and align UUID validation --- CONTRIBUTING.md | 2 +- examples/coordinated-retry-merged.json | 14 +- schema/ctrf.schema.json | 11 +- spec/ctrf.md | 31 +++-- tests/normative/provenance.test.json | 183 ++++++++++++++++++++++--- tests/provenance_semantics.py | 52 ++++++- 6 files changed, 256 insertions(+), 37 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c6c276a..874d7cd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -151,7 +151,7 @@ jsonschema validate schema/ctrf.schema.json examples/*.json The `--exclude top_level_examples` flag is used because CTRF maintains examples in the `examples/` directory rather than embedding them in the schema itself. The provenance identity checks supplement JSON Schema validation with UUID -identity comparison and self-reference checks from Section 20. Run them with: +format validation, identity comparison and self-reference checks from Section 20. Run them with: ```bash python3 tests/provenance_semantics.py diff --git a/examples/coordinated-retry-merged.json b/examples/coordinated-retry-merged.json index bdeccc1..15dd10b 100644 --- a/examples/coordinated-retry-merged.json +++ b/examples/coordinated-retry-merged.json @@ -57,11 +57,21 @@ "provenance": { "inputs": [ { - "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "extra": { + "example.coordinator": { + "label": "initial process" + } + } }, { "reportId": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" } - ] + ], + "extra": { + "example.coordinator": { + "label": "coordinated retry merge" + } + } } } diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 3f94349..58927cc 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -35,12 +35,12 @@ "type": "string" }, "provenance": { - "description": "Direct CTRF document sources from which this report was derived; does not establish completeness or supersession", + "description": "Direct CTRF document sources from which this report was derived", "type": "object", "required": [ "inputs" ], "properties": { "inputs": { - "description": "All distinct direct contributing CTRF source documents, without flattening their ancestry; order has no meaning", + "description": "All distinct direct contributing CTRF source documents", "type": "array", "minItems": 1, "uniqueItems": true, @@ -48,15 +48,18 @@ "type": "object", "required": [ "reportId" ], "propertyNames": { - "const": "reportId" + "enum": [ "reportId", "extra" ] }, "properties": { "reportId": { "description": "The reportId of the contributing document", "type": "string", - "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", "format": "uuid", "minLength": 1 + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" } }, "additionalProperties": false diff --git a/spec/ctrf.md b/spec/ctrf.md index 67f25f5..5be648b 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -2316,7 +2316,7 @@ Unknown fields MUST NOT appear in `provenance` except within its `extra` object. All distinct direct CTRF source documents from which the current document was derived. **Requirements:** -`inputs` MUST be present when `provenance` is present. It MUST be a non-empty array of objects. Each input object MUST contain only a required `reportId`, which MUST be a non-empty string and a valid UUID. +`inputs` MUST be present when `provenance` is present. It MUST be a non-empty array of objects. Each input object MUST contain a required `reportId`, which MUST be a non-empty string and a valid UUID. An input MAY also contain an `extra` object for metadata about that source relationship. Unknown fields MUST NOT appear in an input object except within its `extra` object. Input order has no semantic meaning. Each distinct source document MUST appear exactly once, regardless of how many times its content was used. UUID comparison MUST ignore letter case. An input MUST NOT identify the current document itself when the current document has a `reportId`. @@ -2328,7 +2328,7 @@ A derived report SHOULD receive a new `reportId`, distinct from its inputs, cons Input references neither embed nor locate their sources. Following a chain requires access to the referenced documents through a consumer's storage or lookup mechanism. An unavailable ancestor does not invalidate a direct reference. Consumers MUST NOT interpret an input reference alone as proof of completeness, supersession, authenticity, or artifact integrity. A UUID is an identity, not a content digest or retrieval URL. -The JSON Schema checks the object structure, UUID format, non-empty input list and identical duplicate entries. Case-insensitive identity uniqueness and self-reference require semantic checks in addition to JSON Schema validation. +The JSON Schema defines the object structure, UUID format, non-empty input list and identical duplicate entries. Validators that do not assert UUID formats need supplemental UUID validation. Identity uniqueness is based on reportId regardless of extra metadata; case-insensitive identity comparison and self-reference require semantic checks in addition to JSON Schema validation. ### 20.2. `extra` @@ -2336,7 +2336,7 @@ The JSON Schema checks the object structure, UUID format, non-empty input list a An optional object for provenance-related extension metadata, following Section 4's extension rules. **Requirements:** -`extra` MAY contain arbitrary fields. It is the ONLY permitted extension point directly within `provenance`. Input entries do not provide an extension point in this version. +`extra` MAY contain arbitrary fields. It is the ONLY permitted extension point directly within `provenance`. Each input object also permits its own optional `extra` object. Both locations follow the extension rules in Section 4. This model is CTRF-native and does not claim conformance to an external provenance or attestation standard. Future integrations can supply additional metadata or separate linked documents without changing the meaning of direct inputs. Expected-input manifests, completeness, supersession, activity details, source lookup and verification are outside this version's provenance contract. @@ -2502,12 +2502,12 @@ to this specification. "type": "string" }, "provenance": { - "description": "Direct CTRF document sources from which this report was derived; does not establish completeness or supersession", + "description": "Direct CTRF document sources from which this report was derived", "type": "object", "required": [ "inputs" ], "properties": { "inputs": { - "description": "All distinct direct contributing CTRF source documents, without flattening their ancestry; order has no meaning", + "description": "All distinct direct contributing CTRF source documents", "type": "array", "minItems": 1, "uniqueItems": true, @@ -2515,15 +2515,18 @@ to this specification. "type": "object", "required": [ "reportId" ], "propertyNames": { - "const": "reportId" + "enum": [ "reportId", "extra" ] }, "properties": { "reportId": { "description": "The reportId of the contributing document", "type": "string", - "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", "format": "uuid", "minLength": 1 + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" } }, "additionalProperties": false @@ -4303,12 +4306,22 @@ Each block below matches its named standalone example. "provenance": { "inputs": [ { - "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "extra": { + "example.coordinator": { + "label": "initial process" + } + } }, { "reportId": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" } - ] + ], + "extra": { + "example.coordinator": { + "label": "coordinated retry merge" + } + } } } ``` diff --git a/tests/normative/provenance.test.json b/tests/normative/provenance.test.json index 171b1df..f41db09 100644 --- a/tests/normative/provenance.test.json +++ b/tests/normative/provenance.test.json @@ -327,8 +327,8 @@ } }, { - "description": "Per-input extra is not defined in this version", - "valid": false, + "description": "Per-input extra metadata is allowed", + "valid": true, "data": { "reportFormat": "CTRF", "specVersion": "0.3.0", @@ -367,7 +367,11 @@ "inputs": [ { "reportId": "550e8400-e29b-41d4-a716-446655440100", - "extra": {} + "extra": { + "example.source": { + "label": "initial" + } + } } ] }, @@ -553,7 +557,9 @@ ] }, "provenance": { - "inputs": [ "550e8400-e29b-41d4-a716-446655440100" ] + "inputs": [ + "550e8400-e29b-41d4-a716-446655440100" + ] }, "runId": "coordinated-checkout-run" } @@ -604,7 +610,7 @@ } }, { - "description": "Invalid UUID is rejected", + "description": "Empty UUID is rejected", "valid": false, "data": { "reportFormat": "CTRF", @@ -643,7 +649,7 @@ "provenance": { "inputs": [ { - "reportId": "not-a-uuid" + "reportId": "" } ] }, @@ -651,7 +657,59 @@ } }, { - "description": "Empty UUID is rejected", + "description": "Provenance extra is optional and namespaced", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + } + ], + "extra": { + "example.tool": { + "note": "derived" + } + } + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Input extra rejects non-object null", "valid": false, "data": { "reportFormat": "CTRF", @@ -690,7 +748,8 @@ "provenance": { "inputs": [ { - "reportId": "" + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "extra": null } ] }, @@ -698,8 +757,8 @@ } }, { - "description": "Provenance extra is optional and namespaced", - "valid": true, + "description": "Input extra rejects non-object []", + "valid": false, "data": { "reportFormat": "CTRF", "specVersion": "0.3.0", @@ -737,14 +796,106 @@ "provenance": { "inputs": [ { - "reportId": "550e8400-e29b-41d4-a716-446655440100" + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "extra": [] } - ], - "extra": { - "example.tool": { - "note": "derived" + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Input extra rejects non-object 42", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" } - } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "extra": 42 + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Input extra rejects non-object \"metadata\"", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "extra": "metadata" + } + ] }, "runId": "coordinated-checkout-run" } diff --git a/tests/provenance_semantics.py b/tests/provenance_semantics.py index 747e0e1..4ac9477 100644 --- a/tests/provenance_semantics.py +++ b/tests/provenance_semantics.py @@ -1,4 +1,4 @@ -"""Reference checks for Section 20 identity rules, after JSON Schema validation. +"""Reference checks for Section 20 UUID and identity rules, after shape validation. These checks complement the normative JSON Schema corpus. They do not verify source availability, all-input inclusion, graph completeness or authenticity. @@ -12,14 +12,27 @@ from uuid import UUID +def provenance_uuid(value): + """Use the same UUID syntax as format:uuid, including case-insensitive hex.""" + if not isinstance(value, str): + raise ValueError("invalid provenance UUID") + try: + parsed = UUID(value) + except ValueError as error: + raise ValueError("invalid provenance UUID") from error + if str(parsed) != value.lower(): + raise ValueError("invalid provenance UUID") + return parsed + + def check_provenance_identities(report): - """Reject duplicate UUID identities and self-reference in a schema-valid report.""" + """Check UUID format, identity uniqueness and self-reference after shape validation.""" if "provenance" not in report: return - own_id = UUID(report["reportId"]) if "reportId" in report else None + own_id = provenance_uuid(report["reportId"]) if "reportId" in report else None seen = set() for source in report["provenance"]["inputs"]: - source_id = UUID(source["reportId"]) + source_id = provenance_uuid(source["reportId"]) if source_id in seen: raise ValueError("duplicate provenance input identity") if source_id == own_id: @@ -45,7 +58,8 @@ def test_output_identity_remains_optional(self): check_provenance_identities(report) def test_distinct_direct_sources_and_unavailable_ancestors_are_valid(self): - self.assertEqual(self.merge["provenance"]["inputs"], + self.assertEqual([{"reportId": source["reportId"]} + for source in self.merge["provenance"]["inputs"]], [{"reportId": self.initial["reportId"]}, {"reportId": self.rerun["reportId"]}]) check_provenance_identities(self.merge) @@ -76,6 +90,34 @@ def test_case_variant_self_reference_is_invalid(self): with self.assertRaisesRegex(ValueError, "current document"): check_provenance_identities(self.merge) + def test_duplicate_identity_with_different_metadata_is_invalid(self): + inputs = self.merge["provenance"]["inputs"] + inputs[0]["extra"] = {"example.source": {"label": "first"}} + inputs.append({"reportId": inputs[0]["reportId"], + "extra": {"example.source": {"label": "second"}}}) + with self.assertRaisesRegex(ValueError, "duplicate"): + check_provenance_identities(self.merge) + + def test_metadata_at_both_levels_is_valid(self): + self.merge["provenance"]["extra"] = {"example.merge": {"note": "combined"}} + self.merge["provenance"]["inputs"][0]["extra"] = { + "example.source": {"note": "initial process"}} + check_provenance_identities(self.merge) + + def test_uppercase_uuid_is_valid(self): + self.merge["provenance"]["inputs"][0]["reportId"] = ( + self.merge["provenance"]["inputs"][0]["reportId"].upper()) + check_provenance_identities(self.merge) + + def test_malformed_input_uuid_is_invalid(self): + for value in ("not-a-uuid", "", "aaaaaaaaaaaa4aaa8aaaaaaaaaaaaaaa", + "{aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa}", + "urn:uuid:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"): + with self.subTest(value=value): + self.merge["provenance"]["inputs"] = [{"reportId": value}] + with self.assertRaisesRegex(ValueError, "invalid provenance UUID"): + check_provenance_identities(self.merge) + def test_multi_hop_uses_immediate_source(self): downstream = copy.deepcopy(self.merge) downstream["reportId"] = "dddddddd-dddd-4ddd-8ddd-dddddddddddd" From 2685e2e26a4cf31d1646c442dd48300bbf83d32f Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:14:05 +0000 Subject: [PATCH 5/6] Keep basic provenance guidance and checks focused --- .github/workflows/test.yml | 3 - CONTRIBUTING.md | 7 -- spec/ctrf.md | 12 +-- tests/normative/provenance.test.json | 2 +- tests/provenance_semantics.py | 138 --------------------------- 5 files changed, 5 insertions(+), 157 deletions(-) delete mode 100644 tests/provenance_semantics.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3305f36..bb7d4b2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,9 +28,6 @@ jobs: - name: Run conformance tests (normative) run: jsonschema test tests/normative --json > ctrf-normative.json - - name: Check provenance identity semantics - run: python3 tests/provenance_semantics.py - - name: Run informative tests run: jsonschema test tests/informative --json > ctrf-informative.json continue-on-error: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 874d7cd..2821b57 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -150,13 +150,6 @@ jsonschema validate schema/ctrf.schema.json examples/*.json The `--exclude top_level_examples` flag is used because CTRF maintains examples in the `examples/` directory rather than embedding them in the schema itself. -The provenance identity checks supplement JSON Schema validation with UUID -format validation, identity comparison and self-reference checks from Section 20. Run them with: - -```bash -python3 tests/provenance_semantics.py -``` - ### Adding Tests for Changes When contributing changes, you SHOULD add or update tests: diff --git a/spec/ctrf.md b/spec/ctrf.md index 5be648b..86148d0 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -2306,8 +2306,6 @@ It is the ONLY permitted extension point within the baseline object. The optional top-level `provenance` object records direct document lineage for a derived CTRF report. Its `inputs` identify the existing CTRF source documents whose content contributed to the output, including content used to compute derived values. A report merely consulted without contributing content is not a provenance input. -Provenance describes observed derivation. It does not assert that all expected shards or reports arrived, that the output replaces its inputs, or that overlapping inputs can safely be counted together. It does not attribute individual tests, attempts, attachments, or errors to a source. A schema-valid report is not evidence of complete or verified lineage. - Unknown fields MUST NOT appear in `provenance` except within its `extra` object. ### 20.1. `inputs` @@ -2322,13 +2320,11 @@ Input order has no semantic meaning. Each distinct source document MUST appear e Producers MUST list all contributing direct CTRF source documents. If any such source lacks a `reportId`, producers MUST omit `provenance`; they MUST NOT invent an identity for an already-emitted source or list only the identified subset. Omission does not establish that a report is an original source. -Producers MUST reference the immediate source documents rather than copying their `provenance.inputs` or flattening their ancestry. If reports A and B produce C, and C produces D, C lists A and B; D lists C. If A also contributes directly to D, D lists both C and A. These links do not resolve overlapping content or prescribe merge/deduplication behavior. +Producers MUST reference the immediate source documents rather than copying their `provenance.inputs` or flattening their ancestry. If reports A and B produce C, and C produces D, C lists A and B; D lists C. If A also contributes directly to D, D lists both C and A. A derived report SHOULD receive a new `reportId`, distinct from its inputs, consistent with Section 5.3. Reports without provenance or document identity remain valid. Retransmission of the same unchanged artifact SHOULD preserve its `reportId` and provenance. Logical-run correlation remains separate; provenance does not change the single-logical-run scope of `results`. -Input references neither embed nor locate their sources. Following a chain requires access to the referenced documents through a consumer's storage or lookup mechanism. An unavailable ancestor does not invalidate a direct reference. Consumers MUST NOT interpret an input reference alone as proof of completeness, supersession, authenticity, or artifact integrity. A UUID is an identity, not a content digest or retrieval URL. - -The JSON Schema defines the object structure, UUID format, non-empty input list and identical duplicate entries. Validators that do not assert UUID formats need supplemental UUID validation. Identity uniqueness is based on reportId regardless of extra metadata; case-insensitive identity comparison and self-reference require semantic checks in addition to JSON Schema validation. +Input references do not embed source reports. Consumers need access to those reports to follow multi-hop lineage. Input identity is determined by `reportId`, regardless of `extra` metadata. ### 20.2. `extra` @@ -2338,7 +2334,7 @@ An optional object for provenance-related extension metadata, following Section **Requirements:** `extra` MAY contain arbitrary fields. It is the ONLY permitted extension point directly within `provenance`. Each input object also permits its own optional `extra` object. Both locations follow the extension rules in Section 4. -This model is CTRF-native and does not claim conformance to an external provenance or attestation standard. Future integrations can supply additional metadata or separate linked documents without changing the meaning of direct inputs. Expected-input manifests, completeness, supersession, activity details, source lookup and verification are outside this version's provenance contract. +Completeness and report supersession are outside this provenance model. See Appendix D.8 for a merged report whose provenance references both retained source reports. @@ -2431,7 +2427,7 @@ Consumers: --- -Consumers MAY follow `provenance.inputs` to inspect document lineage when source reports are available. They MUST apply the interpretation limits in Section 20; lineage alone is not a consumer selection or deduplication policy. +Consumers MAY follow `provenance.inputs` to inspect document lineage when source reports are available. ## 23. Security Considerations diff --git a/tests/normative/provenance.test.json b/tests/normative/provenance.test.json index f41db09..0469f4b 100644 --- a/tests/normative/provenance.test.json +++ b/tests/normative/provenance.test.json @@ -1,6 +1,6 @@ { "target": "../../schema/ctrf.schema.json", - "$comment": "Provenance schema constraints; semantic identity rules are checked separately by tests/provenance_semantics.py", + "$comment": "Provenance schema constraints; UUID identity uniqueness and self-reference are defined by the written specification", "tests": [ { "description": "Provenance requires inputs", diff --git a/tests/provenance_semantics.py b/tests/provenance_semantics.py deleted file mode 100644 index 4ac9477..0000000 --- a/tests/provenance_semantics.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Reference checks for Section 20 UUID and identity rules, after shape validation. - -These checks complement the normative JSON Schema corpus. They do not verify -source availability, all-input inclusion, graph completeness or authenticity. -Run with: python3 tests/provenance_semantics.py -""" - -import copy -import json -import unittest -from pathlib import Path -from uuid import UUID - - -def provenance_uuid(value): - """Use the same UUID syntax as format:uuid, including case-insensitive hex.""" - if not isinstance(value, str): - raise ValueError("invalid provenance UUID") - try: - parsed = UUID(value) - except ValueError as error: - raise ValueError("invalid provenance UUID") from error - if str(parsed) != value.lower(): - raise ValueError("invalid provenance UUID") - return parsed - - -def check_provenance_identities(report): - """Check UUID format, identity uniqueness and self-reference after shape validation.""" - if "provenance" not in report: - return - own_id = provenance_uuid(report["reportId"]) if "reportId" in report else None - seen = set() - for source in report["provenance"]["inputs"]: - source_id = provenance_uuid(source["reportId"]) - if source_id in seen: - raise ValueError("duplicate provenance input identity") - if source_id == own_id: - raise ValueError("provenance input identifies the current document") - seen.add(source_id) - - -class ProvenanceIdentityTests(unittest.TestCase): - def setUp(self): - examples = Path(__file__).resolve().parents[1] / "examples" - self.merge = json.loads((examples / "coordinated-retry-merged.json").read_text()) - self.initial = json.loads((examples / "coordinated-retry-initial.json").read_text()) - self.rerun = json.loads((examples / "coordinated-retry-rerun.json").read_text()) - - def test_omission_is_valid(self): - report = copy.deepcopy(self.merge) - del report["provenance"] - check_provenance_identities(report) - - def test_output_identity_remains_optional(self): - report = copy.deepcopy(self.merge) - del report["reportId"] - check_provenance_identities(report) - - def test_distinct_direct_sources_and_unavailable_ancestors_are_valid(self): - self.assertEqual([{"reportId": source["reportId"]} - for source in self.merge["provenance"]["inputs"]], - [{"reportId": self.initial["reportId"]}, - {"reportId": self.rerun["reportId"]}]) - check_provenance_identities(self.merge) - - def test_input_order_has_no_meaning(self): - self.merge["provenance"]["inputs"].reverse() - check_provenance_identities(self.merge) - - def test_exact_duplicate_is_invalid(self): - inputs = self.merge["provenance"]["inputs"] - inputs.append(copy.deepcopy(inputs[0])) - with self.assertRaisesRegex(ValueError, "duplicate"): - check_provenance_identities(self.merge) - - def test_case_variant_duplicate_is_invalid(self): - inputs = self.merge["provenance"]["inputs"] - inputs.append({"reportId": inputs[0]["reportId"].upper()}) - with self.assertRaisesRegex(ValueError, "duplicate"): - check_provenance_identities(self.merge) - - def test_self_reference_is_invalid(self): - self.merge["provenance"]["inputs"] = [{"reportId": self.merge["reportId"]}] - with self.assertRaisesRegex(ValueError, "current document"): - check_provenance_identities(self.merge) - - def test_case_variant_self_reference_is_invalid(self): - self.merge["provenance"]["inputs"] = [{"reportId": self.merge["reportId"].upper()}] - with self.assertRaisesRegex(ValueError, "current document"): - check_provenance_identities(self.merge) - - def test_duplicate_identity_with_different_metadata_is_invalid(self): - inputs = self.merge["provenance"]["inputs"] - inputs[0]["extra"] = {"example.source": {"label": "first"}} - inputs.append({"reportId": inputs[0]["reportId"], - "extra": {"example.source": {"label": "second"}}}) - with self.assertRaisesRegex(ValueError, "duplicate"): - check_provenance_identities(self.merge) - - def test_metadata_at_both_levels_is_valid(self): - self.merge["provenance"]["extra"] = {"example.merge": {"note": "combined"}} - self.merge["provenance"]["inputs"][0]["extra"] = { - "example.source": {"note": "initial process"}} - check_provenance_identities(self.merge) - - def test_uppercase_uuid_is_valid(self): - self.merge["provenance"]["inputs"][0]["reportId"] = ( - self.merge["provenance"]["inputs"][0]["reportId"].upper()) - check_provenance_identities(self.merge) - - def test_malformed_input_uuid_is_invalid(self): - for value in ("not-a-uuid", "", "aaaaaaaaaaaa4aaa8aaaaaaaaaaaaaaa", - "{aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa}", - "urn:uuid:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"): - with self.subTest(value=value): - self.merge["provenance"]["inputs"] = [{"reportId": value}] - with self.assertRaisesRegex(ValueError, "invalid provenance UUID"): - check_provenance_identities(self.merge) - - def test_multi_hop_uses_immediate_source(self): - downstream = copy.deepcopy(self.merge) - downstream["reportId"] = "dddddddd-dddd-4ddd-8ddd-dddddddddddd" - downstream["provenance"]["inputs"] = [ - {"reportId": "cccccccc-cccc-4ccc-8ccc-cccccccccccc"} - ] - check_provenance_identities(downstream) - - def test_ancestor_and_parent_may_both_contribute_directly(self): - downstream = copy.deepcopy(self.merge) - downstream["reportId"] = "dddddddd-dddd-4ddd-8ddd-dddddddddddd" - downstream["provenance"]["inputs"].append( - {"reportId": "cccccccc-cccc-4ccc-8ccc-cccccccccccc"}) - check_provenance_identities(downstream) - - -if __name__ == "__main__": - unittest.main() From 52397befad1d2d0dd8d67e15a15dbfc0064172d1 Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:21:01 +0000 Subject: [PATCH 6/6] Keep provenance example free of extension metadata --- examples/coordinated-retry-merged.json | 14 ++------------ spec/ctrf.md | 14 ++------------ 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/examples/coordinated-retry-merged.json b/examples/coordinated-retry-merged.json index 15dd10b..bdeccc1 100644 --- a/examples/coordinated-retry-merged.json +++ b/examples/coordinated-retry-merged.json @@ -57,21 +57,11 @@ "provenance": { "inputs": [ { - "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", - "extra": { - "example.coordinator": { - "label": "initial process" - } - } + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" }, { "reportId": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" } - ], - "extra": { - "example.coordinator": { - "label": "coordinated retry merge" - } - } + ] } } diff --git a/spec/ctrf.md b/spec/ctrf.md index 86148d0..d42ac60 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -4302,22 +4302,12 @@ Each block below matches its named standalone example. "provenance": { "inputs": [ { - "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", - "extra": { - "example.coordinator": { - "label": "initial process" - } - } + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" }, { "reportId": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" } - ], - "extra": { - "example.coordinator": { - "label": "coordinated retry merge" - } - } + ] } } ```