diff --git a/CHANGELOG.md b/CHANGELOG.md index c3bfa57..6fbc235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Optional document `provenance` in specification 0.3.0, recording direct source report identities for derived reports, with explicit lineage and interpretation rules. + - Optional `results.errors` in specification 0.3.0 for global error messages, traces and metadata, separate from individual test results and counts. - Optional `assertionFailures` on final test results and retry history, with producer-formatted comparisons, individual diagnostics and `extra` metadata. diff --git a/examples/coordinated-retry-merged.json b/examples/coordinated-retry-merged.json index 2e2a6c4..bdeccc1 100644 --- a/examples/coordinated-retry-merged.json +++ b/examples/coordinated-retry-merged.json @@ -53,5 +53,15 @@ ] } ] + }, + "provenance": { + "inputs": [ + { + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + }, + { + "reportId": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" + } + ] } } diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 9ce1fbe..58927cc 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -34,6 +34,44 @@ "description": "Tool or system that produced this CTRF document", "type": "string" }, + "provenance": { + "description": "Direct CTRF document sources from which this report was derived", + "type": "object", + "required": [ "inputs" ], + "properties": { + "inputs": { + "description": "All distinct direct contributing CTRF source documents", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "object", + "required": [ "reportId" ], + "propertyNames": { + "enum": [ "reportId", "extra" ] + }, + "properties": { + "reportId": { + "description": "The reportId of the contributing document", + "type": "string", + "format": "uuid", + "minLength": 1 + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + } + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + }, "extra": { "description": "Extension point for arbitrary metadata", "type": "object" diff --git a/spec/ctrf.md b/spec/ctrf.md index f8e1e89..d42ac60 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -145,7 +145,7 @@ Consumers MAY reconstruct hierarchical views for presentation purposes, but such ### 2.4. Identity Model -CTRF provides a layered identity model that supports stable identifiers for test cases, correlation of documents belonging to the same logical run, deduplication, aggregation, and future provenance support. +CTRF provides a layered identity model that supports stable identifiers for test cases, correlation of documents belonging to the same logical run, deduplication, aggregation, and document provenance. Identity fields are OPTIONAL. When provided, they enable: @@ -343,6 +343,9 @@ A structure describing a metric’s `current` value, the corresponding value fro **Attachment**: An external file or resource referenced by a test case or attempt history entry, such as a screenshot, log file, or other artifact. +**Derived Report**: +A CTRF document produced by transforming or combining content from one or more existing CTRF documents, including content used to compute derived values. + --- ## 4. Overall Document Structure @@ -574,7 +577,7 @@ Each layer addresses a distinct identification concern: All identity fields are OPTIONAL. Producers SHOULD assign them when the corresponding correlation is meaningful for their workflow. -This identity model provides primitives that can be used by future provenance and lineage extensions. It does not itself define derivation, parent/child report relationships, transformation history, or trust metadata. +The identity model provides the identifiers used by the optional document provenance model in Section 20. Identity alone does not establish derivation, transformation history, or trust. #### 4.9.1. Uniqueness @@ -746,7 +749,18 @@ If present, it MUST follow the structure described in Section 19 (Baseline Objec --- -### 5.10. `extra` +### 5.10. `provenance` + +**Description:** +Identifies the direct CTRF source documents from which this document was derived. + +**Requirements:** +`provenance` is OPTIONAL. +If present, it MUST follow the structure described in Section 20 (Provenance Object). + +--- + +### 5.11. `extra` **Description:** An extensibility object containing arbitrary metadata. @@ -2288,7 +2302,45 @@ An extensibility object containing arbitrary metadata related to the baseline. It MAY contain arbitrary fields. It is the ONLY permitted extension point within the baseline object. -## 20. Versioning +## 20. Provenance Object + +The optional top-level `provenance` object records direct document lineage for a derived CTRF report. Its `inputs` identify the existing CTRF source documents whose content contributed to the output, including content used to compute derived values. A report merely consulted without contributing content is not a provenance input. + +Unknown fields MUST NOT appear in `provenance` except within its `extra` object. + +### 20.1. `inputs` + +**Description:** +All distinct direct CTRF source documents from which the current document was derived. + +**Requirements:** +`inputs` MUST be present when `provenance` is present. It MUST be a non-empty array of objects. Each input object MUST contain a required `reportId`, which MUST be a non-empty string and a valid UUID. An input MAY also contain an `extra` object for metadata about that source relationship. Unknown fields MUST NOT appear in an input object except within its `extra` object. + +Input order has no semantic meaning. Each distinct source document MUST appear exactly once, regardless of how many times its content was used. UUID comparison MUST ignore letter case. An input MUST NOT identify the current document itself when the current document has a `reportId`. + +Producers MUST list all contributing direct CTRF source documents. If any such source lacks a `reportId`, producers MUST omit `provenance`; they MUST NOT invent an identity for an already-emitted source or list only the identified subset. Omission does not establish that a report is an original source. + +Producers MUST reference the immediate source documents rather than copying their `provenance.inputs` or flattening their ancestry. If reports A and B produce C, and C produces D, C lists A and B; D lists C. If A also contributes directly to D, D lists both C and A. + +A derived report SHOULD receive a new `reportId`, distinct from its inputs, consistent with Section 5.3. Reports without provenance or document identity remain valid. Retransmission of the same unchanged artifact SHOULD preserve its `reportId` and provenance. Logical-run correlation remains separate; provenance does not change the single-logical-run scope of `results`. + +Input references do not embed source reports. Consumers need access to those reports to follow multi-hop lineage. Input identity is determined by `reportId`, regardless of `extra` metadata. + +### 20.2. `extra` + +**Description:** +An optional object for provenance-related extension metadata, following Section 4's extension rules. + +**Requirements:** +`extra` MAY contain arbitrary fields. It is the ONLY permitted extension point directly within `provenance`. Each input object also permits its own optional `extra` object. Both locations follow the extension rules in Section 4. + +Completeness and report supersession are outside this provenance model. + +See Appendix D.8 for a merged report whose provenance references both retained source reports. + +--- + +## 21. Versioning CTRF follows **Semantic Versioning**. @@ -2320,15 +2372,16 @@ The `specVersion` field in a CTRF document MUST follow these versioning rules. --- -## 21. Conformance Requirements +## 22. Conformance Requirements -### 21.1. Producer Conformance +### 22.1. Producer Conformance Producers: - MUST output UTF-8 JSON - MUST set `reportFormat = "CTRF"` - MUST include all required fields +- MUST follow the provenance requirements in Section 20 when emitting provenance - MUST follow rules for retry histories - MUST NOT introduce fields outside `extra` objects - MUST NOT use extensions to redefine, weaken, or contradict fields defined in this specification @@ -2351,7 +2404,7 @@ Producers SHOULD: --- -## 21.2. Consumer Conformance +## 22.2. Consumer Conformance Consumers: @@ -2374,7 +2427,9 @@ Consumers: --- -## 22. Security Considerations +Consumers MAY follow `provenance.inputs` to inspect document lineage when source reports are available. + +## 23. Security Considerations CTRF documents may contain sensitive information such as: @@ -2391,7 +2446,7 @@ Consumers SHOULD consider size limits when processing attachments or screenshots --- -## 23. IANA Considerations +## 24. IANA Considerations This document registers **no** IANA resources. @@ -2442,6 +2497,44 @@ to this specification. "description": "Tool or system that produced this CTRF document", "type": "string" }, + "provenance": { + "description": "Direct CTRF document sources from which this report was derived", + "type": "object", + "required": [ "inputs" ], + "properties": { + "inputs": { + "description": "All distinct direct contributing CTRF source documents", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "object", + "required": [ "reportId" ], + "propertyNames": { + "enum": [ "reportId", "extra" ] + }, + "properties": { + "reportId": { + "description": "The reportId of the contributing document", + "type": "string", + "format": "uuid", + "minLength": 1 + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + } + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + }, "extra": { "description": "Extension point for arbitrary metadata", "type": "object" @@ -4050,6 +4143,8 @@ A coordinator executes cases A and B. A passes; B fails. It then starts another All three documents share one `runId` and have distinct `reportId` values. Case B retains one `testId` and `executionId` across processes; each attempt has its own `attemptId`. The merged result preserves those attempt identities. +The merged report’s provenance lists the two raw reports as its direct inputs. It records their contribution without claiming completeness or supersession. A subsequent report derived only from this merged document would reference the merged document, rather than copying its input list. + The raw reports contain the attempts observed by each process. The retry process need not repeat history it did not observe; its optional `retries`, `retryAttempts` and `flaky` fields are omitted here. The coordinator knows the relationship between the attempts and records that history in the merged report. The merged report contains two test results, rather than the three entries obtained by concatenating the raw reports. Case B's final duration is 150 ms, with the earlier 200 ms attempt in history, rather than a summed duration of 350 ms. It is flaky because it passed after a failed attempt. @@ -4203,6 +4298,16 @@ Each block below matches its named standalone example. ] } ] + }, + "provenance": { + "inputs": [ + { + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + }, + { + "reportId": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" + } + ] } } ``` diff --git a/tests/normative/additional-properties.test.json b/tests/normative/additional-properties.test.json index a8782c5..96664f3 100644 --- a/tests/normative/additional-properties.test.json +++ b/tests/normative/additional-properties.test.json @@ -255,6 +255,77 @@ ] } } + }, + { + "description": "Provenance object with unknown property", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + }, + "provenance": { + "unknownField": true + } + } + }, + { + "description": "Provenance input object with unknown property", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440000", + "unknownField": true + } + ] + } + } } ] } diff --git a/tests/normative/provenance.test.json b/tests/normative/provenance.test.json new file mode 100644 index 0000000..0469f4b --- /dev/null +++ b/tests/normative/provenance.test.json @@ -0,0 +1,904 @@ +{ + "target": "../../schema/ctrf.schema.json", + "$comment": "Provenance schema constraints; UUID identity uniqueness and self-reference are defined by the written specification", + "tests": [ + { + "description": "Provenance requires inputs", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": {}, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Extra alone does not replace required inputs", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "extra": { + "example.tool": {} + } + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Inputs cannot be empty", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Duplicate identical input references are rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + }, + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Multiple roots are represented by distinct direct inputs", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + }, + { + "reportId": "550e8400-e29b-41d4-a716-446655440101" + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "A single immediate source is sufficient", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440999" + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Unknown input fields are rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100", + "uri": "https://example.com/source" + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Per-input extra metadata is allowed", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100", + "extra": { + "example.source": { + "label": "initial" + } + } + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Unknown provenance fields are rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + } + ], + "operation": "merge" + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Single root is not part of the basic model", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + } + ], + "rootReportId": "550e8400-e29b-41d4-a716-446655440100" + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Null input is rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + null + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "String input is rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + "550e8400-e29b-41d4-a716-446655440100" + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Empty input object is rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + {} + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Empty UUID is rejected", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "" + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Provenance extra is optional and namespaced", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "550e8400-e29b-41d4-a716-446655440100" + } + ], + "extra": { + "example.tool": { + "note": "derived" + } + } + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Input extra rejects non-object null", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "extra": null + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Input extra rejects non-object []", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "extra": [] + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Input extra rejects non-object 42", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "extra": 42 + } + ] + }, + "runId": "coordinated-checkout-run" + } + }, + { + "description": "Input extra rejects non-object \"metadata\"", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "reportId": "550e8400-e29b-41d4-a716-446655440999", + "timestamp": "2025-11-24T12:30:00Z", + "generatedBy": "github-test-reporter", + "results": { + "tool": { + "name": "playwright" + }, + "summary": { + "tests": 2, + "passed": 1, + "failed": 1, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459220000 + }, + "tests": [ + { + "name": "checkout completes", + "status": "passed", + "duration": 240 + }, + { + "name": "dashboard renders metrics", + "status": "failed", + "duration": 980, + "message": "Element not found: .dashboard-metrics" + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "extra": "metadata" + } + ] + }, + "runId": "coordinated-checkout-run" + } + } + ] +} diff --git a/tests/normative/required-properties.test.json b/tests/normative/required-properties.test.json index fa314f2..15ab30a 100644 --- a/tests/normative/required-properties.test.json +++ b/tests/normative/required-properties.test.json @@ -398,6 +398,41 @@ }] } } + }, + { + "description": "Provenance input object missing required reportId property", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + }, + "provenance": { + "inputs": [ + {} + ] + } + } } ] } diff --git a/tests/normative/string-constraints.test.json b/tests/normative/string-constraints.test.json index dff9e85..b580aea 100644 --- a/tests/normative/string-constraints.test.json +++ b/tests/normative/string-constraints.test.json @@ -231,6 +231,43 @@ ] } } + }, + { + "description": "Provenance input with empty reportId violating minLength constraint", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + }, + "provenance": { + "inputs": [ + { + "reportId": "" + } + ] + } + } } ] } diff --git a/tests/normative/type-validation.test.json b/tests/normative/type-validation.test.json index 8ab0c1d..d57c499 100644 --- a/tests/normative/type-validation.test.json +++ b/tests/normative/type-validation.test.json @@ -1419,6 +1419,109 @@ ] } } + }, + { + "description": "Document with provenance as array instead of object", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "provenance": [], + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + } + } + }, + { + "description": "Provenance with inputs as object instead of array", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "provenance": { + "inputs": { + "reportId": "550e8400-e29b-41d4-a716-446655440000" + } + }, + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + } + } + }, + { + "description": "Provenance input with reportId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "provenance": { + "inputs": [ + { + "reportId": 12345 + } + ] + }, + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100 + } + ] + } + } } ] }