diff --git a/README.md b/README.md index 99c7110..9ec8756 100644 --- a/README.md +++ b/README.md @@ -191,12 +191,16 @@ const testById = findTest(report, { id: 'test-uuid' }) ```typescript import { merge } from 'ctrf' -// Merge multiple reports into one +// Merge reports declaring the same specification version into one const merged = merge([report1, report2, report3], { deduplicateTests: true, // Remove duplicate tests by ID }) ``` +Merging preserves the declared specification version and rejects mixed versions. +Convert and validate inputs explicitly before combining different versions. Global +errors are retained in input order, separately from test counts. + ### ID Generation ```typescript @@ -255,8 +259,8 @@ import { } from 'ctrf' REPORT_FORMAT // 'CTRF' -CURRENT_SPEC_VERSION // '0.2.0' -SUPPORTED_SPEC_VERSIONS // ['0.0.1', '0.0.2', '0.0.3', '0.0.4', '0.1.0', '0.2.0'] +CURRENT_SPEC_VERSION // '0.3.0' +SUPPORTED_SPEC_VERSIONS // ['0.0.1', '0.0.2', '0.0.3', '0.0.4', '0.1.0', '0.2.0', '0.3.0'] TEST_STATUSES // ['passed', 'failed', 'skipped', 'pending', 'other'] CTRF_NAMESPACE // UUID namespace for deterministic IDs ``` @@ -296,6 +300,6 @@ const v0_0_2Schema = getSchema('0.0.2') const latestSchema = getSchema('latest') // Get version info -const version = getCurrentSpecVersion() // '0.2.0' -const supported = getSupportedSpecVersions() // ['0.0.1', '0.0.2', '0.0.3', '0.0.4', '0.1.0', '0.2.0'] +const version = getCurrentSpecVersion() // '0.3.0' +const supported = getSupportedSpecVersions() // ['0.0.1', '0.0.2', '0.0.3', '0.0.4', '0.1.0', '0.2.0', '0.3.0'] ``` diff --git a/docs/classes/ReportBuilder.md b/docs/classes/ReportBuilder.md index d4277d0..5ecbb5c 100644 --- a/docs/classes/ReportBuilder.md +++ b/docs/classes/ReportBuilder.md @@ -6,7 +6,7 @@ # Class: ReportBuilder -Defined in: [builder.ts:50](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L50) +Defined in: [builder.ts:52](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L52) ## Example @@ -31,7 +31,7 @@ const report = new ReportBuilder() > **new ReportBuilder**(`options?`): `ReportBuilder` -Defined in: [builder.ts:64](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L64) +Defined in: [builder.ts:67](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L67) #### Parameters @@ -49,7 +49,7 @@ Defined in: [builder.ts:64](https://github.com/ctrf-io/ctrf-js/blob/main/src/bui > **specVersion**(`version`): `this` -Defined in: [builder.ts:76](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L76) +Defined in: [builder.ts:79](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L79) Set the spec version. @@ -69,7 +69,7 @@ Set the spec version. > **reportId**(`uuid?`): `this` -Defined in: [builder.ts:85](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L85) +Defined in: [builder.ts:88](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L88) Set or generate the report ID. @@ -91,7 +91,7 @@ UUID to use, or undefined to auto-generate > **runId**(`id`): `this` -Defined in: [builder.ts:93](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L93) +Defined in: [builder.ts:96](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L96) Set the logical test run ID shared across related CTRF documents. @@ -111,7 +111,7 @@ Set the logical test run ID shared across related CTRF documents. > **timestamp**(`date?`): `this` -Defined in: [builder.ts:102](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L102) +Defined in: [builder.ts:105](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L105) Set the timestamp. @@ -133,7 +133,7 @@ Date to use, or undefined for current time > **generatedBy**(`name`): `this` -Defined in: [builder.ts:116](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L116) +Defined in: [builder.ts:119](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L119) Set the generator name. @@ -153,7 +153,7 @@ Set the generator name. > **tool**(`tool`): `this` -Defined in: [builder.ts:124](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L124) +Defined in: [builder.ts:127](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L127) Set the tool information. @@ -173,7 +173,7 @@ Set the tool information. > **environment**(`env`): `this` -Defined in: [builder.ts:132](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L132) +Defined in: [builder.ts:135](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L135) Set the environment information. @@ -193,7 +193,7 @@ Set the environment information. > **addTest**(`test`): `this` -Defined in: [builder.ts:140](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L140) +Defined in: [builder.ts:143](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L143) Add a single test. @@ -213,7 +213,7 @@ Add a single test. > **addTests**(`tests`): `this` -Defined in: [builder.ts:148](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L148) +Defined in: [builder.ts:151](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L151) Add multiple tests. @@ -229,11 +229,31 @@ Add multiple tests. *** +### addError() + +> **addError**(`error`): `this` + +Defined in: [builder.ts:159](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L159) + +Add a global error without changing test counts. + +#### Parameters + +##### error + +[`GlobalError`](../interfaces/GlobalError.md) + +#### Returns + +`this` + +*** + ### insights() > **insights**(`insights`): `this` -Defined in: [builder.ts:156](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L156) +Defined in: [builder.ts:167](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L167) Set run-level insights. @@ -253,7 +273,7 @@ Set run-level insights. > **baseline**(`baseline`): `this` -Defined in: [builder.ts:164](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L164) +Defined in: [builder.ts:175](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L175) Set the baseline reference. @@ -273,7 +293,7 @@ Set the baseline reference. > **extra**(`data`): `this` -Defined in: [builder.ts:172](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L172) +Defined in: [builder.ts:183](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L183) Set extra metadata. @@ -293,7 +313,7 @@ Set extra metadata. > **summaryOverrides**(`overrides`): `this` -Defined in: [builder.ts:181](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L181) +Defined in: [builder.ts:192](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L192) Override specific summary fields. Useful when you want to set specific timing or counts. @@ -314,7 +334,7 @@ Useful when you want to set specific timing or counts. > **build**(): [`CTRFReport`](../interfaces/CTRFReport.md) -Defined in: [builder.ts:190](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L190) +Defined in: [builder.ts:201](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L201) Build and return the CTRF report. diff --git a/docs/classes/TestBuilder.md b/docs/classes/TestBuilder.md index 0b727aa..bab213e 100644 --- a/docs/classes/TestBuilder.md +++ b/docs/classes/TestBuilder.md @@ -6,7 +6,7 @@ # Class: TestBuilder -Defined in: [builder.ts:265](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L265) +Defined in: [builder.ts:278](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L278) ## Example @@ -25,7 +25,7 @@ const test = new TestBuilder() > **new TestBuilder**(`options?`): `TestBuilder` -Defined in: [builder.ts:301](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L301) +Defined in: [builder.ts:316](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L316) #### Parameters @@ -43,7 +43,7 @@ Defined in: [builder.ts:301](https://github.com/ctrf-io/ctrf-js/blob/main/src/bu > **id**(`uuid?`): `this` -Defined in: [builder.ts:307](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L307) +Defined in: [builder.ts:322](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L322) Set or generate the test ID. @@ -65,7 +65,7 @@ UUID to use, or undefined to auto-generate based on properties > **testId**(`id`): `this` -Defined in: [builder.ts:315](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L315) +Defined in: [builder.ts:330](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L330) Set the stable logical test case ID. @@ -85,7 +85,7 @@ Set the stable logical test case ID. > **executionId**(`id`): `this` -Defined in: [builder.ts:323](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L323) +Defined in: [builder.ts:338](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L338) Set the identifier for this specific test execution. @@ -105,7 +105,7 @@ Set the identifier for this specific test execution. > **attemptId**(`id`): `this` -Defined in: [builder.ts:331](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L331) +Defined in: [builder.ts:346](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L346) Set the attempt ID. @@ -125,7 +125,7 @@ Set the attempt ID. > **name**(`name`): `this` -Defined in: [builder.ts:342](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L342) +Defined in: [builder.ts:357](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L357) Set the test name. @@ -141,11 +141,51 @@ Set the test name. *** +### description() + +> **description**(`description`): `this` + +Defined in: [builder.ts:365](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L365) + +Set the human-readable test-case description. + +#### Parameters + +##### description + +`string` + +#### Returns + +`this` + +*** + +### addAssertionFailure() + +> **addAssertionFailure**(`failure`): `this` + +Defined in: [builder.ts:373](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L373) + +Add an assertion failure attributable to the final attempt. + +#### Parameters + +##### failure + +[`AssertionFailure`](../interfaces/AssertionFailure.md) + +#### Returns + +`this` + +*** + ### status() > **status**(`status`): `this` -Defined in: [builder.ts:350](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L350) +Defined in: [builder.ts:381](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L381) Set the test status. @@ -165,7 +205,7 @@ Set the test status. > **duration**(`ms`): `this` -Defined in: [builder.ts:358](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L358) +Defined in: [builder.ts:389](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L389) Set the duration in milliseconds. @@ -185,7 +225,7 @@ Set the duration in milliseconds. > **start**(`timestamp`): `this` -Defined in: [builder.ts:366](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L366) +Defined in: [builder.ts:397](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L397) Set the start timestamp. @@ -205,7 +245,7 @@ Set the start timestamp. > **stop**(`timestamp`): `this` -Defined in: [builder.ts:374](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L374) +Defined in: [builder.ts:405](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L405) Set the stop timestamp. @@ -225,7 +265,7 @@ Set the stop timestamp. > **suite**(`suites`): `this` -Defined in: [builder.ts:382](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L382) +Defined in: [builder.ts:413](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L413) Set the suite hierarchy. @@ -245,7 +285,7 @@ Set the suite hierarchy. > **message**(`msg`): `this` -Defined in: [builder.ts:390](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L390) +Defined in: [builder.ts:421](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L421) Set the error message. @@ -265,7 +305,7 @@ Set the error message. > **trace**(`trace`): `this` -Defined in: [builder.ts:398](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L398) +Defined in: [builder.ts:429](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L429) Set the stack trace. @@ -285,7 +325,7 @@ Set the stack trace. > **snippet**(`code`): `this` -Defined in: [builder.ts:406](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L406) +Defined in: [builder.ts:437](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L437) Set the code snippet. @@ -305,7 +345,7 @@ Set the code snippet. > **ai**(`analysis`): `this` -Defined in: [builder.ts:414](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L414) +Defined in: [builder.ts:445](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L445) Set AI-generated analysis. @@ -325,7 +365,7 @@ Set AI-generated analysis. > **line**(`num`): `this` -Defined in: [builder.ts:422](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L422) +Defined in: [builder.ts:453](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L453) Set the line number. @@ -345,7 +385,7 @@ Set the line number. > **rawStatus**(`status`): `this` -Defined in: [builder.ts:430](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L430) +Defined in: [builder.ts:461](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L461) Set the raw status from the test framework. @@ -365,7 +405,7 @@ Set the raw status from the test framework. > **tags**(`tags`): `this` -Defined in: [builder.ts:438](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L438) +Defined in: [builder.ts:469](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L469) Set tags. @@ -385,7 +425,7 @@ Set tags. > **labels**(`labels`): `this` -Defined in: [builder.ts:446](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L446) +Defined in: [builder.ts:477](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L477) Set structured labels. @@ -405,7 +445,7 @@ Set structured labels. > **type**(`type`): `this` -Defined in: [builder.ts:454](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L454) +Defined in: [builder.ts:485](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L485) Set test type. @@ -425,7 +465,7 @@ Set test type. > **filePath**(`path`): `this` -Defined in: [builder.ts:462](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L462) +Defined in: [builder.ts:493](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L493) Set file path. @@ -445,7 +485,7 @@ Set file path. > **retries**(`count`): `this` -Defined in: [builder.ts:470](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L470) +Defined in: [builder.ts:501](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L501) Set the number of times the test was re-executed after its initial attempt. @@ -465,7 +505,7 @@ Set the number of times the test was re-executed after its initial attempt. > **addAttemptHistoryEntry**(`attempt`): `this` -Defined in: [builder.ts:482](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L482) +Defined in: [builder.ts:513](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L513) Add an attempt completed before the final attempt. @@ -489,7 +529,7 @@ number of history entries. > **addRetryAttempt**(`attempt`): `this` -Defined in: [builder.ts:495](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L495) +Defined in: [builder.ts:526](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L526) Backward-compatible alias for [addAttemptHistoryEntry](#addattempthistoryentry). @@ -513,7 +553,7 @@ Use `addAttemptHistoryEntry()`. > **flaky**(`isFlaky?`): `this` -Defined in: [builder.ts:502](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L502) +Defined in: [builder.ts:533](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L533) Mark as flaky. @@ -533,7 +573,7 @@ Mark as flaky. > **stdout**(`lines`): `this` -Defined in: [builder.ts:510](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L510) +Defined in: [builder.ts:541](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L541) Set stdout. @@ -553,7 +593,7 @@ Set stdout. > **stderr**(`lines`): `this` -Defined in: [builder.ts:518](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L518) +Defined in: [builder.ts:549](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L549) Set stderr. @@ -573,9 +613,9 @@ Set stderr. > **threadId**(`id`): `this` -Defined in: [builder.ts:526](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L526) +Defined in: [builder.ts:557](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L557) -Set thread ID. +Set the thread or worker that executed the final attempt. #### Parameters @@ -593,7 +633,7 @@ Set thread ID. > **browser**(`name`): `this` -Defined in: [builder.ts:534](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L534) +Defined in: [builder.ts:565](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L565) Set browser name. @@ -613,7 +653,7 @@ Set browser name. > **device**(`name`): `this` -Defined in: [builder.ts:542](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L542) +Defined in: [builder.ts:573](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L573) Set device name. @@ -633,7 +673,7 @@ Set device name. > **screenshot**(`base64`): `this` -Defined in: [builder.ts:550](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L550) +Defined in: [builder.ts:581](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L581) Set screenshot (base64). @@ -653,7 +693,7 @@ Set screenshot (base64). > **addAttachment**(`attachment`): `this` -Defined in: [builder.ts:558](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L558) +Defined in: [builder.ts:589](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L589) Add an attachment. @@ -673,7 +713,7 @@ Add an attachment. > **parameters**(`params`): `this` -Defined in: [builder.ts:569](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L569) +Defined in: [builder.ts:600](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L600) Set parameters. @@ -693,7 +733,7 @@ Set parameters. > **addStep**(`step`): `this` -Defined in: [builder.ts:577](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L577) +Defined in: [builder.ts:608](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L608) Add a step. @@ -713,7 +753,7 @@ Add a step. > **insights**(`insights`): `this` -Defined in: [builder.ts:588](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L588) +Defined in: [builder.ts:619](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L619) Set test-level insights. @@ -733,7 +773,7 @@ Set test-level insights. > **extra**(`data`): `this` -Defined in: [builder.ts:596](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L596) +Defined in: [builder.ts:627](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L627) Set extra metadata. @@ -753,7 +793,7 @@ Set extra metadata. > **build**(): [`Test`](../interfaces/Test.md) -Defined in: [builder.ts:605](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L605) +Defined in: [builder.ts:636](https://github.com/ctrf-io/ctrf-js/blob/main/src/builder.ts#L636) Build and return the Test object. diff --git a/docs/functions/merge.md b/docs/functions/merge.md index 7a00fb4..1be4ec3 100644 --- a/docs/functions/merge.md +++ b/docs/functions/merge.md @@ -8,7 +8,7 @@ > **merge**(`reports`, `options?`): [`CTRFReport`](../interfaces/CTRFReport.md) -Defined in: [merge.ts:38](https://github.com/ctrf-io/ctrf-js/blob/main/src/merge.ts#L38) +Defined in: [merge.ts:42](https://github.com/ctrf-io/ctrf-js/blob/main/src/merge.ts#L42) ## Parameters @@ -32,7 +32,7 @@ A new merged CTRFReport ## Throws -Error if no reports are provided +Error if no reports are provided or specification versions differ ## Example diff --git a/docs/globals.md b/docs/globals.md index 5abb581..115858f 100644 --- a/docs/globals.md +++ b/docs/globals.md @@ -19,6 +19,8 @@ A complete TypeScript implementation for working with CTRF (Common Test Report F - [Tool](interfaces/Tool.md) - [Summary](interfaces/Summary.md) - [Test](interfaces/Test.md) +- [AssertionFailure](interfaces/AssertionFailure.md) +- [GlobalError](interfaces/GlobalError.md) - [TestStatus](type-aliases/TestStatus.md) - [AttemptHistoryEntry](interfaces/AttemptHistoryEntry.md) - [~~RetryAttempt~~](type-aliases/RetryAttempt.md) @@ -166,6 +168,10 @@ A test is considered flaky if: Merge multiple CTRF reports into a single report. Useful for combining results from parallel or sharded test runs. +Reports must declare the same specification version; merging does not convert +historical reports to a newer contract. Global errors are concatenated in input +order and do not contribute to test counts. + - [merge](functions/merge.md) ## Query & Filter diff --git a/docs/interfaces/AssertionFailure.md b/docs/interfaces/AssertionFailure.md new file mode 100644 index 0000000..ec1bf10 --- /dev/null +++ b/docs/interfaces/AssertionFailure.md @@ -0,0 +1,61 @@ +[**CTRF**](../README.md) + +*** + +[CTRF](../globals.md) / AssertionFailure + +# Interface: AssertionFailure + +Defined in: [types.ts:191](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L191) + +Details of an individual assertion failure. At least one property is required. + +## Properties + +### message? + +> `optional` **message?**: `string` + +Defined in: [types.ts:193](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L193) + +Message associated with this assertion failure + +*** + +### trace? + +> `optional` **trace?**: `string` + +Defined in: [types.ts:195](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L195) + +Stack trace associated with this assertion failure + +*** + +### expected? + +> `optional` **expected?**: `string` + +Defined in: [types.ts:197](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L197) + +String representation of the expected value + +*** + +### actual? + +> `optional` **actual?**: `string` + +Defined in: [types.ts:199](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L199) + +String representation of the actual value + +*** + +### extra? + +> `optional` **extra?**: `Record`\<`string`, `unknown`\> + +Defined in: [types.ts:201](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L201) + +Arbitrary metadata diff --git a/docs/interfaces/Attachment.md b/docs/interfaces/Attachment.md index 89aca1a..e46209a 100644 --- a/docs/interfaces/Attachment.md +++ b/docs/interfaces/Attachment.md @@ -6,7 +6,7 @@ # Interface: Attachment -Defined in: [types.ts:246](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L246) +Defined in: [types.ts:289](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L289) File attachment @@ -16,7 +16,7 @@ File attachment > `optional` **attachmentId?**: `string` -Defined in: [types.ts:248](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L248) +Defined in: [types.ts:291](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L291) Identifier for this attachment reference instance @@ -26,7 +26,7 @@ Identifier for this attachment reference instance > **name**: `string` -Defined in: [types.ts:250](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L250) +Defined in: [types.ts:293](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L293) Attachment name @@ -36,7 +36,7 @@ Attachment name > **contentType**: `string` -Defined in: [types.ts:252](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L252) +Defined in: [types.ts:295](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L295) MIME content type @@ -46,9 +46,9 @@ MIME content type > **path**: `string` -Defined in: [types.ts:254](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L254) +Defined in: [types.ts:297](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L297) -Path to the attachment file +Opaque attachment reference (URL or file path); portable references are recommended *** @@ -56,6 +56,6 @@ Path to the attachment file > `optional` **extra?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:256](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L256) +Defined in: [types.ts:299](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L299) Custom metadata diff --git a/docs/interfaces/AttemptHistoryEntry.md b/docs/interfaces/AttemptHistoryEntry.md index 1c59afb..b2859c7 100644 --- a/docs/interfaces/AttemptHistoryEntry.md +++ b/docs/interfaces/AttemptHistoryEntry.md @@ -6,7 +6,7 @@ # Interface: AttemptHistoryEntry -Defined in: [types.ts:200](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L200) +Defined in: [types.ts:239](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L239) An attempt completed before the final attempt represented by the test object. Despite the `retryAttempts` field name, the history includes the initial @@ -18,7 +18,7 @@ attempt when a retry occurred and excludes the final attempt. > **attempt**: `number` -Defined in: [types.ts:202](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L202) +Defined in: [types.ts:241](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L241) Original sequence number for this attempt (1 = initial execution) @@ -28,7 +28,7 @@ Original sequence number for this attempt (1 = initial execution) > `optional` **attemptId?**: `string` -Defined in: [types.ts:204](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L204) +Defined in: [types.ts:243](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L243) Identifier for this individual attempt @@ -38,7 +38,7 @@ Identifier for this individual attempt > **status**: [`TestStatus`](../type-aliases/TestStatus.md) -Defined in: [types.ts:206](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L206) +Defined in: [types.ts:245](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L245) Status of this attempt @@ -48,7 +48,7 @@ Status of this attempt > `optional` **duration?**: `number` -Defined in: [types.ts:208](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L208) +Defined in: [types.ts:247](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L247) Duration of this attempt in milliseconds @@ -58,7 +58,7 @@ Duration of this attempt in milliseconds > `optional` **message?**: `string` -Defined in: [types.ts:210](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L210) +Defined in: [types.ts:249](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L249) Error message @@ -68,17 +68,37 @@ Error message > `optional` **trace?**: `string` -Defined in: [types.ts:212](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L212) +Defined in: [types.ts:251](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L251) Stack trace *** +### assertionFailures? + +> `optional` **assertionFailures?**: [`AssertionFailure`](AssertionFailure.md)[] + +Defined in: [types.ts:253](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L253) + +Assertion failures attributable to this attempt + +*** + +### threadId? + +> `optional` **threadId?**: `string` + +Defined in: [types.ts:255](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L255) + +Thread or worker that executed this attempt; qualify across processes or hosts + +*** + ### line? > `optional` **line?**: `number` -Defined in: [types.ts:214](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L214) +Defined in: [types.ts:257](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L257) Line number @@ -88,7 +108,7 @@ Line number > `optional` **snippet?**: `string` -Defined in: [types.ts:216](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L216) +Defined in: [types.ts:259](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L259) Code snippet @@ -98,7 +118,7 @@ Code snippet > `optional` **stdout?**: `string`[] -Defined in: [types.ts:218](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L218) +Defined in: [types.ts:261](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L261) Standard output @@ -108,7 +128,7 @@ Standard output > `optional` **stderr?**: `string`[] -Defined in: [types.ts:220](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L220) +Defined in: [types.ts:263](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L263) Standard error @@ -118,7 +138,7 @@ Standard error > `optional` **start?**: `number` -Defined in: [types.ts:222](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L222) +Defined in: [types.ts:265](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L265) Start timestamp @@ -128,7 +148,7 @@ Start timestamp > `optional` **stop?**: `number` -Defined in: [types.ts:224](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L224) +Defined in: [types.ts:267](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L267) Stop timestamp @@ -138,7 +158,7 @@ Stop timestamp > `optional` **attachments?**: [`Attachment`](Attachment.md)[] -Defined in: [types.ts:226](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L226) +Defined in: [types.ts:269](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L269) Attachments for this attempt @@ -148,6 +168,6 @@ Attachments for this attempt > `optional` **extra?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:228](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L228) +Defined in: [types.ts:271](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L271) Custom metadata diff --git a/docs/interfaces/Baseline.md b/docs/interfaces/Baseline.md index 0361514..eb792f4 100644 --- a/docs/interfaces/Baseline.md +++ b/docs/interfaces/Baseline.md @@ -6,7 +6,7 @@ # Interface: Baseline -Defined in: [types.ts:386](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L386) +Defined in: [types.ts:429](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L429) Reference to a baseline report @@ -16,7 +16,7 @@ Reference to a baseline report > **reportId**: `string` -Defined in: [types.ts:388](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L388) +Defined in: [types.ts:431](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L431) Report ID of the baseline report @@ -26,7 +26,7 @@ Report ID of the baseline report > `optional` **timestamp?**: `string` -Defined in: [types.ts:390](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L390) +Defined in: [types.ts:433](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L433) Timestamp of the baseline report @@ -36,7 +36,7 @@ Timestamp of the baseline report > `optional` **source?**: `string` -Defined in: [types.ts:392](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L392) +Defined in: [types.ts:435](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L435) Source description (e.g., 'main-branch', 'previous-run') @@ -46,7 +46,7 @@ Source description (e.g., 'main-branch', 'previous-run') > `optional` **buildNumber?**: `number` -Defined in: [types.ts:394](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L394) +Defined in: [types.ts:437](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L437) Build number of the baseline @@ -56,7 +56,7 @@ Build number of the baseline > `optional` **buildName?**: `string` -Defined in: [types.ts:396](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L396) +Defined in: [types.ts:439](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L439) Build name of the baseline @@ -66,7 +66,7 @@ Build name of the baseline > `optional` **buildUrl?**: `string` -Defined in: [types.ts:398](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L398) +Defined in: [types.ts:441](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L441) Build URL of the baseline @@ -76,7 +76,7 @@ Build URL of the baseline > `optional` **commit?**: `string` -Defined in: [types.ts:400](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L400) +Defined in: [types.ts:443](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L443) Git commit of the baseline @@ -86,6 +86,6 @@ Git commit of the baseline > `optional` **extra?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:402](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L402) +Defined in: [types.ts:445](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L445) Custom metadata diff --git a/docs/interfaces/CTRFReport.md b/docs/interfaces/CTRFReport.md index 923f481..be7fc6e 100644 --- a/docs/interfaces/CTRFReport.md +++ b/docs/interfaces/CTRFReport.md @@ -48,7 +48,7 @@ Unique identifier for this report (UUID v4) Defined in: [types.ts:25](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L25) -Identifier shared by all documents belonging to the same logical test run +Logical-run identifier shared across coordinated module, shard and retry-process reports *** diff --git a/docs/interfaces/Environment.md b/docs/interfaces/Environment.md index 3ad5338..778e6fa 100644 --- a/docs/interfaces/Environment.md +++ b/docs/interfaces/Environment.md @@ -6,7 +6,7 @@ # Interface: Environment -Defined in: [types.ts:278](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L278) +Defined in: [types.ts:321](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L321) Environment information @@ -16,7 +16,7 @@ Environment information > `optional` **reportName?**: `string` -Defined in: [types.ts:280](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L280) +Defined in: [types.ts:323](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L323) Custom report name @@ -26,7 +26,7 @@ Custom report name > `optional` **appName?**: `string` -Defined in: [types.ts:282](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L282) +Defined in: [types.ts:325](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L325) Application name @@ -36,7 +36,7 @@ Application name > `optional` **appVersion?**: `string` -Defined in: [types.ts:284](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L284) +Defined in: [types.ts:327](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L327) Application version @@ -46,7 +46,7 @@ Application version > `optional` **buildId?**: `string` -Defined in: [types.ts:286](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L286) +Defined in: [types.ts:329](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L329) Build identifier @@ -56,7 +56,7 @@ Build identifier > `optional` **buildName?**: `string` -Defined in: [types.ts:288](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L288) +Defined in: [types.ts:331](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L331) Build name @@ -66,7 +66,7 @@ Build name > `optional` **buildNumber?**: `number` -Defined in: [types.ts:290](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L290) +Defined in: [types.ts:333](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L333) Build number @@ -76,7 +76,7 @@ Build number > `optional` **buildUrl?**: `string` -Defined in: [types.ts:292](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L292) +Defined in: [types.ts:335](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L335) Build URL @@ -86,7 +86,7 @@ Build URL > `optional` **repositoryName?**: `string` -Defined in: [types.ts:294](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L294) +Defined in: [types.ts:337](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L337) Repository name @@ -96,7 +96,7 @@ Repository name > `optional` **repositoryUrl?**: `string` -Defined in: [types.ts:296](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L296) +Defined in: [types.ts:339](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L339) Repository URL @@ -106,7 +106,7 @@ Repository URL > `optional` **commit?**: `string` -Defined in: [types.ts:298](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L298) +Defined in: [types.ts:341](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L341) Git commit SHA @@ -116,7 +116,7 @@ Git commit SHA > `optional` **branchName?**: `string` -Defined in: [types.ts:300](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L300) +Defined in: [types.ts:343](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L343) Git branch name @@ -126,7 +126,7 @@ Git branch name > `optional` **osPlatform?**: `string` -Defined in: [types.ts:302](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L302) +Defined in: [types.ts:345](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L345) Operating system platform @@ -136,7 +136,7 @@ Operating system platform > `optional` **osRelease?**: `string` -Defined in: [types.ts:304](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L304) +Defined in: [types.ts:347](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L347) Operating system release @@ -146,7 +146,7 @@ Operating system release > `optional` **osVersion?**: `string` -Defined in: [types.ts:306](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L306) +Defined in: [types.ts:349](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L349) Operating system version @@ -156,7 +156,7 @@ Operating system version > `optional` **testEnvironment?**: `string` -Defined in: [types.ts:308](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L308) +Defined in: [types.ts:351](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L351) Test environment name @@ -166,7 +166,7 @@ Test environment name > `optional` **shardId?**: `string` -Defined in: [types.ts:310](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L310) +Defined in: [types.ts:353](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L353) Shard or partition that produced this document @@ -176,7 +176,7 @@ Shard or partition that produced this document > `optional` **healthy?**: `boolean` -Defined in: [types.ts:312](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L312) +Defined in: [types.ts:355](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L355) Whether the environment is healthy @@ -186,6 +186,6 @@ Whether the environment is healthy > `optional` **extra?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:314](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L314) +Defined in: [types.ts:357](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L357) Custom metadata diff --git a/docs/interfaces/FilterCriteria.md b/docs/interfaces/FilterCriteria.md index a874b72..6d47c2d 100644 --- a/docs/interfaces/FilterCriteria.md +++ b/docs/interfaces/FilterCriteria.md @@ -6,7 +6,7 @@ # Interface: FilterCriteria -Defined in: [types.ts:454](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L454) +Defined in: [types.ts:497](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L497) Criteria for filtering and finding tests. @@ -16,7 +16,7 @@ Criteria for filtering and finding tests. > `optional` **id?**: `string` -Defined in: [types.ts:456](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L456) +Defined in: [types.ts:499](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L499) Filter by legacy test ID (UUID) @@ -26,7 +26,7 @@ Filter by legacy test ID (UUID) > `optional` **testId?**: `string` -Defined in: [types.ts:458](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L458) +Defined in: [types.ts:501](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L501) Filter by stable logical test case ID @@ -36,7 +36,7 @@ Filter by stable logical test case ID > `optional` **executionId?**: `string` -Defined in: [types.ts:460](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L460) +Defined in: [types.ts:503](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L503) Filter by a specific test execution ID @@ -46,7 +46,7 @@ Filter by a specific test execution ID > `optional` **name?**: `string` -Defined in: [types.ts:462](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L462) +Defined in: [types.ts:505](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L505) Filter by test name @@ -56,7 +56,7 @@ Filter by test name > `optional` **status?**: [`TestStatus`](../type-aliases/TestStatus.md) \| [`TestStatus`](../type-aliases/TestStatus.md)[] -Defined in: [types.ts:464](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L464) +Defined in: [types.ts:507](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L507) Filter by status @@ -66,7 +66,7 @@ Filter by status > `optional` **tags?**: `string` \| `string`[] -Defined in: [types.ts:466](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L466) +Defined in: [types.ts:509](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L509) Filter by tags @@ -76,7 +76,7 @@ Filter by tags > `optional` **suite?**: `string` \| `string`[] -Defined in: [types.ts:468](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L468) +Defined in: [types.ts:511](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L511) Filter by suite @@ -86,7 +86,7 @@ Filter by suite > `optional` **flaky?**: `boolean` -Defined in: [types.ts:470](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L470) +Defined in: [types.ts:513](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L513) Filter by flaky flag @@ -96,7 +96,7 @@ Filter by flaky flag > `optional` **browser?**: `string` -Defined in: [types.ts:472](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L472) +Defined in: [types.ts:515](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L515) Filter by browser @@ -106,6 +106,6 @@ Filter by browser > `optional` **device?**: `string` -Defined in: [types.ts:474](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L474) +Defined in: [types.ts:517](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L517) Filter by device diff --git a/docs/interfaces/GlobalError.md b/docs/interfaces/GlobalError.md new file mode 100644 index 0000000..4fb5cb8 --- /dev/null +++ b/docs/interfaces/GlobalError.md @@ -0,0 +1,42 @@ +[**CTRF**](../README.md) + +*** + +[CTRF](../globals.md) / GlobalError + +# Interface: GlobalError + +Defined in: [types.ts:210](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L210) + +Error associated with the run rather than an individual test result. +At least one property is required. Global errors do not add test results. + +## Properties + +### message? + +> `optional` **message?**: `string` + +Defined in: [types.ts:212](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L212) + +Message associated with this global error + +*** + +### trace? + +> `optional` **trace?**: `string` + +Defined in: [types.ts:214](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L214) + +Stack trace associated with this global error + +*** + +### extra? + +> `optional` **extra?**: `Record`\<`string`, `unknown`\> + +Defined in: [types.ts:216](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L216) + +Arbitrary metadata diff --git a/docs/interfaces/Insights.md b/docs/interfaces/Insights.md index 411b4f8..1d28ebb 100644 --- a/docs/interfaces/Insights.md +++ b/docs/interfaces/Insights.md @@ -6,7 +6,7 @@ # Interface: Insights -Defined in: [types.ts:326](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L326) +Defined in: [types.ts:369](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L369) Run-level insights computed from historical data @@ -16,7 +16,7 @@ Run-level insights computed from historical data > `optional` **passRate?**: [`MetricDelta`](MetricDelta.md) -Defined in: [types.ts:328](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L328) +Defined in: [types.ts:371](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L371) Pass rate metric @@ -26,7 +26,7 @@ Pass rate metric > `optional` **failRate?**: [`MetricDelta`](MetricDelta.md) -Defined in: [types.ts:330](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L330) +Defined in: [types.ts:373](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L373) Fail rate metric @@ -36,7 +36,7 @@ Fail rate metric > `optional` **flakyRate?**: [`MetricDelta`](MetricDelta.md) -Defined in: [types.ts:332](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L332) +Defined in: [types.ts:375](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L375) Flaky rate metric @@ -46,7 +46,7 @@ Flaky rate metric > `optional` **averageRunDuration?**: [`MetricDelta`](MetricDelta.md) -Defined in: [types.ts:334](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L334) +Defined in: [types.ts:377](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L377) Average run duration metric @@ -56,7 +56,7 @@ Average run duration metric > `optional` **p95RunDuration?**: [`MetricDelta`](MetricDelta.md) -Defined in: [types.ts:336](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L336) +Defined in: [types.ts:379](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L379) 95th percentile run duration metric @@ -66,7 +66,7 @@ Defined in: [types.ts:336](https://github.com/ctrf-io/ctrf-js/blob/main/src/type > `optional` **averageTestDuration?**: [`MetricDelta`](MetricDelta.md) -Defined in: [types.ts:338](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L338) +Defined in: [types.ts:381](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L381) Average test duration metric @@ -76,7 +76,7 @@ Average test duration metric > `optional` **runsAnalyzed?**: `number` -Defined in: [types.ts:340](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L340) +Defined in: [types.ts:383](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L383) Number of historical runs analyzed @@ -86,6 +86,6 @@ Number of historical runs analyzed > `optional` **extra?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:342](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L342) +Defined in: [types.ts:385](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L385) Custom metadata diff --git a/docs/interfaces/InsightsOptions.md b/docs/interfaces/InsightsOptions.md index 0c395d4..be1c148 100644 --- a/docs/interfaces/InsightsOptions.md +++ b/docs/interfaces/InsightsOptions.md @@ -6,7 +6,7 @@ # Interface: InsightsOptions -Defined in: [types.ts:482](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L482) +Defined in: [types.ts:525](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L525) Options for insights calculation @@ -16,7 +16,7 @@ Options for insights calculation > `optional` **baseline?**: [`CTRFReport`](CTRFReport.md) -Defined in: [types.ts:484](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L484) +Defined in: [types.ts:527](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L527) Baseline report for comparison @@ -26,6 +26,6 @@ Baseline report for comparison > `optional` **window?**: `number` -Defined in: [types.ts:486](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L486) +Defined in: [types.ts:529](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L529) Number of historical reports to analyze diff --git a/docs/interfaces/MergeOptions.md b/docs/interfaces/MergeOptions.md index be1ac40..8ef48db 100644 --- a/docs/interfaces/MergeOptions.md +++ b/docs/interfaces/MergeOptions.md @@ -6,7 +6,7 @@ # Interface: MergeOptions -Defined in: [types.ts:440](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L440) +Defined in: [types.ts:483](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L483) Options for merging reports @@ -16,7 +16,7 @@ Options for merging reports > `optional` **deduplicateTests?**: `boolean` -Defined in: [types.ts:442](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L442) +Defined in: [types.ts:485](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L485) Remove duplicate tests by executionId, testId, or legacy id @@ -26,7 +26,7 @@ Remove duplicate tests by executionId, testId, or legacy id > `optional` **mergeSummary?**: `boolean` -Defined in: [types.ts:444](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L444) +Defined in: [types.ts:487](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L487) Recalculate summary from merged tests @@ -36,6 +36,6 @@ Recalculate summary from merged tests > `optional` **preserveEnvironment?**: `"first"` \| `"last"` \| `"merge"` -Defined in: [types.ts:446](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L446) +Defined in: [types.ts:489](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L489) Strategy for handling environments diff --git a/docs/interfaces/MetricDelta.md b/docs/interfaces/MetricDelta.md index cc7be07..4fb4a6e 100644 --- a/docs/interfaces/MetricDelta.md +++ b/docs/interfaces/MetricDelta.md @@ -6,7 +6,7 @@ # Interface: MetricDelta -Defined in: [types.ts:372](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L372) +Defined in: [types.ts:415](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L415) Metric with current value, baseline, and change @@ -16,7 +16,7 @@ Metric with current value, baseline, and change > `optional` **current?**: `number` -Defined in: [types.ts:374](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L374) +Defined in: [types.ts:417](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L417) Current value @@ -26,7 +26,7 @@ Current value > `optional` **baseline?**: `number` -Defined in: [types.ts:376](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L376) +Defined in: [types.ts:419](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L419) Baseline value for comparison @@ -36,6 +36,6 @@ Baseline value for comparison > `optional` **change?**: `number` -Defined in: [types.ts:378](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L378) +Defined in: [types.ts:421](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L421) Change from baseline (current - baseline) diff --git a/docs/interfaces/ParseOptions.md b/docs/interfaces/ParseOptions.md index 228f76b..e3d247f 100644 --- a/docs/interfaces/ParseOptions.md +++ b/docs/interfaces/ParseOptions.md @@ -6,7 +6,7 @@ # Interface: ParseOptions -Defined in: [types.ts:528](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L528) +Defined in: [types.ts:571](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L571) Options for parsing JSON @@ -16,6 +16,6 @@ Options for parsing JSON > `optional` **validate?**: `boolean` -Defined in: [types.ts:530](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L530) +Defined in: [types.ts:573](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L573) Validate after parsing diff --git a/docs/interfaces/ReportBuilderOptions.md b/docs/interfaces/ReportBuilderOptions.md index 61e514d..b5e9b7c 100644 --- a/docs/interfaces/ReportBuilderOptions.md +++ b/docs/interfaces/ReportBuilderOptions.md @@ -6,7 +6,7 @@ # Interface: ReportBuilderOptions -Defined in: [types.ts:494](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L494) +Defined in: [types.ts:537](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L537) Options for ReportBuilder @@ -16,7 +16,7 @@ Options for ReportBuilder > `optional` **autoGenerateId?**: `boolean` -Defined in: [types.ts:496](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L496) +Defined in: [types.ts:539](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L539) Automatically generate report ID @@ -26,6 +26,6 @@ Automatically generate report ID > `optional` **autoTimestamp?**: `boolean` -Defined in: [types.ts:498](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L498) +Defined in: [types.ts:541](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L541) Automatically set timestamp diff --git a/docs/interfaces/Results.md b/docs/interfaces/Results.md index 42b011d..2732fbb 100644 --- a/docs/interfaces/Results.md +++ b/docs/interfaces/Results.md @@ -42,11 +42,21 @@ Array of individual test results *** +### errors? + +> `optional` **errors?**: [`GlobalError`](GlobalError.md)[] + +Defined in: [types.ts:53](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L53) + +Global errors, separate from test results and summary counts + +*** + ### environment? > `optional` **environment?**: [`Environment`](Environment.md) -Defined in: [types.ts:53](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L53) +Defined in: [types.ts:55](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L55) Environment information @@ -56,6 +66,6 @@ Environment information > `optional` **extra?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:55](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L55) +Defined in: [types.ts:57](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L57) Custom metadata diff --git a/docs/interfaces/Step.md b/docs/interfaces/Step.md index 2b403d7..6c1d93f 100644 --- a/docs/interfaces/Step.md +++ b/docs/interfaces/Step.md @@ -6,7 +6,7 @@ # Interface: Step -Defined in: [types.ts:264](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L264) +Defined in: [types.ts:307](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L307) Test step @@ -16,7 +16,7 @@ Test step > **name**: `string` -Defined in: [types.ts:266](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L266) +Defined in: [types.ts:309](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L309) Step name @@ -26,7 +26,7 @@ Step name > **status**: [`TestStatus`](../type-aliases/TestStatus.md) -Defined in: [types.ts:268](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L268) +Defined in: [types.ts:311](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L311) Step status @@ -36,6 +36,6 @@ Step status > `optional` **extra?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:270](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L270) +Defined in: [types.ts:313](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L313) Custom metadata diff --git a/docs/interfaces/StringifyOptions.md b/docs/interfaces/StringifyOptions.md index 3f47196..3730ce3 100644 --- a/docs/interfaces/StringifyOptions.md +++ b/docs/interfaces/StringifyOptions.md @@ -6,7 +6,7 @@ # Interface: StringifyOptions -Defined in: [types.ts:538](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L538) +Defined in: [types.ts:581](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L581) Options for stringifying to JSON @@ -16,7 +16,7 @@ Options for stringifying to JSON > `optional` **pretty?**: `boolean` -Defined in: [types.ts:540](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L540) +Defined in: [types.ts:583](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L583) Pretty print with indentation @@ -26,6 +26,6 @@ Pretty print with indentation > `optional` **indent?**: `number` -Defined in: [types.ts:542](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L542) +Defined in: [types.ts:585](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L585) Number of spaces for indentation (default: 2) diff --git a/docs/interfaces/Summary.md b/docs/interfaces/Summary.md index ed4ee12..659fd93 100644 --- a/docs/interfaces/Summary.md +++ b/docs/interfaces/Summary.md @@ -6,7 +6,7 @@ # Interface: Summary -Defined in: [types.ts:77](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L77) +Defined in: [types.ts:79](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L79) Aggregated test statistics @@ -16,7 +16,7 @@ Aggregated test statistics > **tests**: `number` -Defined in: [types.ts:79](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L79) +Defined in: [types.ts:81](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L81) Total number of tests @@ -26,7 +26,7 @@ Total number of tests > **passed**: `number` -Defined in: [types.ts:81](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L81) +Defined in: [types.ts:83](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L83) Number of passed tests @@ -36,7 +36,7 @@ Number of passed tests > **failed**: `number` -Defined in: [types.ts:83](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L83) +Defined in: [types.ts:85](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L85) Number of failed tests @@ -46,7 +46,7 @@ Number of failed tests > **skipped**: `number` -Defined in: [types.ts:85](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L85) +Defined in: [types.ts:87](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L87) Number of skipped tests @@ -56,7 +56,7 @@ Number of skipped tests > **pending**: `number` -Defined in: [types.ts:87](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L87) +Defined in: [types.ts:89](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L89) Number of pending tests @@ -66,7 +66,7 @@ Number of pending tests > **other**: `number` -Defined in: [types.ts:89](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L89) +Defined in: [types.ts:91](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L91) Number of tests with other status @@ -76,7 +76,7 @@ Number of tests with other status > `optional` **flaky?**: `number` -Defined in: [types.ts:91](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L91) +Defined in: [types.ts:93](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L93) Number of flaky tests @@ -86,7 +86,7 @@ Number of flaky tests > `optional` **suites?**: `number` -Defined in: [types.ts:93](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L93) +Defined in: [types.ts:95](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L95) Number of test suites @@ -96,7 +96,7 @@ Number of test suites > **start**: `number` -Defined in: [types.ts:95](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L95) +Defined in: [types.ts:97](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L97) Start timestamp (Unix epoch milliseconds) @@ -106,7 +106,7 @@ Start timestamp (Unix epoch milliseconds) > **stop**: `number` -Defined in: [types.ts:97](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L97) +Defined in: [types.ts:99](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L99) Stop timestamp (Unix epoch milliseconds) @@ -116,7 +116,7 @@ Stop timestamp (Unix epoch milliseconds) > `optional` **duration?**: `number` -Defined in: [types.ts:99](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L99) +Defined in: [types.ts:101](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L101) Total duration in milliseconds @@ -126,6 +126,6 @@ Total duration in milliseconds > `optional` **extra?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:101](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L101) +Defined in: [types.ts:103](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L103) Custom metadata diff --git a/docs/interfaces/SummaryOptions.md b/docs/interfaces/SummaryOptions.md index 7a0164f..028e449 100644 --- a/docs/interfaces/SummaryOptions.md +++ b/docs/interfaces/SummaryOptions.md @@ -6,7 +6,7 @@ # Interface: SummaryOptions -Defined in: [types.ts:516](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L516) +Defined in: [types.ts:559](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L559) Options for calculating summary @@ -16,7 +16,7 @@ Options for calculating summary > `optional` **start?**: `number` -Defined in: [types.ts:518](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L518) +Defined in: [types.ts:561](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L561) Start timestamp @@ -26,6 +26,6 @@ Start timestamp > `optional` **stop?**: `number` -Defined in: [types.ts:520](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L520) +Defined in: [types.ts:563](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L563) Stop timestamp diff --git a/docs/interfaces/Test.md b/docs/interfaces/Test.md index 6bfebeb..b78c2b5 100644 --- a/docs/interfaces/Test.md +++ b/docs/interfaces/Test.md @@ -6,7 +6,7 @@ # Interface: Test -Defined in: [types.ts:109](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L109) +Defined in: [types.ts:111](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L111) Individual test result @@ -16,9 +16,9 @@ Individual test result > `optional` **id?**: `string` -Defined in: [types.ts:111](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L111) +Defined in: [types.ts:113](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L113) -Legacy test identifier (UUID); new producers should prefer testId +Deprecated test-case identifier (UUID), scheduled for removal in CTRF 1.0.0; prefer optional testId *** @@ -26,9 +26,9 @@ Legacy test identifier (UUID); new producers should prefer testId > `optional` **testId?**: `string` -Defined in: [types.ts:113](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L113) +Defined in: [types.ts:115](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L115) -Stable identifier for the logical test case +Stable test-case identifier across runs within the producer's documented scope *** @@ -36,9 +36,9 @@ Stable identifier for the logical test case > `optional` **executionId?**: `string` -Defined in: [types.ts:115](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L115) +Defined in: [types.ts:117](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L117) -Identifier for this specific execution of the test case +Identifier for one test-case execution lifecycle within a run, including coordinated retries *** @@ -46,7 +46,7 @@ Identifier for this specific execution of the test case > `optional` **attemptId?**: `string` -Defined in: [types.ts:117](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L117) +Defined in: [types.ts:119](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L119) Identifier for this individual attempt @@ -56,17 +56,27 @@ Identifier for this individual attempt > **name**: `string` -Defined in: [types.ts:119](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L119) +Defined in: [types.ts:121](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L121) Test name *** +### description? + +> `optional` **description?**: `string` + +Defined in: [types.ts:123](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L123) + +Human-readable description of the test case + +*** + ### status > **status**: [`TestStatus`](../type-aliases/TestStatus.md) -Defined in: [types.ts:121](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L121) +Defined in: [types.ts:125](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L125) Test execution status @@ -76,9 +86,9 @@ Test execution status > **duration**: `number` -Defined in: [types.ts:123](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L123) +Defined in: [types.ts:127](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L127) -Test duration in milliseconds +Duration of the final attempt in milliseconds, excluding earlier attempts *** @@ -86,7 +96,7 @@ Test duration in milliseconds > `optional` **start?**: `number` -Defined in: [types.ts:125](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L125) +Defined in: [types.ts:129](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L129) Start timestamp (Unix epoch milliseconds) @@ -96,7 +106,7 @@ Start timestamp (Unix epoch milliseconds) > `optional` **stop?**: `number` -Defined in: [types.ts:127](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L127) +Defined in: [types.ts:131](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L131) Stop timestamp (Unix epoch milliseconds) @@ -106,7 +116,7 @@ Stop timestamp (Unix epoch milliseconds) > `optional` **suite?**: `string`[] -Defined in: [types.ts:129](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L129) +Defined in: [types.ts:133](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L133) Test suite hierarchy @@ -116,7 +126,7 @@ Test suite hierarchy > `optional` **message?**: `string` -Defined in: [types.ts:131](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L131) +Defined in: [types.ts:135](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L135) Error message (for failed tests) @@ -126,17 +136,27 @@ Error message (for failed tests) > `optional` **trace?**: `string` -Defined in: [types.ts:133](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L133) +Defined in: [types.ts:137](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L137) Stack trace (for failed tests) *** +### assertionFailures? + +> `optional` **assertionFailures?**: [`AssertionFailure`](AssertionFailure.md)[] + +Defined in: [types.ts:139](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L139) + +Assertion failures attributable to the final attempt + +*** + ### snippet? > `optional` **snippet?**: `string` -Defined in: [types.ts:135](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L135) +Defined in: [types.ts:141](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L141) Code snippet where failure occurred @@ -146,7 +166,7 @@ Code snippet where failure occurred > `optional` **ai?**: `string` -Defined in: [types.ts:137](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L137) +Defined in: [types.ts:143](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L143) AI-generated analysis or suggestion @@ -156,7 +176,7 @@ AI-generated analysis or suggestion > `optional` **line?**: `number` -Defined in: [types.ts:139](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L139) +Defined in: [types.ts:145](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L145) Line number where test is defined or failed @@ -166,7 +186,7 @@ Line number where test is defined or failed > `optional` **rawStatus?**: `string` -Defined in: [types.ts:141](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L141) +Defined in: [types.ts:147](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L147) Original status from the test framework @@ -176,7 +196,7 @@ Original status from the test framework > `optional` **tags?**: `string`[] -Defined in: [types.ts:143](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L143) +Defined in: [types.ts:149](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L149) Tags for categorization @@ -186,7 +206,7 @@ Tags for categorization > `optional` **labels?**: `Record`\<`string`, [`LabelValue`](../type-aliases/LabelValue.md)\> -Defined in: [types.ts:145](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L145) +Defined in: [types.ts:151](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L151) Structured key-value metadata; values may be scalar or multi-valued @@ -196,7 +216,7 @@ Structured key-value metadata; values may be scalar or multi-valued > `optional` **type?**: `string` -Defined in: [types.ts:147](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L147) +Defined in: [types.ts:153](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L153) Test type (e.g., 'unit', 'integration', 'e2e') @@ -206,7 +226,7 @@ Test type (e.g., 'unit', 'integration', 'e2e') > `optional` **filePath?**: `string` -Defined in: [types.ts:149](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L149) +Defined in: [types.ts:155](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L155) Path to the test file @@ -216,7 +236,7 @@ Path to the test file > `optional` **retries?**: `number` -Defined in: [types.ts:151](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L151) +Defined in: [types.ts:157](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L157) Number of times the test was re-executed after its initial attempt @@ -226,7 +246,7 @@ Number of times the test was re-executed after its initial attempt > `optional` **retryAttempts?**: [`AttemptHistoryEntry`](AttemptHistoryEntry.md)[] -Defined in: [types.ts:153](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L153) +Defined in: [types.ts:159](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L159) Ordered history of attempts completed before the final attempt @@ -236,7 +256,7 @@ Ordered history of attempts completed before the final attempt > `optional` **flaky?**: `boolean` -Defined in: [types.ts:155](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L155) +Defined in: [types.ts:161](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L161) Whether the test is flaky @@ -246,7 +266,7 @@ Whether the test is flaky > `optional` **stdout?**: `string`[] -Defined in: [types.ts:157](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L157) +Defined in: [types.ts:163](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L163) Standard output captured during test @@ -256,7 +276,7 @@ Standard output captured during test > `optional` **stderr?**: `string`[] -Defined in: [types.ts:159](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L159) +Defined in: [types.ts:165](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L165) Standard error captured during test @@ -266,9 +286,9 @@ Standard error captured during test > `optional` **threadId?**: `string` -Defined in: [types.ts:161](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L161) +Defined in: [types.ts:167](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L167) -Thread/worker ID that ran this test +Thread or worker that executed the final attempt; qualify across processes or hosts *** @@ -276,7 +296,7 @@ Thread/worker ID that ran this test > `optional` **browser?**: `string` -Defined in: [types.ts:163](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L163) +Defined in: [types.ts:169](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L169) Browser name (for browser tests) @@ -286,7 +306,7 @@ Browser name (for browser tests) > `optional` **device?**: `string` -Defined in: [types.ts:165](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L165) +Defined in: [types.ts:171](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L171) Device name (for device tests) @@ -296,7 +316,7 @@ Device name (for device tests) > `optional` **screenshot?**: `string` -Defined in: [types.ts:167](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L167) +Defined in: [types.ts:173](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L173) Base64 encoded screenshot @@ -306,7 +326,7 @@ Base64 encoded screenshot > `optional` **attachments?**: [`Attachment`](Attachment.md)[] -Defined in: [types.ts:169](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L169) +Defined in: [types.ts:175](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L175) File attachments @@ -316,7 +336,7 @@ File attachments > `optional` **parameters?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:171](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L171) +Defined in: [types.ts:177](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L177) Test parameters (for parameterized tests) @@ -326,7 +346,7 @@ Test parameters (for parameterized tests) > `optional` **steps?**: [`Step`](Step.md)[] -Defined in: [types.ts:173](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L173) +Defined in: [types.ts:179](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L179) Test steps @@ -336,7 +356,7 @@ Test steps > `optional` **insights?**: [`TestInsights`](TestInsights.md) -Defined in: [types.ts:175](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L175) +Defined in: [types.ts:181](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L181) Test-level insights @@ -346,6 +366,6 @@ Test-level insights > `optional` **extra?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:177](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L177) +Defined in: [types.ts:183](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L183) Custom metadata diff --git a/docs/interfaces/TestBuilderOptions.md b/docs/interfaces/TestBuilderOptions.md index ddb2883..85cee4a 100644 --- a/docs/interfaces/TestBuilderOptions.md +++ b/docs/interfaces/TestBuilderOptions.md @@ -6,7 +6,7 @@ # Interface: TestBuilderOptions -Defined in: [types.ts:506](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L506) +Defined in: [types.ts:549](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L549) Options for TestBuilder @@ -16,6 +16,6 @@ Options for TestBuilder > `optional` **autoGenerateId?**: `boolean` -Defined in: [types.ts:508](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L508) +Defined in: [types.ts:551](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L551) Automatically generate test ID diff --git a/docs/interfaces/TestInsights.md b/docs/interfaces/TestInsights.md index 681ec1e..f0ded91 100644 --- a/docs/interfaces/TestInsights.md +++ b/docs/interfaces/TestInsights.md @@ -6,7 +6,7 @@ # Interface: TestInsights -Defined in: [types.ts:350](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L350) +Defined in: [types.ts:393](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L393) Test-level insights computed from historical data @@ -16,7 +16,7 @@ Test-level insights computed from historical data > `optional` **passRate?**: [`MetricDelta`](MetricDelta.md) -Defined in: [types.ts:352](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L352) +Defined in: [types.ts:395](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L395) Pass rate metric @@ -26,7 +26,7 @@ Pass rate metric > `optional` **failRate?**: [`MetricDelta`](MetricDelta.md) -Defined in: [types.ts:354](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L354) +Defined in: [types.ts:397](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L397) Fail rate metric @@ -36,7 +36,7 @@ Fail rate metric > `optional` **flakyRate?**: [`MetricDelta`](MetricDelta.md) -Defined in: [types.ts:356](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L356) +Defined in: [types.ts:399](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L399) Flaky rate metric @@ -46,7 +46,7 @@ Flaky rate metric > `optional` **averageTestDuration?**: [`MetricDelta`](MetricDelta.md) -Defined in: [types.ts:358](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L358) +Defined in: [types.ts:401](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L401) Average test duration metric @@ -56,7 +56,7 @@ Average test duration metric > `optional` **p95TestDuration?**: [`MetricDelta`](MetricDelta.md) -Defined in: [types.ts:360](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L360) +Defined in: [types.ts:403](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L403) 95th percentile test duration metric @@ -66,7 +66,7 @@ Defined in: [types.ts:360](https://github.com/ctrf-io/ctrf-js/blob/main/src/type > `optional` **executedInRuns?**: `number` -Defined in: [types.ts:362](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L362) +Defined in: [types.ts:405](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L405) Number of runs this test was executed in @@ -76,6 +76,6 @@ Number of runs this test was executed in > `optional` **extra?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:364](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L364) +Defined in: [types.ts:407](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L407) Custom metadata diff --git a/docs/interfaces/Tool.md b/docs/interfaces/Tool.md index 551d2ed..a5aded6 100644 --- a/docs/interfaces/Tool.md +++ b/docs/interfaces/Tool.md @@ -6,7 +6,7 @@ # Interface: Tool -Defined in: [types.ts:63](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L63) +Defined in: [types.ts:65](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L65) Test tool information @@ -16,7 +16,7 @@ Test tool information > **name**: `string` -Defined in: [types.ts:65](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L65) +Defined in: [types.ts:67](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L67) Name of the test tool (e.g., 'jest', 'playwright') @@ -26,7 +26,7 @@ Name of the test tool (e.g., 'jest', 'playwright') > `optional` **version?**: `string` -Defined in: [types.ts:67](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L67) +Defined in: [types.ts:69](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L69) Version of the test tool @@ -36,6 +36,6 @@ Version of the test tool > `optional` **extra?**: `Record`\<`string`, `unknown`\> -Defined in: [types.ts:69](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L69) +Defined in: [types.ts:71](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L71) Custom metadata diff --git a/docs/interfaces/ValidateOptions.md b/docs/interfaces/ValidateOptions.md index b05f018..1e198d5 100644 --- a/docs/interfaces/ValidateOptions.md +++ b/docs/interfaces/ValidateOptions.md @@ -6,7 +6,7 @@ # Interface: ValidateOptions -Defined in: [types.ts:550](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L550) +Defined in: [types.ts:593](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L593) Options for validation @@ -16,6 +16,6 @@ Options for validation > `optional` **specVersion?**: [`SchemaSelector`](../type-aliases/SchemaSelector.md) -Defined in: [types.ts:552](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L552) +Defined in: [types.ts:595](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L595) Specific spec version to validate against, or `latest` (default) diff --git a/docs/interfaces/ValidationErrorDetail.md b/docs/interfaces/ValidationErrorDetail.md index 3b63ed7..ce62f13 100644 --- a/docs/interfaces/ValidationErrorDetail.md +++ b/docs/interfaces/ValidationErrorDetail.md @@ -6,7 +6,7 @@ # Interface: ValidationErrorDetail -Defined in: [types.ts:426](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L426) +Defined in: [types.ts:469](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L469) Details of a validation error @@ -16,7 +16,7 @@ Details of a validation error > **message**: `string` -Defined in: [types.ts:428](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L428) +Defined in: [types.ts:471](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L471) Human-readable error message @@ -26,7 +26,7 @@ Human-readable error message > **path**: `string` -Defined in: [types.ts:430](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L430) +Defined in: [types.ts:473](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L473) JSON path to the error location @@ -36,6 +36,6 @@ JSON path to the error location > **keyword**: `string` -Defined in: [types.ts:432](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L432) +Defined in: [types.ts:475](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L475) JSON Schema or semantic validation keyword that failed diff --git a/docs/interfaces/ValidationResult.md b/docs/interfaces/ValidationResult.md index 65d4e44..9e20416 100644 --- a/docs/interfaces/ValidationResult.md +++ b/docs/interfaces/ValidationResult.md @@ -6,7 +6,7 @@ # Interface: ValidationResult -Defined in: [types.ts:414](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L414) +Defined in: [types.ts:457](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L457) Result of schema validation @@ -16,7 +16,7 @@ Result of schema validation > **valid**: `boolean` -Defined in: [types.ts:416](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L416) +Defined in: [types.ts:459](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L459) Whether the report is valid @@ -26,6 +26,6 @@ Whether the report is valid > **errors**: [`ValidationErrorDetail`](ValidationErrorDetail.md)[] -Defined in: [types.ts:418](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L418) +Defined in: [types.ts:461](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L461) Array of schema or semantic validation errors diff --git a/docs/type-aliases/LabelPrimitive.md b/docs/type-aliases/LabelPrimitive.md index 2107264..3bcac0d 100644 --- a/docs/type-aliases/LabelPrimitive.md +++ b/docs/type-aliases/LabelPrimitive.md @@ -8,6 +8,6 @@ > **LabelPrimitive** = `string` \| `number` \| `boolean` -Defined in: [types.ts:188](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L188) +Defined in: [types.ts:227](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L227) Primitive value supported by a test label diff --git a/docs/type-aliases/LabelValue.md b/docs/type-aliases/LabelValue.md index 0b46228..e987323 100644 --- a/docs/type-aliases/LabelValue.md +++ b/docs/type-aliases/LabelValue.md @@ -8,6 +8,6 @@ > **LabelValue** = [`LabelPrimitive`](LabelPrimitive.md) \| \[[`LabelPrimitive`](LabelPrimitive.md), `...LabelPrimitive[]`\] -Defined in: [types.ts:191](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L191) +Defined in: [types.ts:230](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L230) Scalar or non-empty multi-valued test label value diff --git a/docs/type-aliases/RetryAttempt.md b/docs/type-aliases/RetryAttempt.md index 79e9893..e35b7fe 100644 --- a/docs/type-aliases/RetryAttempt.md +++ b/docs/type-aliases/RetryAttempt.md @@ -8,7 +8,7 @@ > **RetryAttempt** = [`AttemptHistoryEntry`](../interfaces/AttemptHistoryEntry.md) -Defined in: [types.ts:239](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L239) +Defined in: [types.ts:282](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L282) Backward-compatible name for an attempt history entry. diff --git a/docs/type-aliases/SchemaSelector.md b/docs/type-aliases/SchemaSelector.md index 820ba4f..d3737ff 100644 --- a/docs/type-aliases/SchemaSelector.md +++ b/docs/type-aliases/SchemaSelector.md @@ -8,6 +8,6 @@ > **SchemaSelector** = [`SupportedSpecVersion`](SupportedSpecVersion.md) \| `"latest"` -Defined in: [constants.ts:34](https://github.com/ctrf-io/ctrf-js/blob/main/src/constants.ts#L34) +Defined in: [constants.ts:35](https://github.com/ctrf-io/ctrf-js/blob/main/src/constants.ts#L35) -A published CTRF specification version or the current-version alias +A CTRF specification version or the current-version alias diff --git a/docs/type-aliases/SupportedSpecVersion.md b/docs/type-aliases/SupportedSpecVersion.md index f17533a..45eb89a 100644 --- a/docs/type-aliases/SupportedSpecVersion.md +++ b/docs/type-aliases/SupportedSpecVersion.md @@ -8,6 +8,6 @@ > **SupportedSpecVersion** = *typeof* [`SUPPORTED_SPEC_VERSIONS`](../variables/SUPPORTED_SPEC_VERSIONS.md)\[`number`\] -Defined in: [constants.ts:31](https://github.com/ctrf-io/ctrf-js/blob/main/src/constants.ts#L31) +Defined in: [constants.ts:32](https://github.com/ctrf-io/ctrf-js/blob/main/src/constants.ts#L32) -A published CTRF specification version supported by this package +A CTRF specification version supported by this package diff --git a/docs/type-aliases/TestStatus.md b/docs/type-aliases/TestStatus.md index 777a742..df952f5 100644 --- a/docs/type-aliases/TestStatus.md +++ b/docs/type-aliases/TestStatus.md @@ -8,6 +8,6 @@ > **TestStatus** = `"passed"` \| `"failed"` \| `"skipped"` \| `"pending"` \| `"other"` -Defined in: [types.ts:185](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L185) +Defined in: [types.ts:224](https://github.com/ctrf-io/ctrf-js/blob/main/src/types.ts#L224) Test status enum diff --git a/docs/variables/CTRF_NAMESPACE.md b/docs/variables/CTRF_NAMESPACE.md index b1625bc..bccdc70 100644 --- a/docs/variables/CTRF_NAMESPACE.md +++ b/docs/variables/CTRF_NAMESPACE.md @@ -8,6 +8,6 @@ > `const` **CTRF\_NAMESPACE**: `"6ba7b810-9dad-11d1-80b4-00c04fd430c8"` = `"6ba7b810-9dad-11d1-80b4-00c04fd430c8"` -Defined in: [constants.ts:37](https://github.com/ctrf-io/ctrf-js/blob/main/src/constants.ts#L37) +Defined in: [constants.ts:38](https://github.com/ctrf-io/ctrf-js/blob/main/src/constants.ts#L38) CTRF namespace UUID for deterministic ID generation diff --git a/docs/variables/CURRENT_SPEC_VERSION.md b/docs/variables/CURRENT_SPEC_VERSION.md index 6fdbd3d..6f11d8d 100644 --- a/docs/variables/CURRENT_SPEC_VERSION.md +++ b/docs/variables/CURRENT_SPEC_VERSION.md @@ -6,7 +6,7 @@ # Variable: CURRENT\_SPEC\_VERSION -> `const` **CURRENT\_SPEC\_VERSION**: `"0.2.0"` = `"0.2.0"` +> `const` **CURRENT\_SPEC\_VERSION**: `"0.3.0"` = `"0.3.0"` Defined in: [constants.ts:9](https://github.com/ctrf-io/ctrf-js/blob/main/src/constants.ts#L9) diff --git a/docs/variables/SUPPORTED_SPEC_VERSIONS.md b/docs/variables/SUPPORTED_SPEC_VERSIONS.md index 5283b26..182c1b5 100644 --- a/docs/variables/SUPPORTED_SPEC_VERSIONS.md +++ b/docs/variables/SUPPORTED_SPEC_VERSIONS.md @@ -6,7 +6,7 @@ # Variable: SUPPORTED\_SPEC\_VERSIONS -> `const` **SUPPORTED\_SPEC\_VERSIONS**: readonly \[`"0.0.1"`, `"0.0.2"`, `"0.0.3"`, `"0.0.4"`, `"0.1.0"`, `"0.2.0"`\] +> `const` **SUPPORTED\_SPEC\_VERSIONS**: readonly \[`"0.0.1"`, `"0.0.2"`, `"0.0.3"`, `"0.0.4"`, `"0.1.0"`, `"0.2.0"`, `"0.3.0"`\] Defined in: [constants.ts:21](https://github.com/ctrf-io/ctrf-js/blob/main/src/constants.ts#L21) diff --git a/src/builder.ts b/src/builder.ts index ae8df06..bf3be74 100644 --- a/src/builder.ts +++ b/src/builder.ts @@ -6,6 +6,8 @@ import type { CTRFReport, Results, + AssertionFailure, + GlobalError, Tool, Summary, Test, @@ -56,6 +58,7 @@ export class ReportBuilder { private _tool?: Tool; private _environment?: Environment; private _tests: Test[] = []; + private _errors: GlobalError[] = []; private _insights?: Insights; private _baseline?: Baseline; private _extra?: Record; @@ -150,6 +153,14 @@ export class ReportBuilder { return this; } + /** + * Add a global error without changing test counts. + */ + addError(error: GlobalError): this { + this._errors.push(error); + return this; + } + /** * Set run-level insights. */ @@ -205,6 +216,8 @@ export class ReportBuilder { tests: this._tests, }; + if (this._errors.length > 0) results.errors = [...this._errors]; + if (this._environment) { results.environment = this._environment; } @@ -268,6 +281,8 @@ export class TestBuilder { private _executionId?: string; private _attemptId?: string; private _name?: string; + private _description?: string; + private _assertionFailures: AssertionFailure[] = []; private _status?: TestStatus; private _duration?: number; private _start?: number; @@ -344,6 +359,22 @@ export class TestBuilder { return this; } + /** + * Set the human-readable test-case description. + */ + description(description: string): this { + this._description = description; + return this; + } + + /** + * Add an assertion failure attributable to the final attempt. + */ + addAssertionFailure(failure: AssertionFailure): this { + this._assertionFailures.push(failure); + return this; + } + /** * Set the test status. */ @@ -521,7 +552,7 @@ export class TestBuilder { } /** - * Set thread ID. + * Set the thread or worker that executed the final attempt. */ threadId(id: string): this { this._threadId = id; @@ -685,6 +716,11 @@ export class TestBuilder { if (this._attemptId !== undefined) test.attemptId = this._attemptId; + if (this._description !== undefined) test.description = this._description; + if (this._assertionFailures.length > 0) { + test.assertionFailures = [...this._assertionFailures]; + } + // Add optional fields only if set if (this._start !== undefined) test.start = this._start; if (this._stop !== undefined) test.stop = this._stop; @@ -704,7 +740,7 @@ export class TestBuilder { if (this._flaky !== undefined) test.flaky = this._flaky; if (this._stdout) test.stdout = this._stdout; if (this._stderr) test.stderr = this._stderr; - if (this._threadId) test.threadId = this._threadId; + if (this._threadId !== undefined) test.threadId = this._threadId; if (this._browser) test.browser = this._browser; if (this._device) test.device = this._device; if (this._screenshot) test.screenshot = this._screenshot; diff --git a/src/constants.test.ts b/src/constants.test.ts index 1edeeab..8ad8d59 100644 --- a/src/constants.test.ts +++ b/src/constants.test.ts @@ -19,8 +19,8 @@ describe("constants", () => { expect(CURRENT_SPEC_VERSION).toMatch(/^\d+\.\d+\.\d+$/); }); - it("should identify the latest published specification", () => { - expect(CURRENT_SPEC_VERSION).toBe("0.2.0"); + it("should identify the current specification", () => { + expect(CURRENT_SPEC_VERSION).toBe("0.3.0"); }); }); @@ -43,7 +43,7 @@ describe("constants", () => { expect(SUPPORTED_SPEC_VERSIONS).toContain(CURRENT_SPEC_VERSION); }); - it("should contain every published CTRF specification version", () => { + it("should contain every supported CTRF specification version", () => { expect(SUPPORTED_SPEC_VERSIONS).toEqual([ "0.0.1", "0.0.2", @@ -51,6 +51,7 @@ describe("constants", () => { "0.0.4", "0.1.0", "0.2.0", + "0.3.0", ]); }); }); diff --git a/src/constants.ts b/src/constants.ts index 3a3e1e3..c08c63d 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -6,7 +6,7 @@ export const REPORT_FORMAT = "CTRF" as const; /** Current spec version */ -export const CURRENT_SPEC_VERSION = "0.2.0"; +export const CURRENT_SPEC_VERSION = "0.3.0"; /** All valid test statuses */ export const TEST_STATUSES = [ @@ -25,12 +25,13 @@ export const SUPPORTED_SPEC_VERSIONS = [ "0.0.4", "0.1.0", "0.2.0", + "0.3.0", ] as const; -/** A published CTRF specification version supported by this package */ +/** A CTRF specification version supported by this package */ export type SupportedSpecVersion = (typeof SUPPORTED_SPEC_VERSIONS)[number]; -/** A published CTRF specification version or the current-version alias */ +/** A CTRF specification version or the current-version alias */ export type SchemaSelector = SupportedSpecVersion | "latest"; /** CTRF namespace UUID for deterministic ID generation */ diff --git a/src/ctrf-schema-0.3.0.json b/src/ctrf-schema-0.3.0.json new file mode 100644 index 0000000..7b002bd --- /dev/null +++ b/src/ctrf-schema-0.3.0.json @@ -0,0 +1,817 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CTRF JSON Schema", + "description": "Common Test Report Format - an open standard JSON format for test results reports", + "type": "object", + "required": [ "results", "reportFormat", "specVersion" ], + "properties": { + "reportFormat": { + "description": "Document format identifier. Must be 'CTRF'", + "const": "CTRF" + }, + "specVersion": { + "description": "CTRF specification version in SemVer format (MAJOR.MINOR.PATCH)", + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "reportId": { + "description": "Unique identifier for this CTRF document instance (UUID). Identifies the serialized artifact", + "type": "string", + "format": "uuid", + "minLength": 1 + }, + "runId": { + "description": "Identifier for the logical test run. Shared across coordinated module, shard and retry-process reports, including their merged report when produced", + "type": "string", + "minLength": 1 + }, + "timestamp": { + "description": "Report generation time (RFC 3339 / ISO 8601)", + "type": "string", + "format": "date-time" + }, + "generatedBy": { + "description": "Tool or system that produced this CTRF document", + "type": "string" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + }, + "results": { + "description": "Results of a single test execution run", + "type": "object", + "required": [ "tool", "summary", "tests" ], + "properties": { + "tool": { + "description": "Tool or framework that produced the test results", + "type": "object", + "required": [ "name" ], + "properties": { + "name": { + "description": "Name of the testing tool or framework", + "type": "string", + "minLength": 1 + }, + "version": { + "description": "Version of the testing tool", + "type": "string" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + }, + "summary": { + "description": "Aggregated statistics and timing for the test run", + "type": "object", + "required": [ + "tests", + "passed", + "failed", + "skipped", + "pending", + "other", + "start", + "stop" + ], + "properties": { + "tests": { + "description": "Total number of test cases executed", + "type": "integer" + }, + "passed": { + "description": "Count of test cases with status 'passed'", + "type": "integer" + }, + "failed": { + "description": "Count of test cases with status 'failed'", + "type": "integer" + }, + "skipped": { + "description": "Count of test cases with status 'skipped'", + "type": "integer" + }, + "pending": { + "description": "Count of test cases with status 'pending'", + "type": "integer" + }, + "other": { + "description": "Count of test cases with status 'other'", + "type": "integer" + }, + "flaky": { + "description": "Count of flaky test cases (passed after failed attempts)", + "type": "integer" + }, + "suites": { + "description": "Number of test suites in the run", + "type": "integer" + }, + "start": { + "description": "Run start time (milliseconds since Unix epoch)", + "type": "integer" + }, + "stop": { + "description": "Run end time (milliseconds since Unix epoch)", + "type": "integer" + }, + "duration": { + "description": "Total run duration (milliseconds)", + "type": "integer" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + }, + "tests": { + "description": "List of test case results for the run", + "type": "array", + "items": { + "description": "Individual test case result", + "type": "object", + "required": [ "name", "status", "duration" ], + "properties": { + "id": { + "description": "Deprecated test-case identifier (UUID), scheduled for removal in CTRF 1.0.0. Still accepted in this version; prefer optional testId when providing case identity", + "type": "string", + "format": "uuid", + "minLength": 1 + }, + "testId": { + "description": "Stable identifier for the test case across runs within the producer's documented scope", + "type": "string", + "minLength": 1 + }, + "executionId": { + "description": "Identifier for one execution lifecycle of a test case within a run, including coordinated retries. UUID recommended", + "type": "string", + "minLength": 1 + }, + "attemptId": { + "description": "Unique identifier for this individual attempt. UUID recommended", + "type": "string", + "minLength": 1 + }, + "name": { + "description": "Name or title of the test case", + "type": "string", + "minLength": 1 + }, + "description": { + "description": "Human-readable description of the test case", + "type": "string" + }, + "status": { + "description": "Final outcome of the test case", + "enum": [ "passed", "failed", "skipped", "pending", "other" ] + }, + "duration": { + "description": "Execution time of the final attempt represented by the test object (milliseconds), not the sum of earlier attempts", + "type": "integer" + }, + "start": { + "description": "Test case execution start time (milliseconds since Unix epoch)", + "type": "integer" + }, + "stop": { + "description": "Test case execution end time (milliseconds since Unix epoch)", + "type": "integer" + }, + "suite": { + "description": "Suite hierarchy for the test case, from top-level to immediate parent", + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "message": { + "description": "Error or failure message", + "type": "string" + }, + "trace": { + "description": "Stack trace or failure trace information", + "type": "string" + }, + "assertionFailures": { + "description": "Assertion failures attributable to this attempt", + "type": "array", + "minItems": 1, + "items": { + "description": "Details of an individual assertion failure", + "type": "object", + "minProperties": 1, + "properties": { + "message": { + "description": "Message associated with this assertion failure", + "type": "string" + }, + "trace": { + "description": "Stack trace associated with this assertion failure", + "type": "string" + }, + "expected": { + "description": "String representation of the expected value", + "type": "string" + }, + "actual": { + "description": "String representation of the actual value", + "type": "string" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + } + }, + "snippet": { + "description": "Code snippet associated with the failure", + "type": "string" + }, + "ai": { + "description": "AI-generated diagnostic data or suggestions related to the test result", + "type": "string" + }, + "line": { + "description": "Line number of the test definition", + "type": "integer" + }, + "rawStatus": { + "description": "Original status from source tool before normalization", + "type": "string" + }, + "tags": { + "description": "Simple, keyless classifications used to categorize the test case", + "type": "array", + "items": { + "type": "string" + } + }, + "labels": { + "description": "Structured key-value metadata describing the test case. Values may be scalar primitives or non-empty arrays of primitive values", + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "array", + "minItems": 1, + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + } + } + ] + } + }, + "type": { + "description": "Test case classification (e.g., 'unit', 'integration', 'e2e')", + "type": "string" + }, + "filePath": { + "description": "Path to the file defining this test case", + "type": "string" + }, + "retries": { + "description": "Number of times the test case was re-executed after its initial attempt", + "type": "integer", + "minimum": 0 + }, + "retryAttempts": { + "description": "Ordered history of attempts completed before the final attempt represented by the test object", + "type": "array", + "minItems": 1, + "items": { + "description": "Attempt completed before the final attempt", + "type": "object", + "required": [ "attempt", "status" ], + "properties": { + "attempt": { + "description": "Original sequence number for this attempt (1 = initial execution)", + "type": "integer", + "minimum": 1 + }, + "attemptId": { + "description": "Unique identifier for this individual attempt. UUID recommended", + "type": "string", + "minLength": 1 + }, + "status": { + "description": "Outcome of this attempt", + "enum": [ + "passed", + "failed", + "skipped", + "pending", + "other" + ] + }, + "duration": { + "description": "Attempt execution time (milliseconds)", + "type": "integer", + "minimum": 0 + }, + "message": { + "description": "Error or failure message for this attempt", + "type": "string" + }, + "trace": { + "description": "Stack trace for this attempt", + "type": "string" + }, + "assertionFailures": { + "description": "Assertion failures attributable to this attempt", + "type": "array", + "minItems": 1, + "items": { + "description": "Details of an individual assertion failure", + "type": "object", + "minProperties": 1, + "properties": { + "message": { + "description": "Message associated with this assertion failure", + "type": "string" + }, + "trace": { + "description": "Stack trace associated with this assertion failure", + "type": "string" + }, + "expected": { + "description": "String representation of the expected value", + "type": "string" + }, + "actual": { + "description": "String representation of the actual value", + "type": "string" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + } + }, + "line": { + "description": "Line number associated with failure", + "type": "integer" + }, + "snippet": { + "description": "Code snippet for this attempt", + "type": "string" + }, + "stdout": { + "description": "Standard output from this attempt, one string per output line", + "type": "array", + "items": { + "type": "string" + } + }, + "stderr": { + "description": "Standard error from this attempt, one string per output line", + "type": "array", + "items": { + "type": "string" + } + }, + "start": { + "description": "Attempt start time (milliseconds since Unix epoch)", + "type": "integer" + }, + "stop": { + "description": "Attempt end time (milliseconds since Unix epoch)", + "type": "integer" + }, + "threadId": { + "description": "Thread or worker that executed this attempt", + "type": "string" + }, + "attachments": { + "description": "Artifacts from this attempt", + "type": "array", + "items": { + "description": "External file or resource reference", + "type": "object", + "required": [ "name", "contentType", "path" ], + "properties": { + "attachmentId": { + "description": "Unique identifier for this attachment reference instance. UUID recommended", + "type": "string", + "minLength": 1 + }, + "name": { + "description": "Display name of the attachment", + "type": "string" + }, + "contentType": { + "description": "MIME type of the attachment", + "type": "string" + }, + "path": { + "description": "Opaque reference to the attachment (URL or file path)", + "type": "string" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + } + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + } + }, + "flaky": { + "description": "True if the test case passed after one or more failed attempts", + "type": "boolean" + }, + "stdout": { + "description": "Standard output, one string per output line", + "type": "array", + "items": { + "type": "string" + } + }, + "stderr": { + "description": "Standard error, one string per output line", + "type": "array", + "items": { + "type": "string" + } + }, + "threadId": { + "description": "Thread or worker that executed the final attempt represented by the test object", + "type": "string" + }, + "browser": { + "description": "Browser used for browser-based tests", + "type": "string" + }, + "device": { + "description": "Device or device profile used", + "type": "string" + }, + "screenshot": { + "description": "Single base64-encoded screenshot image", + "type": "string" + }, + "attachments": { + "description": "Additional artifacts (screenshots, logs, videos, etc.)", + "type": "array", + "items": { + "description": "External file or resource reference", + "type": "object", + "required": [ "name", "contentType", "path" ], + "properties": { + "attachmentId": { + "description": "Unique identifier for this attachment reference instance. UUID recommended", + "type": "string", + "minLength": 1 + }, + "name": { + "description": "Display name of the attachment", + "type": "string" + }, + "contentType": { + "description": "MIME type of the attachment", + "type": "string" + }, + "path": { + "description": "Opaque reference to the attachment (URL or file path)", + "type": "string" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + } + }, + "parameters": { + "description": "Test parameters, input values, or contextual data relevant to the execution", + "type": "object" + }, + "steps": { + "description": "Steps or sub-operations performed during test execution", + "type": "array", + "items": { + "description": "Individual test step", + "type": "object", + "required": [ "name", "status" ], + "properties": { + "name": { + "description": "Name of the step", + "type": "string" + }, + "status": { + "description": "Outcome of the step", + "enum": [ + "passed", + "failed", + "skipped", + "pending", + "other" + ] + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + } + }, + "insights": { + "description": "Derived metrics for this test case across runs", + "type": "object", + "properties": { + "passRate": { + "description": "Pass rate metric with baseline comparison", + "$ref": "#/definitions/metricDelta" + }, + "failRate": { + "description": "Fail rate metric with baseline comparison", + "$ref": "#/definitions/metricDelta" + }, + "flakyRate": { + "description": "Flaky rate metric with baseline comparison", + "$ref": "#/definitions/metricDelta" + }, + "averageTestDuration": { + "description": "Average duration metric with baseline comparison", + "$ref": "#/definitions/metricDelta" + }, + "p95TestDuration": { + "description": "95th percentile duration with baseline comparison", + "$ref": "#/definitions/metricDelta" + }, + "executedInRuns": { + "description": "Number of runs this test case was executed in", + "type": "integer" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + } + }, + "environment": { + "description": "Execution environment, system configuration, and build context", + "type": "object", + "properties": { + "reportName": { + "description": "Human-readable name for this report", + "type": "string" + }, + "appName": { + "description": "Name of the application under test", + "type": "string" + }, + "appVersion": { + "description": "Version of the application under test", + "type": "string" + }, + "buildId": { + "description": "Unique identifier for the CI/CD build", + "type": "string" + }, + "buildName": { + "description": "Name of the CI/CD build or pipeline", + "type": "string" + }, + "buildNumber": { + "description": "Sequential build number", + "type": "integer" + }, + "buildUrl": { + "description": "URL to the CI/CD build", + "type": "string" + }, + "repositoryName": { + "description": "Name of the source code repository", + "type": "string" + }, + "repositoryUrl": { + "description": "URL of the source code repository", + "type": "string" + }, + "commit": { + "description": "Git commit SHA or VCS revision identifier", + "type": "string" + }, + "branchName": { + "description": "Git branch or VCS branch name", + "type": "string" + }, + "osPlatform": { + "description": "Operating system platform (e.g., 'linux', 'darwin', 'win32')", + "type": "string" + }, + "osRelease": { + "description": "Operating system release version", + "type": "string" + }, + "osVersion": { + "description": "Operating system version string", + "type": "string" + }, + "testEnvironment": { + "description": "Logical test environment (e.g., 'staging', 'production')", + "type": "string" + }, + "shardId": { + "description": "Shard or partition label identifying which part of a distributed run produced this document", + "type": "string", + "minLength": 1 + }, + "healthy": { + "description": "Indicates if the run is considered healthy", + "type": "boolean" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + }, + "errors": { + "description": "Errors associated with the test run rather than an individual test result", + "type": "array", + "minItems": 1, + "items": { + "description": "Details of a global error", + "type": "object", + "minProperties": 1, + "properties": { + "message": { + "description": "Message associated with this global error", + "type": "string" + }, + "trace": { + "description": "Stack trace associated with this global error", + "type": "string" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + } + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + }, + "insights": { + "description": "Aggregated metrics computed across multiple test runs", + "type": "object", + "properties": { + "passRate": { + "description": "Overall pass rate with baseline comparison", + "$ref": "#/definitions/metricDelta" + }, + "failRate": { + "description": "Overall fail rate with baseline comparison", + "$ref": "#/definitions/metricDelta" + }, + "flakyRate": { + "description": "Overall flaky rate with baseline comparison", + "$ref": "#/definitions/metricDelta" + }, + "averageRunDuration": { + "description": "Average run duration with baseline comparison", + "$ref": "#/definitions/metricDelta" + }, + "p95RunDuration": { + "description": "95th percentile run duration with baseline comparison", + "$ref": "#/definitions/metricDelta" + }, + "averageTestDuration": { + "description": "Average duration of test case executions with baseline comparison", + "$ref": "#/definitions/metricDelta" + }, + "runsAnalyzed": { + "description": "Number of historical runs analyzed for insights", + "type": "integer" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + }, + "baseline": { + "description": "Reference to a previous report used for comparison", + "type": "object", + "required": [ "reportId" ], + "properties": { + "reportId": { + "description": "UUID of the baseline report", + "type": "string", + "format": "uuid", + "minLength": 1 + }, + "timestamp": { + "description": "Generation time of the baseline report", + "type": "string", + "format": "date-time" + }, + "source": { + "description": "Origin or location of the baseline report", + "type": "string" + }, + "buildNumber": { + "description": "Build number of the baseline run", + "type": "integer" + }, + "buildName": { + "description": "Build name of the baseline run", + "type": "string" + }, + "buildUrl": { + "description": "URL to the baseline build", + "type": "string", + "format": "uri" + }, + "commit": { + "description": "Git commit SHA of the baseline run", + "type": "string" + }, + "extra": { + "description": "Extension point for arbitrary metadata", + "type": "object" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "definitions": { + "metricDelta": { + "description": "Metric value with baseline comparison", + "type": "object", + "properties": { + "current": { + "description": "Current metric value", + "type": "number" + }, + "baseline": { + "description": "Baseline metric value for comparison", + "type": "number" + }, + "change": { + "description": "Computed difference between current and baseline", + "type": "number" + } + }, + "additionalProperties": false + } + } +} diff --git a/src/index.ts b/src/index.ts index be306e6..f69453c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,8 @@ export type { CTRFReport, Results, + AssertionFailure, + GlobalError, Tool, Summary, Test, diff --git a/src/merge.ts b/src/merge.ts index ad1e2de..439df41 100644 --- a/src/merge.ts +++ b/src/merge.ts @@ -9,7 +9,7 @@ import type { Environment, MergeOptions, } from "./types.js"; -import { REPORT_FORMAT, CURRENT_SPEC_VERSION } from "./constants.js"; +import { REPORT_FORMAT } from "./constants.js"; import { calculateSummary } from "./summary.js"; import { generateReportId } from "./id.js"; @@ -19,10 +19,14 @@ import { generateReportId } from "./id.js"; * Merge multiple CTRF reports into a single report. * Useful for combining results from parallel or sharded test runs. * + * Reports must declare the same specification version; merging does not convert + * historical reports to a newer contract. Global errors are concatenated in input + * order and do not contribute to test counts. + * * @param reports - Array of CTRF reports to merge * @param options - Merge options (deduplication, environment handling) * @returns A new merged CTRFReport - * @throws Error if no reports are provided + * @throws Error if no reports are provided or specification versions differ * * @example * ```typescript @@ -47,6 +51,13 @@ export function merge( return { ...reports[0] }; } + const specVersion = reports[0].specVersion; + if (reports.some((report) => report.specVersion !== specVersion)) { + throw new Error( + "Cannot merge reports with different specification versions", + ); + } + const { deduplicateTests = false, mergeSummary = true, @@ -106,7 +117,7 @@ export function merge( const tool = reports[0].results.tool; const merged: CTRFReport = { reportFormat: REPORT_FORMAT, - specVersion: CURRENT_SPEC_VERSION, + specVersion, reportId: generateReportId(), timestamp: new Date().toISOString(), results: { @@ -121,6 +132,9 @@ export function merge( merged.runId = sharedRunId; } + const errors = reports.flatMap((report) => report.results.errors ?? []); + if (errors.length > 0) merged.results.errors = errors; + if (environment && Object.keys(environment).length > 0) { merged.results.environment = environment; } diff --git a/src/spec-0.3.0.test.ts b/src/spec-0.3.0.test.ts new file mode 100644 index 0000000..29d33db --- /dev/null +++ b/src/spec-0.3.0.test.ts @@ -0,0 +1,229 @@ +import { describe, expect, it } from "vitest"; +import { + ReportBuilder, + TestBuilder, + addInsights, + filterTests, + merge, + parse, + stringify, + validate, + validateStrict, + type AssertionFailure, + type GlobalError, +} from "./index.js"; + +function reportWithDiagnostics() { + const assertion: AssertionFailure = { expected: "200", actual: "500" }; + const error: GlobalError = { message: "Worker initialization failed" }; + return new ReportBuilder() + .tool({ name: "runner" }) + .addError(error) + .addTest( + new TestBuilder() + .name("checkout") + .description("") + .status("failed") + .duration(100) + .threadId("") + .stdout([]) + .stderr([]) + .addAssertionFailure(assertion) + .addAttemptHistoryEntry({ + attempt: 1, + status: "failed", + threadId: "host-a/worker-1", + assertionFailures: [{ message: "Request timed out" }], + }) + .build(), + ) + .build(); +} + +describe("specification 0.3.0", () => { + it("builds final/history diagnostics without inflating counts", () => { + const report = reportWithDiagnostics(); + expect(report.specVersion).toBe("0.3.0"); + expect(report.results.summary).toMatchObject({ tests: 1, failed: 1 }); + expect(report.results.tests[0]).toMatchObject({ + description: "", + threadId: "", + stdout: [], + stderr: [], + retries: 1, + assertionFailures: [{ expected: "200", actual: "500" }], + retryAttempts: [{ threadId: "host-a/worker-1" }], + }); + expect(() => validateStrict(report)).not.toThrow(); + }); + + it("omits optional collections when no diagnostics were added", () => { + const report = new ReportBuilder() + .tool({ name: "runner" }) + .addTest( + new TestBuilder().name("ok").status("passed").duration(0).build(), + ) + .build(); + expect(report.results).not.toHaveProperty("errors"); + expect(report.results.tests[0]).not.toHaveProperty("assertionFailures"); + expect(report.results.tests[0]).not.toHaveProperty("description"); + expect(validate(report).valid).toBe(true); + }); + + it("supports global errors with zero reported tests", () => { + const report = new ReportBuilder() + .tool({ name: "runner" }) + .addError({ trace: "InitializationError" }) + .build(); + expect(report.results.summary.tests).toBe(0); + expect(validate(report).valid).toBe(true); + }); + + it("preserves diagnostics through serialization, filtering and enrichment", () => { + const report = reportWithDiagnostics(); + const parsed = parse(stringify(report), { validate: true }); + expect(parsed).toEqual(report); + expect(filterTests(parsed, { status: "failed" })).toEqual( + report.results.tests, + ); + for (const result of [addInsights(parsed, [])]) { + expect(result.results.errors).toEqual(report.results.errors); + expect(result.results.tests[0]).toMatchObject(report.results.tests[0]); + expect(validate(result).valid).toBe(true); + } + }); + + it.each([true, false])( + "retains all global errors when mergeSummary=%s", + (mergeSummary) => { + const first = reportWithDiagnostics(); + const second = reportWithDiagnostics(); + second.results.errors = [{ message: "Second worker failed" }]; + const merged = merge([first, second], { mergeSummary }); + expect(merged.results.errors).toEqual([ + { message: "Worker initialization failed" }, + { message: "Second worker failed" }, + ]); + expect(merged.results.summary).toMatchObject({ tests: 2, failed: 2 }); + expect(merged.results.tests).toEqual([ + first.results.tests[0], + second.results.tests[0], + ]); + expect(validate(merged).valid).toBe(true); + expect(first.results.errors).toHaveLength(1); + }, + ); + + it("keeps arbitrary extension values opaque", () => { + const report = reportWithDiagnostics(); + report.extra = { + "vendor.data": [null, 1, "value"], + "ctrf.example": { extensionVersion: "99.0.0" }, + }; + expect(validate(report).valid).toBe(true); + expect(parse(stringify(report)).extra).toEqual(report.extra); + }); + + for (const location of ["final", "history", "global"] as const) { + const setDiagnostics = ( + report: ReturnType, + value: unknown, + ) => { + if (location === "global") report.results.errors = value as GlobalError[]; + else if (location === "final") + report.results.tests[0].assertionFailures = value as AssertionFailure[]; + else + (report.results.tests[0].retryAttempts ?? [])[0].assertionFailures = + value as AssertionFailure[]; + }; + + it.each( + [ + [], + [{}], + [null], + [{ message: 1 }], + [{ trace: false }], + [{ unknown: "value" }], + [{ extra: [] }], + ].map((value) => [value]), + )(`rejects invalid ${location} diagnostic collection %j`, (value) => { + const report = reportWithDiagnostics(); + setDiagnostics(report, value); + expect(validate(report).valid).toBe(false); + }); + it.each( + [[{ message: "" }], [{ trace: "" }], [{ extra: {} }]].map((value) => [ + value, + ]), + )(`accepts populated ${location} entries with empty values %j`, (value) => { + const report = reportWithDiagnostics(); + setDiagnostics(report, value); + expect(validate(report).valid).toBe(true); + }); + } + + it.each([123, null, false, {}, []].map((value) => [value]))( + "rejects non-string history threadId %j", + (value) => { + const report = reportWithDiagnostics(); + (report.results.tests[0].retryAttempts ?? [])[0].threadId = + value as never; + expect(validate(report).valid).toBe(false); + }, + ); + + it.each([123, null, false, {}, []].map((value) => [value]))( + "rejects non-string description %j", + (value) => { + const report = reportWithDiagnostics(); + report.results.tests[0].description = value as never; + expect(validate(report).valid).toBe(false); + }, + ); + + it.each(["expected", "actual"] as const)( + "requires string assertion %s", + (key) => { + const report = reportWithDiagnostics(); + (report.results.tests[0].assertionFailures ?? [])[0][key] = 123 as never; + expect(validate(report).valid).toBe(false); + (report.results.tests[0].assertionFailures ?? [])[0][key] = ""; + expect(validate(report).valid).toBe(true); + }, + ); + + it("continues enforcing retry count and sequence semantics", () => { + const report = reportWithDiagnostics(); + report.results.tests[0].retries = 2; + expect(validate(report).errors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ keyword: "retryHistory" }), + ]), + ); + report.results.tests[0].retries = 1; + (report.results.tests[0].retryAttempts ?? [])[0].attempt = 2; + expect(validate(report).valid).toBe(false); + }); + + it("does not relabel legacy reports or implicitly convert mixed versions", () => { + const legacy = new ReportBuilder() + .specVersion("0.0.3") + .tool({ name: "runner" }) + .addTest({ name: "legacy", status: "passed", duration: 0, retries: 1 }) + .build(); + expect(validate(legacy, { specVersion: "0.0.3" }).valid).toBe(true); + const result = merge([legacy, structuredClone(legacy)]); + expect(result.specVersion).toBe("0.0.3"); + expect(validate(result, { specVersion: "0.0.3" }).valid).toBe(true); + expect(() => merge([legacy, reportWithDiagnostics()])).toThrow( + "different specification versions", + ); + }); + + it("keeps 0.3.0 additions out of historical schemas", () => { + expect( + validate(reportWithDiagnostics(), { specVersion: "0.2.0" }).valid, + ).toBe(false); + }); +}); diff --git a/src/types.ts b/src/types.ts index 8a2b1c3..373d2fb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -21,7 +21,7 @@ export interface CTRFReport { specVersion: string; /** Unique identifier for this report (UUID v4) */ reportId?: string; - /** Identifier shared by all documents belonging to the same logical test run */ + /** Logical-run identifier shared across coordinated module, shard and retry-process reports */ runId?: string; /** ISO 8601 timestamp when the report was generated */ timestamp?: string; @@ -49,6 +49,8 @@ export interface Results { summary: Summary; /** Array of individual test results */ tests: Test[]; + /** Global errors, separate from test results and summary counts */ + errors?: GlobalError[]; /** Environment information */ environment?: Environment; /** Custom metadata */ @@ -107,19 +109,21 @@ export interface Summary { * @group Core Types */ export interface Test { - /** Legacy test identifier (UUID); new producers should prefer testId */ + /** Deprecated test-case identifier (UUID), scheduled for removal in CTRF 1.0.0; prefer optional testId */ id?: string; - /** Stable identifier for the logical test case */ + /** Stable test-case identifier across runs within the producer's documented scope */ testId?: string; - /** Identifier for this specific execution of the test case */ + /** Identifier for one test-case execution lifecycle within a run, including coordinated retries */ executionId?: string; /** Identifier for this individual attempt */ attemptId?: string; /** Test name */ name: string; + /** Human-readable description of the test case */ + description?: string; /** Test execution status */ status: TestStatus; - /** Test duration in milliseconds */ + /** Duration of the final attempt in milliseconds, excluding earlier attempts */ duration: number; /** Start timestamp (Unix epoch milliseconds) */ start?: number; @@ -131,6 +135,8 @@ export interface Test { message?: string; /** Stack trace (for failed tests) */ trace?: string; + /** Assertion failures attributable to the final attempt */ + assertionFailures?: AssertionFailure[]; /** Code snippet where failure occurred */ snippet?: string; /** AI-generated analysis or suggestion */ @@ -157,7 +163,7 @@ export interface Test { stdout?: string[]; /** Standard error captured during test */ stderr?: string[]; - /** Thread/worker ID that ran this test */ + /** Thread or worker that executed the final attempt; qualify across processes or hosts */ threadId?: string; /** Browser name (for browser tests) */ browser?: string; @@ -177,6 +183,39 @@ export interface Test { extra?: Record; } +/** + * Details of an individual assertion failure. At least one property is required. + * + * @group Core Types + */ +export interface AssertionFailure { + /** Message associated with this assertion failure */ + message?: string; + /** Stack trace associated with this assertion failure */ + trace?: string; + /** String representation of the expected value */ + expected?: string; + /** String representation of the actual value */ + actual?: string; + /** Arbitrary metadata */ + extra?: Record; +} + +/** + * Error associated with the run rather than an individual test result. + * At least one property is required. Global errors do not add test results. + * + * @group Core Types + */ +export interface GlobalError { + /** Message associated with this global error */ + message?: string; + /** Stack trace associated with this global error */ + trace?: string; + /** Arbitrary metadata */ + extra?: Record; +} + /** * Test status enum * @@ -210,6 +249,10 @@ export interface AttemptHistoryEntry { message?: string; /** Stack trace */ trace?: string; + /** Assertion failures attributable to this attempt */ + assertionFailures?: AssertionFailure[]; + /** Thread or worker that executed this attempt; qualify across processes or hosts */ + threadId?: string; /** Line number */ line?: number; /** Code snippet */ @@ -250,7 +293,7 @@ export interface Attachment { name: string; /** MIME content type */ contentType: string; - /** Path to the attachment file */ + /** Opaque attachment reference (URL or file path); portable references are recommended */ path: string; /** Custom metadata */ extra?: Record; diff --git a/src/validate.test.ts b/src/validate.test.ts index 0000b95..5d7f386 100644 --- a/src/validate.test.ts +++ b/src/validate.test.ts @@ -182,7 +182,7 @@ describe("validate", () => { expect(validate(report, { specVersion: "0.2.0" }).valid).toBe(false); }); - it("should treat latest as 0.2.0", () => { + it("should treat latest as 0.3.0", () => { const invalidLatestReport = { ...validReport, results: { @@ -199,7 +199,7 @@ describe("validate", () => { }; expect(validate(invalidLatestReport, { specVersion: "latest" })).toEqual( - validate(invalidLatestReport, { specVersion: "0.2.0" }), + validate(invalidLatestReport, { specVersion: "0.3.0" }), ); }); diff --git a/src/validate.ts b/src/validate.ts index 8c2f167..e14c225 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -29,7 +29,7 @@ function validateRetryHistories( // Versions before 0.0.4 used the earlier retry model. In those versions, // matching history length was recommended rather than required, and the // presence and ordering constraints below were not normative requirements. - if (version !== "0.0.4" && version !== "0.1.0" && version !== "0.2.0") + if (version === "0.0.1" || version === "0.0.2" || version === "0.0.3") return errors; for (const [testIndex, test] of report.results.tests.entries()) {