From 6048c8e1b5a5c04272e39d9348f3d8ecb78351ae Mon Sep 17 00:00:00 2001 From: Kevin Buffardi Date: Wed, 29 Jul 2026 22:15:58 -0700 Subject: [PATCH 1/3] fix: preserve actionable pytest failure details Keep custom assertion messages intact and retain default pytest diagnostics for failed Python tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- executor.js | 155 ++++++++++++++++++++++------- package.json | 2 +- test/executor.test.js | 222 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 341 insertions(+), 38 deletions(-) create mode 100644 test/executor.test.js diff --git a/executor.js b/executor.js index 79d8964..dcfcee5 100644 --- a/executor.js +++ b/executor.js @@ -243,54 +243,135 @@ ${testCode} * @param {string} output - The output from pytest. * @returns {object} - The test summary. */ +function hasCustomAssertionMessage(assertionSource) { + const expression = assertionSource.replace(/^>\s*assert\s+/, ''); + let depth = 0; + let quote; + let escaped = false; + + for (const character of expression) { + if (quote) { + if (escaped) { + escaped = false; + } else if (character === '\\') { + escaped = true; + } else if (character === quote) { + quote = undefined; + } + } else if (character === '"' || character === "'") { + quote = character; + } else if (character === '(' || character === '[' || character === '{') { + depth += 1; + } else if (character === ')' || character === ']' || character === '}') { + depth -= 1; + } else if (character === ',' && depth === 0) { + return true; + } + } + + return false; +} + function parsePytestOutput(output, stdout = '', stderr = '') { - let total_tests = 0; + output = output.toString(); + const lines = output.split(/\r?\n/); + const failures = []; let passed_tests = 0; let failed_tests = 0; - let failures = []; + let currentFailure; - const match = output.match(/(\d+) passed, (\d+) failed/); - if (match) { - passed_tests = parseInt(match[1]); - failed_tests = parseInt(match[2]); - total_tests = passed_tests + failed_tests; - } else { - const singlePassMatch = output.match(/(\d+) passed/); - if (singlePassMatch) { - passed_tests = parseInt(singlePassMatch[1]); - total_tests = passed_tests; + const addFailure = () => { + if (!currentFailure?.errorLines.length) { + return; } - const singleFailMatch = output.match(/(\d+) failed/); - if (singleFailMatch) { - failed_tests = parseInt(singleFailMatch[1]); - total_tests += failed_tests; + + const messageLines = currentFailure.errorLines.map((line) => + line.replace(/^\s*E {0,7}/, '') + ); + const firstMessageLine = messageLines[0]; + const usesCustomAssertionMessage = + hasCustomAssertionMessage(currentFailure.assertionSource) && + firstMessageLine.startsWith('AssertionError:'); + const error_message = usesCustomAssertionMessage + ? [firstMessageLine.replace(/^AssertionError:\s*/, ''), ...messageLines.slice(1)].join('\n') + : messageLines.join('\n'); + const comparison = firstMessageLine + .replace(/^AssertionError:\s*/, '') + .match(/^assert\s+(.+?)\s*==\s*(.+)$/); + + failures.push({ + test_case: currentFailure.testCase || `Test ${failures.length + 1}`, + expected: comparison?.[2]?.trim() || '', + received: comparison?.[1]?.trim() || '', + error_message, + rawout: `${stdout}\n${stderr}`, + }); + }; + + for (const line of lines) { + const failureHeader = line.match(/^_{5,}\s*(.*?)\s*_{5,}\s*$/); + if (failureHeader) { + addFailure(); + currentFailure = { + testCase: failureHeader[1].trim(), + assertionSource: '', + errorLines: [], + finishedErrors: false, + }; + continue; } - } - - const failureBlocks = output.split(/={10,} FAILURES ={10,}/)[1]?.split(/={10,}/)[0] || ''; + if (!currentFailure) { + continue; + } - const matches = [...failureBlocks.matchAll( - /_{5,}\s*(.*?)\s*_{5,}[\s\S]*?>\s*assert\s+(.*?)\s*?\nE\s+assert\s+(.*?)\s*?(?:\nE\s+\+\s+where\s+(.*?)\s+=)?/g - )]; - - matches.forEach((match, index) => { - const test_case = match[1]?.trim() || `Test ${index + 1}`; - const assertionLine = match[2]?.trim(); - const failedExpr = match[3]?.trim(); - const evaluated = match[4]?.trim() || ''; - + if (/^={3,}|^!{3,}/.test(line)) { + addFailure(); + currentFailure = undefined; + continue; + } + + if (/^>\s*assert\b/.test(line)) { + currentFailure.assertionSource = line; + } + + if (/^\s*E(?:\s|$)/.test(line) && !currentFailure.finishedErrors) { + currentFailure.errorLines.push(line); + } else if (currentFailure.errorLines.length) { + currentFailure.finishedErrors = true; + } + } + addFailure(); + + const summaryLine = [...lines] + .reverse() + .find((line) => /^=+.*\b(?:passed|failed|errors?)\b.*=+$/.test(line)); + const summaryCounts = summaryLine + ? [...summaryLine.matchAll(/(\d+)\s+(passed|failed|errors?)\b/g)] + : []; + for (const [, count, status] of summaryCounts) { + if (status === 'passed') { + passed_tests += Number(count); + } else { + failed_tests += Number(count); + } + } + + if (failed_tests === 0 && failures.length > 0) { + failed_tests = failures.length; + } + if (failed_tests > failures.length && failures.length === 0) { failures.push({ - test_case, - expected: failedExpr.split('==')[1]?.trim() || '', - received: evaluated || failedExpr.split('==')[0]?.trim(), - error_message: `Assertion failed: ${assertionLine}`, - rawout: `${stdout}\n${stderr}` + test_case: 'Pytest collection', + expected: '', + received: '', + error_message: 'Pytest reported a failure without diagnostic details.', + rawout: `${stdout}\n${stderr}`, }); - }); + } return { - tests_run: total_tests, + tests_run: passed_tests + failed_tests, passed: passed_tests, failed: failed_tests, failure_details: failures, @@ -659,4 +740,4 @@ async function cleanupDir(dirPath) { } } -module.exports = { executeCode }; +module.exports = { executeCode, parsePytestOutput }; diff --git a/package.json b/package.json index 8625d81..dd7980c 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "node test/executor.test.js" }, "keywords": [], "author": "", diff --git a/test/executor.test.js b/test/executor.test.js new file mode 100644 index 0000000..55d14fa --- /dev/null +++ b/test/executor.test.js @@ -0,0 +1,222 @@ +const assert = require('assert'); +const { parsePytestOutput } = require('../executor'); + +const testCases = [ + { + name: 'returns only a custom assertion message', + output: `============================= test session starts ============================== +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): +> assert hasattr(program, "HatName"), "there should be a variable named exactly HatName" +E AssertionError: there should be a variable named exactly HatName + +test_program.py:5: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_hat_variables - AssertionError: there should be a variable named exactly HatName +============================== 1 failed in 0.02s ===============================`, + expected: { + tests_run: 1, + passed: 0, + failed: 1, + messages: ['there should be a variable named exactly HatName'], + expectedValues: [''], + receivedValues: [''], + }, + }, + { + name: 'preserves multiline custom assertion messages', + output: `=================================== FAILURES =================================== +________________________________ test_message _________________________________ + + def test_message(): +> assert False, "first instruction\\nsecond instruction" +E AssertionError: first instruction +E second instruction + +test_program.py:2: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_message - AssertionError: first instruction +============================== 1 failed in 0.01s ===============================`, + expected: { + tests_run: 1, + passed: 0, + failed: 1, + messages: ['first instruction\nsecond instruction'], + expectedValues: [''], + receivedValues: [''], + }, + }, + { + name: 'preserves all default assertion detail lines', + output: `=================================== FAILURES =================================== +__________________________________ test_value __________________________________ + + def test_value(): +> assert "actual" == "expected" +E AssertionError: assert 'actual' == 'expected' +E - expected +E + actual + +test_program.py:2: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_value - AssertionError: assert 'actual' == 'expected' +============================== 1 failed in 0.01s ===============================`, + expected: { + tests_run: 1, + passed: 0, + failed: 1, + messages: ["AssertionError: assert 'actual' == 'expected'\n - expected\n + actual"], + expectedValues: ["'expected'"], + receivedValues: ["'actual'"], + }, + }, + { + name: 'preserves exception type and message', + output: `=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): +> assert program.HatName == "Veracruz" +E AttributeError: module 'program' has no attribute 'HatName' + +test_program.py:5: AttributeError +=========================== short test summary info ============================ +FAILED test_program.py::test_hat_variables - AttributeError: module 'program' has no attribute 'HatName' +============================== 1 failed in 0.02s ===============================`, + expected: { + tests_run: 1, + passed: 0, + failed: 1, + messages: ["AttributeError: module 'program' has no attribute 'HatName'"], + expectedValues: [''], + receivedValues: [''], + }, + }, + { + name: 'returns distinct details for multiple failures', + output: `=================================== FAILURES =================================== +_________________________________ test_first __________________________________ + + def test_first(): +> assert False, "first failure" +E AssertionError: first failure + +test_program.py:2: AssertionError +_________________________________ test_second _________________________________ + + def test_second(): +> raise ValueError("second failure") +E ValueError: second failure + +test_program.py:5: ValueError +=========================== short test summary info ============================ +FAILED test_program.py::test_first - AssertionError: first failure +FAILED test_program.py::test_second - ValueError: second failure +============================== 2 failed in 0.01s ===============================`, + expected: { + tests_run: 2, + passed: 0, + failed: 2, + messages: ['first failure', 'ValueError: second failure'], + expectedValues: ['', ''], + receivedValues: ['', ''], + }, + }, + { + name: 'counts collection errors as one failure', + output: `==================================== ERRORS ==================================== +___________________ ERROR collecting test_program.py ___________________ +test_program.py:1: in + assert ( +E SyntaxError: '(' was never closed +=========================== short test summary info ============================ +ERROR test_program.py +!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!! +=============================== 1 error in 0.03s ===============================`, + expected: { + tests_run: 1, + passed: 0, + failed: 1, + messages: ["SyntaxError: '(' was never closed"], + expectedValues: [''], + receivedValues: [''], + }, + }, + { + name: 'counts mixed failed tests and errors', + output: `=================================== FAILURES =================================== +_________________________________ test_value __________________________________ + + def test_value(): +> assert False, "assertion failure" +E AssertionError: assertion failure + +test_program.py:2: AssertionError +==================================== ERRORS ==================================== +_____________________ ERROR at setup of test_setup_error ______________________ + + @pytest.fixture + def broken_fixture(): +> raise RuntimeError("setup failure") +E RuntimeError: setup failure + +test_program.py:6: RuntimeError +=========================== short test summary info ============================ +FAILED test_program.py::test_value - AssertionError: assertion failure +ERROR test_program.py::test_setup_error - RuntimeError: setup failure +===================== 1 passed, 1 failed, 1 error in 0.01s =====================`, + expected: { + tests_run: 3, + passed: 1, + failed: 2, + messages: ['assertion failure', 'RuntimeError: setup failure'], + expectedValues: ['', ''], + receivedValues: ['', ''], + }, + }, + { + name: 'counts passing tests', + output: `============================= test session starts ============================== +collected 2 items + +test_program.py .. [100%] + +============================== 2 passed in 0.01s ===============================`, + expected: { + tests_run: 2, + passed: 2, + failed: 0, + messages: [], + expectedValues: [], + receivedValues: [], + }, + }, +]; + +for (const testCase of testCases) { + const result = parsePytestOutput(testCase.output, testCase.output, ''); + + assert.strictEqual(result.tests_run, testCase.expected.tests_run, testCase.name); + assert.strictEqual(result.passed, testCase.expected.passed, testCase.name); + assert.strictEqual(result.failed, testCase.expected.failed, testCase.name); + assert.strictEqual(result.failure_details.length, testCase.expected.failed, testCase.name); + assert.deepStrictEqual( + result.failure_details.map((failure) => failure.error_message), + testCase.expected.messages, + testCase.name + ); + assert.deepStrictEqual( + result.failure_details.map((failure) => failure.expected), + testCase.expected.expectedValues, + testCase.name + ); + assert.deepStrictEqual( + result.failure_details.map((failure) => failure.received), + testCase.expected.receivedValues, + testCase.name + ); +} + +console.log(`Passed ${testCases.length} pytest parser regression tests.`); From d1b1adaf8a550001051849484583e52c1f249b38 Mon Sep 17 00:00:00 2001 From: Kevin Buffardi Date: Thu, 30 Jul 2026 11:44:21 -0700 Subject: [PATCH 2/3] fix: run pytest with full verbosity Ensure production requests complete assertion diagnostics through a tested invocation contract. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- executor.js | 14 +++++++++++--- test/executor.test.js | 13 ++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/executor.js b/executor.js index 94f10f6..ec2889a 100644 --- a/executor.js +++ b/executor.js @@ -114,6 +114,13 @@ function configureExecution(language, code, uniqueDir) { return config; } +function buildPytestInvocation(uniqueDir) { + return { + command: 'pytest', + args: ['-vv', path.join(uniqueDir, 'test_program.py')], + }; +} + /** * Writes the provided code to a file. * @param {string} uniqueDir - The directory to write the file. @@ -570,8 +577,9 @@ async function executeCode(language, code, stdin, expectedOutput, runTests = fal let output; if (runTests && testCode) { if (language.toLowerCase() === 'python') { - executionConfig.runCommand = 'pytest'; - executionConfig.runArgs = [path.join(uniqueDir, 'test_program.py')]; + const pytestInvocation = buildPytestInvocation(uniqueDir); + executionConfig.runCommand = pytestInvocation.command; + executionConfig.runArgs = pytestInvocation.args; } else if (language.toLowerCase() === 'cpp') { executionConfig.runCommand = path.join(uniqueDir, 'runner'); executionConfig.runArgs = []; @@ -844,4 +852,4 @@ async function cleanupDir(dirPath) { } } -module.exports = { executeCode, parsePytestOutput }; +module.exports = { buildPytestInvocation, executeCode, parsePytestOutput }; diff --git a/test/executor.test.js b/test/executor.test.js index 2089b8c..1adedf7 100644 --- a/test/executor.test.js +++ b/test/executor.test.js @@ -1,5 +1,16 @@ const assert = require('assert'); -const { parsePytestOutput } = require('../executor'); +const path = require('path'); +const { buildPytestInvocation, parsePytestOutput } = require('../executor'); + +const pytestDirectory = path.join('tmp', 'pytest-invocation'); +assert.deepStrictEqual( + buildPytestInvocation(pytestDirectory), + { + command: 'pytest', + args: ['-vv', path.join(pytestDirectory, 'test_program.py')], + }, + 'configures Python tests to request complete pytest diagnostics' +); const testCases = [ { From c315590c7ed5ce70dc9e6d3c0bc7280f765a29da Mon Sep 17 00:00:00 2001 From: Kevin Buffardi Date: Thu, 30 Jul 2026 11:54:28 -0700 Subject: [PATCH 3/3] test: verify complete pytest outcomes Exercise real verbose pytest output and preserve response-state behavior across success, failures, collection errors, no tests, and process failures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- executor.js | 65 ++++++++----- package.json | 4 +- test/executor.test.js | 166 +++++++++++++++++++++++++++++++- test/pytest-integration.test.js | 58 +++++++++++ 4 files changed, 266 insertions(+), 27 deletions(-) create mode 100644 test/pytest-integration.test.js diff --git a/executor.js b/executor.js index ec2889a..9b1b5a7 100644 --- a/executor.js +++ b/executor.js @@ -465,6 +465,40 @@ function parsePytestOutput(stdout = '', stderr = '', exitCode = null) { }; } +function buildPythonTestResponse(response, output) { + const testResults = parsePytestOutput( + output.stdout, + output.stderr, + output.exitCode ?? null + ); + const keepGenericRuntimeError = output.exitCode == null || ( + output.exitCode !== 0 && + output.exitCode !== 1 && + output.exitCode !== 5 && + testResults.errors === 0 + ); + const runtime_error = testResults.runtime_error || ( + keepGenericRuntimeError ? response.runtime_error : '' + ); + const hasUnexpectedPytestExecutionError = Boolean(runtime_error) && ( + testResults.failed === 0 && + testResults.errors === 0 && + !testResults.no_tests_collected + ); + + return { + ...response, + ...testResults, + runtime_error, + state: ( + testResults.failed === 0 && + testResults.errors === 0 && + !testResults.no_tests_collected && + !hasUnexpectedPytestExecutionError + ) ? 'passed' : 'failed', + }; +} + function parseCppTestOutput(output, stdout = '', stderr = '') { output = output.toString(); let total_tests = 0; @@ -639,29 +673,7 @@ async function executeCode(language, code, stdin, expectedOutput, runTests = fal if (runTests && testCode) { if (language.toLowerCase() === 'python') { - const testResults = parsePytestOutput(output.stdout, output.stderr, output.exitCode ?? null); - const keepGenericRuntimeError = output.exitCode == null || ( - output.exitCode !== 0 && - output.exitCode !== 1 && - output.exitCode !== 5 && - testResults.errors === 0 - ); - const runtime_error = testResults.runtime_error || (keepGenericRuntimeError ? response.runtime_error : ''); - const hasUnexpectedPytestExecutionError = Boolean(runtime_error) && ( - testResults.failed === 0 && - testResults.errors === 0 && - !testResults.no_tests_collected - ); - - response = { ...response, ...testResults }; - response.runtime_error = runtime_error; - response.state = ( - testResults.failed === 0 && - testResults.errors === 0 && - !testResults.no_tests_collected && - !hasUnexpectedPytestExecutionError - ) ? 'passed' : 'failed'; - return response; + return buildPythonTestResponse(response, output); } if (language.toLowerCase() === 'cpp') { @@ -852,4 +864,9 @@ async function cleanupDir(dirPath) { } } -module.exports = { buildPytestInvocation, executeCode, parsePytestOutput }; +module.exports = { + buildPytestInvocation, + buildPythonTestResponse, + executeCode, + parsePytestOutput, +}; diff --git a/package.json b/package.json index dd7980c..b4e62e0 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,9 @@ "version": "1.0.0", "main": "index.js", "scripts": { - "test": "node test/executor.test.js" + "test": "npm run test:unit && npm run test:integration", + "test:unit": "node test/executor.test.js", + "test:integration": "node test/pytest-integration.test.js" }, "keywords": [], "author": "", diff --git a/test/executor.test.js b/test/executor.test.js index 1adedf7..d5b0c9f 100644 --- a/test/executor.test.js +++ b/test/executor.test.js @@ -1,6 +1,10 @@ const assert = require('assert'); const path = require('path'); -const { buildPytestInvocation, parsePytestOutput } = require('../executor'); +const { + buildPytestInvocation, + buildPythonTestResponse, + parsePytestOutput, +} = require('../executor'); const pytestDirectory = path.join('tmp', 'pytest-invocation'); assert.deepStrictEqual( @@ -243,4 +247,162 @@ for (const testCase of testCases) { ); } -console.log(`Passed ${testCases.length} pytest parser regression tests.`); +const responseCases = [ + { + name: 'maps a successful pytest exit to a passing response', + output: { + stdout: '============================== 2 passed in 0.01s ==============================', + stderr: '', + exitCode: 0, + }, + runtimeError: '', + expected: { + state: 'passed', + exit_code: 0, + tests_run: 2, + passed: 2, + failed: 0, + errors: 0, + no_tests_collected: false, + runtime_error: '', + details: 0, + }, + }, + { + name: 'maps an assertion exit without retaining the generic process error', + output: { + stdout: testCases[0].output, + stderr: '', + exitCode: 1, + }, + runtimeError: 'Execution failed with code 1', + expected: { + state: 'failed', + exit_code: 1, + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + runtime_error: '', + details: 1, + }, + }, + { + name: 'maps collection errors to parsed pytest diagnostics', + output: { + stdout: testCases[5].output, + stderr: '', + exitCode: 2, + }, + runtimeError: 'Execution failed with code 2', + expected: { + state: 'failed', + exit_code: 2, + tests_run: 0, + passed: 0, + failed: 0, + errors: 1, + no_tests_collected: false, + runtime_error: "SyntaxError: '(' was never closed", + details: 1, + }, + }, + { + name: 'maps no collected tests to the canonical failed response', + output: { + stdout: '============================ no tests ran in 0.01s ============================', + stderr: '', + exitCode: 5, + }, + runtimeError: 'Execution failed with code 5', + expected: { + state: 'failed', + exit_code: 5, + tests_run: 0, + passed: 0, + failed: 0, + errors: 0, + no_tests_collected: true, + runtime_error: 'Pytest did not collect any tests', + details: 1, + }, + }, + { + name: 'retains an unexpected pytest process error', + output: { + stdout: '', + stderr: 'pytest: internal error', + exitCode: 3, + }, + runtimeError: 'Execution failed with code 3', + expected: { + state: 'failed', + exit_code: 3, + tests_run: 0, + passed: 0, + failed: 0, + errors: 0, + no_tests_collected: false, + runtime_error: 'Execution failed with code 3', + details: 0, + }, + }, + { + name: 'retains a terminated pytest process error', + output: { + stdout: '', + stderr: '', + exitCode: null, + }, + runtimeError: 'Execution terminated: terminated by signal SIGTERM', + expected: { + state: 'failed', + exit_code: null, + tests_run: 0, + passed: 0, + failed: 0, + errors: 0, + no_tests_collected: false, + runtime_error: 'Execution terminated: terminated by signal SIGTERM', + details: 0, + }, + }, +]; + +for (const responseCase of responseCases) { + const result = buildPythonTestResponse( + { + state: 'failed', + exit_code: responseCase.output.exitCode, + runtime_error: responseCase.runtimeError, + }, + responseCase.output + ); + + for (const field of [ + 'state', + 'exit_code', + 'tests_run', + 'passed', + 'failed', + 'errors', + 'no_tests_collected', + 'runtime_error', + ]) { + assert.strictEqual( + result[field], + responseCase.expected[field], + `${responseCase.name}: ${field}` + ); + } + assert.strictEqual( + result.failure_details.length, + responseCase.expected.details, + `${responseCase.name}: failure details` + ); +} + +console.log( + `Passed ${testCases.length} pytest parser and ${responseCases.length} response regression tests.` +); diff --git a/test/pytest-integration.test.js b/test/pytest-integration.test.js new file mode 100644 index 0000000..8a43398 --- /dev/null +++ b/test/pytest-integration.test.js @@ -0,0 +1,58 @@ +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { buildPytestInvocation, parsePytestOutput } = require('../executor'); + +const tempDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'codeval-pytest-')); +const testPath = path.join(tempDirectory, 'test_program.py'); +const versionResult = spawnSync('pytest', ['--version'], { encoding: 'utf8' }); +const pytestVersion = (versionResult.stdout || versionResult.stderr).trim(); + +try { + const testCode = ` +def test_long_collection_diff(): + actual = {"items": [f"actual-{index:03d}" for index in range(80)]} + expected = {"items": [f"expected-{index:03d}" for index in range(80)]} + assert actual == expected +`; + fs.writeFileSync(testPath, testCode); + + const defaultResult = spawnSync('pytest', [testPath], { + cwd: tempDirectory, + encoding: 'utf8', + }); + assert.strictEqual(defaultResult.status, 1, `default pytest failed unexpectedly (${pytestVersion})`); + assert.match( + defaultResult.stdout, + /truncated|use ['"]?-v{1,2}['"]? to get more diff/i, + `test data did not trigger default pytest truncation (${pytestVersion})` + ); + + const invocation = buildPytestInvocation(tempDirectory); + const verboseResult = spawnSync(invocation.command, invocation.args, { + cwd: tempDirectory, + encoding: 'utf8', + }); + assert.strictEqual(verboseResult.status, 1, `verbose pytest failed unexpectedly (${pytestVersion})`); + assert.match(verboseResult.stdout, /Full diff:/, `full diff was not emitted (${pytestVersion})`); + assert.match(verboseResult.stdout, /actual-000/, `start of actual diff is missing (${pytestVersion})`); + assert.match(verboseResult.stdout, /actual-079/, `end of actual diff is missing (${pytestVersion})`); + assert.match(verboseResult.stdout, /expected-000/, `start of expected diff is missing (${pytestVersion})`); + assert.match(verboseResult.stdout, /expected-079/, `end of expected diff is missing (${pytestVersion})`); + assert.doesNotMatch(verboseResult.stdout, /Use -v to get more diff/i); + assert.doesNotMatch(verboseResult.stdout, /Full output truncated/i); + + const parsed = parsePytestOutput(verboseResult.stdout, verboseResult.stderr, verboseResult.status); + assert.strictEqual(parsed.failed, 1); + assert.strictEqual(parsed.failure_details.length, 1); + assert.match(parsed.failure_details[0].error_message, /actual-000/); + assert.match(parsed.failure_details[0].error_message, /actual-079/); + assert.match(parsed.failure_details[0].error_message, /expected-000/); + assert.match(parsed.failure_details[0].error_message, /expected-079/); +} finally { + fs.rmSync(tempDirectory, { recursive: true, force: true }); +} + +console.log(`Passed pytest integration regression with ${pytestVersion}.`);