diff --git a/executor.js b/executor.js index f951a30..9b1b5a7 100644 --- a/executor.js +++ b/executor.js @@ -114,6 +114,13 @@ function configureExecution(language, code, uniqueDir) { return config; } +function buildPytestInvocation(uniqueDir) { + return { + command: 'pytest', + args: ['-vv', path.join(uniqueDir, 'test_program.py')], + }; +} + /** * Writes the provided code to a file. * @param {string} uniqueDir - The directory to write the file. @@ -303,6 +310,35 @@ function extractPytestErrorMessage(output, stderr = '') { return 'Pytest error during collection or execution'; } +function hasCustomAssertionMessage(assertionSource) { + const expression = assertionSource.replace(/^>\s*assert\s+/, ''); + let depth = 0; + let quote; + let escaped = false; + + for (const character of expression) { + if (quote) { + if (escaped) { + escaped = false; + } else if (character === '\\') { + escaped = true; + } else if (character === quote) { + quote = undefined; + } + } else if (character === '"' || character === "'") { + quote = character; + } else if (character === '(' || character === '[' || character === '{') { + depth += 1; + } else if (character === ')' || character === ']' || character === '}') { + depth -= 1; + } else if (character === ',' && depth === 0) { + return true; + } + } + + return false; +} + function parsePytestOutput(stdout = '', stderr = '', exitCode = null) { const summary = extractPytestSummary(stdout); const rawout = buildRawOutput(stdout, stderr); @@ -311,34 +347,83 @@ function parsePytestOutput(stdout = '', stderr = '', exitCode = null) { const errors = extractPytestCount(summary, 'error(?:s)?'); const no_tests_collected = exitCode === 5 || /\bno tests ran\b/.test(summary); const failures = []; + let currentFailure; - const failureBlocks = stdout.split(/={10,} FAILURES ={10,}/)[1]?.split(/={10,}/)[0] || ''; - const matches = [...failureBlocks.matchAll( - /_{5,}\s*(.*?)\s*_{5,}[\s\S]*?>\s*assert\s+(.*?)\s*?\nE\s+assert\s+(.*?)\s*?(?:\nE\s+\+\s+where\s+(.*?)\s+=)?/g - )]; + const addFailure = () => { + if (!currentFailure?.errorLines.length) { + return; + } - matches.forEach((match, index) => { - const test_case = match[1]?.trim() || `Test ${index + 1}`; - const assertionLine = match[2]?.trim(); - const failedExpr = match[3]?.trim(); - const evaluated = match[4]?.trim() || ''; + const messageLines = currentFailure.errorLines.map((line) => + line.replace(/^\s*E {0,7}/, '') + ); + const firstMessageLine = messageLines[0]; + const usesCustomAssertionMessage = + hasCustomAssertionMessage(currentFailure.assertionSource) && + firstMessageLine.startsWith('AssertionError:'); + const comparison = firstMessageLine + .replace(/^AssertionError:\s*/, '') + .match(/^assert\s+(.+?)\s*==\s*(.+)$/); failures.push({ - test_case, - expected: failedExpr.split('==')[1]?.trim() || '', - received: evaluated || failedExpr.split('==')[0]?.trim(), - error_message: `Assertion failed: ${assertionLine}`, + test_case: currentFailure.testCase || `Test ${failures.length + 1}`, + expected: comparison?.[2]?.trim() || '', + received: comparison?.[1]?.trim() || '', + error_message: usesCustomAssertionMessage + ? [firstMessageLine.replace(/^AssertionError:\s*/, ''), ...messageLines.slice(1)].join('\n') + : messageLines.join('\n'), rawout, + isError: currentFailure.isError, }); - }); + }; + + for (const line of stdout.split(/\r?\n/)) { + const failureHeader = line.match(/^_{5,}\s*(.*?)\s*_{5,}\s*$/); + if (failureHeader) { + addFailure(); + currentFailure = { + testCase: failureHeader[1].trim(), + assertionSource: '', + errorLines: [], + finishedErrors: false, + isError: /\bERROR\b/.test(failureHeader[1]), + }; + continue; + } + + if (!currentFailure) { + continue; + } - if (failed_tests > 0 && failures.length === 0) { + if (/^={3,}|^!{3,}/.test(line)) { + addFailure(); + currentFailure = undefined; + continue; + } + + if (/^>\s*assert\b/.test(line)) { + currentFailure.assertionSource = line; + } + + if (/^\s*E(?:\s|$)/.test(line) && !currentFailure.finishedErrors) { + currentFailure.errorLines.push(line); + } else if (currentFailure.errorLines.length) { + currentFailure.finishedErrors = true; + } + } + addFailure(); + + const parsedFailures = failures.filter((failure) => !failure.isError).length; + const parsedErrors = failures.length - parsedFailures; + + if (failed_tests > parsedFailures) { failures.push({ test_case: extractPytestShortSummaryTarget(stdout, 'FAILED') || 'pytest assertion failure', expected: '', received: '', error_message: 'Pytest reported one or more failed assertions', rawout, + isError: false, }); } @@ -346,13 +431,16 @@ function parsePytestOutput(stdout = '', stderr = '', exitCode = null) { if (errors > 0) { runtime_error = extractPytestErrorMessage(stdout, stderr); - failures.push({ - test_case: extractPytestShortSummaryTarget(stdout, 'ERROR') || 'pytest collection/execution', - expected: '', - received: '', - error_message: runtime_error, - rawout, - }); + if (errors > parsedErrors) { + failures.push({ + test_case: extractPytestShortSummaryTarget(stdout, 'ERROR') || 'pytest collection/execution', + expected: '', + received: '', + error_message: runtime_error, + rawout, + isError: true, + }); + } } else if (no_tests_collected) { runtime_error = 'Pytest did not collect any tests'; failures.push({ @@ -361,6 +449,7 @@ function parsePytestOutput(stdout = '', stderr = '', exitCode = null) { received: '0 collected tests', error_message: runtime_error, rawout, + isError: true, }); } @@ -371,11 +460,45 @@ function parsePytestOutput(stdout = '', stderr = '', exitCode = null) { errors, no_tests_collected, exit_code: exitCode, - failure_details: failures, + failure_details: failures.map(({ isError, ...failure }) => failure), runtime_error, }; } +function buildPythonTestResponse(response, output) { + const testResults = parsePytestOutput( + output.stdout, + output.stderr, + output.exitCode ?? null + ); + const keepGenericRuntimeError = output.exitCode == null || ( + output.exitCode !== 0 && + output.exitCode !== 1 && + output.exitCode !== 5 && + testResults.errors === 0 + ); + const runtime_error = testResults.runtime_error || ( + keepGenericRuntimeError ? response.runtime_error : '' + ); + const hasUnexpectedPytestExecutionError = Boolean(runtime_error) && ( + testResults.failed === 0 && + testResults.errors === 0 && + !testResults.no_tests_collected + ); + + return { + ...response, + ...testResults, + runtime_error, + state: ( + testResults.failed === 0 && + testResults.errors === 0 && + !testResults.no_tests_collected && + !hasUnexpectedPytestExecutionError + ) ? 'passed' : 'failed', + }; +} + function parseCppTestOutput(output, stdout = '', stderr = '') { output = output.toString(); let total_tests = 0; @@ -488,8 +611,9 @@ async function executeCode(language, code, stdin, expectedOutput, runTests = fal let output; if (runTests && testCode) { if (language.toLowerCase() === 'python') { - executionConfig.runCommand = 'pytest'; - executionConfig.runArgs = [path.join(uniqueDir, 'test_program.py')]; + const pytestInvocation = buildPytestInvocation(uniqueDir); + executionConfig.runCommand = pytestInvocation.command; + executionConfig.runArgs = pytestInvocation.args; } else if (language.toLowerCase() === 'cpp') { executionConfig.runCommand = path.join(uniqueDir, 'runner'); executionConfig.runArgs = []; @@ -549,29 +673,7 @@ async function executeCode(language, code, stdin, expectedOutput, runTests = fal if (runTests && testCode) { if (language.toLowerCase() === 'python') { - const testResults = parsePytestOutput(output.stdout, output.stderr, output.exitCode ?? null); - const keepGenericRuntimeError = output.exitCode == null || ( - output.exitCode !== 0 && - output.exitCode !== 1 && - output.exitCode !== 5 && - testResults.errors === 0 - ); - const runtime_error = testResults.runtime_error || (keepGenericRuntimeError ? response.runtime_error : ''); - const hasUnexpectedPytestExecutionError = Boolean(runtime_error) && ( - testResults.failed === 0 && - testResults.errors === 0 && - !testResults.no_tests_collected - ); - - response = { ...response, ...testResults }; - response.runtime_error = runtime_error; - response.state = ( - testResults.failed === 0 && - testResults.errors === 0 && - !testResults.no_tests_collected && - !hasUnexpectedPytestExecutionError - ) ? 'passed' : 'failed'; - return response; + return buildPythonTestResponse(response, output); } if (language.toLowerCase() === 'cpp') { @@ -762,4 +864,9 @@ async function cleanupDir(dirPath) { } } -module.exports = { executeCode }; +module.exports = { + buildPytestInvocation, + buildPythonTestResponse, + executeCode, + parsePytestOutput, +}; diff --git a/package.json b/package.json index 8625d81..b4e62e0 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,9 @@ "version": "1.0.0", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "npm run test:unit && npm run test:integration", + "test:unit": "node test/executor.test.js", + "test:integration": "node test/pytest-integration.test.js" }, "keywords": [], "author": "", diff --git a/test/executor.test.js b/test/executor.test.js new file mode 100644 index 0000000..d5b0c9f --- /dev/null +++ b/test/executor.test.js @@ -0,0 +1,408 @@ +const assert = require('assert'); +const path = require('path'); +const { + buildPytestInvocation, + buildPythonTestResponse, + parsePytestOutput, +} = require('../executor'); + +const pytestDirectory = path.join('tmp', 'pytest-invocation'); +assert.deepStrictEqual( + buildPytestInvocation(pytestDirectory), + { + command: 'pytest', + args: ['-vv', path.join(pytestDirectory, 'test_program.py')], + }, + 'configures Python tests to request complete pytest diagnostics' +); + +const testCases = [ + { + name: 'returns only a custom assertion message', + output: `============================= test session starts ============================== +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): +> assert hasattr(program, "HatName"), "there should be a variable named exactly HatName" +E AssertionError: there should be a variable named exactly HatName + +test_program.py:5: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_hat_variables - AssertionError: there should be a variable named exactly HatName +============================== 1 failed in 0.02s ===============================`, + expected: { + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + messages: ['there should be a variable named exactly HatName'], + expectedValues: [''], + receivedValues: [''], + }, + }, + { + name: 'preserves multiline custom assertion messages', + output: `=================================== FAILURES =================================== +________________________________ test_message _________________________________ + + def test_message(): +> assert False, "first instruction\\nsecond instruction" +E AssertionError: first instruction +E second instruction + +test_program.py:2: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_message - AssertionError: first instruction +============================== 1 failed in 0.01s ===============================`, + expected: { + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + messages: ['first instruction\nsecond instruction'], + expectedValues: [''], + receivedValues: [''], + }, + }, + { + name: 'preserves all default assertion detail lines', + output: `=================================== FAILURES =================================== +__________________________________ test_value __________________________________ + + def test_value(): +> assert "actual" == "expected" +E AssertionError: assert 'actual' == 'expected' +E - expected +E + actual + +test_program.py:2: AssertionError +=========================== short test summary info ============================ +FAILED test_program.py::test_value - AssertionError: assert 'actual' == 'expected' +============================== 1 failed in 0.01s ===============================`, + expected: { + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + messages: ["AssertionError: assert 'actual' == 'expected'\n - expected\n + actual"], + expectedValues: ["'expected'"], + receivedValues: ["'actual'"], + }, + }, + { + name: 'preserves exception type and message', + output: `=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): +> assert program.HatName == "Veracruz" +E AttributeError: module 'program' has no attribute 'HatName' + +test_program.py:5: AttributeError +=========================== short test summary info ============================ +FAILED test_program.py::test_hat_variables - AttributeError: module 'program' has no attribute 'HatName' +============================== 1 failed in 0.02s ===============================`, + expected: { + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + messages: ["AttributeError: module 'program' has no attribute 'HatName'"], + expectedValues: [''], + receivedValues: [''], + }, + }, + { + name: 'returns distinct details for multiple failures', + output: `=================================== FAILURES =================================== +_________________________________ test_first __________________________________ + + def test_first(): +> assert False, "first failure" +E AssertionError: first failure + +test_program.py:2: AssertionError +_________________________________ test_second _________________________________ + + def test_second(): +> raise ValueError("second failure") +E ValueError: second failure + +test_program.py:5: ValueError +=========================== short test summary info ============================ +FAILED test_program.py::test_first - AssertionError: first failure +FAILED test_program.py::test_second - ValueError: second failure +============================== 2 failed in 0.01s ===============================`, + expected: { + tests_run: 2, + passed: 0, + failed: 2, + errors: 0, + messages: ['first failure', 'ValueError: second failure'], + expectedValues: ['', ''], + receivedValues: ['', ''], + }, + }, + { + name: 'counts collection errors as one failure', + output: `==================================== ERRORS ==================================== +___________________ ERROR collecting test_program.py ___________________ +test_program.py:1: in + assert ( +E SyntaxError: '(' was never closed +=========================== short test summary info ============================ +ERROR test_program.py +!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!! +=============================== 1 error in 0.03s ===============================`, + expected: { + tests_run: 0, + passed: 0, + failed: 0, + errors: 1, + messages: ["SyntaxError: '(' was never closed"], + expectedValues: [''], + receivedValues: [''], + }, + }, + { + name: 'counts mixed failed tests and errors', + output: `=================================== FAILURES =================================== +_________________________________ test_value __________________________________ + + def test_value(): +> assert False, "assertion failure" +E AssertionError: assertion failure + +test_program.py:2: AssertionError +==================================== ERRORS ==================================== +_____________________ ERROR at setup of test_setup_error ______________________ + + @pytest.fixture + def broken_fixture(): +> raise RuntimeError("setup failure") +E RuntimeError: setup failure + +test_program.py:6: RuntimeError +=========================== short test summary info ============================ +FAILED test_program.py::test_value - AssertionError: assertion failure +ERROR test_program.py::test_setup_error - RuntimeError: setup failure +===================== 1 passed, 1 failed, 1 error in 0.01s =====================`, + expected: { + tests_run: 2, + passed: 1, + failed: 1, + errors: 1, + messages: ['assertion failure', 'RuntimeError: setup failure'], + expectedValues: ['', ''], + receivedValues: ['', ''], + }, + }, + { + name: 'counts passing tests', + output: `============================= test session starts ============================== +collected 2 items + +test_program.py .. [100%] + +============================== 2 passed in 0.01s ===============================`, + expected: { + tests_run: 2, + passed: 2, + failed: 0, + errors: 0, + messages: [], + expectedValues: [], + receivedValues: [], + }, + }, +]; + +for (const testCase of testCases) { + const result = parsePytestOutput(testCase.output); + + assert.strictEqual(result.tests_run, testCase.expected.tests_run, testCase.name); + assert.strictEqual(result.passed, testCase.expected.passed, testCase.name); + assert.strictEqual(result.failed, testCase.expected.failed, testCase.name); + assert.strictEqual(result.errors, testCase.expected.errors, testCase.name); + assert.strictEqual( + result.failure_details.length, + testCase.expected.failed + testCase.expected.errors, + testCase.name + ); + assert.deepStrictEqual( + result.failure_details.map((failure) => failure.error_message), + testCase.expected.messages, + testCase.name + ); + assert.deepStrictEqual( + result.failure_details.map((failure) => failure.expected), + testCase.expected.expectedValues, + testCase.name + ); + assert.deepStrictEqual( + result.failure_details.map((failure) => failure.received), + testCase.expected.receivedValues, + testCase.name + ); +} + +const responseCases = [ + { + name: 'maps a successful pytest exit to a passing response', + output: { + stdout: '============================== 2 passed in 0.01s ==============================', + stderr: '', + exitCode: 0, + }, + runtimeError: '', + expected: { + state: 'passed', + exit_code: 0, + tests_run: 2, + passed: 2, + failed: 0, + errors: 0, + no_tests_collected: false, + runtime_error: '', + details: 0, + }, + }, + { + name: 'maps an assertion exit without retaining the generic process error', + output: { + stdout: testCases[0].output, + stderr: '', + exitCode: 1, + }, + runtimeError: 'Execution failed with code 1', + expected: { + state: 'failed', + exit_code: 1, + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + runtime_error: '', + details: 1, + }, + }, + { + name: 'maps collection errors to parsed pytest diagnostics', + output: { + stdout: testCases[5].output, + stderr: '', + exitCode: 2, + }, + runtimeError: 'Execution failed with code 2', + expected: { + state: 'failed', + exit_code: 2, + tests_run: 0, + passed: 0, + failed: 0, + errors: 1, + no_tests_collected: false, + runtime_error: "SyntaxError: '(' was never closed", + details: 1, + }, + }, + { + name: 'maps no collected tests to the canonical failed response', + output: { + stdout: '============================ no tests ran in 0.01s ============================', + stderr: '', + exitCode: 5, + }, + runtimeError: 'Execution failed with code 5', + expected: { + state: 'failed', + exit_code: 5, + tests_run: 0, + passed: 0, + failed: 0, + errors: 0, + no_tests_collected: true, + runtime_error: 'Pytest did not collect any tests', + details: 1, + }, + }, + { + name: 'retains an unexpected pytest process error', + output: { + stdout: '', + stderr: 'pytest: internal error', + exitCode: 3, + }, + runtimeError: 'Execution failed with code 3', + expected: { + state: 'failed', + exit_code: 3, + tests_run: 0, + passed: 0, + failed: 0, + errors: 0, + no_tests_collected: false, + runtime_error: 'Execution failed with code 3', + details: 0, + }, + }, + { + name: 'retains a terminated pytest process error', + output: { + stdout: '', + stderr: '', + exitCode: null, + }, + runtimeError: 'Execution terminated: terminated by signal SIGTERM', + expected: { + state: 'failed', + exit_code: null, + tests_run: 0, + passed: 0, + failed: 0, + errors: 0, + no_tests_collected: false, + runtime_error: 'Execution terminated: terminated by signal SIGTERM', + details: 0, + }, + }, +]; + +for (const responseCase of responseCases) { + const result = buildPythonTestResponse( + { + state: 'failed', + exit_code: responseCase.output.exitCode, + runtime_error: responseCase.runtimeError, + }, + responseCase.output + ); + + for (const field of [ + 'state', + 'exit_code', + 'tests_run', + 'passed', + 'failed', + 'errors', + 'no_tests_collected', + 'runtime_error', + ]) { + assert.strictEqual( + result[field], + responseCase.expected[field], + `${responseCase.name}: ${field}` + ); + } + assert.strictEqual( + result.failure_details.length, + responseCase.expected.details, + `${responseCase.name}: failure details` + ); +} + +console.log( + `Passed ${testCases.length} pytest parser and ${responseCases.length} response regression tests.` +); diff --git a/test/pytest-integration.test.js b/test/pytest-integration.test.js new file mode 100644 index 0000000..8a43398 --- /dev/null +++ b/test/pytest-integration.test.js @@ -0,0 +1,58 @@ +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { buildPytestInvocation, parsePytestOutput } = require('../executor'); + +const tempDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'codeval-pytest-')); +const testPath = path.join(tempDirectory, 'test_program.py'); +const versionResult = spawnSync('pytest', ['--version'], { encoding: 'utf8' }); +const pytestVersion = (versionResult.stdout || versionResult.stderr).trim(); + +try { + const testCode = ` +def test_long_collection_diff(): + actual = {"items": [f"actual-{index:03d}" for index in range(80)]} + expected = {"items": [f"expected-{index:03d}" for index in range(80)]} + assert actual == expected +`; + fs.writeFileSync(testPath, testCode); + + const defaultResult = spawnSync('pytest', [testPath], { + cwd: tempDirectory, + encoding: 'utf8', + }); + assert.strictEqual(defaultResult.status, 1, `default pytest failed unexpectedly (${pytestVersion})`); + assert.match( + defaultResult.stdout, + /truncated|use ['"]?-v{1,2}['"]? to get more diff/i, + `test data did not trigger default pytest truncation (${pytestVersion})` + ); + + const invocation = buildPytestInvocation(tempDirectory); + const verboseResult = spawnSync(invocation.command, invocation.args, { + cwd: tempDirectory, + encoding: 'utf8', + }); + assert.strictEqual(verboseResult.status, 1, `verbose pytest failed unexpectedly (${pytestVersion})`); + assert.match(verboseResult.stdout, /Full diff:/, `full diff was not emitted (${pytestVersion})`); + assert.match(verboseResult.stdout, /actual-000/, `start of actual diff is missing (${pytestVersion})`); + assert.match(verboseResult.stdout, /actual-079/, `end of actual diff is missing (${pytestVersion})`); + assert.match(verboseResult.stdout, /expected-000/, `start of expected diff is missing (${pytestVersion})`); + assert.match(verboseResult.stdout, /expected-079/, `end of expected diff is missing (${pytestVersion})`); + assert.doesNotMatch(verboseResult.stdout, /Use -v to get more diff/i); + assert.doesNotMatch(verboseResult.stdout, /Full output truncated/i); + + const parsed = parsePytestOutput(verboseResult.stdout, verboseResult.stderr, verboseResult.status); + assert.strictEqual(parsed.failed, 1); + assert.strictEqual(parsed.failure_details.length, 1); + assert.match(parsed.failure_details[0].error_message, /actual-000/); + assert.match(parsed.failure_details[0].error_message, /actual-079/); + assert.match(parsed.failure_details[0].error_message, /expected-000/); + assert.match(parsed.failure_details[0].error_message, /expected-079/); +} finally { + fs.rmSync(tempDirectory, { recursive: true, force: true }); +} + +console.log(`Passed pytest integration regression with ${pytestVersion}.`);