fix: clarify Python pytest failure messages

executor.js

@@ -303,6 +303,35 @@ function extractPytestErrorMessage(output, stderr = '') {
   return 'Pytest error during collection or execution';
 }
 
+function hasCustomAssertionMessage(assertionSource) {
+  const expression = assertionSource.replace(/^>\s*assert\s+/, '');
+  let depth = 0;
+  let quote;
+  let escaped = false;
+
+  for (const character of expression) {
+    if (quote) {
+      if (escaped) {
+        escaped = false;
+      } else if (character === '\\') {
+        escaped = true;
+      } else if (character === quote) {
+        quote = undefined;
+      }
+    } else if (character === '"' || character === "'") {
+      quote = character;
+    } else if (character === '(' || character === '[' || character === '{') {
+      depth += 1;
+    } else if (character === ')' || character === ']' || character === '}') {
+      depth -= 1;
+    } else if (character === ',' && depth === 0) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 function parsePytestOutput(stdout = '', stderr = '', exitCode = null) {
   const summary = extractPytestSummary(stdout);
   const rawout = buildRawOutput(stdout, stderr);
@@ -311,48 +340,100 @@ function parsePytestOutput(stdout = '', stderr = '', exitCode = null) {
   const errors = extractPytestCount(summary, 'error(?:s)?');
   const no_tests_collected = exitCode === 5 || /\bno tests ran\b/.test(summary);
   const failures = [];
+  let currentFailure;
 
-  const failureBlocks = stdout.split(/={10,} FAILURES ={10,}/)[1]?.split(/={10,}/)[0] || '';
-  const matches = [...failureBlocks.matchAll(
-    /_{5,}\s*(.*?)\s*_{5,}[\s\S]*?>\s*assert\s+(.*?)\s*?\nE\s+assert\s+(.*?)\s*?(?:\nE\s+\+\s+where\s+(.*?)\s+=)?/g
-  )];
+  const addFailure = () => {
+    if (!currentFailure?.errorLines.length) {
+      return;
+    }
 
-  matches.forEach((match, index) => {
-    const test_case = match[1]?.trim() || `Test ${index + 1}`;
-    const assertionLine = match[2]?.trim();
-    const failedExpr = match[3]?.trim();
-    const evaluated = match[4]?.trim() || '';
+    const messageLines = currentFailure.errorLines.map((line) =>
+      line.replace(/^\s*E {0,7}/, '')
+    );
+    const firstMessageLine = messageLines[0];
+    const usesCustomAssertionMessage =
+      hasCustomAssertionMessage(currentFailure.assertionSource) &&
+      firstMessageLine.startsWith('AssertionError:');
+    const comparison = firstMessageLine
+      .replace(/^AssertionError:\s*/, '')
+      .match(/^assert\s+(.+?)\s*==\s*(.+)$/);
 
     failures.push({
-      test_case,
-      expected: failedExpr.split('==')[1]?.trim() || '',
-      received: evaluated || failedExpr.split('==')[0]?.trim(),
-      error_message: `Assertion failed: ${assertionLine}`,
+      test_case: currentFailure.testCase || `Test ${failures.length + 1}`,
+      expected: comparison?.[2]?.trim() || '',
+      received: comparison?.[1]?.trim() || '',
+      error_message: usesCustomAssertionMessage
+        ? [firstMessageLine.replace(/^AssertionError:\s*/, ''), ...messageLines.slice(1)].join('\n')
+        : messageLines.join('\n'),
       rawout,
+      isError: currentFailure.isError,
     });
-  });
+  };
+
+  for (const line of stdout.split(/\r?\n/)) {
+    const failureHeader = line.match(/^_{5,}\s*(.*?)\s*_{5,}\s*$/);
+    if (failureHeader) {
+      addFailure();
+      currentFailure = {
+        testCase: failureHeader[1].trim(),
+        assertionSource: '',
+        errorLines: [],
+        finishedErrors: false,
+        isError: /\bERROR\b/.test(failureHeader[1]),
+      };
+      continue;
+    }
+
+    if (!currentFailure) {
+      continue;
+    }
+
+    if (/^={3,}|^!{3,}/.test(line)) {
+      addFailure();
+      currentFailure = undefined;
+      continue;
+    }
+
+    if (/^>\s*assert\b/.test(line)) {
+      currentFailure.assertionSource = line;
+    }
+
+    if (/^\s*E(?:\s|$)/.test(line) && !currentFailure.finishedErrors) {
+      currentFailure.errorLines.push(line);
+    } else if (currentFailure.errorLines.length) {
+      currentFailure.finishedErrors = true;
+    }
+  }
+  addFailure();
 
-  if (failed_tests > 0 && failures.length === 0) {
+  const parsedFailures = failures.filter((failure) => !failure.isError).length;
+  const parsedErrors = failures.length - parsedFailures;
+
+  if (failed_tests > parsedFailures) {
     failures.push({
       test_case: extractPytestShortSummaryTarget(stdout, 'FAILED') || 'pytest assertion failure',
       expected: '',
       received: '',
       error_message: 'Pytest reported one or more failed assertions',
       rawout,
+      isError: false,
     });
   }
 
   let runtime_error = '';
 
   if (errors > 0) {
     runtime_error = extractPytestErrorMessage(stdout, stderr);
-    failures.push({
-      test_case: extractPytestShortSummaryTarget(stdout, 'ERROR') || 'pytest collection/execution',
-      expected: '',
-      received: '',
-      error_message: runtime_error,
-      rawout,
-    });
+    if (errors > parsedErrors) {
+      failures.push({
+        test_case: extractPytestShortSummaryTarget(stdout, 'ERROR') || 'pytest collection/execution',
+        expected: '',
+        received: '',
+        error_message: runtime_error,
+        rawout,
+        isError: true,
+      });
+    }
   } else if (no_tests_collected) {
     runtime_error = 'Pytest did not collect any tests';
     failures.push({
@@ -361,6 +442,7 @@ function parsePytestOutput(stdout = '', stderr = '', exitCode = null) {
       received: '0 collected tests',
       error_message: runtime_error,
       rawout,
+      isError: true,
     });
   }
 
@@ -371,7 +453,7 @@ function parsePytestOutput(stdout = '', stderr = '', exitCode = null) {
     errors,
     no_tests_collected,
     exit_code: exitCode,
-    failure_details: failures,
+    failure_details: failures.map(({ isError, ...failure }) => failure),
     runtime_error,
   };
 }
@@ -762,4 +844,4 @@ async function cleanupDir(dirPath) {
   }
 }
 
-module.exports = { executeCode };
+module.exports = { executeCode, parsePytestOutput };

package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "node test/executor.test.js"
   },
   "keywords": [],
   "author": "",