test: vendor Python parity fixtures (binary, parser, truncation)

landing/lib/filter.js

@@ -525,7 +525,7 @@ function truncateText(text, maxBytes, maxTokens) {
  * @param {number} maxBytes
  * @returns {string}
  */
-function truncateUtf8(text, maxBytes) {
+export function truncateUtf8(text, maxBytes) {
   const bytes = new TextEncoder().encode(text);
   if (bytes.length <= maxBytes || maxBytes <= 0) return maxBytes <= 0 ? '' : text;
   let end = maxBytes;

src/filter.d.ts

@@ -29,6 +29,14 @@ export function parseRelevanceAnswer(raw: unknown): {
  * @returns {number}
  */
 export function countTokens(text: string): number;
+/**
+ * UTF-8 prefix cut on a code-point boundary. Never emits U+FFFD: a character
+ * that does not fit entirely inside `maxBytes` is dropped.
+ * @param {string} text
+ * @param {number} maxBytes
+ * @returns {string}
+ */
+export function truncateUtf8(text: string, maxBytes: number): string;
 /**
  * True when `content` looks like a binary file (NUL, invalid UTF-8, or a
  * dense run of non-text controls). The whole payload is validated. A tail

src/filter.js

@@ -525,7 +525,7 @@ function truncateText(text, maxBytes, maxTokens) {
  * @param {number} maxBytes
  * @returns {string}
  */
-function truncateUtf8(text, maxBytes) {
+export function truncateUtf8(text, maxBytes) {
   const bytes = new TextEncoder().encode(text);
   if (bytes.length <= maxBytes || maxBytes <= 0) return maxBytes <= 0 ? '' : text;
   let end = maxBytes;

test/fixtures/parity/binary.json

@@ -0,0 +1,15 @@
+{
+  "description": "Binary detection on the first 8192 bytes. An incomplete trailing UTF-8 sequence of at most 3 bytes is still text. NUL and genuinely invalid bytes are binary.",
+  "cases": [
+    {"id": "nul", "hex": "6100", "expect_binary": true},
+    {"id": "invalid-ff", "hex": "61ff62", "expect_binary": true},
+    {"id": "bad-continuation", "hex": "c341", "expect_binary": true},
+    {"id": "valid-2byte", "hex": "c3a9", "expect_binary": false},
+    {"id": "valid-3byte", "hex": "e282ac", "expect_binary": false},
+    {"id": "valid-4byte", "hex": "f09f9880", "expect_binary": false},
+    {"id": "cut-2byte-at-8192", "fill": "a", "length": 8192, "tail_hex": "c3", "expect_binary": false},
+    {"id": "cut-3byte-at-8192", "fill": "a", "length": 8192, "tail_hex": "e282", "expect_binary": false},
+    {"id": "cut-4byte-at-8192", "fill": "a", "length": 8192, "tail_hex": "f09f98", "expect_binary": false},
+    {"id": "invalid-at-end-of-window", "fill": "a", "length": 8192, "tail_hex": "ff", "expect_binary": true}
+  ]
+}

test/fixtures/parity/parser.json

@@ -0,0 +1,21 @@
+{
+  "description": "Leading standalone yes/no after whitespace, quotes, markdown bold, backticks, and Answer:/Decision: prefixes. Think tags are stripped first. Anything else is malformed (null).",
+  "cases": [
+    {"id": "plain-yes", "raw": "yes", "decision": "yes"},
+    {"id": "plain-no", "raw": "NO", "decision": "no"},
+    {"id": "punctuation", "raw": "Yes.", "decision": "yes"},
+    {"id": "quoted-yes", "raw": "\"yes\"", "decision": "yes"},
+    {"id": "quoted-no", "raw": "'no'", "decision": "no"},
+    {"id": "bold-yes", "raw": "**yes**", "decision": "yes"},
+    {"id": "backtick-no", "raw": "`no`", "decision": "no"},
+    {"id": "answer-prefix", "raw": "Answer: yes, the parser lives here", "decision": "yes"},
+    {"id": "decision-prefix", "raw": "Decision: no", "decision": "no"},
+    {"id": "negated", "raw": "There is no doubt this file is required", "decision": null},
+    {"id": "explanatory", "raw": "This file is unrelated to the task", "decision": null},
+    {"id": "nope", "raw": "nope", "decision": null},
+    {"id": "leading-yes-later-no", "raw": "yes for the helper, no for the tests", "decision": "yes"},
+    {"id": "think-then-yes", "raw": "<think>no, skip it</think>\nyes", "decision": "yes"},
+    {"id": "empty", "raw": "", "decision": null},
+    {"id": "whitespace", "raw": "  \n  yes — needed", "decision": "yes"}
+  ]
+}

test/fixtures/parity/truncation.json

@@ -0,0 +1,18 @@
+{
+  "description": "UTF-8 truncation on a code-point boundary. Never emit U+FFFD. Shared with the JS filter.",
+  "cases": [
+    {"id": "2byte-max-1", "text": "éX", "max_bytes": 1, "expected": ""},
+    {"id": "2byte-max-2", "text": "éX", "max_bytes": 2, "expected": "é"},
+    {"id": "2byte-max-3", "text": "éX", "max_bytes": 3, "expected": "éX"},
+    {"id": "3byte-max-1", "text": "€X", "max_bytes": 1, "expected": ""},
+    {"id": "3byte-max-2", "text": "€X", "max_bytes": 2, "expected": ""},
+    {"id": "3byte-max-3", "text": "€X", "max_bytes": 3, "expected": "€"},
+    {"id": "3byte-max-4", "text": "€X", "max_bytes": 4, "expected": "€X"},
+    {"id": "4byte-max-1", "text": "😀X", "max_bytes": 1, "expected": ""},
+    {"id": "4byte-max-2", "text": "😀X", "max_bytes": 2, "expected": ""},
+    {"id": "4byte-max-3", "text": "😀X", "max_bytes": 3, "expected": ""},
+    {"id": "4byte-max-4", "text": "😀X", "max_bytes": 4, "expected": "😀"},
+    {"id": "4byte-max-5", "text": "😀X", "max_bytes": 5, "expected": "😀X"},
+    {"id": "ascii-max-1", "text": "ab", "max_bytes": 1, "expected": "a"}
+  ]
+}

test/parity_fixtures.test.js

@@ -0,0 +1,77 @@
+/**
+ * Shared parity fixtures vendored byte-for-byte from 255BITS/patchling-py
+ * (tests/fixtures/parity/, commit 533749a). The Python suite runs the same
+ * cases in tests/test_relevance.py. Each file's sha256 is pinned so a drift
+ * on either side shows up as a failure here.
+ */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { isBinaryContent, parseRelevanceAnswer } from '../src/index.js';
+import { truncateUtf8 } from '../src/filter.js';
+
+const FIXTURE_SHA256 = {
+  'binary.json': 'babcc94fe1dd990f0a69c41819ed9f76a185756ff28fa2d0e14651a190314934',
+  'parser.json': '809a3522c79781da2aa2f10168bafd770aa781dc29c215b1dd885689047d3c7b',
+  'truncation.json': 'c1d9db9b70687c24068f13a0b2a0edcf84ca3c5830e166336ff4da3c35b5c0df',
+};
+
+function loadFixture(name) {
+  const bytes = readFileSync(new URL(`./fixtures/parity/${name}`, import.meta.url));
+  return { bytes, data: JSON.parse(bytes.toString('utf8')) };
+}
+
+const fixtures = Object.fromEntries(
+  Object.keys(FIXTURE_SHA256).map((name) => [name, loadFixture(name)]),
+);
+
+for (const [name, sha256] of Object.entries(FIXTURE_SHA256)) {
+  test(`${name} bytes match the recorded sha256`, () => {
+    assert.equal(createHash('sha256').update(fixtures[name].bytes).digest('hex'), sha256);
+  });
+}
+
+// Same construction as patchling-py test_parity_binary_cut_fixtures:
+// fill * (length - len(tail)) + tail, so `length` is the total size and the
+// tail sits at the very end of that window.
+function binaryCaseBytes(c) {
+  if (c.hex !== undefined) return new Uint8Array(Buffer.from(c.hex, 'hex'));
+  const tail = Buffer.from(c.tail_hex, 'hex');
+  const fill = Buffer.from(c.fill ?? 'a', 'ascii');
+  assert.equal(fill.length, 1, `${c.id}: fill must be one ASCII byte`);
+  assert.ok(tail.length <= c.length, `${c.id}: tail longer than length`);
+  const pad = Buffer.alloc(c.length - tail.length, fill);
+  const bytes = Buffer.concat([pad, tail]);
+  assert.equal(bytes.length, c.length, `${c.id}: sample must be exactly ${c.length} bytes`);
+  return new Uint8Array(bytes);
+}
+
+test('binary.json: every case matches isBinaryContent', () => {
+  const { cases } = fixtures['binary.json'].data;
+  assert.ok(cases.length > 0);
+  for (const c of cases) {
+    const sample = binaryCaseBytes(c);
+    if (c.length !== undefined) assert.equal(sample.length, 8192, `${c.id}: windowed cases are 8192 bytes`);
+    assert.equal(isBinaryContent(sample), c.expect_binary, c.id);
+  }
+});
+
+test('parser.json: every case matches parseRelevanceAnswer', () => {
+  const { cases } = fixtures['parser.json'].data;
+  assert.ok(cases.length > 0);
+  for (const c of cases) {
+    assert.equal(parseRelevanceAnswer(c.raw).decision, c.decision, c.id);
+  }
+});
+
+test('truncation.json: every case matches truncateUtf8 and never emits U+FFFD', () => {
+  const { cases } = fixtures['truncation.json'].data;
+  assert.ok(cases.length > 0);
+  for (const c of cases) {
+    const out = truncateUtf8(c.text, c.max_bytes);
+    assert.equal(out, c.expected, c.id);
+    assert.ok(!out.includes('\uFFFD'), c.id);
+    assert.ok(Buffer.byteLength(out, 'utf8') <= Math.max(c.max_bytes, 0), c.id);
+  }
+});